Split merge.py into three pieces.

1 files changed, 130 insertions, 0 deletions

diff --git a/tools/merge_dist.py b/tools/merge_dist.py
new file mode 100644
index 0000000..2b2f259
--- /dev/null
+++ b/tools/merge_dist.py
@@ -0,0 +1,130 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2014-2015, NORDUnet A/S.
+# See LICENSE for licensing information.
+
+import sys
+import urllib2
+import base64
+import select
+from certtools import timing_point, check_sth_signature, \
+     get_public_key_from_file
+from mergetools import get_curpos, get_logorder, chunks, get_missingentries, \
+     sendsth, create_sth_signature, hexencode, sendlog, sendentry, read_chain
+
+def merge_dist(args, config, localconfig, sth_in):
+    paths = localconfig["paths"]
+    own_key = (localconfig["nodename"],
+               "%s/%s-private.pem" % (paths["privatekeys"],
+                                      localconfig["nodename"]))
+    frontendnodes = config["frontendnodes"]
+    signingnodes = config["signingnodes"]
+    ctbaseurl = config["baseurl"]
+    logpublickey = get_public_key_from_file(paths["logpublickey"])
+    mergedb = paths["mergedb"]
+    chainsdir = mergedb + "/chains"
+    logorderfile = mergedb + "/logorder"
+    timing = timing_point()
+
+    logorder = get_logorder(logorderfile)
+    timing_point(timing, "get logorder")
+
+    (tree_size, root_hash, timestamp) = sth_in
+    tree_head_signature = None
+    for signingnode in signingnodes:
+        try:
+            tree_head_signature = \
+              create_sth_signature(tree_size, timestamp,
+                                   root_hash,
+                                   "https://%s/" % signingnode["address"],
+                                   key=own_key)
+            break
+        except urllib2.URLError, err:
+            print >>sys.stderr, err
+            sys.stderr.flush()
+    if tree_head_signature == None:
+        print >>sys.stderr, "Could not contact any signing nodes"
+        sys.exit(1)
+
+    sth = {"tree_size": tree_size, "timestamp": timestamp,
+           "sha256_root_hash": base64.b64encode(root_hash),
+           "tree_head_signature": base64.b64encode(tree_head_signature)}
+
+    check_sth_signature(ctbaseurl, sth, publickey=logpublickey)
+
+    timing_point(timing, "build sth")
+
+    if args.timing:
+        print >>sys.stderr, timing["deltatimes"]
+        sys.stderr.flush()
+
+    print hexencode(root_hash)
+    sys.stdout.flush()
+
+    for frontendnode in frontendnodes:
+        nodeaddress = "https://%s/" % frontendnode["address"]
+        nodename = frontendnode["name"]
+        timing = timing_point()
+        print >>sys.stderr, "distributing for node", nodename
+        sys.stderr.flush()
+        curpos = get_curpos(nodename, nodeaddress, own_key, paths)
+        timing_point(timing, "get curpos")
+        print >>sys.stderr, "current position", curpos
+        sys.stderr.flush()
+        entries = [base64.b64encode(entry) for entry in logorder[curpos:]]
+        print >>sys.stderr, "sending log:",
+        sys.stderr.flush()
+        for chunk in chunks(entries, 1000):
+            for trynumber in range(5, 0, -1):
+                sendlogresult = sendlog(nodename, nodeaddress,
+                                        own_key, paths,
+                                        {"start": curpos, "hashes": chunk})
+                if sendlogresult == None:
+                    if trynumber == 1:
+                        sys.exit(1)
+                    select.select([], [], [], 10.0)
+                    print >>sys.stderr, "tries left:", trynumber
+                    sys.stderr.flush()
+                    continue
+                break
+            if sendlogresult["result"] != "ok":
+                print >>sys.stderr, "sendlog:", sendlogresult
+                sys.exit(1)
+            curpos += len(chunk)
+            print >>sys.stderr, curpos,
+            sys.stderr.flush()
+        print >>sys.stderr
+        timing_point(timing, "sendlog")
+        print >>sys.stderr, "log sent"
+        sys.stderr.flush()
+        missingentries = get_missingentries(nodename, nodeaddress, own_key,
+                                            paths)
+        timing_point(timing, "get missing")
+        print >>sys.stderr, "missing entries:", len(missingentries)
+        sys.stderr.flush()
+        fetched_entries = 0
+        print >>sys.stderr, "fetching missing entries",
+        sys.stderr.flush()
+        for missingentry in missingentries:
+            ehash = base64.b64decode(missingentry)
+            sendentryresult = sendentry(nodename, nodeaddress, own_key, paths,
+                                        read_chain(chainsdir, ehash), ehash)
+            if sendentryresult["result"] != "ok":
+                print >>sys.stderr, "send sth:", sendentryresult
+                sys.exit(1)
+            fetched_entries += 1
+            if fetched_entries % 1000 == 0:
+                print >>sys.stderr, fetched_entries,
+                sys.stderr.flush()
+        print >>sys.stderr
+        sys.stderr.flush()
+        timing_point(timing, "send missing")
+        sendsthresult = sendsth(nodename, nodeaddress, own_key, paths, sth)
+        if sendsthresult["result"] != "ok":
+            print >>sys.stderr, "send sth:", sendsthresult
+            sys.exit(1)
+        timing_point(timing, "send sth")
+        if args.timing:
+            print >>sys.stderr, timing["deltatimes"]
+            sys.stderr.flush()