Merge is now run by shell script tools/merge.

tools/merge run merge_fetch.py, merge_backup.py, merge_sth.py and
merge_dist.py sequentially.

TODO: test backupquorum != 0

1 files changed, 51 insertions, 7 deletions

diff --git a/tools/merge_backup.py b/tools/merge_backup.py
index 27c71a5..48197fc 100644..100755
--- a/tools/merge_backup.py
+++ b/tools/merge_backup.py
@@ -7,12 +7,15 @@
 import sys
 import base64
 import select
-from certtools import timing_point
+from time import sleep
+from certtools import timing_point, build_merkle_tree, write_file, \
+     create_ssl_context
 from mergetools import chunks, backup_sendlog, get_logorder, \
      get_verifiedsize, get_missingentriesforbackup, read_chain, \
-     hexencode, setverifiedsize, sendentry_merge, verifyroot
+     hexencode, setverifiedsize, sendentry_merge, verifyroot, \
+     get_nfetched, parse_args
 
-def merge_backup(args, config, localconfig, sth_in):
+def merge_backup(args, config, localconfig):
     paths = localconfig["paths"]
     own_key = (localconfig["nodename"],
                "%s/%s-private.pem" % (paths["privatekeys"],
@@ -21,12 +24,17 @@ def merge_backup(args, config, localconfig, sth_in):
     mergedb = paths["mergedb"]
     chainsdir = mergedb + "/chains"
     logorderfile = mergedb + "/logorder"
+    currentsizefile = mergedb + "/fetched"
     timing = timing_point()
 
-    logorder = get_logorder(logorderfile)
+    nfetched = get_nfetched(currentsizefile, logorderfile)
+    logorder = get_logorder(logorderfile, nfetched)
+    tree_size = len(logorder)
     timing_point(timing, "get logorder")
 
-    (tree_size, root_hash, _) = sth_in
+    tree = build_merkle_tree(logorder)
+    root_hash = tree[-1][0]
+    timing_point(timing, "build tree")
 
     for secondary in secondaries:
         if secondary["name"] == config["primarymergenode"]:
@@ -40,6 +48,7 @@ def merge_backup(args, config, localconfig, sth_in):
         timing_point(timing, "get verified size")
         print >>sys.stderr, "verified size", verifiedsize
         sys.stderr.flush()
+
         entries = [base64.b64encode(entry) for entry in logorder[verifiedsize:]]
         print >>sys.stderr, "sending log:",
         sys.stderr.flush()
@@ -57,7 +66,7 @@ def merge_backup(args, config, localconfig, sth_in):
                     continue
                 break
             if sendlogresult["result"] != "ok":
-                print >>sys.stderr, "sendlog:", sendlogresult
+                print >>sys.stderr, "backup_sendlog:", sendlogresult
                 sys.exit(1)
             verifiedsize += len(chunk)
             print >>sys.stderr, verifiedsize,
@@ -66,11 +75,13 @@ def merge_backup(args, config, localconfig, sth_in):
         timing_point(timing, "sendlog")
         print >>sys.stderr, "log sent"
         sys.stderr.flush()
+
         missingentries = get_missingentriesforbackup(nodename, nodeaddress,
                                                      own_key, paths)
         timing_point(timing, "get missing")
         print >>sys.stderr, "missing entries:", len(missingentries)
         sys.stderr.flush()
+
         fetched_entries = 0
         print >>sys.stderr, "fetching missing entries",
         sys.stderr.flush()
@@ -81,7 +92,7 @@ def merge_backup(args, config, localconfig, sth_in):
                                               read_chain(chainsdir, ehash),
                                               ehash)
             if sendentryresult["result"] != "ok":
-                print >>sys.stderr, "send sth:", sendentryresult
+                print >>sys.stderr, "sendentry_merge:", sendentryresult
                 sys.exit(1)
             fetched_entries += 1
             if fetched_entries % 1000 == 0:
@@ -90,6 +101,7 @@ def merge_backup(args, config, localconfig, sth_in):
         print >>sys.stderr
         sys.stderr.flush()
         timing_point(timing, "send missing")
+
         verifyrootresult = verifyroot(nodename, nodeaddress, own_key, paths,
                                       tree_size)
         if verifyrootresult["result"] != "ok":
@@ -102,7 +114,39 @@ def merge_backup(args, config, localconfig, sth_in):
             print >>sys.stderr, "  expected", hexencode(root_hash)
             sys.exit(1)
         timing_point(timing, "verifyroot")
+
         setverifiedsize(nodename, nodeaddress, own_key, paths, tree_size)
+        backuppath = mergedb + "/verified." + nodename
+        backupdata = {"tree_size": tree_size,
+                      "sha256_root_hash": hexencode(root_hash)}
+        print >>sys.stderr, "DEBUG: writing to", backuppath, ":", backupdata
+        write_file(backuppath, backupdata)
+
         if args.timing:
             print >>sys.stderr, timing["deltatimes"]
             sys.stderr.flush()
+
+def main():
+    """
+    Read logorder file up until what's indicated by fetched file and
+    build the tree.
+
+    Distribute entries to all secondaries, write tree size and tree head
+    to backup.<secondary> files as each secondary is verified to have
+    the entries.
+
+    Sleep some and start over.
+    """
+    args, config, localconfig = parse_args()
+    paths = localconfig["paths"]
+    create_ssl_context(cafile=paths["https_cacertfile"])
+
+    while True:
+        merge_backup(args, config, localconfig)
+        if args.interval is None:
+            break
+        print >>sys.stderr, "sleeping", args.interval, "seconds"
+        sleep(args.interval)
+
+if __name__ == '__main__':
+    sys.exit(main())