Merge is now run by shell script tools/merge.

tools/merge run merge_fetch.py, merge_backup.py, merge_sth.py and
merge_dist.py sequentially.

TODO: test backupquorum != 0

1 files changed, 36 insertions, 12 deletions

diff --git a/tools/merge_fetch.py b/tools/merge_fetch.py
index a0a0396..1540c34 100644..100755
--- a/tools/merge_fetch.py
+++ b/tools/merge_fetch.py
@@ -6,13 +6,14 @@
 
 import sys
 import struct
-import time
 import subprocess
+from time import sleep
 from mergetools import get_logorder, verify_entry, get_new_entries, \
-     chunks, fsync_logorder, get_entries, write_chain, add_to_logorder
-from certtools import timing_point, build_merkle_tree
+     chunks, fsync_logorder, get_entries, write_chain, add_to_logorder, \
+     hexencode, parse_args
+from certtools import timing_point, write_file, create_ssl_context
 
-def merge_fetch(args, config, localconfig):
+def merge_fetch(_args, config, localconfig):
     paths = localconfig["paths"]
     storagenodes = config["storagenodes"]
     mergedb = paths["mergedb"]
@@ -21,7 +22,6 @@ def merge_fetch(args, config, localconfig):
     own_key = (localconfig["nodename"],
                "%s/%s-private.pem" % (paths["privatekeys"],
                                       localconfig["nodename"]))
-
     timing = timing_point()
 
     logorder = get_logorder(logorderfile)
@@ -48,9 +48,6 @@ def merge_fetch(args, config, localconfig):
     print >>sys.stderr, "adding", len(new_entries), "entries"
     sys.stderr.flush()
 
-    if args.nomerge:
-        sys.exit(0)
-
     for ehash in new_entries:
         for storagenode in storagenodes:
             if ehash in new_entries_per_node[storagenode["name"]]:
@@ -89,9 +86,36 @@ def merge_fetch(args, config, localconfig):
 
     verifycert.communicate(struct.pack("I", 0))
 
-    tree = build_merkle_tree(logorder)
     tree_size = len(logorder)
-    root_hash = tree[-1][0]
-    timestamp = int(time.time() * 1000)
+    if tree_size == 0:
+        return (0, '')
+    else:
+        return (tree_size, logorder[tree_size-1])
+
+def main():
+    """
+    Fetch new entries from all storage nodes.
+
+    Indicate current position by writing the index in the logorder file
+    (0-based) to the 'fetched' file.
+
+    Sleep some and start over.
+    """
+    args, config, localconfig = parse_args()
+    paths = localconfig["paths"]
+    mergedb = paths["mergedb"]
+    currentsizefile = mergedb + "/fetched"
+    create_ssl_context(cafile=paths["https_cacertfile"])
+
+    while True:
+        logsize, last_hash = merge_fetch(args, config, localconfig)
+        currentsize = {"index": logsize - 1, "hash": hexencode(last_hash)}
+        print >>sys.stderr, "DEBUG: writing to", currentsizefile, ":", currentsize
+        write_file(currentsizefile, currentsize)
+        if args.interval is None:
+            break
+        print >>sys.stderr, "sleeping", args.interval, "seconds"
+        sleep(args.interval)
 
-    return (tree_size, root_hash, timestamp)
+if __name__ == '__main__':
+    sys.exit(main())