diff options
Diffstat (limited to 'tools/merge_fetch.py')
| -rwxr-xr-x[-rw-r--r--] | tools/merge_fetch.py | 48 |
1 files changed, 36 insertions, 12 deletions
diff --git a/tools/merge_fetch.py b/tools/merge_fetch.py index a0a0396..1540c34 100644..100755 --- a/tools/merge_fetch.py +++ b/tools/merge_fetch.py @@ -6,13 +6,14 @@ import sys import struct -import time import subprocess +from time import sleep from mergetools import get_logorder, verify_entry, get_new_entries, \ - chunks, fsync_logorder, get_entries, write_chain, add_to_logorder -from certtools import timing_point, build_merkle_tree + chunks, fsync_logorder, get_entries, write_chain, add_to_logorder, \ + hexencode, parse_args +from certtools import timing_point, write_file, create_ssl_context -def merge_fetch(args, config, localconfig): +def merge_fetch(_args, config, localconfig): paths = localconfig["paths"] storagenodes = config["storagenodes"] mergedb = paths["mergedb"] @@ -21,7 +22,6 @@ def merge_fetch(args, config, localconfig): own_key = (localconfig["nodename"], "%s/%s-private.pem" % (paths["privatekeys"], localconfig["nodename"])) - timing = timing_point() logorder = get_logorder(logorderfile) @@ -48,9 +48,6 @@ def merge_fetch(args, config, localconfig): print >>sys.stderr, "adding", len(new_entries), "entries" sys.stderr.flush() - if args.nomerge: - sys.exit(0) - for ehash in new_entries: for storagenode in storagenodes: if ehash in new_entries_per_node[storagenode["name"]]: @@ -89,9 +86,36 @@ def merge_fetch(args, config, localconfig): verifycert.communicate(struct.pack("I", 0)) - tree = build_merkle_tree(logorder) tree_size = len(logorder) - root_hash = tree[-1][0] - timestamp = int(time.time() * 1000) + if tree_size == 0: + return (0, '') + else: + return (tree_size, logorder[tree_size-1]) + +def main(): + """ + Fetch new entries from all storage nodes. + + Indicate current position by writing the index in the logorder file + (0-based) to the 'fetched' file. + + Sleep some and start over. + """ + args, config, localconfig = parse_args() + paths = localconfig["paths"] + mergedb = paths["mergedb"] + currentsizefile = mergedb + "/fetched" + create_ssl_context(cafile=paths["https_cacertfile"]) + + while True: + logsize, last_hash = merge_fetch(args, config, localconfig) + currentsize = {"index": logsize - 1, "hash": hexencode(last_hash)} + print >>sys.stderr, "DEBUG: writing to", currentsizefile, ":", currentsize + write_file(currentsizefile, currentsize) + if args.interval is None: + break + print >>sys.stderr, "sleeping", args.interval, "seconds" + sleep(args.interval) - return (tree_size, root_hash, timestamp) +if __name__ == '__main__': + sys.exit(main()) |