diff options
Diffstat (limited to 'tools/merge_backup.py')
| -rwxr-xr-x[-rw-r--r--] | tools/merge_backup.py | 58 |
1 files changed, 51 insertions, 7 deletions
diff --git a/tools/merge_backup.py b/tools/merge_backup.py index 27c71a5..48197fc 100644..100755 --- a/tools/merge_backup.py +++ b/tools/merge_backup.py @@ -7,12 +7,15 @@ import sys import base64 import select -from certtools import timing_point +from time import sleep +from certtools import timing_point, build_merkle_tree, write_file, \ + create_ssl_context from mergetools import chunks, backup_sendlog, get_logorder, \ get_verifiedsize, get_missingentriesforbackup, read_chain, \ - hexencode, setverifiedsize, sendentry_merge, verifyroot + hexencode, setverifiedsize, sendentry_merge, verifyroot, \ + get_nfetched, parse_args -def merge_backup(args, config, localconfig, sth_in): +def merge_backup(args, config, localconfig): paths = localconfig["paths"] own_key = (localconfig["nodename"], "%s/%s-private.pem" % (paths["privatekeys"], @@ -21,12 +24,17 @@ def merge_backup(args, config, localconfig, sth_in): mergedb = paths["mergedb"] chainsdir = mergedb + "/chains" logorderfile = mergedb + "/logorder" + currentsizefile = mergedb + "/fetched" timing = timing_point() - logorder = get_logorder(logorderfile) + nfetched = get_nfetched(currentsizefile, logorderfile) + logorder = get_logorder(logorderfile, nfetched) + tree_size = len(logorder) timing_point(timing, "get logorder") - (tree_size, root_hash, _) = sth_in + tree = build_merkle_tree(logorder) + root_hash = tree[-1][0] + timing_point(timing, "build tree") for secondary in secondaries: if secondary["name"] == config["primarymergenode"]: @@ -40,6 +48,7 @@ def merge_backup(args, config, localconfig, sth_in): timing_point(timing, "get verified size") print >>sys.stderr, "verified size", verifiedsize sys.stderr.flush() + entries = [base64.b64encode(entry) for entry in logorder[verifiedsize:]] print >>sys.stderr, "sending log:", sys.stderr.flush() @@ -57,7 +66,7 @@ def merge_backup(args, config, localconfig, sth_in): continue break if sendlogresult["result"] != "ok": - print >>sys.stderr, "sendlog:", sendlogresult + print >>sys.stderr, "backup_sendlog:", sendlogresult sys.exit(1) verifiedsize += len(chunk) print >>sys.stderr, verifiedsize, @@ -66,11 +75,13 @@ def merge_backup(args, config, localconfig, sth_in): timing_point(timing, "sendlog") print >>sys.stderr, "log sent" sys.stderr.flush() + missingentries = get_missingentriesforbackup(nodename, nodeaddress, own_key, paths) timing_point(timing, "get missing") print >>sys.stderr, "missing entries:", len(missingentries) sys.stderr.flush() + fetched_entries = 0 print >>sys.stderr, "fetching missing entries", sys.stderr.flush() @@ -81,7 +92,7 @@ def merge_backup(args, config, localconfig, sth_in): read_chain(chainsdir, ehash), ehash) if sendentryresult["result"] != "ok": - print >>sys.stderr, "send sth:", sendentryresult + print >>sys.stderr, "sendentry_merge:", sendentryresult sys.exit(1) fetched_entries += 1 if fetched_entries % 1000 == 0: @@ -90,6 +101,7 @@ def merge_backup(args, config, localconfig, sth_in): print >>sys.stderr sys.stderr.flush() timing_point(timing, "send missing") + verifyrootresult = verifyroot(nodename, nodeaddress, own_key, paths, tree_size) if verifyrootresult["result"] != "ok": @@ -102,7 +114,39 @@ def merge_backup(args, config, localconfig, sth_in): print >>sys.stderr, " expected", hexencode(root_hash) sys.exit(1) timing_point(timing, "verifyroot") + setverifiedsize(nodename, nodeaddress, own_key, paths, tree_size) + backuppath = mergedb + "/verified." + nodename + backupdata = {"tree_size": tree_size, + "sha256_root_hash": hexencode(root_hash)} + print >>sys.stderr, "DEBUG: writing to", backuppath, ":", backupdata + write_file(backuppath, backupdata) + if args.timing: print >>sys.stderr, timing["deltatimes"] sys.stderr.flush() + +def main(): + """ + Read logorder file up until what's indicated by fetched file and + build the tree. + + Distribute entries to all secondaries, write tree size and tree head + to backup.<secondary> files as each secondary is verified to have + the entries. + + Sleep some and start over. + """ + args, config, localconfig = parse_args() + paths = localconfig["paths"] + create_ssl_context(cafile=paths["https_cacertfile"]) + + while True: + merge_backup(args, config, localconfig) + if args.interval is None: + break + print >>sys.stderr, "sleeping", args.interval, "seconds" + sleep(args.interval) + +if __name__ == '__main__': + sys.exit(main()) |