diff options
| -rwxr-xr-x | tools/merge.py | 42 |
1 files changed, 37 insertions, 5 deletions
diff --git a/tools/merge.py b/tools/merge.py index 2b83f54..9e60ca2 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -11,7 +11,7 @@ import urllib import urllib2 import sys import time -from certtools import build_merkle_tree, create_sth_signature, check_sth_signature, get_eckey_from_file +from certtools import build_merkle_tree, create_sth_signature, check_sth_signature, get_eckey_from_file, timing_point parser = argparse.ArgumentParser(description="") parser.add_argument("--baseurl", metavar="url", help="Base URL for CT server", required=True) @@ -19,6 +19,7 @@ parser.add_argument("--frontend", action="append", metavar="url", help="Base URL parser.add_argument("--storage", action="append", metavar="url", help="Base URL for storage server", required=True) parser.add_argument("--mergedb", metavar="dir", help="Merge database directory", required=True) parser.add_argument("--keyfile", metavar="keyfile", help="File containing log key", required=True) +parser.add_argument("--nomerge", action='store_true', help="Don't actually do merge") args = parser.parse_args() ctbaseurl = args.baseurl @@ -151,23 +152,43 @@ def get_missingentries(baseurl): print "ERROR: missingentries", e.read() sys.exit(1) +timing = timing_point() logorder = get_logorder() + +timing_point(timing, "get logorder") + certsinlog = set(logorder) -new_entries = [entry for storagenode in storagenodes for entry in get_new_entries(storagenode)] +new_entries_per_node = {} +new_entries = set() + +for storagenode in storagenodes: + new_entries_per_node[storagenode] = get_new_entries(storagenode) + new_entries.update(set(new_entries_per_node[storagenode])) + +timing_point(timing, "get new entries") + +print "adding", len(new_entries), "entries" + +if args.nomerge: + sys.exit(0) -print "adding entries" added_entries = 0 for new_entry in new_entries: hash = base64.b64decode(new_entry) if hash not in certsinlog: - entry = get_entry(storagenode, hash) + entry = None + for storagenode in storagenodes: + if new_entry in new_entries_per_node[storagenode]: + entry = get_entry(storagenode, hash) + assert entry != None write_chain(hash, entry) add_to_logorder(hash) logorder.append(hash) certsinlog.add(hash) added_entries += 1 +timing_point(timing, "add entries") print "added", added_entries, "entries" tree = build_merkle_tree(logorder) @@ -185,18 +206,29 @@ sth = {"tree_size": tree_size, "timestamp": timestamp, check_sth_signature(ctbaseurl, sth) +timing_point(timing, "build sth") + +print timing["deltatimes"] + print "root hash", base64.b16encode(root_hash) for frontendnode in frontendnodes: + timing = timing_point() print "distributing for node", frontendnode curpos = get_curpos(frontendnode) + timing_point(timing, "get curpos") print "current position", curpos entries = [base64.b64encode(entry) for entry in logorder[curpos:]] sendlog(frontendnode, {"start": curpos, "hashes": entries}) + timing_point(timing, "sendlog") print "log sent" missingentries = get_missingentries(frontendnode) - print "missing entries:", missingentries + timing_point(timing, "get missing") + print "missing entries:", len(missingentries) for missingentry in missingentries: hash = base64.b64decode(missingentry) sendentry(frontendnode, read_chain(hash), hash) + timing_point(timing, "send missing") sendsth(frontendnode, sth) + timing_point(timing, "send sth") + print timing["deltatimes"] |