diff options
Diffstat (limited to 'tools/merge.py')
| -rwxr-xr-x | tools/merge.py | 49 |
1 files changed, 45 insertions, 4 deletions
diff --git a/tools/merge.py b/tools/merge.py index f9c93d9..ce3bf0b 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -16,9 +16,11 @@ import hashlib import urlparse import os import yaml +import select +import struct from certtools import build_merkle_tree, create_sth_signature, \ check_sth_signature, get_eckey_from_file, timing_point, http_request, \ - get_public_key_from_file + get_public_key_from_file, get_leaf_hash, decode_certificate_chain parser = argparse.ArgumentParser(description="") parser.add_argument('--config', help="System configuration", required=True) @@ -61,7 +63,7 @@ def write_chain(key, value): try: os.makedirs(path) except Exception, e: - print e + pass else: path = chainsdir f = open(path + "/" + filename, "w") @@ -131,7 +133,7 @@ def sendlog(node, baseurl, submission): return json.loads(result) except urllib2.HTTPError, e: print "ERROR: sendlog", e.read() - sys.exit(1) + return None except ValueError, e: print "==== FAILED REQUEST ====" print submission @@ -206,6 +208,32 @@ for storagenode in storagenodes: new_entries.update(new_entries_per_node[storagenode["name"]]) entries_to_fetch[storagenode["name"]] = [] +def unpack_entry(entry): + pieces = [] + while len(entry): + (length,) = struct.unpack(">I", entry[0:4]) + data = entry[4:4+length] + entry = entry[4+length:] + pieces.append(data) + return pieces + +import subprocess + +def verify_entry(verifycert, entry, hash): + unpacked = unpack_entry(entry) + mtl = unpacked[0] + assert hash == get_leaf_hash(mtl) + s = struct.pack(">I", len(entry)) + entry + verifycert.stdin.write(s) + result_length_packed = verifycert.stdout.read(4) + (result_length,) = struct.unpack(">I", result_length_packed) + result = verifycert.stdout.read(result_length) + assert len(result) == result_length + (error_code,) = struct.unpack("B", result[0:1]) + if error_code != 0: + print >>sys.stderr, result[1:] + sys.exit(1) + timing_point(timing, "get new entries") new_entries -= certsinlog @@ -221,6 +249,8 @@ for hash in new_entries: entries_to_fetch[storagenode["name"]].append(hash) break +verifycert = subprocess.Popen(["../verifycert.erl"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE) added_entries = 0 for storagenode in storagenodes: @@ -229,6 +259,7 @@ for storagenode in storagenodes: entries = get_entries(storagenode["name"], "https://%s/" % storagenode["address"], chunk) for hash in chunk: entry = entries[hash] + verify_entry(verifycert, entry, hash) write_chain(hash, entry) add_to_logorder(hash) logorder.append(hash) @@ -237,6 +268,8 @@ for storagenode in storagenodes: timing_point(timing, "add entries") print "added", added_entries, "entries" +verifycert.communicate(struct.pack("I", 0)) + tree = build_merkle_tree(logorder) tree_size = len(logorder) root_hash = tree[-1][0] @@ -277,7 +310,15 @@ for frontendnode in frontendnodes: print "current position", curpos entries = [base64.b64encode(entry) for entry in logorder[curpos:]] for chunk in chunks(entries, 1000): - sendlogresult = sendlog(nodename, nodeaddress, {"start": curpos, "hashes": chunk}) + for trynumber in range(5, 0, -1): + sendlogresult = sendlog(nodename, nodeaddress, {"start": curpos, "hashes": chunk}) + if sendlogresult == None: + if trynumber == 1: + sys.exit(1) + select.select([], [], [], 10.0) + print "tries left:", trynumber + continue + break if sendlogresult["result"] != "ok": print "sendlog:", sendlogresult sys.exit(1) |