diff options
Diffstat (limited to 'tools')
| -rwxr-xr-x | tools/fetchallcerts.py | 193 |
1 files changed, 151 insertions, 42 deletions
diff --git a/tools/fetchallcerts.py b/tools/fetchallcerts.py index 14ec1a7..d2afb6a 100755 --- a/tools/fetchallcerts.py +++ b/tools/fetchallcerts.py @@ -15,13 +15,13 @@ import hashlib import itertools from certtools import * import zipfile +import os +import time parser = argparse.ArgumentParser(description='') parser.add_argument('baseurl', help="Base URL for CT server") parser.add_argument('--store', default=None, metavar="dir", help='Store certificates in directory dir') -parser.add_argument('--start', default=0, metavar="n", type=int, help='Start at index n') -parser.add_argument('--single', default=None, metavar="n", type=int, help='Onlyfetch index n') -parser.add_argument('--verify', action='store_true', help='Verify STH') +parser.add_argument('--write-sth', action='store_true', help='Write STH') args = parser.parse_args() def extract_original_entry(entry): @@ -49,67 +49,176 @@ def print_layer(layer): for entry in layer: print base64.b16encode(entry) +def get_hash_from_certfile(cert): + for line in cert.split("\n"): + if line.startswith("-----"): + return None + if line.startswith("Leafhash: "): + return base64.b16decode(line[len("Leafhash: "):]) + return None + +def get_certs_from_zipfiles(zipfiles, firstleaf, lastleaf): + for i in range(firstleaf, lastleaf + 1): + try: + yield zipfiles[i / 10000].read("%08d" % i) + except KeyError: + return + +def get_merkle_hash_64k(store, blocknumber): + hashfilename = "%s/%04x.64khash" % (store, blocknumber) + try: + hash = base64.b16decode(open(hashfilename).read()) + assert len(hash) == 32 + return ("hash", hash) + except IOError: + pass + firstleaf = blocknumber * 65536 + lastleaf = firstleaf + 65535 + firstfile = firstleaf / 10000 + lastfile = lastleaf / 10000 + zipfiles = {} + for i in range(firstfile, lastfile + 1): + try: + zipfiles[i] = zipfile.ZipFile("%s/%04d.zip" % (store, i)) + except IOError: + break + certs = get_certs_from_zipfiles(zipfiles, firstleaf, lastleaf) + layer0 = [get_hash_from_certfile(cert) for cert in certs] + tree = build_merkle_tree(layer0) + calculated_hash = tree[-1][0] + for zf in zipfiles.values(): + zf.close() + if len(layer0) != 65536: + return ("incomplete", (len(layer0), calculated_hash)) + f = open(hashfilename, "w") + f.write(base64.b16encode(calculated_hash)) + f.close() + return ("hash", calculated_hash) + sth = get_sth(args.baseurl) +check_sth_signature(args.baseurl, sth) tree_size = sth["tree_size"] root_hash = base64.decodestring(sth["sha256_root_hash"]) +try: + if args.store: + oldsth = json.load(open(args.store + "/currentsth")) + else: + oldsth = None +except IOError: + oldsth = None + +sth_timestamp = datetime.datetime.fromtimestamp(sth["timestamp"]/1000) +since_timestamp = time.time() - sth["timestamp"]/1000 + +print "Log last updated %s, %d seconds ago" % (sth_timestamp.ctime(), since_timestamp) + print "tree size", tree_size print "root hash", base64.b16encode(root_hash) -if args.single == None: - entries = get_entries_wrapper(args.baseurl, args.start, tree_size - 1) -elif args.single > tree_size - 1: - print "index", args.single, "too large, tree size:", tree_size +if oldsth: + if oldsth["tree_size"] == tree_size: + print "Tree size has not changed" + if oldsth["sha256_root_hash"] != sth["sha256_root_hash"]: + print "Root hash is different even though tree size is the same." + print "Log has violated the append-only property." + print "Old hash:", oldsth["sha256_root_hash"] + print "New hash:", sth["sha256_root_hash"] + sys.exit(1) + if oldsth["timestamp"] == sth["timestamp"]: + print "Timestamp has not changed" + else: + print "Tree size changed, old tree size was", oldsth["tree_size"] + +merkle_64klayer = [] + +if args.store: + ncerts = None + for blocknumber in range(0, (tree_size / 65536) + 1): + (resulttype, result) = get_merkle_hash_64k(args.store, blocknumber) + if resulttype == "incomplete": + (incompletelength, hash) = result + ncerts = blocknumber * 65536 + incompletelength + break + assert resulttype == "hash" + hash = result + merkle_64klayer.append(hash) + print blocknumber * 65536, + sys.stdout.flush() + print + print "ncerts", ncerts else: - entries = get_entries_wrapper(args.baseurl, args.single, args.single) - args.start = args.single + ncerts = 0 -if args.verify: +entries = get_entries_wrapper(args.baseurl, ncerts, tree_size - 1) + +if not args.store: layer0 = [get_leaf_hash(base64.decodestring(entry["leaf_input"])) for entry in entries] tree = build_merkle_tree(layer0) calculated_root_hash = tree[-1][0] - print "calculated root hash", base64.b16encode(calculated_root_hash) - - if calculated_root_hash != root_hash: - print "fetched root hash and calculated root hash different, aborting" - sys.exit(1) - else: currentfilename = None zf = None - for entry, i in itertools.izip(entries, itertools.count(args.start)): + for entry, i in itertools.izip(entries, itertools.count(ncerts)): try: (chain, timestamp, issuer_key_hash) = extract_original_entry(entry) - if args.store: - zipfilename = args.store + "/" + ("%04d.zip" % (i / 10000)) - if zipfilename != currentfilename: - if zf: - zf.close() - zf = zipfile.ZipFile(zipfilename, "w", - compression=zipfile.ZIP_DEFLATED) - currentfilename = zipfilename - s = "" - s += "Timestamp: %s\n" % timestamp - leaf_input = base64.decodestring(entry["leaf_input"]) - leaf_hash = get_leaf_hash(leaf_input) - s += "Leafhash: %s\n" % base64.b16encode(leaf_hash) - if issuer_key_hash: - s += "-----BEGIN PRECERTIFICATE-----\n" - s += base64.encodestring(chain[0]).rstrip() + "\n" - s += "-----END PRECERTIFICATE-----\n" - s += "\n" - chain = chain[1:] - for cert in chain: - s += "-----BEGIN CERTIFICATE-----\n" - s += base64.encodestring(cert).rstrip() + "\n" - s += "-----END CERTIFICATE-----\n" - s += "\n" - zf.writestr("%08d" % i, s) + zipfilename = args.store + "/" + ("%04d.zip" % (i / 10000)) + if zipfilename != currentfilename: + if zf: + zf.close() + zf = zipfile.ZipFile(zipfilename, "a", + compression=zipfile.ZIP_DEFLATED) + currentfilename = zipfilename + s = "" + s += "Timestamp: %s\n" % timestamp + leaf_input = base64.decodestring(entry["leaf_input"]) + leaf_hash = get_leaf_hash(leaf_input) + s += "Leafhash: %s\n" % base64.b16encode(leaf_hash) + if issuer_key_hash: + s += "-----BEGIN PRECERTIFICATE-----\n" + s += base64.encodestring(chain[0]).rstrip() + "\n" + s += "-----END PRECERTIFICATE-----\n" + s += "\n" + chain = chain[1:] + for cert in chain: + s += "-----BEGIN CERTIFICATE-----\n" + s += base64.encodestring(cert).rstrip() + "\n" + s += "-----END CERTIFICATE-----\n" + s += "\n" + zf.writestr("%08d" % i, s) except AssertionError, e: print "error for cert", i, e if zf: zf.close() + for blocknumber in range(ncerts / 65536, (tree_size / 65536) + 1): + (resulttype, result) = get_merkle_hash_64k(args.store, blocknumber) + if resulttype == "incomplete": + (incompletelength, hash) = result + ncerts = blocknumber * 65536 + incompletelength + merkle_64klayer.append(hash) + break + assert resulttype == "hash" + hash = result + merkle_64klayer.append(hash) + print blocknumber * 65536, base64.b16encode(hash) + + tree = build_merkle_tree(merkle_64klayer) + + calculated_root_hash = tree[-1][0] + + assert ncerts == tree_size + +print "calculated root hash", base64.b16encode(calculated_root_hash) + +if calculated_root_hash != root_hash: + print "fetched root hash and calculated root hash different" + sys.exit(1) + +if args.store and args.write_sth: + f = open(args.store + "/currentsth", "w") + f.write(json.dumps(sth)) + f.close() |