fetchallcerts.py: Always calculate full tree

Cache level 16 hashes
Save STH

1 files changed, 151 insertions, 42 deletions

diff --git a/tools/fetchallcerts.py b/tools/fetchallcerts.py
index 14ec1a7..d2afb6a 100755
--- a/tools/fetchallcerts.py
+++ b/tools/fetchallcerts.py
@@ -15,13 +15,13 @@ import hashlib
 import itertools
 from certtools import *
 import zipfile
+import os
+import time
 
 parser = argparse.ArgumentParser(description='')
 parser.add_argument('baseurl', help="Base URL for CT server")
 parser.add_argument('--store', default=None, metavar="dir", help='Store certificates in directory dir')
-parser.add_argument('--start', default=0, metavar="n", type=int, help='Start at index n')
-parser.add_argument('--single', default=None, metavar="n", type=int, help='Onlyfetch index n')
-parser.add_argument('--verify', action='store_true', help='Verify STH')
+parser.add_argument('--write-sth', action='store_true', help='Write STH')
 args = parser.parse_args()
 
 def extract_original_entry(entry):
@@ -49,67 +49,176 @@ def print_layer(layer):
     for entry in layer:
         print base64.b16encode(entry)
 
+def get_hash_from_certfile(cert):
+    for line in cert.split("\n"):
+        if line.startswith("-----"):
+            return None
+        if line.startswith("Leafhash: "):
+            return base64.b16decode(line[len("Leafhash: "):])
+    return None
+
+def get_certs_from_zipfiles(zipfiles, firstleaf, lastleaf):
+    for i in range(firstleaf, lastleaf + 1):
+        try:
+            yield zipfiles[i / 10000].read("%08d" % i)
+        except KeyError:
+            return
+
+def get_merkle_hash_64k(store, blocknumber):
+    hashfilename = "%s/%04x.64khash" % (store, blocknumber)
+    try:
+        hash = base64.b16decode(open(hashfilename).read())
+        assert len(hash) == 32
+        return ("hash", hash)
+    except IOError:
+        pass
+    firstleaf = blocknumber * 65536
+    lastleaf = firstleaf + 65535
+    firstfile = firstleaf / 10000
+    lastfile = lastleaf / 10000
+    zipfiles = {}
+    for i in range(firstfile, lastfile + 1):
+        try:
+            zipfiles[i] = zipfile.ZipFile("%s/%04d.zip" % (store, i))
+        except IOError:
+            break
+    certs = get_certs_from_zipfiles(zipfiles, firstleaf, lastleaf)
+    layer0 = [get_hash_from_certfile(cert) for cert in certs]
+    tree = build_merkle_tree(layer0)
+    calculated_hash = tree[-1][0]
+    for zf in zipfiles.values():
+        zf.close()
+    if len(layer0) != 65536:
+        return ("incomplete", (len(layer0), calculated_hash))
+    f = open(hashfilename, "w")
+    f.write(base64.b16encode(calculated_hash))
+    f.close()
+    return ("hash", calculated_hash)
+
 sth = get_sth(args.baseurl)
+check_sth_signature(args.baseurl, sth)
 tree_size = sth["tree_size"]
 root_hash = base64.decodestring(sth["sha256_root_hash"])
 
+try:
+    if args.store:
+        oldsth = json.load(open(args.store + "/currentsth"))
+    else:
+        oldsth = None
+except IOError:
+    oldsth = None
+
+sth_timestamp = datetime.datetime.fromtimestamp(sth["timestamp"]/1000)
+since_timestamp = time.time() - sth["timestamp"]/1000
+
+print "Log last updated %s, %d seconds ago" % (sth_timestamp.ctime(), since_timestamp)
+
 print "tree size", tree_size
 print "root hash", base64.b16encode(root_hash)
 
-if args.single == None:
-    entries = get_entries_wrapper(args.baseurl, args.start, tree_size - 1)
-elif args.single > tree_size - 1:
-    print "index", args.single, "too large, tree size:", tree_size
+if oldsth:
+    if oldsth["tree_size"] == tree_size:
+        print "Tree size has not changed"
+        if oldsth["sha256_root_hash"] != sth["sha256_root_hash"]:
+            print "Root hash is different even though tree size is the same."
+            print "Log has violated the append-only property."
+            print "Old hash:", oldsth["sha256_root_hash"]
+            print "New hash:", sth["sha256_root_hash"]
+            sys.exit(1)
+        if oldsth["timestamp"] == sth["timestamp"]:
+            print "Timestamp has not changed"
+    else:
+        print "Tree size changed, old tree size was", oldsth["tree_size"]
+
+merkle_64klayer = []
+
+if args.store:
+    ncerts = None
+    for blocknumber in range(0, (tree_size / 65536) + 1):
+        (resulttype, result) = get_merkle_hash_64k(args.store, blocknumber)
+        if resulttype == "incomplete":
+            (incompletelength, hash) = result
+            ncerts = blocknumber * 65536 + incompletelength
+            break
+        assert resulttype == "hash"
+        hash = result
+        merkle_64klayer.append(hash)
+        print blocknumber * 65536,
+        sys.stdout.flush()
+    print
+    print "ncerts", ncerts
 else:
-    entries = get_entries_wrapper(args.baseurl, args.single, args.single)
-    args.start = args.single
+    ncerts = 0
 
-if args.verify:
+entries = get_entries_wrapper(args.baseurl, ncerts, tree_size - 1)
+
+if not args.store:
     layer0 = [get_leaf_hash(base64.decodestring(entry["leaf_input"])) for entry in entries]
 
     tree = build_merkle_tree(layer0)
 
     calculated_root_hash = tree[-1][0]
 
-    print "calculated root hash", base64.b16encode(calculated_root_hash)
-
-    if calculated_root_hash != root_hash:
-        print "fetched root hash and calculated root hash different, aborting"
-        sys.exit(1)
-
 else:
     currentfilename = None
     zf = None
-    for entry, i in itertools.izip(entries, itertools.count(args.start)):
+    for entry, i in itertools.izip(entries, itertools.count(ncerts)):
         try:
             (chain, timestamp, issuer_key_hash) = extract_original_entry(entry)
-            if args.store:
-                zipfilename = args.store + "/" + ("%04d.zip" % (i / 10000))
-                if zipfilename != currentfilename:
-                    if zf:
-                        zf.close()
-                    zf = zipfile.ZipFile(zipfilename, "w",
-                                         compression=zipfile.ZIP_DEFLATED)
-                    currentfilename = zipfilename
-                s = ""
-                s += "Timestamp: %s\n" % timestamp
-                leaf_input = base64.decodestring(entry["leaf_input"])
-                leaf_hash = get_leaf_hash(leaf_input)
-                s += "Leafhash: %s\n" % base64.b16encode(leaf_hash)
-                if issuer_key_hash:
-                    s += "-----BEGIN PRECERTIFICATE-----\n"
-                    s += base64.encodestring(chain[0]).rstrip() + "\n"
-                    s += "-----END PRECERTIFICATE-----\n"
-                    s += "\n"
-                    chain = chain[1:]
-                for cert in chain:
-                    s += "-----BEGIN CERTIFICATE-----\n"
-                    s += base64.encodestring(cert).rstrip() + "\n"
-                    s += "-----END CERTIFICATE-----\n"
-                    s += "\n"
-                zf.writestr("%08d" % i, s)
+            zipfilename = args.store + "/" + ("%04d.zip" % (i / 10000))
+            if zipfilename != currentfilename:
+                if zf:
+                    zf.close()
+                zf = zipfile.ZipFile(zipfilename, "a",
+                                     compression=zipfile.ZIP_DEFLATED)
+                currentfilename = zipfilename
+            s = ""
+            s += "Timestamp: %s\n" % timestamp
+            leaf_input = base64.decodestring(entry["leaf_input"])
+            leaf_hash = get_leaf_hash(leaf_input)
+            s += "Leafhash: %s\n" % base64.b16encode(leaf_hash)
+            if issuer_key_hash:
+                s += "-----BEGIN PRECERTIFICATE-----\n"
+                s += base64.encodestring(chain[0]).rstrip() + "\n"
+                s += "-----END PRECERTIFICATE-----\n"
+                s += "\n"
+                chain = chain[1:]
+            for cert in chain:
+                s += "-----BEGIN CERTIFICATE-----\n"
+                s += base64.encodestring(cert).rstrip() + "\n"
+                s += "-----END CERTIFICATE-----\n"
+                s += "\n"
+            zf.writestr("%08d" % i, s)
         except AssertionError, e:
             print "error for cert", i, e
     if zf:
         zf.close()
 
+    for blocknumber in range(ncerts / 65536, (tree_size / 65536) + 1):
+        (resulttype, result) = get_merkle_hash_64k(args.store, blocknumber)
+        if resulttype == "incomplete":
+            (incompletelength, hash) = result
+            ncerts = blocknumber * 65536 + incompletelength
+            merkle_64klayer.append(hash)
+            break
+        assert resulttype == "hash"
+        hash = result
+        merkle_64klayer.append(hash)
+        print blocknumber * 65536, base64.b16encode(hash)
+
+    tree = build_merkle_tree(merkle_64klayer)
+
+    calculated_root_hash = tree[-1][0]
+
+    assert ncerts == tree_size
+
+print "calculated root hash", base64.b16encode(calculated_root_hash)
+
+if calculated_root_hash != root_hash:
+    print "fetched root hash and calculated root hash different"
+    sys.exit(1)
+
+if args.store and args.write_sth:
+    f = open(args.store + "/currentsth", "w")
+    f.write(json.dumps(sth))
+    f.close()