fetchallcerts.py: handle precerts

submitcert.py: handle .zip files
fetchallcerts.py: Always calculate full tree
fetchallcerts.py: Cache level 16 hashes
fetchallcerts.py: Save STH

1 files changed, 120 insertions, 22 deletions

diff --git a/tools/fetchallcerts.py b/tools/fetchallcerts.py
index 2276e68..866bb43 100644..100755
--- a/tools/fetchallcerts.py
+++ b/tools/fetchallcerts.py
@@ -14,20 +14,25 @@ import struct
 import hashlib
 import itertools
 from certtools import *
+import zipfile
+import os
+import time
 
 parser = argparse.ArgumentParser(description='')
 parser.add_argument('baseurl', help="Base URL for CT server")
 parser.add_argument('--store', default=None, metavar="dir", help='Store certificates in directory dir')
-parser.add_argument('--start', default=0, metavar="n", type=int, help='Start at index n')
-parser.add_argument('--verify', action='store_true', help='Verify STH')
+parser.add_argument('--write-sth', action='store_true', help='Write STH')
 args = parser.parse_args()
 
 def extract_original_entry(entry):
     leaf_input =  base64.decodestring(entry["leaf_input"])
-    (leaf_cert, timestamp) = unpack_mtl(leaf_input)
+    (leaf_cert, timestamp, issuer_key_hash) = unpack_mtl(leaf_input)
     extra_data = base64.decodestring(entry["extra_data"])
+    if issuer_key_hash != None:
+        (precert, extra_data) = extract_precertificate(extra_data)
+        leaf_cert = precert
     certchain = decode_certificate_chain(extra_data)
-    return [leaf_cert] + certchain
+    return ([leaf_cert] + certchain, timestamp, issuer_key_hash)
 
 def get_entries_wrapper(baseurl, start, end):
     fetched_entries = 0
@@ -45,36 +50,129 @@ def print_layer(layer):
         print base64.b16encode(entry)
 
 sth = get_sth(args.baseurl)
+check_sth_signature(args.baseurl, sth)
 tree_size = sth["tree_size"]
 root_hash = base64.decodestring(sth["sha256_root_hash"])
 
+try:
+    if args.store:
+        oldsth = json.load(open(args.store + "/currentsth"))
+    else:
+        oldsth = None
+except IOError:
+    oldsth = None
+
+sth_timestamp = datetime.datetime.fromtimestamp(sth["timestamp"]/1000)
+since_timestamp = time.time() - sth["timestamp"]/1000
+
+print "Log last updated %s, %d seconds ago" % (sth_timestamp.ctime(), since_timestamp)
+
 print "tree size", tree_size
 print "root hash", base64.b16encode(root_hash)
 
-entries = get_entries_wrapper(args.baseurl, args.start, tree_size - 1)
+if oldsth:
+    if oldsth["tree_size"] == tree_size:
+        print "Tree size has not changed"
+        if oldsth["sha256_root_hash"] != sth["sha256_root_hash"]:
+            print "Root hash is different even though tree size is the same."
+            print "Log has violated the append-only property."
+            print "Old hash:", oldsth["sha256_root_hash"]
+            print "New hash:", sth["sha256_root_hash"]
+            sys.exit(1)
+        if oldsth["timestamp"] == sth["timestamp"]:
+            print "Timestamp has not changed"
+    else:
+        print "Tree size changed, old tree size was", oldsth["tree_size"]
 
-if args.verify:
+merkle_64klayer = []
+
+if args.store:
+    ncerts = None
+    for blocknumber in range(0, (tree_size / 65536) + 1):
+        (resulttype, result) = get_merkle_hash_64k(args.store, blocknumber, write_to_cache=True)
+        if resulttype == "incomplete":
+            (incompletelength, hash) = result
+            ncerts = blocknumber * 65536 + incompletelength
+            break
+        assert resulttype == "hash"
+        hash = result
+        merkle_64klayer.append(hash)
+        print blocknumber * 65536,
+        sys.stdout.flush()
+    print
+    print "ncerts", ncerts
+else:
+    ncerts = 0
+
+entries = get_entries_wrapper(args.baseurl, ncerts, tree_size - 1)
+
+if not args.store:
     layer0 = [get_leaf_hash(base64.decodestring(entry["leaf_input"])) for entry in entries]
 
     tree = build_merkle_tree(layer0)
 
     calculated_root_hash = tree[-1][0]
 
-    print "calculated root hash", base64.b16encode(calculated_root_hash)
-
-    if calculated_root_hash != root_hash:
-        print "fetched root hash and calculated root hash different, aborting"
-        sys.exit(1)
-
-elif args.store:
-    for entry, i in itertools.izip(entries, itertools.count(args.start)):
+else:
+    currentfilename = None
+    zf = None
+    for entry, i in itertools.izip(entries, itertools.count(ncerts)):
         try:
-            chain = extract_original_entry(entry)
-            f = open(args.store + "/" + ("%08d" % i), "w")
+            (chain, timestamp, issuer_key_hash) = extract_original_entry(entry)
+            zipfilename = args.store + "/" + ("%04d.zip" % (i / 10000))
+            if zipfilename != currentfilename:
+                if zf:
+                    zf.close()
+                zf = zipfile.ZipFile(zipfilename, "a",
+                                     compression=zipfile.ZIP_DEFLATED)
+                currentfilename = zipfilename
+            s = ""
+            s += "Timestamp: %s\n" % timestamp
+            leaf_input = base64.decodestring(entry["leaf_input"])
+            leaf_hash = get_leaf_hash(leaf_input)
+            s += "Leafhash: %s\n" % base64.b16encode(leaf_hash)
+            if issuer_key_hash:
+                s += "-----BEGIN PRECERTIFICATE-----\n"
+                s += base64.encodestring(chain[0]).rstrip() + "\n"
+                s += "-----END PRECERTIFICATE-----\n"
+                s += "\n"
+                chain = chain[1:]
             for cert in chain:
-                print >> f, "-----BEGIN CERTIFICATE-----"
-                print >> f, base64.encodestring(cert).rstrip()
-                print >> f, "-----END CERTIFICATE-----"
-                print >> f, ""
-        except AssertionError:
-            print "error for cert", i
+                s += "-----BEGIN CERTIFICATE-----\n"
+                s += base64.encodestring(cert).rstrip() + "\n"
+                s += "-----END CERTIFICATE-----\n"
+                s += "\n"
+            zf.writestr("%08d" % i, s)
+        except AssertionError, e:
+            print "error for cert", i, e
+    if zf:
+        zf.close()
+
+    for blocknumber in range(ncerts / 65536, (tree_size / 65536) + 1):
+        (resulttype, result) = get_merkle_hash_64k(args.store, blocknumber, write_to_cache=True)
+        if resulttype == "incomplete":
+            (incompletelength, hash) = result
+            ncerts = blocknumber * 65536 + incompletelength
+            merkle_64klayer.append(hash)
+            break
+        assert resulttype == "hash"
+        hash = result
+        merkle_64klayer.append(hash)
+        print blocknumber * 65536, base64.b16encode(hash)
+
+    tree = build_merkle_tree(merkle_64klayer)
+
+    calculated_root_hash = tree[-1][0]
+
+    assert ncerts == tree_size
+
+print "calculated root hash", base64.b16encode(calculated_root_hash)
+
+if calculated_root_hash != root_hash:
+    print "fetched root hash and calculated root hash different"
+    sys.exit(1)
+
+if args.store and args.write_sth:
+    f = open(args.store + "/currentsth", "w")
+    f.write(json.dumps(sth))
+    f.close()