fetchallcerts.py: handle precerts

2 files changed, 59 insertions, 15 deletions

diff --git a/tools/certtools.py b/tools/certtools.py
index e1ca57a..f6c1cd9 100644
--- a/tools/certtools.py
+++ b/tools/certtools.py
@@ -138,6 +138,10 @@ def get_entries(baseurl, start, end):
         print "ERROR:", e.read()
         sys.exit(1)
 
+def extract_precertificate(precert_chain_entry):
+    (precert, certchain) = unpack_tls_array(precert_chain_entry, 3)
+    return (precert, certchain)
+
 def decode_certificate_chain(packed_certchain):
     (unpacked_certchain, rest) = unpack_tls_array(packed_certchain, 3)
     assert len(rest) == 0
@@ -235,8 +239,13 @@ def unpack_mtl(merkle_tree_leaf):
     leaf_type = merkle_tree_leaf[1:2]
     timestamped_entry = merkle_tree_leaf[2:]
     (timestamp, entry_type) = struct.unpack(">QH", timestamped_entry[0:10])
-    (leafcert, rest_entry) = unpack_tls_array(timestamped_entry[10:], 3)
-    return (leafcert, timestamp)
+    if entry_type == 0:
+        issuer_key_hash = None
+        (leafcert, rest_entry) = unpack_tls_array(timestamped_entry[10:], 3)
+    elif entry_type == 1:
+        issuer_key_hash = timestamped_entry[10:42]
+        (leafcert, rest_entry) = unpack_tls_array(timestamped_entry[42:], 3)
+    return (leafcert, timestamp, issuer_key_hash)
 
 def get_leaf_hash(merkle_tree_leaf):
     leaf_hash = hashlib.sha256()
diff --git a/tools/fetchallcerts.py b/tools/fetchallcerts.py
index 2276e68..14ec1a7 100644..100755
--- a/tools/fetchallcerts.py
+++ b/tools/fetchallcerts.py
@@ -14,20 +14,25 @@ import struct
 import hashlib
 import itertools
 from certtools import *
+import zipfile
 
 parser = argparse.ArgumentParser(description='')
 parser.add_argument('baseurl', help="Base URL for CT server")
 parser.add_argument('--store', default=None, metavar="dir", help='Store certificates in directory dir')
 parser.add_argument('--start', default=0, metavar="n", type=int, help='Start at index n')
+parser.add_argument('--single', default=None, metavar="n", type=int, help='Onlyfetch index n')
 parser.add_argument('--verify', action='store_true', help='Verify STH')
 args = parser.parse_args()
 
 def extract_original_entry(entry):
     leaf_input =  base64.decodestring(entry["leaf_input"])
-    (leaf_cert, timestamp) = unpack_mtl(leaf_input)
+    (leaf_cert, timestamp, issuer_key_hash) = unpack_mtl(leaf_input)
     extra_data = base64.decodestring(entry["extra_data"])
+    if issuer_key_hash != None:
+        (precert, extra_data) = extract_precertificate(extra_data)
+        leaf_cert = precert
     certchain = decode_certificate_chain(extra_data)
-    return [leaf_cert] + certchain
+    return ([leaf_cert] + certchain, timestamp, issuer_key_hash)
 
 def get_entries_wrapper(baseurl, start, end):
     fetched_entries = 0
@@ -51,7 +56,13 @@ root_hash = base64.decodestring(sth["sha256_root_hash"])
 print "tree size", tree_size
 print "root hash", base64.b16encode(root_hash)
 
-entries = get_entries_wrapper(args.baseurl, args.start, tree_size - 1)
+if args.single == None:
+    entries = get_entries_wrapper(args.baseurl, args.start, tree_size - 1)
+elif args.single > tree_size - 1:
+    print "index", args.single, "too large, tree size:", tree_size
+else:
+    entries = get_entries_wrapper(args.baseurl, args.single, args.single)
+    args.start = args.single
 
 if args.verify:
     layer0 = [get_leaf_hash(base64.decodestring(entry["leaf_input"])) for entry in entries]
@@ -66,15 +77,39 @@ if args.verify:
         print "fetched root hash and calculated root hash different, aborting"
         sys.exit(1)
 
-elif args.store:
+else:
+    currentfilename = None
+    zf = None
     for entry, i in itertools.izip(entries, itertools.count(args.start)):
         try:
-            chain = extract_original_entry(entry)
-            f = open(args.store + "/" + ("%08d" % i), "w")
-            for cert in chain:
-                print >> f, "-----BEGIN CERTIFICATE-----"
-                print >> f, base64.encodestring(cert).rstrip()
-                print >> f, "-----END CERTIFICATE-----"
-                print >> f, ""
-        except AssertionError:
-            print "error for cert", i
+            (chain, timestamp, issuer_key_hash) = extract_original_entry(entry)
+            if args.store:
+                zipfilename = args.store + "/" + ("%04d.zip" % (i / 10000))
+                if zipfilename != currentfilename:
+                    if zf:
+                        zf.close()
+                    zf = zipfile.ZipFile(zipfilename, "w",
+                                         compression=zipfile.ZIP_DEFLATED)
+                    currentfilename = zipfilename
+                s = ""
+                s += "Timestamp: %s\n" % timestamp
+                leaf_input = base64.decodestring(entry["leaf_input"])
+                leaf_hash = get_leaf_hash(leaf_input)
+                s += "Leafhash: %s\n" % base64.b16encode(leaf_hash)
+                if issuer_key_hash:
+                    s += "-----BEGIN PRECERTIFICATE-----\n"
+                    s += base64.encodestring(chain[0]).rstrip() + "\n"
+                    s += "-----END PRECERTIFICATE-----\n"
+                    s += "\n"
+                    chain = chain[1:]
+                for cert in chain:
+                    s += "-----BEGIN CERTIFICATE-----\n"
+                    s += base64.encodestring(cert).rstrip() + "\n"
+                    s += "-----END CERTIFICATE-----\n"
+                    s += "\n"
+                zf.writestr("%08d" % i, s)
+        except AssertionError, e:
+            print "error for cert", i, e
+    if zf:
+        zf.close()
+