summaryrefslogtreecommitdiff
path: root/tools/fetchallcerts.py
diff options
context:
space:
mode:
authorMagnus Ahltorp <map@kth.se>2015-02-19 13:39:19 +0100
committerMagnus Ahltorp <map@kth.se>2015-02-19 13:39:19 +0100
commit4ee7e18aa4a31d605bc751a514698641aa9fae4c (patch)
treecbd90ed4877023f88ba692d5266a7632845e703b /tools/fetchallcerts.py
parent33b4dbd785c72eb715fb8e82778d60db292212e1 (diff)
fetchallcerts.py: handle precerts
submitcert.py: handle .zip files fetchallcerts.py: Always calculate full tree fetchallcerts.py: Cache level 16 hashes fetchallcerts.py: Save STH
Diffstat (limited to 'tools/fetchallcerts.py')
-rwxr-xr-x[-rw-r--r--]tools/fetchallcerts.py142
1 files changed, 120 insertions, 22 deletions
diff --git a/tools/fetchallcerts.py b/tools/fetchallcerts.py
index 2276e68..866bb43 100644..100755
--- a/tools/fetchallcerts.py
+++ b/tools/fetchallcerts.py
@@ -14,20 +14,25 @@ import struct
import hashlib
import itertools
from certtools import *
+import zipfile
+import os
+import time
parser = argparse.ArgumentParser(description='')
parser.add_argument('baseurl', help="Base URL for CT server")
parser.add_argument('--store', default=None, metavar="dir", help='Store certificates in directory dir')
-parser.add_argument('--start', default=0, metavar="n", type=int, help='Start at index n')
-parser.add_argument('--verify', action='store_true', help='Verify STH')
+parser.add_argument('--write-sth', action='store_true', help='Write STH')
args = parser.parse_args()
def extract_original_entry(entry):
leaf_input = base64.decodestring(entry["leaf_input"])
- (leaf_cert, timestamp) = unpack_mtl(leaf_input)
+ (leaf_cert, timestamp, issuer_key_hash) = unpack_mtl(leaf_input)
extra_data = base64.decodestring(entry["extra_data"])
+ if issuer_key_hash != None:
+ (precert, extra_data) = extract_precertificate(extra_data)
+ leaf_cert = precert
certchain = decode_certificate_chain(extra_data)
- return [leaf_cert] + certchain
+ return ([leaf_cert] + certchain, timestamp, issuer_key_hash)
def get_entries_wrapper(baseurl, start, end):
fetched_entries = 0
@@ -45,36 +50,129 @@ def print_layer(layer):
print base64.b16encode(entry)
sth = get_sth(args.baseurl)
+check_sth_signature(args.baseurl, sth)
tree_size = sth["tree_size"]
root_hash = base64.decodestring(sth["sha256_root_hash"])
+try:
+ if args.store:
+ oldsth = json.load(open(args.store + "/currentsth"))
+ else:
+ oldsth = None
+except IOError:
+ oldsth = None
+
+sth_timestamp = datetime.datetime.fromtimestamp(sth["timestamp"]/1000)
+since_timestamp = time.time() - sth["timestamp"]/1000
+
+print "Log last updated %s, %d seconds ago" % (sth_timestamp.ctime(), since_timestamp)
+
print "tree size", tree_size
print "root hash", base64.b16encode(root_hash)
-entries = get_entries_wrapper(args.baseurl, args.start, tree_size - 1)
+if oldsth:
+ if oldsth["tree_size"] == tree_size:
+ print "Tree size has not changed"
+ if oldsth["sha256_root_hash"] != sth["sha256_root_hash"]:
+ print "Root hash is different even though tree size is the same."
+ print "Log has violated the append-only property."
+ print "Old hash:", oldsth["sha256_root_hash"]
+ print "New hash:", sth["sha256_root_hash"]
+ sys.exit(1)
+ if oldsth["timestamp"] == sth["timestamp"]:
+ print "Timestamp has not changed"
+ else:
+ print "Tree size changed, old tree size was", oldsth["tree_size"]
-if args.verify:
+merkle_64klayer = []
+
+if args.store:
+ ncerts = None
+ for blocknumber in range(0, (tree_size / 65536) + 1):
+ (resulttype, result) = get_merkle_hash_64k(args.store, blocknumber, write_to_cache=True)
+ if resulttype == "incomplete":
+ (incompletelength, hash) = result
+ ncerts = blocknumber * 65536 + incompletelength
+ break
+ assert resulttype == "hash"
+ hash = result
+ merkle_64klayer.append(hash)
+ print blocknumber * 65536,
+ sys.stdout.flush()
+ print
+ print "ncerts", ncerts
+else:
+ ncerts = 0
+
+entries = get_entries_wrapper(args.baseurl, ncerts, tree_size - 1)
+
+if not args.store:
layer0 = [get_leaf_hash(base64.decodestring(entry["leaf_input"])) for entry in entries]
tree = build_merkle_tree(layer0)
calculated_root_hash = tree[-1][0]
- print "calculated root hash", base64.b16encode(calculated_root_hash)
-
- if calculated_root_hash != root_hash:
- print "fetched root hash and calculated root hash different, aborting"
- sys.exit(1)
-
-elif args.store:
- for entry, i in itertools.izip(entries, itertools.count(args.start)):
+else:
+ currentfilename = None
+ zf = None
+ for entry, i in itertools.izip(entries, itertools.count(ncerts)):
try:
- chain = extract_original_entry(entry)
- f = open(args.store + "/" + ("%08d" % i), "w")
+ (chain, timestamp, issuer_key_hash) = extract_original_entry(entry)
+ zipfilename = args.store + "/" + ("%04d.zip" % (i / 10000))
+ if zipfilename != currentfilename:
+ if zf:
+ zf.close()
+ zf = zipfile.ZipFile(zipfilename, "a",
+ compression=zipfile.ZIP_DEFLATED)
+ currentfilename = zipfilename
+ s = ""
+ s += "Timestamp: %s\n" % timestamp
+ leaf_input = base64.decodestring(entry["leaf_input"])
+ leaf_hash = get_leaf_hash(leaf_input)
+ s += "Leafhash: %s\n" % base64.b16encode(leaf_hash)
+ if issuer_key_hash:
+ s += "-----BEGIN PRECERTIFICATE-----\n"
+ s += base64.encodestring(chain[0]).rstrip() + "\n"
+ s += "-----END PRECERTIFICATE-----\n"
+ s += "\n"
+ chain = chain[1:]
for cert in chain:
- print >> f, "-----BEGIN CERTIFICATE-----"
- print >> f, base64.encodestring(cert).rstrip()
- print >> f, "-----END CERTIFICATE-----"
- print >> f, ""
- except AssertionError:
- print "error for cert", i
+ s += "-----BEGIN CERTIFICATE-----\n"
+ s += base64.encodestring(cert).rstrip() + "\n"
+ s += "-----END CERTIFICATE-----\n"
+ s += "\n"
+ zf.writestr("%08d" % i, s)
+ except AssertionError, e:
+ print "error for cert", i, e
+ if zf:
+ zf.close()
+
+ for blocknumber in range(ncerts / 65536, (tree_size / 65536) + 1):
+ (resulttype, result) = get_merkle_hash_64k(args.store, blocknumber, write_to_cache=True)
+ if resulttype == "incomplete":
+ (incompletelength, hash) = result
+ ncerts = blocknumber * 65536 + incompletelength
+ merkle_64klayer.append(hash)
+ break
+ assert resulttype == "hash"
+ hash = result
+ merkle_64klayer.append(hash)
+ print blocknumber * 65536, base64.b16encode(hash)
+
+ tree = build_merkle_tree(merkle_64klayer)
+
+ calculated_root_hash = tree[-1][0]
+
+ assert ncerts == tree_size
+
+print "calculated root hash", base64.b16encode(calculated_root_hash)
+
+if calculated_root_hash != root_hash:
+ print "fetched root hash and calculated root hash different"
+ sys.exit(1)
+
+if args.store and args.write_sth:
+ f = open(args.store + "/currentsth", "w")
+ f.write(json.dumps(sth))
+ f.close()