#!/usr/bin/env python # -*- coding: utf-8 -*- # Copyright (c) 2014, NORDUnet A/S. # See LICENSE for licensing information. import argparse import urllib2 import urllib import json import base64 import sys import struct import hashlib import itertools from certtools import * import zipfile import os import time parser = argparse.ArgumentParser(description='') parser.add_argument('baseurl', help="Base URL for CT server") parser.add_argument('--store', default=None, metavar="dir", help='Store certificates in directory dir') parser.add_argument('--write-sth', action='store_true', help='Write STH') args = parser.parse_args() def extract_original_entry(entry): leaf_input = base64.decodestring(entry["leaf_input"]) (leaf_cert, timestamp, issuer_key_hash) = unpack_mtl(leaf_input) extra_data = base64.decodestring(entry["extra_data"]) if issuer_key_hash != None: (precert, extra_data) = extract_precertificate(extra_data) leaf_cert = precert certchain = decode_certificate_chain(extra_data) return ([leaf_cert] + certchain, timestamp, issuer_key_hash) def get_entries_wrapper(baseurl, start, end): fetched_entries = 0 while start + fetched_entries < (end + 1): print "fetching from", start + fetched_entries entries = get_entries(baseurl, start + fetched_entries, end)["entries"] if len(entries) == 0: break for entry in entries: fetched_entries += 1 yield entry def print_layer(layer): for entry in layer: print base64.b16encode(entry) def get_hash_from_certfile(cert): for line in cert.split("\n"): if line.startswith("-----"): return None if line.startswith("Leafhash: "): return base64.b16decode(line[len("Leafhash: "):]) return None def get_certs_from_zipfiles(zipfiles, firstleaf, lastleaf): for i in range(firstleaf, lastleaf + 1): try: yield zipfiles[i / 10000].read("%08d" % i) except KeyError: return def get_merkle_hash_64k(store, blocknumber): hashfilename = "%s/%04x.64khash" % (store, blocknumber) try: hash = base64.b16decode(open(hashfilename).read()) assert len(hash) == 32 return ("hash", hash) except IOError: pass firstleaf = blocknumber * 65536 lastleaf = firstleaf + 65535 firstfile = firstleaf / 10000 lastfile = lastleaf / 10000 zipfiles = {} for i in range(firstfile, lastfile + 1): try: zipfiles[i] = zipfile.ZipFile("%s/%04d.zip" % (store, i)) except IOError: break certs = get_certs_from_zipfiles(zipfiles, firstleaf, lastleaf) layer0 = [get_hash_from_certfile(cert) for cert in certs] tree = build_merkle_tree(layer0) calculated_hash = tree[-1][0] for zf in zipfiles.values(): zf.close() if len(layer0) != 65536: return ("incomplete", (len(layer0), calculated_hash)) f = open(hashfilename, "w") f.write(base64.b16encode(calculated_hash)) f.close() return ("hash", calculated_hash) sth = get_sth(args.baseurl) check_sth_signature(args.baseurl, sth) tree_size = sth["tree_size"] root_hash = base64.decodestring(sth["sha256_root_hash"]) try: if args.store: oldsth = json.load(open(args.store + "/currentsth")) else: oldsth = None except IOError: oldsth = None sth_timestamp = datetime.datetime.fromtimestamp(sth["timestamp"]/1000) since_timestamp = time.time() - sth["timestamp"]/1000 print "Log last updated %s, %d seconds ago" % (sth_timestamp.ctime(), since_timestamp) print "tree size", tree_size print "root hash", base64.b16encode(root_hash) if oldsth: if oldsth["tree_size"] == tree_size: print "Tree size has not changed" if oldsth["sha256_root_hash"] != sth["sha256_root_hash"]: print "Root hash is different even though tree size is the same." print "Log has violated the append-only property." print "Old hash:", oldsth["sha256_root_hash"] print "New hash:", sth["sha256_root_hash"] sys.exit(1) if oldsth["timestamp"] == sth["timestamp"]: print "Timestamp has not changed" else: print "Tree size changed, old tree size was", oldsth["tree_size"] merkle_64klayer = [] if args.store: ncerts = None for blocknumber in range(0, (tree_size / 65536) + 1): (resulttype, result) = get_merkle_hash_64k(args.store, blocknumber) if resulttype == "incomplete": (incompletelength, hash) = result ncerts = blocknumber * 65536 + incompletelength break assert resulttype == "hash" hash = result merkle_64klayer.append(hash) print blocknumber * 65536, sys.stdout.flush() print print "ncerts", ncerts else: ncerts = 0 entries = get_entries_wrapper(args.baseurl, ncerts, tree_size - 1) if not args.store: layer0 = [get_leaf_hash(base64.decodestring(entry["leaf_input"])) for entry in entries] tree = build_merkle_tree(layer0) calculated_root_hash = tree[-1][0] else: currentfilename = None zf = None for entry, i in itertools.izip(entries, itertools.count(ncerts)): try: (chain, timestamp, issuer_key_hash) = extract_original_entry(entry) zipfilename = args.store + "/" + ("%04d.zip" % (i / 10000)) if zipfilename != currentfilename: if zf: zf.close() zf = zipfile.ZipFile(zipfilename, "a", compression=zipfile.ZIP_DEFLATED) currentfilename = zipfilename s = "" s += "Timestamp: %s\n" % timestamp leaf_input = base64.decodestring(entry["leaf_input"]) leaf_hash = get_leaf_hash(leaf_input) s += "Leafhash: %s\n" % base64.b16encode(leaf_hash) if issuer_key_hash: s += "-----BEGIN PRECERTIFICATE-----\n" s += base64.encodestring(chain[0]).rstrip() + "\n" s += "-----END PRECERTIFICATE-----\n" s += "\n" chain = chain[1:] for cert in chain: s += "-----BEGIN CERTIFICATE-----\n" s += base64.encodestring(cert).rstrip() + "\n" s += "-----END CERTIFICATE-----\n" s += "\n" zf.writestr("%08d" % i, s) except AssertionError, e: print "error for cert", i, e if zf: zf.close() for blocknumber in range(ncerts / 65536, (tree_size / 65536) + 1): (resulttype, result) = get_merkle_hash_64k(args.store, blocknumber) if resulttype == "incomplete": (incompletelength, hash) = result ncerts = blocknumber * 65536 + incompletelength merkle_64klayer.append(hash) break assert resulttype == "hash" hash = result merkle_64klayer.append(hash) print blocknumber * 65536, base64.b16encode(hash) tree = build_merkle_tree(merkle_64klayer) calculated_root_hash = tree[-1][0] assert ncerts == tree_size print "calculated root hash", base64.b16encode(calculated_root_hash) if calculated_root_hash != root_hash: print "fetched root hash and calculated root hash different" sys.exit(1) if args.store and args.write_sth: f = open(args.store + "/currentsth", "w") f.write(json.dumps(sth)) f.close()