From d01f3c4b8ca72d722aff8ac6ac4346e089f319a6 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Tue, 18 Nov 2014 15:46:23 +0100 Subject: Catch ctrl-c more correctly. Catch SystemExit from add_chain and exit in main process instead --- tools/submitcert.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) (limited to 'tools') diff --git a/tools/submitcert.py b/tools/submitcert.py index 80a3e37..2bb0c1a 100755 --- a/tools/submitcert.py +++ b/tools/submitcert.py @@ -13,6 +13,7 @@ import hashlib import itertools from certtools import * import os +import signal from multiprocessing import Pool @@ -32,7 +33,10 @@ def submitcert(certfile): certs = get_certs_from_file(certfile) timing_point(timing, "readcerts") - result = add_chain(baseurl, {"chain":map(base64.b64encode, certs)}) + try: + result = add_chain(baseurl, {"chain":map(base64.b64encode, certs)}) + except SystemExit: + return None timing_point(timing, "addchain") @@ -42,10 +46,13 @@ def submitcert(certfile): timing_point(timing, "checksig") except AssertionError, e: print "ERROR:", e - sys.exit(1) + return None + except urllib2.HTTPError, e: + print "ERROR:", e + return None except ecdsa.keys.BadSignatureError, e: print "ERROR: bad signature" - sys.exit(1) + return None if lookup_in_log: @@ -91,8 +98,15 @@ def submitcert(certfile): timing_point(timing, "lookup") return timing["deltatimes"] -p = Pool(1) - -for timing in p.imap_unordered(submitcert, certfiles): - print timing - +p = Pool(1, lambda: signal.signal(signal.SIGINT, signal.SIG_IGN)) + +try: + for timing in p.imap_unordered(submitcert, certfiles): + if timing == None: + p.terminate() + p.join() + sys.exit(1) + print timing +except KeyboardInterrupt: + p.terminate() + p.join() -- cgit v1.1 From 7d9413dcea9645cfa0f8887ac7536682ff5777f0 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Wed, 28 Jan 2015 12:01:41 +0100 Subject: merge.py: add call to storage/getentry since fetchnewentries no longer gives us the actual entry --- tools/merge.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) (limited to 'tools') diff --git a/tools/merge.py b/tools/merge.py index e007d7c..1cc75ef 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -54,6 +54,22 @@ def get_new_entries(baseurl): print "ERROR: fetchnewentries", e.read() sys.exit(1) +def get_entry(baseurl, hash): + try: + params = urllib.urlencode({"hash":base64.b64encode(hash)}) + result = urllib2.urlopen(baseurl + "ct/storage/getentry?" + params).read() + parsed_result = json.loads(result) + if parsed_result.get(u"result") == u"ok": + entries = parsed_result[u"entries"] + assert len(entries) == 1 + assert base64.b64decode(entries[0]["hash"]) == hash + return base64.b64decode(entries[0]["entry"]) + print "ERROR: getentry", parsed_result + sys.exit(1) + except urllib2.HTTPError, e: + print "ERROR: getentry", e.read() + sys.exit(1) + def get_curpos(baseurl): try: result = urllib2.urlopen(baseurl + "ct/frontend/currentposition").read() @@ -135,9 +151,9 @@ new_entries = [entry for storagenode in storagenodes for entry in get_new_entrie print "adding entries" added_entries = 0 for new_entry in new_entries: - hash = base64.b64decode(new_entry["hash"]) - entry = base64.b64decode(new_entry["entry"]) + hash = base64.b64decode(new_entry) if hash not in certsinlog: + entry = get_entry(storagenode, hash) write_chain(hash, entry) add_to_logorder(hash) logorder.append(hash) -- cgit v1.1 From 832dba2e1cd4e6e6ff9e8ea7afa98eebb44595e5 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Wed, 28 Jan 2015 13:42:06 +0100 Subject: Move hardcoded merge parameters to command line --- tools/certtools.py | 37 +++++++++++++++++++++++-------------- tools/merge.py | 28 ++++++++++++++++------------ tools/testcase1.py | 17 ++++++++++------- 3 files changed, 49 insertions(+), 33 deletions(-) (limited to 'tools') diff --git a/tools/certtools.py b/tools/certtools.py index cbb4ff7..af94fb8 100644 --- a/tools/certtools.py +++ b/tools/certtools.py @@ -42,22 +42,31 @@ def get_cert_info(s): result[key] = value return result -def get_certs_from_file(certfile): - certs = [] - cert = "" - incert = False - for line in open(certfile): +def get_pemlike(filename, marker): + entries = [] + entry = "" + inentry = False + + for line in open(filename): line = line.strip() - if line == "-----BEGIN CERTIFICATE-----": - cert = "" - incert = True - elif line == "-----END CERTIFICATE-----": - certs.append(base64.decodestring(cert)) - incert = False - elif incert: - cert += line - return certs + if line == "-----BEGIN " + marker + "-----": + entry = "" + inentry = True + elif line == "-----END " + marker + "-----": + entries.append(base64.decodestring(entry)) + inentry = False + elif inentry: + entry += line + return entries + +def get_certs_from_file(certfile): + return get_pemlike(certfile, "CERTIFICATE") + +def get_eckey_from_file(keyfile): + keys = get_pemlike(keyfile, "EC PRIVATE KEY") + assert len(keys) == 1 + return keys[0] def get_root_cert(issuer): accepted_certs = \ diff --git a/tools/merge.py b/tools/merge.py index 1cc75ef..2b83f54 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -4,20 +4,29 @@ # Copyright (c) 2014, NORDUnet A/S. # See LICENSE for licensing information. +import argparse import json import base64 import urllib import urllib2 import sys import time -from certtools import build_merkle_tree, create_sth_signature, check_sth_signature +from certtools import build_merkle_tree, create_sth_signature, check_sth_signature, get_eckey_from_file -ctbaseurl = "https://127.0.0.1:8080/" -frontendnodes = ["https://127.0.0.1:8082/"] -storagenodes = ["https://127.0.0.1:8081/"] +parser = argparse.ArgumentParser(description="") +parser.add_argument("--baseurl", metavar="url", help="Base URL for CT server", required=True) +parser.add_argument("--frontend", action="append", metavar="url", help="Base URL for frontend server", required=True) +parser.add_argument("--storage", action="append", metavar="url", help="Base URL for storage server", required=True) +parser.add_argument("--mergedb", metavar="dir", help="Merge database directory", required=True) +parser.add_argument("--keyfile", metavar="keyfile", help="File containing log key", required=True) +args = parser.parse_args() -chainsdir = "../rel/mergedb/chains" -logorderfile = "../rel/mergedb/logorder" +ctbaseurl = args.baseurl +frontendnodes = args.frontend +storagenodes = args.storage + +chainsdir = args.mergedb + "/chains" +logorderfile = args.mergedb + "/logorder" def parselogrow(row): return base64.b16decode(row) @@ -165,12 +174,7 @@ tree = build_merkle_tree(logorder) tree_size = len(logorder) root_hash = tree[-1][0] timestamp = int(time.time() * 1000) -privatekey = base64.decodestring( - "MHcCAQEEIMM/FjZ4FSzfENTTwGpTve6CP+IVr" - "Y7p8OKV634uJI/foAoGCCqGSM49AwEHoUQDQg" - "AE4qWq6afhBUi0OdcWUYhyJLNXTkGqQ9PMS5l" - "qoCgkV2h1ZvpNjBH2u8UbgcOQwqDo66z6BWQJ" - "GolozZYmNHE2kQ==") +privatekey = get_eckey_from_file(args.keyfile) tree_head_signature = create_sth_signature(tree_size, timestamp, root_hash, privatekey) diff --git a/tools/testcase1.py b/tools/testcase1.py index 639cd69..415d475 100755 --- a/tools/testcase1.py +++ b/tools/testcase1.py @@ -120,16 +120,19 @@ def get_and_check_entry(timestamp, chain, leaf_index): len(submittedcertchain), len(submittedcertchain)) +def merge(): + return subprocess.call(["./merge.py", "--baseurl", "https://127.0.0.1:8080/", "--frontend", "https://127.0.0.1:8082/", "--storage", "https://127.0.0.1:8081/", "--mergedb", "../rel/mergedb", "--keyfile", "../rel/test/eckey.pem"]) + print_and_check_tree_size(0) -mergeresult = subprocess.call(["./merge.py"]) +mergeresult = merge() assert_equal(mergeresult, 0, "merge", quiet=True) testgroup("cert1") result1 = do_add_chain(cc1) -mergeresult = subprocess.call(["./merge.py"]) +mergeresult = merge() assert_equal(mergeresult, 0, "merge", quiet=True) print_and_check_tree_size(1) @@ -138,7 +141,7 @@ result2 = do_add_chain(cc1) assert_equal(result2["timestamp"], result1["timestamp"], "timestamp") -mergeresult = subprocess.call(["./merge.py"]) +mergeresult = merge() assert_equal(mergeresult, 0, "merge", quiet=True) print_and_check_tree_size(1) @@ -152,7 +155,7 @@ testgroup("cert2") result3 = do_add_chain(cc2) -mergeresult = subprocess.call(["./merge.py"]) +mergeresult = merge() assert_equal(mergeresult, 0, "merge", quiet=True) print_and_check_tree_size(2) @@ -164,7 +167,7 @@ testgroup("cert3") result4 = do_add_chain(cc3) -mergeresult = subprocess.call(["./merge.py"]) +mergeresult = merge() assert_equal(mergeresult, 0, "merge", quiet=True) print_and_check_tree_size(3) @@ -177,7 +180,7 @@ testgroup("cert4") result5 = do_add_chain(cc4) -mergeresult = subprocess.call(["./merge.py"]) +mergeresult = merge() assert_equal(mergeresult, 0, "merge", quiet=True) print_and_check_tree_size(4) @@ -191,7 +194,7 @@ testgroup("cert5") result6 = do_add_chain(cc5) -mergeresult = subprocess.call(["./merge.py"]) +mergeresult = merge() assert_equal(mergeresult, 0, "merge", quiet=True) print_and_check_tree_size(5) -- cgit v1.1 From 5f16e0b43df17d556905e0885abed28cef8b8e29 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Thu, 19 Feb 2015 13:23:08 +0100 Subject: merge.py: Only ask node that actually has the entry. Fetch multiple entries from storage node. Chunk sendlog. --- tools/merge.py | 91 ++++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 70 insertions(+), 21 deletions(-) (limited to 'tools') diff --git a/tools/merge.py b/tools/merge.py index 2b83f54..f4a007d 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -11,7 +11,7 @@ import urllib import urllib2 import sys import time -from certtools import build_merkle_tree, create_sth_signature, check_sth_signature, get_eckey_from_file +from certtools import build_merkle_tree, create_sth_signature, check_sth_signature, get_eckey_from_file, timing_point parser = argparse.ArgumentParser(description="") parser.add_argument("--baseurl", metavar="url", help="Base URL for CT server", required=True) @@ -19,6 +19,7 @@ parser.add_argument("--frontend", action="append", metavar="url", help="Base URL parser.add_argument("--storage", action="append", metavar="url", help="Base URL for storage server", required=True) parser.add_argument("--mergedb", metavar="dir", help="Merge database directory", required=True) parser.add_argument("--keyfile", metavar="keyfile", help="File containing log key", required=True) +parser.add_argument("--nomerge", action='store_true', help="Don't actually do merge") args = parser.parse_args() ctbaseurl = args.baseurl @@ -56,23 +57,23 @@ def get_new_entries(baseurl): result = urllib2.urlopen(baseurl + "ct/storage/fetchnewentries").read() parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": - return parsed_result[u"entries"] + return [base64.b64decode(entry) for entry in parsed_result[u"entries"]] print "ERROR: fetchnewentries", parsed_result sys.exit(1) except urllib2.HTTPError, e: print "ERROR: fetchnewentries", e.read() sys.exit(1) -def get_entry(baseurl, hash): +def get_entries(baseurl, hashes): try: - params = urllib.urlencode({"hash":base64.b64encode(hash)}) + params = urllib.urlencode({"hash":[base64.b64encode(hash) for hash in hashes]}, doseq=True) result = urllib2.urlopen(baseurl + "ct/storage/getentry?" + params).read() parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": - entries = parsed_result[u"entries"] - assert len(entries) == 1 - assert base64.b64decode(entries[0]["hash"]) == hash - return base64.b64decode(entries[0]["entry"]) + entries = dict([(base64.b64decode(entry["hash"]), base64.b64decode(entry["entry"])) for entry in parsed_result[u"entries"]]) + assert len(entries) == len(hashes) + assert set(entries.keys()) == set(hashes) + return entries print "ERROR: getentry", parsed_result sys.exit(1) except urllib2.HTTPError, e: @@ -151,23 +152,56 @@ def get_missingentries(baseurl): print "ERROR: missingentries", e.read() sys.exit(1) +def chunks(l, n): + return [l[i:i+n] for i in range(0, len(l), n)] + +timing = timing_point() logorder = get_logorder() + +timing_point(timing, "get logorder") + certsinlog = set(logorder) -new_entries = [entry for storagenode in storagenodes for entry in get_new_entries(storagenode)] +new_entries_per_node = {} +new_entries = set() +entries_to_fetch = {} + +for storagenode in storagenodes: + print "getting new entries from", storagenode + new_entries_per_node[storagenode] = set(get_new_entries(storagenode)) + new_entries.update(new_entries_per_node[storagenode]) + entries_to_fetch[storagenode] = [] + +timing_point(timing, "get new entries") + +new_entries -= certsinlog + +print "adding", len(new_entries), "entries" + +if args.nomerge: + sys.exit(0) + +for hash in new_entries: + for storagenode in storagenodes: + if hash in new_entries_per_node[storagenode]: + entries_to_fetch[storagenode].append(hash) + break + -print "adding entries" added_entries = 0 -for new_entry in new_entries: - hash = base64.b64decode(new_entry) - if hash not in certsinlog: - entry = get_entry(storagenode, hash) - write_chain(hash, entry) - add_to_logorder(hash) - logorder.append(hash) - certsinlog.add(hash) - added_entries += 1 +for storagenode in storagenodes: + print "getting", len(entries_to_fetch[storagenode]), "entries from", storagenode + for chunk in chunks(entries_to_fetch[storagenode], 100): + entries = get_entries(storagenode, chunk) + for hash in chunk: + entry = entries[hash] + write_chain(hash, entry) + add_to_logorder(hash) + logorder.append(hash) + certsinlog.add(hash) + added_entries += 1 +timing_point(timing, "add entries") print "added", added_entries, "entries" tree = build_merkle_tree(logorder) @@ -185,18 +219,33 @@ sth = {"tree_size": tree_size, "timestamp": timestamp, check_sth_signature(ctbaseurl, sth) +timing_point(timing, "build sth") + +print timing["deltatimes"] + print "root hash", base64.b16encode(root_hash) for frontendnode in frontendnodes: + timing = timing_point() print "distributing for node", frontendnode curpos = get_curpos(frontendnode) + timing_point(timing, "get curpos") print "current position", curpos entries = [base64.b64encode(entry) for entry in logorder[curpos:]] - sendlog(frontendnode, {"start": curpos, "hashes": entries}) + for chunk in chunks(entries, 1000): + sendlog(frontendnode, {"start": curpos, "hashes": chunk}) + curpos += len(chunk) + print curpos, + sys.stdout.flush() + timing_point(timing, "sendlog") print "log sent" missingentries = get_missingentries(frontendnode) - print "missing entries:", missingentries + timing_point(timing, "get missing") + print "missing entries:", len(missingentries) for missingentry in missingentries: hash = base64.b64decode(missingentry) sendentry(frontendnode, read_chain(hash), hash) + timing_point(timing, "send missing") sendsth(frontendnode, sth) + timing_point(timing, "send sth") + print timing["deltatimes"] -- cgit v1.1 From b24f6ae5a129b5f0cbbffcf07b5482eda8a3eef2 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Sun, 8 Feb 2015 01:53:40 +0100 Subject: Wait after first submission. Continue on http error 400. Print submission rate and number of submissions every 1000 submissions. --- tools/certtools.py | 4 +++- tools/submitcert.py | 39 +++++++++++++++++++++++++++++++++------ 2 files changed, 36 insertions(+), 7 deletions(-) (limited to 'tools') diff --git a/tools/certtools.py b/tools/certtools.py index af94fb8..e1ca57a 100644 --- a/tools/certtools.py +++ b/tools/certtools.py @@ -117,7 +117,9 @@ def add_chain(baseurl, submission): json.dumps(submission)).read() return json.loads(result) except urllib2.HTTPError, e: - print "ERROR:", e.read() + print "ERROR", e.code,":", e.read() + if e.code == 400: + return None sys.exit(1) except ValueError, e: print "==== FAILED REQUEST ====" diff --git a/tools/submitcert.py b/tools/submitcert.py index 2bb0c1a..1b87b53 100755 --- a/tools/submitcert.py +++ b/tools/submitcert.py @@ -14,6 +14,7 @@ import itertools from certtools import * import os import signal +import select from multiprocessing import Pool @@ -36,22 +37,28 @@ def submitcert(certfile): try: result = add_chain(baseurl, {"chain":map(base64.b64encode, certs)}) except SystemExit: + print "EXIT:", certfile + select.select([], [], [], 1.0) return None timing_point(timing, "addchain") + if result == None: + print "ERROR for certfile", certfile + return timing["deltatimes"] + try: if check_sig: check_sct_signature(baseurl, certs[0], result) timing_point(timing, "checksig") except AssertionError, e: - print "ERROR:", e + print "ERROR:", certfile, e return None except urllib2.HTTPError, e: - print "ERROR:", e + print "ERROR:", certfile, e return None except ecdsa.keys.BadSignatureError, e: - print "ERROR: bad signature" + print "ERROR: bad signature", certfile return None if lookup_in_log: @@ -98,15 +105,35 @@ def submitcert(certfile): timing_point(timing, "lookup") return timing["deltatimes"] -p = Pool(1, lambda: signal.signal(signal.SIGINT, signal.SIG_IGN)) +p = Pool(16, lambda: signal.signal(signal.SIGINT, signal.SIG_IGN)) + +nsubmitted = 0 +lastprinted = 0 +starttime = datetime.datetime.now() + +print len(certfiles), "certs" + +submitcert(certfiles[0]) +nsubmitted += 1 +select.select([], [], [], 3.0) try: - for timing in p.imap_unordered(submitcert, certfiles): + for timing in p.imap_unordered(submitcert, certfiles[1:]): if timing == None: + print "error" + print "submitted", nsubmitted p.terminate() p.join() sys.exit(1) - print timing + nsubmitted += 1 + deltatime = datetime.datetime.now() - starttime + deltatime_f = deltatime.seconds + deltatime.microseconds / 1000000.0 + rate = nsubmitted / deltatime_f + if nsubmitted > lastprinted + len(certfiles) / 10: + print nsubmitted, "rate %.1f" % rate + lastprinted = nsubmitted + #print timing, "rate %.1f" % rate + print "submitted", nsubmitted except KeyboardInterrupt: p.terminate() p.join() -- cgit v1.1 From e7e1bd58569f4e86c9ce1a7055b72ca0165a4538 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Sun, 8 Feb 2015 23:23:38 +0100 Subject: fetchallcerts.py: Store certificates. --- tools/fetchallcerts.py | 55 +++++++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 25 deletions(-) (limited to 'tools') diff --git a/tools/fetchallcerts.py b/tools/fetchallcerts.py index dad5241..2276e68 100644 --- a/tools/fetchallcerts.py +++ b/tools/fetchallcerts.py @@ -18,6 +18,8 @@ from certtools import * parser = argparse.ArgumentParser(description='') parser.add_argument('baseurl', help="Base URL for CT server") parser.add_argument('--store', default=None, metavar="dir", help='Store certificates in directory dir') +parser.add_argument('--start', default=0, metavar="n", type=int, help='Start at index n') +parser.add_argument('--verify', action='store_true', help='Verify STH') args = parser.parse_args() def extract_original_entry(entry): @@ -28,14 +30,15 @@ def extract_original_entry(entry): return [leaf_cert] + certchain def get_entries_wrapper(baseurl, start, end): - fetched_entries = [] - while start + len(fetched_entries) < (end + 1): - print "fetching from", start + len(fetched_entries) - entries = get_entries(baseurl, start + len(fetched_entries), end)["entries"] + fetched_entries = 0 + while start + fetched_entries < (end + 1): + print "fetching from", start + fetched_entries + entries = get_entries(baseurl, start + fetched_entries, end)["entries"] if len(entries) == 0: break - fetched_entries.extend(entries) - return fetched_entries + for entry in entries: + fetched_entries += 1 + yield entry def print_layer(layer): for entry in layer: @@ -48,28 +51,30 @@ root_hash = base64.decodestring(sth["sha256_root_hash"]) print "tree size", tree_size print "root hash", base64.b16encode(root_hash) -entries = get_entries_wrapper(args.baseurl, 0, tree_size - 1) +entries = get_entries_wrapper(args.baseurl, args.start, tree_size - 1) -print "fetched", len(entries), "entries" +if args.verify: + layer0 = [get_leaf_hash(base64.decodestring(entry["leaf_input"])) for entry in entries] -layer0 = [get_leaf_hash(base64.decodestring(entry["leaf_input"])) for entry in entries] + tree = build_merkle_tree(layer0) -tree = build_merkle_tree(layer0) + calculated_root_hash = tree[-1][0] -calculated_root_hash = tree[-1][0] + print "calculated root hash", base64.b16encode(calculated_root_hash) -print "calculated root hash", base64.b16encode(calculated_root_hash) + if calculated_root_hash != root_hash: + print "fetched root hash and calculated root hash different, aborting" + sys.exit(1) -if calculated_root_hash != root_hash: - print "fetched root hash and calculated root hash different, aborting" - sys.exit(1) - -if args.store: - for entry, i in zip(entries, range(0, len(entries))): - chain = extract_original_entry(entry) - f = open(args.store + "/" + ("%06d" % i), "w") - for cert in chain: - print >> f, "-----BEGIN CERTIFICATE-----" - print >> f, base64.encodestring(cert).rstrip() - print >> f, "-----END CERTIFICATE-----" - print >> f, "" +elif args.store: + for entry, i in itertools.izip(entries, itertools.count(args.start)): + try: + chain = extract_original_entry(entry) + f = open(args.store + "/" + ("%08d" % i), "w") + for cert in chain: + print >> f, "-----BEGIN CERTIFICATE-----" + print >> f, base64.encodestring(cert).rstrip() + print >> f, "-----END CERTIFICATE-----" + print >> f, "" + except AssertionError: + print "error for cert", i -- cgit v1.1 From ec31631bb097be9780c7355d4183bfd5050c5af4 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Thu, 19 Feb 2015 13:39:19 +0100 Subject: fetchallcerts.py: handle precerts submitcert.py: handle .zip files fetchallcerts.py: Always calculate full tree fetchallcerts.py: Cache level 16 hashes fetchallcerts.py: Save STH --- tools/certtools.py | 93 ++++++++++++++++++++++++++++++-- tools/fetchallcerts.py | 142 +++++++++++++++++++++++++++++++++++++++++-------- tools/submitcert.py | 51 +++++++++++++----- 3 files changed, 249 insertions(+), 37 deletions(-) mode change 100644 => 100755 tools/fetchallcerts.py (limited to 'tools') diff --git a/tools/certtools.py b/tools/certtools.py index e1ca57a..6a144c9 100644 --- a/tools/certtools.py +++ b/tools/certtools.py @@ -11,6 +11,8 @@ import sys import hashlib import ecdsa import datetime +import cStringIO +import zipfile publickeys = { "https://ct.googleapis.com/pilot/": @@ -44,11 +46,14 @@ def get_cert_info(s): def get_pemlike(filename, marker): + return get_pemlike_from_file(open(filename), marker) + +def get_pemlike_from_file(f, marker): entries = [] entry = "" inentry = False - for line in open(filename): + for line in f: line = line.strip() if line == "-----BEGIN " + marker + "-----": entry = "" @@ -63,6 +68,10 @@ def get_pemlike(filename, marker): def get_certs_from_file(certfile): return get_pemlike(certfile, "CERTIFICATE") +def get_certs_from_string(s): + f = cStringIO.StringIO(s) + return get_pemlike_from_file(f, "CERTIFICATE") + def get_eckey_from_file(keyfile): keys = get_pemlike(keyfile, "EC PRIVATE KEY") assert len(keys) == 1 @@ -138,6 +147,10 @@ def get_entries(baseurl, start, end): print "ERROR:", e.read() sys.exit(1) +def extract_precertificate(precert_chain_entry): + (precert, certchain) = unpack_tls_array(precert_chain_entry, 3) + return (precert, certchain) + def decode_certificate_chain(packed_certchain): (unpacked_certchain, rest) = unpack_tls_array(packed_certchain, 3) assert len(rest) == 0 @@ -235,8 +248,13 @@ def unpack_mtl(merkle_tree_leaf): leaf_type = merkle_tree_leaf[1:2] timestamped_entry = merkle_tree_leaf[2:] (timestamp, entry_type) = struct.unpack(">QH", timestamped_entry[0:10]) - (leafcert, rest_entry) = unpack_tls_array(timestamped_entry[10:], 3) - return (leafcert, timestamp) + if entry_type == 0: + issuer_key_hash = None + (leafcert, rest_entry) = unpack_tls_array(timestamped_entry[10:], 3) + elif entry_type == 1: + issuer_key_hash = timestamped_entry[10:42] + (leafcert, rest_entry) = unpack_tls_array(timestamped_entry[42:], 3) + return (leafcert, timestamp, issuer_key_hash) def get_leaf_hash(merkle_tree_leaf): leaf_hash = hashlib.sha256() @@ -284,3 +302,72 @@ def build_merkle_tree(layer0): current_layer = next_merkle_layer(current_layer) layers.append(current_layer) return layers + +def print_inclusion_proof(proof): + audit_path = proof[u'audit_path'] + n = proof[u'leaf_index'] + level = 0 + for s in audit_path: + entry = base64.b16encode(base64.b64decode(s)) + n ^= 1 + print level, n, entry + n >>= 1 + level += 1 + +def get_one_cert(store, i): + filename = i / 10000 + zf = zipfile.ZipFile("%s/%04d.zip" % (store, i / 10000)) + cert = zf.read("%08d" % i) + zf.close() + return cert + +def get_hash_from_certfile(cert): + for line in cert.split("\n"): + if line.startswith("-----"): + return None + if line.startswith("Leafhash: "): + return base64.b16decode(line[len("Leafhash: "):]) + return None + +def get_proof(store, tree_size, n): + hash = get_hash_from_certfile(get_one_cert(store, n)) + return get_proof_by_hash(args.baseurl, hash, tree_size) + +def get_certs_from_zipfiles(zipfiles, firstleaf, lastleaf): + for i in range(firstleaf, lastleaf + 1): + try: + yield zipfiles[i / 10000].read("%08d" % i) + except KeyError: + return + +def get_merkle_hash_64k(store, blocknumber, write_to_cache=False): + hashfilename = "%s/%04x.64khash" % (store, blocknumber) + try: + hash = base64.b16decode(open(hashfilename).read()) + assert len(hash) == 32 + return ("hash", hash) + except IOError: + pass + firstleaf = blocknumber * 65536 + lastleaf = firstleaf + 65535 + firstfile = firstleaf / 10000 + lastfile = lastleaf / 10000 + zipfiles = {} + for i in range(firstfile, lastfile + 1): + try: + zipfiles[i] = zipfile.ZipFile("%s/%04d.zip" % (store, i)) + except IOError: + break + certs = get_certs_from_zipfiles(zipfiles, firstleaf, lastleaf) + layer0 = [get_hash_from_certfile(cert) for cert in certs] + tree = build_merkle_tree(layer0) + calculated_hash = tree[-1][0] + for zf in zipfiles.values(): + zf.close() + if len(layer0) != 65536: + return ("incomplete", (len(layer0), calculated_hash)) + if write_to_cache: + f = open(hashfilename, "w") + f.write(base64.b16encode(calculated_hash)) + f.close() + return ("hash", calculated_hash) diff --git a/tools/fetchallcerts.py b/tools/fetchallcerts.py old mode 100644 new mode 100755 index 2276e68..866bb43 --- a/tools/fetchallcerts.py +++ b/tools/fetchallcerts.py @@ -14,20 +14,25 @@ import struct import hashlib import itertools from certtools import * +import zipfile +import os +import time parser = argparse.ArgumentParser(description='') parser.add_argument('baseurl', help="Base URL for CT server") parser.add_argument('--store', default=None, metavar="dir", help='Store certificates in directory dir') -parser.add_argument('--start', default=0, metavar="n", type=int, help='Start at index n') -parser.add_argument('--verify', action='store_true', help='Verify STH') +parser.add_argument('--write-sth', action='store_true', help='Write STH') args = parser.parse_args() def extract_original_entry(entry): leaf_input = base64.decodestring(entry["leaf_input"]) - (leaf_cert, timestamp) = unpack_mtl(leaf_input) + (leaf_cert, timestamp, issuer_key_hash) = unpack_mtl(leaf_input) extra_data = base64.decodestring(entry["extra_data"]) + if issuer_key_hash != None: + (precert, extra_data) = extract_precertificate(extra_data) + leaf_cert = precert certchain = decode_certificate_chain(extra_data) - return [leaf_cert] + certchain + return ([leaf_cert] + certchain, timestamp, issuer_key_hash) def get_entries_wrapper(baseurl, start, end): fetched_entries = 0 @@ -45,36 +50,129 @@ def print_layer(layer): print base64.b16encode(entry) sth = get_sth(args.baseurl) +check_sth_signature(args.baseurl, sth) tree_size = sth["tree_size"] root_hash = base64.decodestring(sth["sha256_root_hash"]) +try: + if args.store: + oldsth = json.load(open(args.store + "/currentsth")) + else: + oldsth = None +except IOError: + oldsth = None + +sth_timestamp = datetime.datetime.fromtimestamp(sth["timestamp"]/1000) +since_timestamp = time.time() - sth["timestamp"]/1000 + +print "Log last updated %s, %d seconds ago" % (sth_timestamp.ctime(), since_timestamp) + print "tree size", tree_size print "root hash", base64.b16encode(root_hash) -entries = get_entries_wrapper(args.baseurl, args.start, tree_size - 1) +if oldsth: + if oldsth["tree_size"] == tree_size: + print "Tree size has not changed" + if oldsth["sha256_root_hash"] != sth["sha256_root_hash"]: + print "Root hash is different even though tree size is the same." + print "Log has violated the append-only property." + print "Old hash:", oldsth["sha256_root_hash"] + print "New hash:", sth["sha256_root_hash"] + sys.exit(1) + if oldsth["timestamp"] == sth["timestamp"]: + print "Timestamp has not changed" + else: + print "Tree size changed, old tree size was", oldsth["tree_size"] -if args.verify: +merkle_64klayer = [] + +if args.store: + ncerts = None + for blocknumber in range(0, (tree_size / 65536) + 1): + (resulttype, result) = get_merkle_hash_64k(args.store, blocknumber, write_to_cache=True) + if resulttype == "incomplete": + (incompletelength, hash) = result + ncerts = blocknumber * 65536 + incompletelength + break + assert resulttype == "hash" + hash = result + merkle_64klayer.append(hash) + print blocknumber * 65536, + sys.stdout.flush() + print + print "ncerts", ncerts +else: + ncerts = 0 + +entries = get_entries_wrapper(args.baseurl, ncerts, tree_size - 1) + +if not args.store: layer0 = [get_leaf_hash(base64.decodestring(entry["leaf_input"])) for entry in entries] tree = build_merkle_tree(layer0) calculated_root_hash = tree[-1][0] - print "calculated root hash", base64.b16encode(calculated_root_hash) - - if calculated_root_hash != root_hash: - print "fetched root hash and calculated root hash different, aborting" - sys.exit(1) - -elif args.store: - for entry, i in itertools.izip(entries, itertools.count(args.start)): +else: + currentfilename = None + zf = None + for entry, i in itertools.izip(entries, itertools.count(ncerts)): try: - chain = extract_original_entry(entry) - f = open(args.store + "/" + ("%08d" % i), "w") + (chain, timestamp, issuer_key_hash) = extract_original_entry(entry) + zipfilename = args.store + "/" + ("%04d.zip" % (i / 10000)) + if zipfilename != currentfilename: + if zf: + zf.close() + zf = zipfile.ZipFile(zipfilename, "a", + compression=zipfile.ZIP_DEFLATED) + currentfilename = zipfilename + s = "" + s += "Timestamp: %s\n" % timestamp + leaf_input = base64.decodestring(entry["leaf_input"]) + leaf_hash = get_leaf_hash(leaf_input) + s += "Leafhash: %s\n" % base64.b16encode(leaf_hash) + if issuer_key_hash: + s += "-----BEGIN PRECERTIFICATE-----\n" + s += base64.encodestring(chain[0]).rstrip() + "\n" + s += "-----END PRECERTIFICATE-----\n" + s += "\n" + chain = chain[1:] for cert in chain: - print >> f, "-----BEGIN CERTIFICATE-----" - print >> f, base64.encodestring(cert).rstrip() - print >> f, "-----END CERTIFICATE-----" - print >> f, "" - except AssertionError: - print "error for cert", i + s += "-----BEGIN CERTIFICATE-----\n" + s += base64.encodestring(cert).rstrip() + "\n" + s += "-----END CERTIFICATE-----\n" + s += "\n" + zf.writestr("%08d" % i, s) + except AssertionError, e: + print "error for cert", i, e + if zf: + zf.close() + + for blocknumber in range(ncerts / 65536, (tree_size / 65536) + 1): + (resulttype, result) = get_merkle_hash_64k(args.store, blocknumber, write_to_cache=True) + if resulttype == "incomplete": + (incompletelength, hash) = result + ncerts = blocknumber * 65536 + incompletelength + merkle_64klayer.append(hash) + break + assert resulttype == "hash" + hash = result + merkle_64klayer.append(hash) + print blocknumber * 65536, base64.b16encode(hash) + + tree = build_merkle_tree(merkle_64klayer) + + calculated_root_hash = tree[-1][0] + + assert ncerts == tree_size + +print "calculated root hash", base64.b16encode(calculated_root_hash) + +if calculated_root_hash != root_hash: + print "fetched root hash and calculated root hash different" + sys.exit(1) + +if args.store and args.write_sth: + f = open(args.store + "/currentsth", "w") + f.write(json.dumps(sth)) + f.close() diff --git a/tools/submitcert.py b/tools/submitcert.py index 1b87b53..04b6ebe 100755 --- a/tools/submitcert.py +++ b/tools/submitcert.py @@ -15,6 +15,7 @@ from certtools import * import os import signal import select +import zipfile from multiprocessing import Pool @@ -29,13 +30,13 @@ if certfilepath[-1] == "/": else: certfiles = [certfilepath] -def submitcert(certfile): +def submitcert((certfile, cert)): timing = timing_point() - certs = get_certs_from_file(certfile) + certchain = get_certs_from_string(cert) timing_point(timing, "readcerts") try: - result = add_chain(baseurl, {"chain":map(base64.b64encode, certs)}) + result = add_chain(baseurl, {"chain":map(base64.b64encode, certchain)}) except SystemExit: print "EXIT:", certfile select.select([], [], [], 1.0) @@ -49,7 +50,7 @@ def submitcert(certfile): try: if check_sig: - check_sct_signature(baseurl, certs[0], result) + check_sct_signature(baseurl, certchain[0], result) timing_point(timing, "checksig") except AssertionError, e: print "ERROR:", certfile, e @@ -63,7 +64,7 @@ def submitcert(certfile): if lookup_in_log: - merkle_tree_leaf = pack_mtl(result["timestamp"], certs[0]) + merkle_tree_leaf = pack_mtl(result["timestamp"], certchain[0]) leaf_hash = get_leaf_hash(merkle_tree_leaf) @@ -84,14 +85,14 @@ def submitcert(certfile): certchain = decode_certificate_chain(base64.decodestring(extra_data)) - submittedcertchain = certs[1:] + submittedcertchain = certchain[1:] for (submittedcert, fetchedcert, i) in zip(submittedcertchain, certchain, itertools.count(1)): print "cert", i, "in chain is the same:", submittedcert == fetchedcert if len(certchain) == len(submittedcertchain) + 1: - last_issuer = get_cert_info(certs[-1])["issuer"] + last_issuer = get_cert_info(certchain[-1])["issuer"] root_subject = get_cert_info(certchain[-1])["subject"] print "issuer of last cert in submitted chain and " \ "subject of last cert in fetched chain is the same:", \ @@ -105,20 +106,46 @@ def submitcert(certfile): timing_point(timing, "lookup") return timing["deltatimes"] +def get_ncerts(certfiles): + n = 0 + for certfile in certfiles: + if certfile.endswith(".zip"): + zf = zipfile.ZipFile(certfile) + n += len(zf.namelist()) + zf.close() + else: + n += 1 + return n + +def get_all_certificates(certfiles): + for certfile in certfiles: + if certfile.endswith(".zip"): + zf = zipfile.ZipFile(certfile) + for name in zf.namelist(): + yield (name, zf.read(name)) + zf.close() + else: + yield (certfile, open(certfile).read()) + p = Pool(16, lambda: signal.signal(signal.SIGINT, signal.SIG_IGN)) nsubmitted = 0 lastprinted = 0 -starttime = datetime.datetime.now() -print len(certfiles), "certs" +ncerts = get_ncerts(certfiles) -submitcert(certfiles[0]) +print ncerts, "certs" + +certs = get_all_certificates(certfiles) + +submitcert(certs.next()) nsubmitted += 1 select.select([], [], [], 3.0) +starttime = datetime.datetime.now() + try: - for timing in p.imap_unordered(submitcert, certfiles[1:]): + for timing in p.imap_unordered(submitcert, certs): if timing == None: print "error" print "submitted", nsubmitted @@ -129,7 +156,7 @@ try: deltatime = datetime.datetime.now() - starttime deltatime_f = deltatime.seconds + deltatime.microseconds / 1000000.0 rate = nsubmitted / deltatime_f - if nsubmitted > lastprinted + len(certfiles) / 10: + if nsubmitted > lastprinted + ncerts / 10: print nsubmitted, "rate %.1f" % rate lastprinted = nsubmitted #print timing, "rate %.1f" % rate -- cgit v1.1 From 31589e4fe5485cd05c4f28b759d90d4ab754123e Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Sun, 15 Feb 2015 15:23:29 +0100 Subject: Implement function to fetch consistency proof Implement function to calculate tree head from disk Implement function to calculate an intermediate node from disk --- tools/certtools.py | 108 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 99 insertions(+), 9 deletions(-) (limited to 'tools') diff --git a/tools/certtools.py b/tools/certtools.py index 6a144c9..31045b9 100644 --- a/tools/certtools.py +++ b/tools/certtools.py @@ -105,6 +105,17 @@ def get_proof_by_hash(baseurl, hash, tree_size): print "ERROR:", e.read() sys.exit(1) +def get_consistency_proof(baseurl, tree_size1, tree_size2): + try: + params = urllib.urlencode({"first":tree_size1, + "second":tree_size2}) + result = \ + urllib2.urlopen(baseurl + "ct/v1/get-sth-consistency?" + params).read() + return json.loads(result)["consistency"] + except urllib2.HTTPError, e: + print "ERROR:", e.read() + sys.exit(1) + def tls_array(data, length_len): length_bytes = struct.pack(">Q", len(data))[-length_len:] return length_bytes + data @@ -340,16 +351,24 @@ def get_certs_from_zipfiles(zipfiles, firstleaf, lastleaf): except KeyError: return -def get_merkle_hash_64k(store, blocknumber, write_to_cache=False): - hashfilename = "%s/%04x.64khash" % (store, blocknumber) - try: - hash = base64.b16decode(open(hashfilename).read()) - assert len(hash) == 32 - return ("hash", hash) - except IOError: - pass +def get_merkle_hash_64k(store, blocknumber, write_to_cache=False, treesize=None): firstleaf = blocknumber * 65536 lastleaf = firstleaf + 65535 + if treesize != None: + assert firstleaf < treesize + usecache = lastleaf < treesize + lastleaf = min(lastleaf, treesize - 1) + else: + usecache = True + + hashfilename = "%s/%04x.64khash" % (store, blocknumber) + if usecache: + try: + hash = base64.b16decode(open(hashfilename).read()) + assert len(hash) == 32 + return ("hash", hash) + except IOError: + pass firstfile = firstleaf / 10000 lastfile = lastleaf / 10000 zipfiles = {} @@ -364,10 +383,81 @@ def get_merkle_hash_64k(store, blocknumber, write_to_cache=False): calculated_hash = tree[-1][0] for zf in zipfiles.values(): zf.close() - if len(layer0) != 65536: + if len(layer0) != lastleaf - firstleaf + 1: return ("incomplete", (len(layer0), calculated_hash)) if write_to_cache: f = open(hashfilename, "w") f.write(base64.b16encode(calculated_hash)) f.close() return ("hash", calculated_hash) + +def get_tree_head(store, treesize): + merkle_64klayer = [] + + for blocknumber in range(0, (treesize / 65536) + 1): + (resulttype, result) = get_merkle_hash_64k(store, blocknumber, treesize=treesize) + if resulttype == "incomplete": + print >>sys.stderr, "Couldn't read until tree size", treesize + (incompletelength, hash) = result + print >>sys.stderr, "Stopped at", blocknumber * 65536 + incompletelength + sys.exit(1) + assert resulttype == "hash" + hash = result + merkle_64klayer.append(hash) + #print >>sys.stderr, print blocknumber * 65536, + sys.stdout.flush() + tree = build_merkle_tree(merkle_64klayer) + calculated_root_hash = tree[-1][0] + return calculated_root_hash + +def get_intermediate_hash(store, treesize, level, index): + if level >= 16: + merkle_64klayer = [] + + levelsize = (2**(level-16)) + + for blocknumber in range(index * levelsize, (index + 1) * levelsize): + if blocknumber * (2 ** 16) >= treesize: + break + #print "looking at block", blocknumber + (resulttype, result) = get_merkle_hash_64k(store, blocknumber, treesize=treesize) + if resulttype == "incomplete": + print >>sys.stderr, "Couldn't read until tree size", treesize + (incompletelength, hash) = result + print >>sys.stderr, "Stopped at", blocknumber * 65536 + incompletelength + sys.exit(1) + assert resulttype == "hash" + hash = result + #print "block hash", base64.b16encode(hash) + merkle_64klayer.append(hash) + #print >>sys.stderr, print blocknumber * 65536, + sys.stdout.flush() + tree = build_merkle_tree(merkle_64klayer) + return tree[-1][0] + else: + levelsize = 2 ** level + firstleaf = index * levelsize + lastleaf = firstleaf + levelsize - 1 + #print "firstleaf", firstleaf + #print "lastleaf", lastleaf + assert firstleaf < treesize + lastleaf = min(lastleaf, treesize - 1) + #print "modified lastleaf", lastleaf + firstfile = firstleaf / 10000 + lastfile = lastleaf / 10000 + #print "files", firstfile, lastfile + zipfiles = {} + for i in range(firstfile, lastfile + 1): + try: + zipfiles[i] = zipfile.ZipFile("%s/%04d.zip" % (store, i)) + except IOError: + break + certs = get_certs_from_zipfiles(zipfiles, firstleaf, lastleaf) + layer0 = [get_hash_from_certfile(cert) for cert in certs] + #print "layer0", repr(layer0) + tree = build_merkle_tree(layer0) + calculated_hash = tree[-1][0] + for zf in zipfiles.values(): + zf.close() + assert len(layer0) == lastleaf - firstleaf + 1 + return calculated_hash -- cgit v1.1 From 74a4460cba73877830b73742be76cd2bf0d5f47b Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Thu, 19 Feb 2015 16:23:25 +0100 Subject: Added verification of consistency proofs --- tools/certtools.py | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++ tools/fetchallcerts.py | 6 ++++ 2 files changed, 95 insertions(+) (limited to 'tools') diff --git a/tools/certtools.py b/tools/certtools.py index 31045b9..5b556cf 100644 --- a/tools/certtools.py +++ b/tools/certtools.py @@ -461,3 +461,92 @@ def get_intermediate_hash(store, treesize, level, index): zf.close() assert len(layer0) == lastleaf - firstleaf + 1 return calculated_hash + +def bits(n): + p = 0 + while n > 0: + n >>= 1 + p += 1 + return p + +def merkle_height(n): + if n == 0: + return 1 + return bits(n - 1) + +def node_above((pathp, pathl), levels=1): + return (pathp >> levels, pathl + levels) + +def node_even((pathp, pathl)): + return pathp & 1 == 0 + +def node_odd((pathp, pathl)): + return pathp & 1 == 1 + +def node_lower((path1p, path1l), (path2p, path2l)): + return path1l < path2l + +def node_higher((path1p, path1l), (path2p, path2l)): + return path1l > path2l + +def node_level((path1p, path1l)): + return path1l + +def node_outside((path1p, path1l), (path2p, path2l)): + assert path1l == path2l + return path1p > path2p + +def combine_two_hashes((path1, hash1), (path2, hash2), treesize): + assert not node_higher(path1, path2) + edge_node = (treesize - 1, 0) + + if node_lower(path1, path2): + assert path1 == node_above(edge_node, levels=node_level(path1)) + while node_even(path1): + path1 = node_above(path1) + + assert node_above(path1) == node_above(path2) + assert (node_even(path1) and node_odd(path2)) or (node_odd(path1) and node_even(path2)) + + if node_outside(path2, node_above(edge_node, levels=node_level(path2))): + return (node_above(path1), hash1) + + if node_even(path1): + newhash = internal_hash((hash1, hash2)) + else: + newhash = internal_hash((hash2, hash1)) + + return (node_above(path1), newhash) + +def path_as_string(pos, level, treesize): + height = merkle_height(treesize) + path = "{0:0{width}b}".format(pos, width=height - level) + if height == level: + return "" + return path + +def nodes_for_pos(pos, treesize): + height = merkle_height(treesize) + nodes = [] + level = 0 + while pos > 0 and pos & 1 == 0: + pos >>= 1 + level += 1 + if pos & 1: + nodes.append((pos ^ 1, level)) + #print pos, level + while level < height: + pos_level0 = pos * (2 ** level) + #print pos, level + if pos_level0 < treesize: + nodes.append((pos, level)) + pos >>= 1 + pos ^= 1 + level += 1 + return nodes + +def verify_consistency_proof(consistency_proof, first, second): + chain = zip(nodes_for_pos(first, second), consistency_proof) + (_, hash) = reduce(lambda e1, e2: combine_two_hashes(e1, e2, second), chain) + (_, oldhash) = reduce(lambda e1, e2: combine_two_hashes(e1, e2, first), chain) + return (oldhash, hash) diff --git a/tools/fetchallcerts.py b/tools/fetchallcerts.py index 866bb43..39ffd64 100755 --- a/tools/fetchallcerts.py +++ b/tools/fetchallcerts.py @@ -168,6 +168,12 @@ else: print "calculated root hash", base64.b16encode(calculated_root_hash) +if oldsth and oldsth["tree_size"] > 0 and oldsth["tree_size"] != tree_size: + consistency_proof = [base64.decodestring(entry) for entry in get_consistency_proof(args.baseurl, oldsth["tree_size"], tree_size)] + (old_treehead, new_treehead) = verify_consistency_proof(consistency_proof, oldsth["tree_size"], tree_size) + assert old_treehead == base64.b64decode(oldsth["sha256_root_hash"]) + assert new_treehead == base64.b64decode(sth["sha256_root_hash"]) + if calculated_root_hash != root_hash: print "fetched root hash and calculated root hash different" sys.exit(1) -- cgit v1.1 From 8fa65d708aa1bb44609c2fdc755520e21328b175 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Thu, 19 Feb 2015 19:14:13 +0100 Subject: Move public keys to separate file --- tools/certkeys.py | 14 ++++++++++++++ tools/certtools.py | 15 +-------------- 2 files changed, 15 insertions(+), 14 deletions(-) create mode 100644 tools/certkeys.py (limited to 'tools') diff --git a/tools/certkeys.py b/tools/certkeys.py new file mode 100644 index 0000000..3c459e9 --- /dev/null +++ b/tools/certkeys.py @@ -0,0 +1,14 @@ + +publickeys = { + "https://ct.googleapis.com/pilot/": + "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfahLEimAoz2t01p3uMziiLOl/fHTD" + "M0YDOhBRuiBARsV4UvxG2LdNgoIGLrtCzWE0J5APC2em4JlvR8EEEFMoA==", + + "https://127.0.0.1:8080/": + "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE4qWq6afhBUi0OdcWUYhyJLNXTkGqQ9" + "PMS5lqoCgkV2h1ZvpNjBH2u8UbgcOQwqDo66z6BWQJGolozZYmNHE2kQ==", + + "https://flimsy.ct.nordu.net/": + "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE4qWq6afhBUi0OdcWUYhyJLNXTkGqQ9" + "PMS5lqoCgkV2h1ZvpNjBH2u8UbgcOQwqDo66z6BWQJGolozZYmNHE2kQ==", +} diff --git a/tools/certtools.py b/tools/certtools.py index 5b556cf..11e09bb 100644 --- a/tools/certtools.py +++ b/tools/certtools.py @@ -13,20 +13,7 @@ import ecdsa import datetime import cStringIO import zipfile - -publickeys = { - "https://ct.googleapis.com/pilot/": - "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfahLEimAoz2t01p3uMziiLOl/fHTD" - "M0YDOhBRuiBARsV4UvxG2LdNgoIGLrtCzWE0J5APC2em4JlvR8EEEFMoA==", - - "https://127.0.0.1:8080/": - "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE4qWq6afhBUi0OdcWUYhyJLNXTkGqQ9" - "PMS5lqoCgkV2h1ZvpNjBH2u8UbgcOQwqDo66z6BWQJGolozZYmNHE2kQ==", - - "https://flimsy.ct.nordu.net/": - "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE4qWq6afhBUi0OdcWUYhyJLNXTkGqQ9" - "PMS5lqoCgkV2h1ZvpNjBH2u8UbgcOQwqDo66z6BWQJGolozZYmNHE2kQ==", -} +from certkeys import publickeys def get_cert_info(s): p = subprocess.Popen( -- cgit v1.1 From 81a35a696d813ac5803afe602e549b56a9dfd9c4 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Fri, 20 Feb 2015 01:58:17 +0100 Subject: testcase1: Actually verify inclusion proof --- tools/certtools.py | 26 ++++++++++++++++++++++++-- tools/testcase1.py | 11 +++++++++-- 2 files changed, 33 insertions(+), 4 deletions(-) (limited to 'tools') diff --git a/tools/certtools.py b/tools/certtools.py index 11e09bb..fdff0e1 100644 --- a/tools/certtools.py +++ b/tools/certtools.py @@ -512,10 +512,11 @@ def path_as_string(pos, level, treesize): return "" return path -def nodes_for_pos(pos, treesize): +def nodes_for_subtree(subtreesize, treesize): height = merkle_height(treesize) nodes = [] level = 0 + pos = subtreesize while pos > 0 and pos & 1 == 0: pos >>= 1 level += 1 @@ -532,8 +533,29 @@ def nodes_for_pos(pos, treesize): level += 1 return nodes +def nodes_for_index(pos, treesize): + height = merkle_height(treesize) + nodes = [] + level = 0 + pos ^= 1 + #print pos, level + while level < height: + pos_level0 = pos * (2 ** level) + #print pos, level + if pos_level0 < treesize: + nodes.append((pos, level)) + pos >>= 1 + pos ^= 1 + level += 1 + return nodes + def verify_consistency_proof(consistency_proof, first, second): - chain = zip(nodes_for_pos(first, second), consistency_proof) + chain = zip(nodes_for_subtree(first, second), consistency_proof) (_, hash) = reduce(lambda e1, e2: combine_two_hashes(e1, e2, second), chain) (_, oldhash) = reduce(lambda e1, e2: combine_two_hashes(e1, e2, first), chain) return (oldhash, hash) + +def verify_inclusion_proof(inclusion_proof, index, treesize, leafhash): + chain = zip([(index, 0)] + nodes_for_index(index, treesize), [leafhash] + inclusion_proof) + (_, hash) = reduce(lambda e1, e2: combine_two_hashes(e1, e2, treesize), chain) + return hash diff --git a/tools/testcase1.py b/tools/testcase1.py index 415d475..ce322f1 100755 --- a/tools/testcase1.py +++ b/tools/testcase1.py @@ -84,8 +84,15 @@ def get_and_validate_proof(timestamp, chain, leaf_index, nentries): leaf_hash = get_leaf_hash(merkle_tree_leaf) sth = get_sth(baseurl) proof = get_proof_by_hash(baseurl, leaf_hash, sth["tree_size"]) - assert_equal(proof["leaf_index"], leaf_index, "leaf_index") - assert_equal(len(proof["audit_path"]), nentries, "audit_path length") + leaf_index = proof["leaf_index"] + inclusion_proof = [base64.b64decode(e) for e in proof["audit_path"]] + assert_equal(leaf_index, leaf_index, "leaf_index") + assert_equal(len(inclusion_proof), nentries, "audit_path length") + + calc_root_hash = verify_inclusion_proof(inclusion_proof, leaf_index, sth["tree_size"], leaf_hash) + root_hash = base64.b64decode(sth["sha256_root_hash"]) + + assert_equal(root_hash, calc_root_hash, "verified root hash", nodata=True) get_and_check_entry(timestamp, chain, leaf_index) def get_and_check_entry(timestamp, chain, leaf_index): -- cgit v1.1 From db418ce9f59dc2e2861fd7b2398f94c4faf509e9 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Sun, 22 Feb 2015 00:35:24 +0100 Subject: Added tool for drawing merkle trees and extracting node hashes --- tools/treeinfo.py | 123 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100755 tools/treeinfo.py (limited to 'tools') diff --git a/tools/treeinfo.py b/tools/treeinfo.py new file mode 100755 index 0000000..036aeb2 --- /dev/null +++ b/tools/treeinfo.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2014, NORDUnet A/S. +# See LICENSE for licensing information. + +import argparse +import urllib2 +import urllib +import json +import base64 +import sys +import struct +import hashlib +import itertools +from certtools import * +import zipfile +import os +import time + +parser = argparse.ArgumentParser(description='') +parser.add_argument('--store', default=None, metavar="dir", help='Certificates directory') + +parser.add_argument('--head', action='store_true', help="Calculate tree head") +parser.add_argument('--printnode', action='store_true', help="Print tree node") + +parser.add_argument('--treesize', type=int, default=None, metavar="treesize", help="Tree size") +parser.add_argument('--level', type=int, default=None, metavar="level", help="Level") +parser.add_argument('--index', type=int, default=None, metavar="index", help="Index") + +parser.add_argument('--follow', action='store_true', help="Follow upwards") + +parser.add_argument('--dot', default=None, metavar="file", help='Output data in dot format') + +args = parser.parse_args() + +def index_to_root(index, treesize, level=0): + path = (index, level) + height = merkle_height(treesize) + nodes = [] + while node_level(path) < height: + nodes.append(path) + path = node_above(path) + return nodes + +def set_tree_node(tree, level, index, value, overwrite=True): + if not overwrite and index in levels.setdefault(level, {}): + return + levels.setdefault(level, {})[index] = value + +def draw_path(tree, startlevel, startindex, treesize, colors): + height = merkle_height(treesize) + nodes = index_to_root(startindex, treesize, level=startlevel) + + for (index, level) in nodes: + if level == 0: + set_tree_node(tree, level, index, colors[0]) + else: + set_tree_node(tree, level, index, colors[1]) + index ^= 1 + levelsize = 2 ** level + firstleaf = index * levelsize + if firstleaf < treesize: + set_tree_node(tree, level, index, "", overwrite=False) + set_tree_node(tree, height, 0, colors[1]) + + +if args.head: + treehead = get_tree_head(args.store, args.treesize) + print base64.b16encode(treehead) +elif args.dot: + levels = {} + if args.index >= args.treesize: + sys.exit(1) + dotfile = open(args.dot, "w") + print >>dotfile, 'graph "" {' + print >>dotfile, 'ordering=out;' + print >>dotfile, 'node [style=filled];' + + height = merkle_height(args.treesize) + + draw_path(levels, 0, args.treesize - 1, args.treesize, ["0.600 0.500 0.900", "0.600 0.300 0.900"]) + + draw_path(levels, args.level, args.index, args.treesize, ["0.300 0.500 0.900", "0.300 0.200 0.900"]) + + for l in sorted(levels.keys(), reverse=True): + for i in sorted(levels[l].keys()): + print >>dotfile, "l%di%d [color=\"%s\" label=\"%s\"];" % (l, i, levels[l][i], path_as_string(i, l, args.treesize)) + if height != l: + print >>dotfile, "l%di%d -- l%di%d;" % (l + 1, i / 2, l, i) + if i & 1 == 0: + print >>dotfile, "ml%di%d [shape=point style=invis];" % (l, i) + print >>dotfile, "l%di%d -- ml%di%d [weight=100 style=invis];" % (l + 1, i / 2, l, i) + print >>dotfile, "}" + dotfile.close() +elif args.printnode: + index = args.index + level = args.level + if args.index >= args.treesize: + sys.exit(1) + height = merkle_height(args.treesize) + nodes = index_to_root(index, args.treesize, level=level) + + for (index, level) in nodes: + print level, index + if args.store: + print base64.b16encode(get_intermediate_hash(args.store, args.treesize, level, index)) + + if not args.follow: + sys.exit(0) + + index ^= 1 + levelsize = 2 ** level + + firstleaf = index * levelsize + if firstleaf < args.treesize: + print level, index + if args.store: + print base64.b16encode(get_intermediate_hash(args.store, args.treesize, level, index)) + + print height, 0 + if args.store: + print base64.b16encode(get_intermediate_hash(args.store, args.treesize, height, 0)) -- cgit v1.1 From bdfa89bcf0b8f65554baabda52b107a2ab36690a Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 23 Feb 2015 12:01:20 +0100 Subject: Add consistency proof checking to testcase1 Fix consistency proof checking when first size is power of 2 --- tools/certtools.py | 5 ++++- tools/fetchallcerts.py | 2 +- tools/testcase1.py | 24 +++++++++++++++++++++++- 3 files changed, 28 insertions(+), 3 deletions(-) (limited to 'tools') diff --git a/tools/certtools.py b/tools/certtools.py index fdff0e1..2fb1492 100644 --- a/tools/certtools.py +++ b/tools/certtools.py @@ -549,8 +549,11 @@ def nodes_for_index(pos, treesize): level += 1 return nodes -def verify_consistency_proof(consistency_proof, first, second): +def verify_consistency_proof(consistency_proof, first, second, oldhash_input): + if 2 ** bits(first - 1) == first: + consistency_proof = [oldhash_input] + consistency_proof chain = zip(nodes_for_subtree(first, second), consistency_proof) + assert len(nodes_for_subtree(first, second)) == len(consistency_proof) (_, hash) = reduce(lambda e1, e2: combine_two_hashes(e1, e2, second), chain) (_, oldhash) = reduce(lambda e1, e2: combine_two_hashes(e1, e2, first), chain) return (oldhash, hash) diff --git a/tools/fetchallcerts.py b/tools/fetchallcerts.py index 39ffd64..398c563 100755 --- a/tools/fetchallcerts.py +++ b/tools/fetchallcerts.py @@ -170,7 +170,7 @@ print "calculated root hash", base64.b16encode(calculated_root_hash) if oldsth and oldsth["tree_size"] > 0 and oldsth["tree_size"] != tree_size: consistency_proof = [base64.decodestring(entry) for entry in get_consistency_proof(args.baseurl, oldsth["tree_size"], tree_size)] - (old_treehead, new_treehead) = verify_consistency_proof(consistency_proof, oldsth["tree_size"], tree_size) + (old_treehead, new_treehead) = verify_consistency_proof(consistency_proof, oldsth["tree_size"], tree_size, base64.b64decode(oldsth["sha256_root_hash"])) assert old_treehead == base64.b64decode(oldsth["sha256_root_hash"]) assert new_treehead == base64.b64decode(sth["sha256_root_hash"]) diff --git a/tools/testcase1.py b/tools/testcase1.py index ce322f1..a41a783 100755 --- a/tools/testcase1.py +++ b/tools/testcase1.py @@ -95,6 +95,15 @@ def get_and_validate_proof(timestamp, chain, leaf_index, nentries): assert_equal(root_hash, calc_root_hash, "verified root hash", nodata=True) get_and_check_entry(timestamp, chain, leaf_index) +def get_and_validate_consistency_proof(sth1, sth2, size1, size2): + consistency_proof = [base64.decodestring(entry) for entry in get_consistency_proof(baseurl, size1, size2)] + (old_treehead, new_treehead) = verify_consistency_proof(consistency_proof, size1, size2, sth1) + #print repr(sth1), repr(old_treehead) + #print repr(sth2), repr(new_treehead) + assert_equal(old_treehead, sth1, "sth1", nodata=True) + assert_equal(new_treehead, sth2, "sth2", nodata=True) + + def get_and_check_entry(timestamp, chain, leaf_index): entries = get_entries(baseurl, leaf_index, leaf_index) assert_equal(len(entries), 1, "get_entries", quiet=True) @@ -118,7 +127,6 @@ def get_and_check_entry(timestamp, chain, leaf_index): print_success("fetched chain has an appended root cert") else: print_error("fetched chain has an extra entry") - failures += 1 elif len(certchain) == len(submittedcertchain): print_success("cert chains are the same length") else: @@ -142,7 +150,10 @@ result1 = do_add_chain(cc1) mergeresult = merge() assert_equal(mergeresult, 0, "merge", quiet=True) +size_sth = {} + print_and_check_tree_size(1) +size_sth[1] = base64.b64decode(get_sth(baseurl)["sha256_root_hash"]) result2 = do_add_chain(cc1) @@ -152,6 +163,9 @@ mergeresult = merge() assert_equal(mergeresult, 0, "merge", quiet=True) print_and_check_tree_size(1) +size1_v2_sth = base64.b64decode(get_sth(baseurl)["sha256_root_hash"]) + +assert_equal(size_sth[1], size1_v2_sth, "sth", nodata=True) # TODO: add invalid cert and check that it generates an error # and that treesize still is 1 @@ -166,6 +180,7 @@ mergeresult = merge() assert_equal(mergeresult, 0, "merge", quiet=True) print_and_check_tree_size(2) +size_sth[2] = base64.b64decode(get_sth(baseurl)["sha256_root_hash"]) get_and_validate_proof(result1["timestamp"], cc1, 0, 1) get_and_validate_proof(result3["timestamp"], cc2, 1, 1) @@ -178,6 +193,7 @@ mergeresult = merge() assert_equal(mergeresult, 0, "merge", quiet=True) print_and_check_tree_size(3) +size_sth[3] = base64.b64decode(get_sth(baseurl)["sha256_root_hash"]) get_and_validate_proof(result1["timestamp"], cc1, 0, 2) get_and_validate_proof(result3["timestamp"], cc2, 1, 2) @@ -191,6 +207,7 @@ mergeresult = merge() assert_equal(mergeresult, 0, "merge", quiet=True) print_and_check_tree_size(4) +size_sth[4] = base64.b64decode(get_sth(baseurl)["sha256_root_hash"]) get_and_validate_proof(result1["timestamp"], cc1, 0, 2) get_and_validate_proof(result3["timestamp"], cc2, 1, 2) @@ -205,6 +222,7 @@ mergeresult = merge() assert_equal(mergeresult, 0, "merge", quiet=True) print_and_check_tree_size(5) +size_sth[5] = base64.b64decode(get_sth(baseurl)["sha256_root_hash"]) get_and_validate_proof(result1["timestamp"], cc1, 0, 3) get_and_validate_proof(result3["timestamp"], cc2, 1, 3) @@ -212,6 +230,10 @@ get_and_validate_proof(result4["timestamp"], cc3, 2, 3) get_and_validate_proof(result5["timestamp"], cc4, 3, 3) get_and_validate_proof(result6["timestamp"], cc5, 4, 1) +for first_size in range(1, 5): + for second_size in range(first_size + 1, 6): + get_and_validate_consistency_proof(size_sth[first_size], size_sth[second_size], first_size, second_size) + print "-------" if failures: print failures, "failed tests" if failures != 1 else "failed test" -- cgit v1.1 From f3042ca10909fa6d97e30d1226072e158cf04161 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 23 Feb 2015 12:37:35 +0100 Subject: Add saving and verification of SCT to python tools --- tools/submitcert.py | 62 ++++++++++++++++++++++---------- tools/verifysct.py | 101 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 19 deletions(-) create mode 100755 tools/verifysct.py (limited to 'tools') diff --git a/tools/submitcert.py b/tools/submitcert.py index 04b6ebe..9f0be67 100755 --- a/tools/submitcert.py +++ b/tools/submitcert.py @@ -3,6 +3,7 @@ # Copyright (c) 2014, NORDUnet A/S. # See LICENSE for licensing information. +import argparse import urllib2 import urllib import json @@ -17,19 +18,29 @@ import signal import select import zipfile +parser = argparse.ArgumentParser(description='') +parser.add_argument('baseurl', help="Base URL for CT server") +parser.add_argument('--store', default=None, metavar="dir", help='Get certificates from directory dir') +parser.add_argument('--sct-file', default=None, metavar="file", help='Store SCT:s in file') +parser.add_argument('--parallel', type=int, default=16, metavar="n", help="Number of parallel submits") +parser.add_argument('--check-sct', action='store_true', help="Check SCT signature") +parser.add_argument('--pre-warm', action='store_true', help="Wait 3 seconds after first submit") +args = parser.parse_args() + from multiprocessing import Pool -baseurl = sys.argv[1] -certfilepath = sys.argv[2] +baseurl = args.baseurl +certfilepath = args.store lookup_in_log = False -check_sig = False if certfilepath[-1] == "/": - certfiles = [certfilepath + filename for filename in sorted(os.listdir(certfilepath))] + certfiles = [certfilepath + filename for filename in sorted(os.listdir(certfilepath)) if os.path.isfile(certfilepath + filename)] else: certfiles = [certfilepath] +sth = get_sth(baseurl) + def submitcert((certfile, cert)): timing = timing_point() certchain = get_certs_from_string(cert) @@ -40,27 +51,27 @@ def submitcert((certfile, cert)): except SystemExit: print "EXIT:", certfile select.select([], [], [], 1.0) - return None + return (None, None) timing_point(timing, "addchain") if result == None: print "ERROR for certfile", certfile - return timing["deltatimes"] + return (None, timing["deltatimes"]) try: - if check_sig: + if args.check_sct: check_sct_signature(baseurl, certchain[0], result) timing_point(timing, "checksig") except AssertionError, e: print "ERROR:", certfile, e - return None + return (None, None) except urllib2.HTTPError, e: print "ERROR:", certfile, e - return None + return (None, None) except ecdsa.keys.BadSignatureError, e: print "ERROR: bad signature", certfile - return None + return (None, None) if lookup_in_log: @@ -68,8 +79,6 @@ def submitcert((certfile, cert)): leaf_hash = get_leaf_hash(merkle_tree_leaf) - sth = get_sth(baseurl) - proof = get_proof_by_hash(baseurl, leaf_hash, sth["tree_size"]) leaf_index = proof["leaf_index"] @@ -104,7 +113,7 @@ def submitcert((certfile, cert)): print "and submitted chain has length", len(submittedcertchain) timing_point(timing, "lookup") - return timing["deltatimes"] + return ((certchain[0], result), timing["deltatimes"]) def get_ncerts(certfiles): n = 0 @@ -127,32 +136,47 @@ def get_all_certificates(certfiles): else: yield (certfile, open(certfile).read()) -p = Pool(16, lambda: signal.signal(signal.SIGINT, signal.SIG_IGN)) +def save_sct(sct, sth): + sctlog = open(args.sct_file, "a") + json.dump({"leafcert": base64.b64encode(leafcert), "sct": sct, "sth": sth}, sctlog) + sctlog.write("\n") + sctlog.close() + +p = Pool(args.parallel, lambda: signal.signal(signal.SIGINT, signal.SIG_IGN)) nsubmitted = 0 lastprinted = 0 +print "listing certs" ncerts = get_ncerts(certfiles) print ncerts, "certs" certs = get_all_certificates(certfiles) -submitcert(certs.next()) -nsubmitted += 1 -select.select([], [], [], 3.0) +(result, timing) = submitcert(certs.next()) +if result != None: + nsubmitted += 1 + (leafcert, sct) = result + save_sct(sct, sth) + +if args.pre_warm: + select.select([], [], [], 3.0) starttime = datetime.datetime.now() try: - for timing in p.imap_unordered(submitcert, certs): + for result, timing in p.imap_unordered(submitcert, certs): if timing == None: print "error" print "submitted", nsubmitted p.terminate() p.join() sys.exit(1) - nsubmitted += 1 + if result != None: + nsubmitted += 1 + (leafcert, sct) = result + save_sct(sct, sth) deltatime = datetime.datetime.now() - starttime deltatime_f = deltatime.seconds + deltatime.microseconds / 1000000.0 rate = nsubmitted / deltatime_f diff --git a/tools/verifysct.py b/tools/verifysct.py new file mode 100755 index 0000000..290d471 --- /dev/null +++ b/tools/verifysct.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python + +# Copyright (c) 2014, NORDUnet A/S. +# See LICENSE for licensing information. + +import argparse +import urllib2 +import urllib +import json +import base64 +import sys +import struct +import hashlib +import itertools +from certtools import * +import os +import signal +import select +import zipfile + +parser = argparse.ArgumentParser(description='') +parser.add_argument('baseurl', help="Base URL for CT server") +parser.add_argument('--sct-file', default=None, metavar="dir", help='SCT:s to verify') +parser.add_argument('--parallel', type=int, default=16, metavar="n", help="Number of parallel verifications") +args = parser.parse_args() + +from multiprocessing import Pool + +baseurl = args.baseurl + +def verifysct(sctentry): + timing = timing_point() + + leafcert = base64.b64decode(sctentry["leafcert"]) + try: + check_sct_signature(baseurl, leafcert, sctentry["sct"]) + timing_point(timing, "checksig") + except AssertionError, e: + print "ERROR:", e + return (None, None) + except urllib2.HTTPError, e: + print "ERROR:", e + return (None, None) + except ecdsa.keys.BadSignatureError, e: + print "ERROR: bad signature" + return (None, None) + + merkle_tree_leaf = pack_mtl(sctentry["sct"]["timestamp"], leafcert) + + leaf_hash = get_leaf_hash(merkle_tree_leaf) + + proof = get_proof_by_hash(baseurl, leaf_hash, sctentry["sth"]["tree_size"]) + + #print proof + + leaf_index = proof["leaf_index"] + inclusion_proof = [base64.b64decode(e) for e in proof["audit_path"]] + + calc_root_hash = verify_inclusion_proof(inclusion_proof, leaf_index, sctentry["sth"]["tree_size"], leaf_hash) + + root_hash = base64.b64decode(sctentry["sth"]["sha256_root_hash"]) + if root_hash != calc_root_hash: + print "sth" + print base64.b16encode(root_hash) + print base64.b16encode(calc_root_hash) + assert root_hash == calc_root_hash + + timing_point(timing, "lookup") + return (True, timing["deltatimes"]) + +p = Pool(args.parallel, lambda: signal.signal(signal.SIGINT, signal.SIG_IGN)) + +sctfile = open(args.sct_file) +scts = [json.loads(row) for row in sctfile] + +nverified = 0 +lastprinted = 0 + +starttime = datetime.datetime.now() + +try: + for result, timing in p.imap_unordered(verifysct, scts): + if timing == None: + print "error" + print "verified", nverified + p.terminate() + p.join() + sys.exit(1) + if result != None: + nverified += 1 + deltatime = datetime.datetime.now() - starttime + deltatime_f = deltatime.seconds + deltatime.microseconds / 1000000.0 + rate = nverified / deltatime_f + if nverified > lastprinted + 100: + print nverified, "rate %.1f" % rate + lastprinted = nverified + #print timing, "rate %.1f" % rate + print "verified", nverified +except KeyboardInterrupt: + p.terminate() + p.join() -- cgit v1.1 From 777d5897ba93840675df2beb8737b08c2a551036 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 23 Feb 2015 13:14:21 +0100 Subject: verifysct.py: Fetch fresh STH instead of using old --- tools/verifysct.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'tools') diff --git a/tools/verifysct.py b/tools/verifysct.py index 290d471..699a0ad 100755 --- a/tools/verifysct.py +++ b/tools/verifysct.py @@ -28,6 +28,8 @@ from multiprocessing import Pool baseurl = args.baseurl +sth = get_sth(baseurl) + def verifysct(sctentry): timing = timing_point() @@ -49,16 +51,19 @@ def verifysct(sctentry): leaf_hash = get_leaf_hash(merkle_tree_leaf) - proof = get_proof_by_hash(baseurl, leaf_hash, sctentry["sth"]["tree_size"]) + try: + proof = get_proof_by_hash(baseurl, leaf_hash, sth["tree_size"]) + except SystemExit: + return (None, None) #print proof leaf_index = proof["leaf_index"] inclusion_proof = [base64.b64decode(e) for e in proof["audit_path"]] - calc_root_hash = verify_inclusion_proof(inclusion_proof, leaf_index, sctentry["sth"]["tree_size"], leaf_hash) + calc_root_hash = verify_inclusion_proof(inclusion_proof, leaf_index, sth["tree_size"], leaf_hash) - root_hash = base64.b64decode(sctentry["sth"]["sha256_root_hash"]) + root_hash = base64.b64decode(sth["sha256_root_hash"]) if root_hash != calc_root_hash: print "sth" print base64.b16encode(root_hash) -- cgit v1.1 From 90bd73177964246a0e1a5d6c5e4255dcc8ec700d Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Fri, 27 Feb 2015 13:53:32 +0100 Subject: Require authentication for merge calls --- tools/merge.py | 42 ++++++++++++++++++++++++++++++++---------- tools/testcase1.py | 2 +- 2 files changed, 33 insertions(+), 11 deletions(-) (limited to 'tools') diff --git a/tools/merge.py b/tools/merge.py index f4a007d..6becf7e 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -11,6 +11,9 @@ import urllib import urllib2 import sys import time +import ecdsa +import hashlib +import urlparse from certtools import build_merkle_tree, create_sth_signature, check_sth_signature, get_eckey_from_file, timing_point parser = argparse.ArgumentParser(description="") @@ -19,6 +22,8 @@ parser.add_argument("--frontend", action="append", metavar="url", help="Base URL parser.add_argument("--storage", action="append", metavar="url", help="Base URL for storage server", required=True) parser.add_argument("--mergedb", metavar="dir", help="Merge database directory", required=True) parser.add_argument("--keyfile", metavar="keyfile", help="File containing log key", required=True) +parser.add_argument("--own-keyname", metavar="keyname", help="The key name of the merge node", required=True) +parser.add_argument("--own-keyfile", metavar="keyfile", help="The file containing the private key of the merge node", required=True) parser.add_argument("--nomerge", action='store_true', help="Don't actually do merge") args = parser.parse_args() @@ -52,9 +57,26 @@ def add_to_logorder(key): f.write(base64.b16encode(key) + "\n") f.close() +def http_request(url, data=None): + req = urllib2.Request(url, data) + keyname = args.own_keyname + privatekey = get_eckey_from_file(args.own_keyfile) + sk = ecdsa.SigningKey.from_der(privatekey) + parsed_url = urlparse.urlparse(url) + if data == None: + data = parsed_url.query + method = "GET" + else: + method = "POST" + signature = sk.sign("%s\0%s\0%s" % (method, parsed_url.path, data), hashfunc=hashlib.sha256, + sigencode=ecdsa.util.sigencode_der) + req.add_header('X-Catlfish-Auth', base64.b64encode(signature) + ";key=" + keyname) + result = urllib2.urlopen(req).read() + return result + def get_new_entries(baseurl): try: - result = urllib2.urlopen(baseurl + "ct/storage/fetchnewentries").read() + result = http_request(baseurl + "ct/storage/fetchnewentries") parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": return [base64.b64decode(entry) for entry in parsed_result[u"entries"]] @@ -67,7 +89,7 @@ def get_new_entries(baseurl): def get_entries(baseurl, hashes): try: params = urllib.urlencode({"hash":[base64.b64encode(hash) for hash in hashes]}, doseq=True) - result = urllib2.urlopen(baseurl + "ct/storage/getentry?" + params).read() + result = http_request(baseurl + "ct/storage/getentry?" + params) parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": entries = dict([(base64.b64decode(entry["hash"]), base64.b64decode(entry["entry"])) for entry in parsed_result[u"entries"]]) @@ -82,7 +104,7 @@ def get_entries(baseurl, hashes): def get_curpos(baseurl): try: - result = urllib2.urlopen(baseurl + "ct/frontend/currentposition").read() + result = http_request(baseurl + "ct/frontend/currentposition") parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": return parsed_result[u"position"] @@ -94,8 +116,8 @@ def get_curpos(baseurl): def sendlog(baseurl, submission): try: - result = urllib2.urlopen(baseurl + "ct/frontend/sendlog", - json.dumps(submission)).read() + result = http_request(baseurl + "ct/frontend/sendlog", + json.dumps(submission)) return json.loads(result) except urllib2.HTTPError, e: print "ERROR: sendlog", e.read() @@ -110,8 +132,8 @@ def sendlog(baseurl, submission): def sendentry(baseurl, entry, hash): try: - result = urllib2.urlopen(baseurl + "ct/frontend/sendentry", - json.dumps({"entry":base64.b64encode(entry), "treeleafhash":base64.b64encode(hash)})).read() + result = http_request(baseurl + "ct/frontend/sendentry", + json.dumps({"entry":base64.b64encode(entry), "treeleafhash":base64.b64encode(hash)})) return json.loads(result) except urllib2.HTTPError, e: print "ERROR: sendentry", e.read() @@ -126,8 +148,8 @@ def sendentry(baseurl, entry, hash): def sendsth(baseurl, submission): try: - result = urllib2.urlopen(baseurl + "ct/frontend/sendsth", - json.dumps(submission)).read() + result = http_request(baseurl + "ct/frontend/sendsth", + json.dumps(submission)) return json.loads(result) except urllib2.HTTPError, e: print "ERROR: sendsth", e.read() @@ -142,7 +164,7 @@ def sendsth(baseurl, submission): def get_missingentries(baseurl): try: - result = urllib2.urlopen(baseurl + "ct/frontend/missingentries").read() + result = http_request(baseurl + "ct/frontend/missingentries") parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": return parsed_result[u"entries"] diff --git a/tools/testcase1.py b/tools/testcase1.py index a41a783..c87e8eb 100755 --- a/tools/testcase1.py +++ b/tools/testcase1.py @@ -136,7 +136,7 @@ def get_and_check_entry(timestamp, chain, leaf_index): len(submittedcertchain)) def merge(): - return subprocess.call(["./merge.py", "--baseurl", "https://127.0.0.1:8080/", "--frontend", "https://127.0.0.1:8082/", "--storage", "https://127.0.0.1:8081/", "--mergedb", "../rel/mergedb", "--keyfile", "../rel/test/eckey.pem"]) + return subprocess.call(["./merge.py", "--baseurl", "https://127.0.0.1:8080/", "--frontend", "https://127.0.0.1:8082/", "--storage", "https://127.0.0.1:8081/", "--mergedb", "../rel/mergedb", "--keyfile", "../rel/test/eckey.pem", "--own-keyname", "merge-1", "--own-keyfile", "../rel/privatekeys/merge-1-private.pem"]) print_and_check_tree_size(0) -- cgit v1.1 From f87aba6cf765c3ba3f6b4cc097c63b567966de85 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Fri, 27 Feb 2015 15:49:48 +0100 Subject: Improve tests-start and tests-stop --- tools/halt.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 tools/halt.py (limited to 'tools') diff --git a/tools/halt.py b/tools/halt.py new file mode 100755 index 0000000..cfbf14e --- /dev/null +++ b/tools/halt.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2014, NORDUnet A/S. +# See LICENSE for licensing information. + +import argparse +import subprocess +import sys + +parser = argparse.ArgumentParser(description='') +parser.add_argument('toerl') +parser.add_argument('nodedir') +args = parser.parse_args() + +p = subprocess.Popen( + [args.toerl, args.nodedir], + stdin=subprocess.PIPE) +p.communicate("halt().\n") -- cgit v1.1 From 4e1bcab3f91f975a19710a4350bbee0e9af5168e Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Tue, 3 Mar 2015 14:03:05 +0100 Subject: Move http_request to certtools --- tools/certtools.py | 18 ++++++++++++++++++ tools/merge.py | 35 ++++++++++------------------------- 2 files changed, 28 insertions(+), 25 deletions(-) (limited to 'tools') diff --git a/tools/certtools.py b/tools/certtools.py index 2fb1492..ad90e5c 100644 --- a/tools/certtools.py +++ b/tools/certtools.py @@ -6,6 +6,7 @@ import json import base64 import urllib import urllib2 +import urlparse import struct import sys import hashlib @@ -182,6 +183,23 @@ def check_signature(baseurl, signature, data): vk.verify(unpacked_signature, data, hashfunc=hashlib.sha256, sigdecode=ecdsa.util.sigdecode_der) +def http_request(url, data=None, key=None): + req = urllib2.Request(url, data) + (keyname, keyfile) = key + privatekey = get_eckey_from_file(keyfile) + sk = ecdsa.SigningKey.from_der(privatekey) + parsed_url = urlparse.urlparse(url) + if data == None: + data = parsed_url.query + method = "GET" + else: + method = "POST" + signature = sk.sign("%s\0%s\0%s" % (method, parsed_url.path, data), hashfunc=hashlib.sha256, + sigencode=ecdsa.util.sigencode_der) + req.add_header('X-Catlfish-Auth', base64.b64encode(signature) + ";key=" + keyname) + result = urllib2.urlopen(req).read() + return result + def create_signature(privatekey, data): sk = ecdsa.SigningKey.from_der(privatekey) unpacked_signature = sk.sign(data, hashfunc=hashlib.sha256, diff --git a/tools/merge.py b/tools/merge.py index 6becf7e..c9f99af 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -14,7 +14,7 @@ import time import ecdsa import hashlib import urlparse -from certtools import build_merkle_tree, create_sth_signature, check_sth_signature, get_eckey_from_file, timing_point +from certtools import build_merkle_tree, create_sth_signature, check_sth_signature, get_eckey_from_file, timing_point, http_request parser = argparse.ArgumentParser(description="") parser.add_argument("--baseurl", metavar="url", help="Base URL for CT server", required=True) @@ -34,6 +34,8 @@ storagenodes = args.storage chainsdir = args.mergedb + "/chains" logorderfile = args.mergedb + "/logorder" +own_key = (args.own_keyname, args.own_keyfile) + def parselogrow(row): return base64.b16decode(row) @@ -57,26 +59,9 @@ def add_to_logorder(key): f.write(base64.b16encode(key) + "\n") f.close() -def http_request(url, data=None): - req = urllib2.Request(url, data) - keyname = args.own_keyname - privatekey = get_eckey_from_file(args.own_keyfile) - sk = ecdsa.SigningKey.from_der(privatekey) - parsed_url = urlparse.urlparse(url) - if data == None: - data = parsed_url.query - method = "GET" - else: - method = "POST" - signature = sk.sign("%s\0%s\0%s" % (method, parsed_url.path, data), hashfunc=hashlib.sha256, - sigencode=ecdsa.util.sigencode_der) - req.add_header('X-Catlfish-Auth', base64.b64encode(signature) + ";key=" + keyname) - result = urllib2.urlopen(req).read() - return result - def get_new_entries(baseurl): try: - result = http_request(baseurl + "ct/storage/fetchnewentries") + result = http_request(baseurl + "ct/storage/fetchnewentries", key=own_key) parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": return [base64.b64decode(entry) for entry in parsed_result[u"entries"]] @@ -89,7 +74,7 @@ def get_new_entries(baseurl): def get_entries(baseurl, hashes): try: params = urllib.urlencode({"hash":[base64.b64encode(hash) for hash in hashes]}, doseq=True) - result = http_request(baseurl + "ct/storage/getentry?" + params) + result = http_request(baseurl + "ct/storage/getentry?" + params, key=own_key) parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": entries = dict([(base64.b64decode(entry["hash"]), base64.b64decode(entry["entry"])) for entry in parsed_result[u"entries"]]) @@ -104,7 +89,7 @@ def get_entries(baseurl, hashes): def get_curpos(baseurl): try: - result = http_request(baseurl + "ct/frontend/currentposition") + result = http_request(baseurl + "ct/frontend/currentposition", key=own_key) parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": return parsed_result[u"position"] @@ -117,7 +102,7 @@ def get_curpos(baseurl): def sendlog(baseurl, submission): try: result = http_request(baseurl + "ct/frontend/sendlog", - json.dumps(submission)) + json.dumps(submission), key=own_key) return json.loads(result) except urllib2.HTTPError, e: print "ERROR: sendlog", e.read() @@ -133,7 +118,7 @@ def sendlog(baseurl, submission): def sendentry(baseurl, entry, hash): try: result = http_request(baseurl + "ct/frontend/sendentry", - json.dumps({"entry":base64.b64encode(entry), "treeleafhash":base64.b64encode(hash)})) + json.dumps({"entry":base64.b64encode(entry), "treeleafhash":base64.b64encode(hash)}), key=own_key) return json.loads(result) except urllib2.HTTPError, e: print "ERROR: sendentry", e.read() @@ -149,7 +134,7 @@ def sendentry(baseurl, entry, hash): def sendsth(baseurl, submission): try: result = http_request(baseurl + "ct/frontend/sendsth", - json.dumps(submission)) + json.dumps(submission), key=own_key) return json.loads(result) except urllib2.HTTPError, e: print "ERROR: sendsth", e.read() @@ -164,7 +149,7 @@ def sendsth(baseurl, submission): def get_missingentries(baseurl): try: - result = http_request(baseurl + "ct/frontend/missingentries") + result = http_request(baseurl + "ct/frontend/missingentries", key=own_key) parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": return parsed_result[u"entries"] -- cgit v1.1 From ff18e0fdd57a6b485f427173fe7febee03345037 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Tue, 3 Mar 2015 15:33:39 +0100 Subject: merge.py: use external signing --- tools/certtools.py | 20 ++++++++++++++------ tools/merge.py | 5 ++--- tools/testcase1.py | 2 +- 3 files changed, 17 insertions(+), 10 deletions(-) (limited to 'tools') diff --git a/tools/certtools.py b/tools/certtools.py index ad90e5c..222497f 100644 --- a/tools/certtools.py +++ b/tools/certtools.py @@ -200,10 +200,18 @@ def http_request(url, data=None, key=None): result = urllib2.urlopen(req).read() return result -def create_signature(privatekey, data): - sk = ecdsa.SigningKey.from_der(privatekey) - unpacked_signature = sk.sign(data, hashfunc=hashlib.sha256, - sigencode=ecdsa.util.sigencode_der) +def get_signature(baseurl, data, key=None): + try: + params = json.dumps({"plop_version":1, "data": base64.b64encode(data)}) + result = http_request(baseurl + "ct/signing/sth", params, key=key) + parsed_result = json.loads(result) + return base64.b64decode(parsed_result.get(u"result")) + except urllib2.HTTPError, e: + print "ERROR: get_signature", e.read() + sys.exit(1) + +def create_signature(baseurl, data, key=None): + unpacked_signature = get_signature(baseurl, data, key) return encode_signature(4, 3, unpacked_signature) def check_sth_signature(baseurl, sth): @@ -218,14 +226,14 @@ def check_sth_signature(baseurl, sth): check_signature(baseurl, signature, tree_head) -def create_sth_signature(tree_size, timestamp, root_hash, privatekey): +def create_sth_signature(tree_size, timestamp, root_hash, baseurl, key=None): version = struct.pack(">b", 0) signature_type = struct.pack(">b", 1) timestamp_packed = struct.pack(">Q", timestamp) tree_size_packed = struct.pack(">Q", tree_size) tree_head = version + signature_type + timestamp_packed + tree_size_packed + root_hash - return create_signature(privatekey, tree_head) + return create_signature(baseurl, tree_head, key=key) def check_sct_signature(baseurl, leafcert, sct): publickey = base64.decodestring(publickeys[baseurl]) diff --git a/tools/merge.py b/tools/merge.py index c9f99af..0996ec9 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -21,7 +21,7 @@ parser.add_argument("--baseurl", metavar="url", help="Base URL for CT server", r parser.add_argument("--frontend", action="append", metavar="url", help="Base URL for frontend server", required=True) parser.add_argument("--storage", action="append", metavar="url", help="Base URL for storage server", required=True) parser.add_argument("--mergedb", metavar="dir", help="Merge database directory", required=True) -parser.add_argument("--keyfile", metavar="keyfile", help="File containing log key", required=True) +parser.add_argument("--signing", metavar="url", help="Base URL for signing server", required=True) parser.add_argument("--own-keyname", metavar="keyname", help="The key name of the merge node", required=True) parser.add_argument("--own-keyfile", metavar="keyfile", help="The file containing the private key of the merge node", required=True) parser.add_argument("--nomerge", action='store_true', help="Don't actually do merge") @@ -215,10 +215,9 @@ tree = build_merkle_tree(logorder) tree_size = len(logorder) root_hash = tree[-1][0] timestamp = int(time.time() * 1000) -privatekey = get_eckey_from_file(args.keyfile) tree_head_signature = create_sth_signature(tree_size, timestamp, - root_hash, privatekey) + root_hash, args.signing, key=own_key) sth = {"tree_size": tree_size, "timestamp": timestamp, "sha256_root_hash": base64.b64encode(root_hash), diff --git a/tools/testcase1.py b/tools/testcase1.py index c87e8eb..0c0f728 100755 --- a/tools/testcase1.py +++ b/tools/testcase1.py @@ -136,7 +136,7 @@ def get_and_check_entry(timestamp, chain, leaf_index): len(submittedcertchain)) def merge(): - return subprocess.call(["./merge.py", "--baseurl", "https://127.0.0.1:8080/", "--frontend", "https://127.0.0.1:8082/", "--storage", "https://127.0.0.1:8081/", "--mergedb", "../rel/mergedb", "--keyfile", "../rel/test/eckey.pem", "--own-keyname", "merge-1", "--own-keyfile", "../rel/privatekeys/merge-1-private.pem"]) + return subprocess.call(["./merge.py", "--baseurl", "https://127.0.0.1:8080/", "--frontend", "https://127.0.0.1:8082/", "--storage", "https://127.0.0.1:8081/", "--mergedb", "../rel/mergedb", "--signing", "https://127.0.0.1:8088/", "--own-keyname", "merge-1", "--own-keyfile", "../rel/privatekeys/merge-1-private.pem"]) print_and_check_tree_size(0) -- cgit v1.1 From e0f11a58033d52c70bc76b4b5611cb88485d4653 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Wed, 4 Mar 2015 15:42:59 +0100 Subject: Save STH instead of calculating a new one each time. --- tools/merge.py | 15 ++++++++++++--- tools/testcase1.py | 7 +++++-- 2 files changed, 17 insertions(+), 5 deletions(-) (limited to 'tools') diff --git a/tools/merge.py b/tools/merge.py index 0996ec9..c137f4b 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -239,7 +239,10 @@ for frontendnode in frontendnodes: print "current position", curpos entries = [base64.b64encode(entry) for entry in logorder[curpos:]] for chunk in chunks(entries, 1000): - sendlog(frontendnode, {"start": curpos, "hashes": chunk}) + sendlogresult = sendlog(frontendnode, {"start": curpos, "hashes": chunk}) + if sendlogresult["result"] != "ok": + print "sendlog:", sendlogresult + sys.exit(1) curpos += len(chunk) print curpos, sys.stdout.flush() @@ -250,8 +253,14 @@ for frontendnode in frontendnodes: print "missing entries:", len(missingentries) for missingentry in missingentries: hash = base64.b64decode(missingentry) - sendentry(frontendnode, read_chain(hash), hash) + sendentryresult = sendentry(frontendnode, read_chain(hash), hash) + if sendentryresult["result"] != "ok": + print "send sth:", sendentryresult + sys.exit(1) timing_point(timing, "send missing") - sendsth(frontendnode, sth) + sendsthresult = sendsth(frontendnode, sth) + if sendsthresult["result"] != "ok": + print "send sth:", sendsthresult + sys.exit(1) timing_point(timing, "send sth") print timing["deltatimes"] diff --git a/tools/testcase1.py b/tools/testcase1.py index 0c0f728..73613fb 100755 --- a/tools/testcase1.py +++ b/tools/testcase1.py @@ -138,11 +138,11 @@ def get_and_check_entry(timestamp, chain, leaf_index): def merge(): return subprocess.call(["./merge.py", "--baseurl", "https://127.0.0.1:8080/", "--frontend", "https://127.0.0.1:8082/", "--storage", "https://127.0.0.1:8081/", "--mergedb", "../rel/mergedb", "--signing", "https://127.0.0.1:8088/", "--own-keyname", "merge-1", "--own-keyfile", "../rel/privatekeys/merge-1-private.pem"]) -print_and_check_tree_size(0) - mergeresult = merge() assert_equal(mergeresult, 0, "merge", quiet=True) +print_and_check_tree_size(0) + testgroup("cert1") result1 = do_add_chain(cc1) @@ -230,6 +230,9 @@ get_and_validate_proof(result4["timestamp"], cc3, 2, 3) get_and_validate_proof(result5["timestamp"], cc4, 3, 3) get_and_validate_proof(result6["timestamp"], cc5, 4, 1) +mergeresult = merge() +assert_equal(mergeresult, 0, "merge", quiet=True) + for first_size in range(1, 5): for second_size in range(first_size + 1, 6): get_and_validate_consistency_proof(size_sth[first_size], size_sth[second_size], first_size, second_size) -- cgit v1.1