From f8a36b4a27a976d56448a884b36b4ac2534f09f6 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Thu, 2 Apr 2015 18:26:09 +0200 Subject: Retry sendlog on error --- tools/merge.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tools/merge.py b/tools/merge.py index f9c93d9..5ceb245 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -16,6 +16,7 @@ import hashlib import urlparse import os import yaml +import select from certtools import build_merkle_tree, create_sth_signature, \ check_sth_signature, get_eckey_from_file, timing_point, http_request, \ get_public_key_from_file @@ -61,7 +62,7 @@ def write_chain(key, value): try: os.makedirs(path) except Exception, e: - print e + pass else: path = chainsdir f = open(path + "/" + filename, "w") @@ -131,7 +132,7 @@ def sendlog(node, baseurl, submission): return json.loads(result) except urllib2.HTTPError, e: print "ERROR: sendlog", e.read() - sys.exit(1) + return None except ValueError, e: print "==== FAILED REQUEST ====" print submission @@ -277,7 +278,15 @@ for frontendnode in frontendnodes: print "current position", curpos entries = [base64.b64encode(entry) for entry in logorder[curpos:]] for chunk in chunks(entries, 1000): - sendlogresult = sendlog(nodename, nodeaddress, {"start": curpos, "hashes": chunk}) + for trynumber in range(5, 0, -1): + sendlogresult = sendlog(nodename, nodeaddress, {"start": curpos, "hashes": chunk}) + if sendlogresult == None: + if trynumber == 1: + sys.exit(1) + select.select([], [], [], 10.0) + print "tries left:", trynumber + continue + break if sendlogresult["result"] != "ok": print "sendlog:", sendlogresult sys.exit(1) -- cgit v1.1 From 613e50c433a23aa1282453108859c8e590917729 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 6 Apr 2015 03:14:58 +0200 Subject: Verify that database entry actually contains the certificate --- src/catlfish.erl | 9 ++++++--- tools/merge.py | 34 +++++++++++++++++++++++++++++++++- verifycert.erl | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 4 deletions(-) create mode 100755 verifycert.erl diff --git a/src/catlfish.erl b/src/catlfish.erl index ed75495..d98e741 100644 --- a/src/catlfish.erl +++ b/src/catlfish.erl @@ -5,7 +5,7 @@ -export([add_chain/3, entries/2, entry_and_proof/2]). -export([known_roots/0, update_known_roots/0]). -export([init_cache_table/0]). --export([entryhash_from_entry/1]). +-export([entryhash_from_entry/1, chain_from_entry/1]). -include_lib("eunit/include/eunit.hrl"). -define(PROTOCOL_VERSION, 0). @@ -252,7 +252,7 @@ deserialise_extra_data(ExtraData) -> [E | deserialise_extra_data(Rest)] end. -entryhash_from_entry(Entry) -> +chain_from_entry(Entry) -> {MTLText, ExtraDataPacked} = unpack_entry(Entry), {ExtraData, <<>>} = decode_tls_vector(ExtraDataPacked, 3), MTL = deserialise_mtl(MTLText), @@ -266,7 +266,10 @@ entryhash_from_entry(Entry) -> precert_entry -> Chain end, - crypto:hash(sha256, Data). + Data. + +entryhash_from_entry(Entry) -> + crypto:hash(sha256, chain_from_entry(Entry)). %% Private functions. -spec unpack_entry(binary()) -> {binary(), binary()}. diff --git a/tools/merge.py b/tools/merge.py index 5ceb245..ce3bf0b 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -17,9 +17,10 @@ import urlparse import os import yaml import select +import struct from certtools import build_merkle_tree, create_sth_signature, \ check_sth_signature, get_eckey_from_file, timing_point, http_request, \ - get_public_key_from_file + get_public_key_from_file, get_leaf_hash, decode_certificate_chain parser = argparse.ArgumentParser(description="") parser.add_argument('--config', help="System configuration", required=True) @@ -207,6 +208,32 @@ for storagenode in storagenodes: new_entries.update(new_entries_per_node[storagenode["name"]]) entries_to_fetch[storagenode["name"]] = [] +def unpack_entry(entry): + pieces = [] + while len(entry): + (length,) = struct.unpack(">I", entry[0:4]) + data = entry[4:4+length] + entry = entry[4+length:] + pieces.append(data) + return pieces + +import subprocess + +def verify_entry(verifycert, entry, hash): + unpacked = unpack_entry(entry) + mtl = unpacked[0] + assert hash == get_leaf_hash(mtl) + s = struct.pack(">I", len(entry)) + entry + verifycert.stdin.write(s) + result_length_packed = verifycert.stdout.read(4) + (result_length,) = struct.unpack(">I", result_length_packed) + result = verifycert.stdout.read(result_length) + assert len(result) == result_length + (error_code,) = struct.unpack("B", result[0:1]) + if error_code != 0: + print >>sys.stderr, result[1:] + sys.exit(1) + timing_point(timing, "get new entries") new_entries -= certsinlog @@ -222,6 +249,8 @@ for hash in new_entries: entries_to_fetch[storagenode["name"]].append(hash) break +verifycert = subprocess.Popen(["../verifycert.erl"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE) added_entries = 0 for storagenode in storagenodes: @@ -230,6 +259,7 @@ for storagenode in storagenodes: entries = get_entries(storagenode["name"], "https://%s/" % storagenode["address"], chunk) for hash in chunk: entry = entries[hash] + verify_entry(verifycert, entry, hash) write_chain(hash, entry) add_to_logorder(hash) logorder.append(hash) @@ -238,6 +268,8 @@ for storagenode in storagenodes: timing_point(timing, "add entries") print "added", added_entries, "entries" +verifycert.communicate(struct.pack("I", 0)) + tree = build_merkle_tree(logorder) tree_size = len(logorder) root_hash = tree[-1][0] diff --git a/verifycert.erl b/verifycert.erl new file mode 100755 index 0000000..e501d6d --- /dev/null +++ b/verifycert.erl @@ -0,0 +1,41 @@ +#!/usr/bin/env escript +%% -*- erlang -*- +%%! -pa ebin -pa lib/catlfish-0.2.0-dev.ez/catlfish-0.2.0-dev/ebin -pa lib/lager-2.1.1.ez/lager-2.1.1/ebin + +write_reply(Bin) -> + Length = size(Bin), + file:write(standard_io, <>). + +verify(Certs, DBEntry) -> + try + Chain = catlfish:chain_from_entry(DBEntry), + %% XXX: doesn't verify that MTL is derived from Chain + case x509:normalise_chain(Certs, Chain) of + {ok, _} -> + write_reply(<<0:8>>); + {error, Reason} -> + ReasonBin = list_to_binary(io_lib:format("~p", [Reason])), + write_reply(<<1:8, ReasonBin/binary>>) + end + catch + Type:What -> + [CrashFunction | Stack] = erlang:get_stacktrace(), + ErrorBin = list_to_binary(io_lib:format("Crash: ~p ~p~n~p~n~p~n", [Type, What, CrashFunction, Stack])), + write_reply(<<2:8, ErrorBin/binary>>) + end. + +loop(Certs) -> + {ok, LengthBin} = file:read(standard_io, 4), + <> = list_to_binary(LengthBin), + case Length of + 0 -> + none; + _ -> + {ok, DBEntry} = file:read(standard_io, Length), + verify(Certs, list_to_binary(DBEntry)), + loop(Certs) + end. + +main(_) -> + Certs = x509:read_pemfiles_from_dir("tests/known_roots/"), + loop(Certs). -- cgit v1.1 From f364054054477ed7ac7c15dd1199b7bac2e1cef7 Mon Sep 17 00:00:00 2001 From: Linus Nordberg Date: Tue, 7 Apr 2015 16:36:39 +0200 Subject: catlfish version is 0.6.0-dev. Also, call the list of root certs RootCerts. --- verifycert.erl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/verifycert.erl b/verifycert.erl index e501d6d..b9a3753 100755 --- a/verifycert.erl +++ b/verifycert.erl @@ -1,16 +1,16 @@ #!/usr/bin/env escript %% -*- erlang -*- -%%! -pa ebin -pa lib/catlfish-0.2.0-dev.ez/catlfish-0.2.0-dev/ebin -pa lib/lager-2.1.1.ez/lager-2.1.1/ebin +%%! -pa ebin -pa lib/catlfish-0.6.0-dev.ez/catlfish-0.6.0-dev/ebin -pa lib/lager-2.1.1.ez/lager-2.1.1/ebin write_reply(Bin) -> Length = size(Bin), file:write(standard_io, <>). -verify(Certs, DBEntry) -> +verify(RootCerts, DBEntry) -> try Chain = catlfish:chain_from_entry(DBEntry), %% XXX: doesn't verify that MTL is derived from Chain - case x509:normalise_chain(Certs, Chain) of + case x509:normalise_chain(RootCerts, Chain) of {ok, _} -> write_reply(<<0:8>>); {error, Reason} -> @@ -24,7 +24,7 @@ verify(Certs, DBEntry) -> write_reply(<<2:8, ErrorBin/binary>>) end. -loop(Certs) -> +loop(RootCerts) -> {ok, LengthBin} = file:read(standard_io, 4), <> = list_to_binary(LengthBin), case Length of @@ -32,8 +32,8 @@ loop(Certs) -> none; _ -> {ok, DBEntry} = file:read(standard_io, Length), - verify(Certs, list_to_binary(DBEntry)), - loop(Certs) + verify(RootCerts, list_to_binary(DBEntry)), + loop(RootCerts) end. main(_) -> -- cgit v1.1