diff options
| -rw-r--r-- | Makefile | 9 | ||||
| -rw-r--r-- | NEWS.md | 11 | ||||
| -rw-r--r-- | README.md | 4 | ||||
| -rw-r--r-- | src/catlfish.erl | 117 | ||||
| -rw-r--r-- | src/x509.erl | 15 | ||||
| -rwxr-xr-x | tools/comparecert.py | 78 | ||||
| -rw-r--r-- | tools/mergetools.py | 5 |
7 files changed, 172 insertions, 67 deletions
@@ -92,6 +92,15 @@ tests-run: @(cd $(INSTDIR) && python ../tools/merge.py --config ../test/catlfish-test.cfg --localconfig ../test/catlfish-test-local-merge.cfg) || (echo "Merge failed" ; false) @diff -r -x nursery -x verifiedsize catlfish/tests/mergedb catlfish/tests/mergedb-secondary || (echo "Merge databases not matching" ; false) @(cd $(INSTDIR) && python ../tools/check-sth.py --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem https://localhost:8080/) || (echo "Check failed" ; false) + @(cd $(INSTDIR) && mkdir fetchcertstore) + @(cd $(INSTDIR) && python ../tools/fetchallcerts.py $(BASEURL) --store fetchcertstore --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem) || (echo "Verification failed" ; false) + @(cd $(INSTDIR)/fetchcertstore && unzip 0000.zip) + @(cd $(INSTDIR) && python ../tools/comparecert.py ../tools/testcerts/cert1.txt fetchcertstore/00000000) || (echo "Verification failed" ; false) + @(cd $(INSTDIR) && python ../tools/comparecert.py ../tools/testcerts/cert2.txt fetchcertstore/00000001) || (echo "Verification failed" ; false) + @(cd $(INSTDIR) && python ../tools/comparecert.py ../tools/testcerts/cert3.txt fetchcertstore/00000002) || (echo "Verification failed" ; false) + @(cd $(INSTDIR) && python ../tools/comparecert.py ../tools/testcerts/cert4.txt fetchcertstore/00000003) || (echo "Verification failed" ; false) + @(cd $(INSTDIR) && python ../tools/comparecert.py ../tools/testcerts/cert5.txt fetchcertstore/00000004) || (echo "Verification failed" ; false) + @(cd $(INSTDIR) && python ../tools/comparecert.py ../tools/testcerts/pre1.txt:../tools/testcerts/pre2.txt fetchcertstore/00000005:fetchcertstore/00000006) || (echo "Verification failed" ; false) @(cd $(INSTDIR) && python ../tools/storagegc.py --config ../test/catlfish-test.cfg --localconfig ../test/catlfish-test-local-1.cfg) || (echo "GC failed" ; false) tests-run2: @@ -1,5 +1,12 @@ # Changes in version 0.8.0-dev +## Incompatible changes + +- The file format for persistent storage of log entries have + changed. catlfish-0.8.0 is unable to read a database created by all + previous versions. Previous versions are unable to read a database + created by 0.8.0. + ## Features - Library call for plop verification of entries added. @@ -13,6 +20,10 @@ - A bug with merging submitted root certs, i.e. lacking ExtraData has been fixed (closes CATLFISH-45). - Merge now fsyncs the logorder file (closes CATLFISH-46). +- A chain returned from the log (get-entries) now always contains a + known root cert (closes CATLFISH-55). +- "Extra data" for precerts returned from the log is now conformant + with RFC6962 (closes CATLFISH-56). ## Code cleanup @@ -49,6 +49,10 @@ To submit a test cert and verify the resulting SCT: $ (cd catlfish; ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/pre2.txt --check-sct --sct-file=submittedcerts https://localhost:8080/ --publickey=tests/keys/logkey.pem) +# Unit tests + + $ make check + # Logs and traces Logs can be found in catlfish/log/. diff --git a/src/catlfish.erl b/src/catlfish.erl index e48f788..d940147 100644 --- a/src/catlfish.erl +++ b/src/catlfish.erl @@ -124,20 +124,7 @@ add_to_db(Type, LeafCert, CertChain, EntryHash) -> leaf_type = timestamped_entry, entry = TSE}), MTLHash = ht:leaf_hash(MTLText), - ExtraData = - case Type of - normal -> CertChain; - precert -> [LeafCert | CertChain] - end, - LogEntry = - list_to_binary( - [encode_tls_vector(MTLText, 4), - encode_tls_vector( - encode_tls_vector( - list_to_binary( - [encode_tls_vector(C, 3) || C <- ExtraData]), - 3), - 4)]), + LogEntry = pack_entry(Type, MTLText, LeafCert, CertChain), ok = plop:add(LogEntry, MTLHash, EntryHash), {TSE, MTLHash}. @@ -157,7 +144,7 @@ add_chain(LeafCert, CertChain, Type) -> exit({internalerror, "Rate limiting"}) end; {_Index, MTLHash, DBEntry} -> - {MTLText, _ExtraData} = unpack_entry(DBEntry), + {_Type, MTLText, _Cert, _Chain} = unpack_entry(DBEntry), MTL = deserialise_mtl(MTLText), MTLText = serialise(MTL), % verify FIXME: remove {MTL#mtl.entry, MTLHash} @@ -228,6 +215,18 @@ deserialise_signed_precert_entry(Data) -> tbs_certificate = TBSCertificate}, RestData2}. +serialise_extra_data(Type, Cert, Chain) -> + EncodedChain = encode_tls_vector( + list_to_binary( + [encode_tls_vector(C, 3) || C <- Chain]), 3), + case Type of + normal -> + EncodedChain; + precert -> + list_to_binary( + [encode_tls_vector(Cert, 3), EncodedChain]) + end. + -spec entries(non_neg_integer(), non_neg_integer()) -> {[{entries, list()},...]}. entries(Start, End) -> {[{entries, x_entries(plop:get(Start, End))}]}. @@ -236,8 +235,9 @@ entries(Start, End) -> entry_and_proof(Index, TreeSize) -> case plop:inclusion_and_entry(Index, TreeSize) of {ok, Entry, Path} -> - {MTL, ExtraData} = unpack_entry(Entry), - {[{leaf_input, base64:encode(MTL)}, + {Type, MTLText, Cert, Chain} = unpack_entry(Entry), + ExtraData = serialise_extra_data(Type, Cert, Chain), + {[{leaf_input, base64:encode(MTLText)}, {extra_data, base64:encode(ExtraData)}, {audit_path, [base64:encode(X) || X <- Path]}]}; {notfound, Msg} -> @@ -253,29 +253,6 @@ init_cache_table() -> end, ets:new(?CACHE_TABLE, [set, public, named_table]). -deserialise_extra_data(<<>>) -> - []; -deserialise_extra_data(ExtraData) -> - {E, Rest} = decode_tls_vector(ExtraData, 3), - [E | deserialise_extra_data(Rest)]. - -chain_from_mtl_extradata(MTL, ExtraData) -> - TimestampedEntry = MTL#mtl.entry, - Chain = deserialise_extra_data(ExtraData), - case TimestampedEntry#timestamped_entry.entry_type of - x509_entry -> - SignedEntry = TimestampedEntry#timestamped_entry.signed_entry, - [SignedEntry#signed_x509_entry.asn1_cert | Chain]; - precert_entry -> - Chain - end. - -mtl_and_extra_from_entry(Entry) -> - {MTLText, ExtraDataPacked} = unpack_entry(Entry), - {ExtraData, <<>>} = decode_tls_vector(ExtraDataPacked, 3), - MTL = deserialise_mtl(MTLText), - {MTL, ExtraData}. - verify_mtl(MTL, LeafCert, CertChain) -> Timestamp = MTL#mtl.entry#timestamped_entry.timestamp, EntryType = MTL#mtl.entry#timestamped_entry.entry_type, @@ -293,15 +270,14 @@ verify_entry(Entry) -> RootCerts = known_roots(), verify_entry(Entry, RootCerts). -verify_entry(Entry, RootCerts) -> - {MTL, ExtraData} = mtl_and_extra_from_entry(Entry), - Chain = chain_from_mtl_extradata(MTL, ExtraData), - - case x509:normalise_chain(RootCerts, Chain) of - {ok, [LeafCert|CertChain]} -> - case verify_mtl(MTL, LeafCert, CertChain) of +%% Used from plop. +verify_entry(PackedEntry, RootCerts) -> + {_Type, MTLText, Cert, Chain} = unpack_entry(PackedEntry), + case x509:normalise_chain(RootCerts, [Cert | Chain]) of + {ok, [Cert | FullChain]} -> + case verify_mtl(deserialise_mtl(MTLText), Cert, FullChain) of ok -> - {ok, ht:leaf_hash(serialise(MTL))}; + {ok, ht:leaf_hash(MTLText)}; error -> {error, "MTL verification failed"} end; @@ -309,24 +285,51 @@ verify_entry(Entry, RootCerts) -> {error, Reason} end. -entryhash_from_entry(Entry) -> - {MTL, ExtraData} = mtl_and_extra_from_entry(Entry), - Chain = chain_from_mtl_extradata(MTL, ExtraData), - crypto:hash(sha256, Chain). +%% Used from plop. +entryhash_from_entry(PackedEntry) -> + {_Type, _MTLText, Cert, Chain} = unpack_entry(PackedEntry), + crypto:hash(sha256, [Cert | Chain]). %% Private functions. --spec unpack_entry(binary()) -> {binary(), binary()}. +-spec pack_entry(normal|precert, binary(), binary(), [binary()]) -> binary(). +pack_entry(Type, MTLText, EndEntityCert, CertChain) -> + list_to_binary( + [tlv:encode(<<"MTL1">>, MTLText), + tlv:encode(case Type of + normal -> <<"EEC1">>; + precert -> <<"PRC1">> + end, EndEntityCert), + tlv:encode(<<"CHN1">>, + list_to_binary( + [tlv:encode(<<"X509">>, E) || E <- CertChain]))]). + +-spec unpack_entry(binary()) -> {normal|precert, binary(), binary(), [binary()]}. unpack_entry(Entry) -> - {MTL, Rest} = decode_tls_vector(Entry, 4), - {ExtraData, <<>>} = decode_tls_vector(Rest, 4), - {MTL, ExtraData}. + {<<"MTL1">>, MTLText, Rest1} = tlv:decode(Entry), + {EECType, EndEntityCert, Rest2} = tlv:decode(Rest1), + Type = case EECType of + <<"EEC1">> -> + normal; + <<"PRC1">> -> + precert + end, + {<<"CHN1">>, PackedChain, _Rest3} = tlv:decode(Rest2), % Ignore rest. + Chain = unpack_certchain(PackedChain), + {Type, MTLText, EndEntityCert, Chain}. + +unpack_certchain(<<>>) -> + []; +unpack_certchain(Data) -> + {<<"X509">>, Unpacked, Rest} = tlv:decode(Data), + [Unpacked | unpack_certchain(Rest)]. -spec x_entries([{non_neg_integer(), binary(), binary()}]) -> list(). x_entries([]) -> []; x_entries([H|T]) -> {_Index, _Hash, Entry} = H, - {MTL, ExtraData} = unpack_entry(Entry), + {Type, MTL, Cert, Chain} = unpack_entry(Entry), + ExtraData = serialise_extra_data(Type, Cert, Chain), [{[{leaf_input, base64:encode(MTL)}, {extra_data, base64:encode(ExtraData)}]} | x_entries(T)]. diff --git a/src/x509.erl b/src/x509.erl index 7bbfb8e..279d9b9 100644 --- a/src/x509.erl +++ b/src/x509.erl @@ -71,10 +71,9 @@ detox(LeafDer, ChainDer) -> %% argument is irrelevant. %% %% Return {false, Reason} or {true, ListWithRoot}. Note that -%% ListWithRoot is the empty list when the root of the chain is found -%% amongst the acceptable root certs. Otherwise it contains exactly -%% one element, a CA cert from the acceptable root certs signing the -%% root of the chain. +%% ListWithRoot allways contain exactly one element -- a CA cert from +%% first argument (AcceptableRootCerts) signing the root of the +%% chain. FIXME: Any point in returning this as a list? normalise_chain(_, _, MaxChainLength) when MaxChainLength =< 0 -> %% Chain too long. {false, chain_too_long}; @@ -83,7 +82,7 @@ normalise_chain(AcceptableRootCerts, [TopCert], MaxChainLength) -> case lists:member(TopCert, AcceptableRootCerts) of true -> %% Top cert is part of chain. - {true, []}; + {true, [TopCert]}; false when MaxChainLength =< 1 -> %% Chain too long. {false, chain_too_long}; @@ -485,14 +484,14 @@ chain_test(C0, C1) -> %% Chain too long. ?_assertMatch({false, chain_too_long}, normalise_chain([C1], [C0], 1)), %% Root in chain and in trust store. - ?_assertEqual({true, []}, normalise_chain([C1], [C0, C1], 2)), + ?_assertEqual({true, [C1]}, normalise_chain([C1], [C0, C1], 2)), %% Chain too long. ?_assertMatch({false, chain_too_long}, normalise_chain([C1], [C0, C1], 1)), %% Root not in trust store. ?_assertMatch({false, root_unknown}, normalise_chain([], [C0, C1], 10)), %% Selfsigned. Actually OK. - ?_assertMatch({true, []}, normalise_chain([C0], [C0], 10)), - ?_assertMatch({true, []}, normalise_chain([C0], [C0], 1)), + ?_assertMatch({true, [C0]}, normalise_chain([C0], [C0], 10)), + ?_assertMatch({true, [C0]}, normalise_chain([C0], [C0], 1)), %% Max chain length 0 is not OK. ?_assertMatch({false, chain_too_long}, normalise_chain([C0], [C0], 0)) ]. diff --git a/tools/comparecert.py b/tools/comparecert.py new file mode 100755 index 0000000..81893f7 --- /dev/null +++ b/tools/comparecert.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python + +# Copyright (c) 2014, NORDUnet A/S. +# See LICENSE for licensing information. + +import argparse +import urllib2 +import urllib +import json +import base64 +import sys +import struct +import hashlib +import itertools +from certtools import * +from certtools import * +from precerttools import * +import os +import signal +import select +import zipfile + +def readfile(filename): + contents = open(filename).read() + certchain = get_certs_from_string(contents) + precerts = get_precerts_from_string(contents) + return (certchain, precerts) + +def testcerts(template, test): + (certchain1, precerts1) = template + (certchain2, precerts2) = test + + if precerts1 != precerts2: + return (False, "precerts are different") + + if certchain1 == certchain2: + return (True, "") + + if len(certchain2) == len(certchain1) + 1: + if certchain2[:-1] != certchain1: + return (False, "certchains are different") + last_issuer = get_cert_info(certchain1[-1])["issuer"] + root_subject = get_cert_info(certchain2[-1])["subject"] + if last_issuer == root_subject: + return (True, "fetched chain has an appended root cert") + else: + return (False, "fetched chain has an extra entry") + + return (False, "certchains are different") + +parser = argparse.ArgumentParser(description='') +parser.add_argument('templates', help="Test templates, separated with colon") +parser.add_argument('test', help="Files to test, separated with colon") +args = parser.parse_args() + +templates = [readfile(filename) for filename in args.templates.split(":")] + +tests = [readfile(filename) for filename in args.test.split(":")] + + +for test in tests: + found = False + errors = [] + for template in templates: + (result, message) = testcerts(template, test) + if result: + print message + found = True + templates.remove(template) + break + else: + errors.append(message) + if not found: + print "Matching template not found for test" + for error in errors: + print error + sys.exit(1) +sys.exit(0) diff --git a/tools/mergetools.py b/tools/mergetools.py index 9e84038..9f5feee 100644 --- a/tools/mergetools.py +++ b/tools/mergetools.py @@ -31,8 +31,9 @@ def unpack_entry(entry): pieces = [] while len(entry): (length,) = struct.unpack(">I", entry[0:4]) - data = entry[4:4+length] - entry = entry[4+length:] + type = entry[4:8] + data = entry[8:length] + entry = entry[length:] pieces.append(data) return pieces |