From b5b5c0f43803a2d31fb5bf240e4b70377d367b7a Mon Sep 17 00:00:00 2001 From: Linus Nordberg Date: Sat, 27 Sep 2014 15:44:07 +0200 Subject: wip --- src/catlfish.erl | 151 ++++++++++++++++++++++++++++++++++++++++++++----------- src/v1.erl | 2 +- 2 files changed, 123 insertions(+), 30 deletions(-) diff --git a/src/catlfish.erl b/src/catlfish.erl index 2ba9d58..b6856b8 100644 --- a/src/catlfish.erl +++ b/src/catlfish.erl @@ -7,32 +7,119 @@ -define(PROTOCOL_VERSION, 0). --spec add_chain(binary(), list()) -> list(). +%%-type signature_type() :: certificate_timestamp | tree_hash | test. % uint8 +-type entry_type() :: x509_entry | precert_entry | test. % uint16 +-type leaf_type() :: timestamped_entry | test. % uint8 +-type leaf_version() :: v1 | v2. % uint8 + +-record(mtl, {leaf_version :: leaf_version(), + leaf_type :: leaf_type(), + entry :: timestamped_entry()}). +-type mtl() :: #mtl{}. + +-record(timestamped_entry, {timestamp :: integer(), + entry_type :: entry_type(), + signed_entry :: binary(), + extensions = <<>> :: binary()}). +-type timestamped_entry() :: #timestamped_entry{}. + +-spec serialise(mtl() | timestamped_entry()) -> binary(). +serialise(#timestamped_entry{timestamp = Timestamp} = E) -> + list_to_binary( + [<>, + serialise_entry_type(E#timestamped_entry.entry_type), + encode_tls_vector(E#timestamped_entry.signed_entry, 3), + encode_tls_vector(E#timestamped_entry.extensions, 2)]); +serialise(#mtl{leaf_version = LeafVersion, + leaf_type = LeafType, + entry = TimestampedEntry}) -> + list_to_binary( + [serialise_leaf_version(LeafVersion), + serialise_leaf_type(LeafType), + serialise(TimestampedEntry)]). + +serialise_leaf_version(v1) -> + <<0:8>>; +serialise_leaf_version(v2) -> + <<1:8>>. + +serialise_leaf_type(timestamped_entry) -> + <<0:8>>. +%% serialise_leaf_type(_) -> +%% <<>>. + +serialise_entry_type(x509_entry) -> + <<0:16>>; +serialise_entry_type(precert_entry) -> + <<1:16>>. + +serialise_signature_type(certificate_timestamp) -> + <<0:8>>; +serialise_signature_type(tree_hash) -> + <<1:8>>. + +-spec add_chain(binary(), [binary()]) -> nonempty_string(). add_chain(LeafCert, CertChain) -> - Entry = #plop_entry{type = x509, data = LeafCert}, - EDVectors = [serialise_tls_vector(X, 3) || X <- CertChain], - ExtraData = serialise_tls_vector(list_to_binary(EDVectors), 3), - SPT = plop:add(#timestamped_entry{entry = Entry}, ExtraData), - R = [{sct_version, ?PROTOCOL_VERSION}, - {id, base64:encode(SPT#spt.logid)}, - {timestamp, SPT#spt.timestamp}, - {extensions, base64:encode("")}, - {signature, base64:encode(plop:serialise(SPT#spt.signature))}], - binary_to_list(jiffy:encode({R})). + EntryHash = crypto:hash(sha256, LeafCert), + TimestampedEntry = + case plop:get(EntryHash) of + notfound -> + Timestamp = plop:generate_timestamp(), + TSE = #timestamped_entry{timestamp = Timestamp, + entry_type = x509_entry, + signed_entry = LeafCert}, + MTL = #mtl{leaf_version = v1, + leaf_type = timestamped_entry, + entry = TSE}, + ok = plop:add( + serialise_logentry(Timestamp, LeafCert, CertChain), + ht:leaf_hash(serialise(MTL)), + crypto:hash(sha256, LeafCert)), + TSE; + {_Index, Entry} -> + <> = Entry, + %% TODO: Perform a costly db consistency check against + %% unpacked LogEntry (w/ LeafCert and CertChain) + #timestamped_entry{timestamp = Timestamp, + entry_type = x509_entry, + signed_entry = LeafCert} + end, + SCT_sig = + plop:spt(list_to_binary([<>, + serialise_signature_type(certificate_timestamp), + serialise(TimestampedEntry)])), + binary_to_list( + jiffy:encode( + {[{sct_version, ?PROTOCOL_VERSION}, + {id, base64:encode(plop:logid())}, + {timestamp, TimestampedEntry#timestamped_entry.timestamp}, + {extensions, base64:encode(<<>>)}, + {signature, base64:encode(plop:serialise(SCT_sig))}]})). + +-spec serialise_logentry(integer(), binary(), [binary()]) -> binary(). +serialise_logentry(Timestamp, LeafCert, CertChain) -> + list_to_binary( + [<>, + list_to_binary( + [encode_tls_vector(LeafCert, 3), + encode_tls_vector( + list_to_binary( + [encode_tls_vector(X, 3) || X <- CertChain]), 3)])]). -spec entries(non_neg_integer(), non_neg_integer()) -> list(). entries(Start, End) -> - encode_entries(plop:get(Start, End)). + binary_to_list( + jiffy:encode({[{entries, x_entries(plop:get(Start, End))}]})). -spec entry_and_proof(non_neg_integer(), non_neg_integer()) -> list(). entry_and_proof(Index, TreeSize) -> binary_to_list( jiffy:encode( case plop:inclusion_and_entry(Index, TreeSize) of - {ok, MTL, Extra, Path} -> - {[{leaf_input, base64:encode(plop:serialise(MTL))}, - %% Extra data is already in TLS vector format. - {extra_data, base64:encode(Extra)}, + {ok, {Entry, Path}} -> + {LeafCertVector, CertChainVector} = unpack_entry(Entry), + {[{leaf_input, base64:encode(LeafCertVector)}, + {extra_data, base64:encode(CertChainVector)}, {audit_path, [base64:encode(X) || X <- Path]}]}; {notfound, Msg} -> {[{success, false}, @@ -40,20 +127,26 @@ entry_and_proof(Index, TreeSize) -> end)). %% Private functions. --spec encode_entries([{mtl(), binary()}]) -> list(). -encode_entries(Entries) -> - binary_to_list(jiffy:encode({[{entries, unpack_entries(Entries)}]})). +unpack_entry(Entry) -> + %% FIXME: Do this with some beatiful binary matching. + LeafCertVectorLen = binary:decode_unsigned(binary_part(Entry, 0, 3)), + LeafCertVector = binary_part(Entry, 3, LeafCertVectorLen), + CertChainVectorPos = 3 + LeafCertVectorLen, + CertChainVector = binary_part( + Entry, CertChainVectorPos, + byte_size(Entry) - CertChainVectorPos), + {LeafCertVector, CertChainVector}. --spec unpack_entries([{mtl(), binary()}]) -> list(). -unpack_entries([]) -> +-spec x_entries([{non_neg_integer(), binary(), binary()}]) -> list(). +x_entries([]) -> []; -unpack_entries([H|T]) -> - {MTL, Extra} = H, - LeafInput = base64:encode(plop:serialise(MTL)), - ExtraData = base64:encode(Extra), - [{[{leaf_input, LeafInput}, {extra_data, ExtraData}]} | unpack_entries(T)]. - --spec serialise_tls_vector(binary(), non_neg_integer()) -> binary(). -serialise_tls_vector(Binary, LengthLen) -> +x_entries([H|T]) -> + {_Index, _Hash, Entry} = H, + {LeafCertVector, CertChainVector} = unpack_entry(Entry), + [{[{leaf_input, LeafCertVector}, {extra_data, CertChainVector}]} | + x_entries(T)]. + +-spec encode_tls_vector(binary(), non_neg_integer()) -> binary(). +encode_tls_vector(Binary, LengthLen) -> Length = byte_size(Binary), <>. diff --git a/src/v1.erl b/src/v1.erl index 304b0a8..b58516d 100644 --- a/src/v1.erl +++ b/src/v1.erl @@ -82,7 +82,7 @@ binary_to_list( jiffy:encode( case plop:inclusion(Hash, TreeSize) of - {ok, Index, Path} -> + {ok, {Index, Path}} -> {[{leaf_index, Index}, {audit_path, [base64:encode(X) || X <- Path]}]}; -- cgit v1.1 From 9ffb69e264ebd826f8aa1aa5e1a77b3a4626c106 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Sat, 27 Sep 2014 18:34:49 +0200 Subject: Fix api problems --- src/catlfish.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/catlfish.erl b/src/catlfish.erl index b6856b8..2f95cfb 100644 --- a/src/catlfish.erl +++ b/src/catlfish.erl @@ -76,8 +76,8 @@ add_chain(LeafCert, CertChain) -> ht:leaf_hash(serialise(MTL)), crypto:hash(sha256, LeafCert)), TSE; - {_Index, Entry} -> - <> = Entry, + {_Index, _MTLHash, Entry} -> + <> = Entry, %% TODO: Perform a costly db consistency check against %% unpacked LogEntry (w/ LeafCert and CertChain) #timestamped_entry{timestamp = Timestamp, @@ -91,7 +91,7 @@ add_chain(LeafCert, CertChain) -> binary_to_list( jiffy:encode( {[{sct_version, ?PROTOCOL_VERSION}, - {id, base64:encode(plop:logid())}, + {id, base64:encode(plop:get_logid())}, {timestamp, TimestampedEntry#timestamped_entry.timestamp}, {extensions, base64:encode(<<>>)}, {signature, base64:encode(plop:serialise(SCT_sig))}]})). -- cgit v1.1 From b93ed6bf66b7ccadd17d5ca0f9bbf14dfd63cb24 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Sat, 27 Sep 2014 18:35:48 +0200 Subject: Make each included application in the release explicit --- reltool.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/reltool.config b/reltool.config index 434345c..80b374d 100644 --- a/reltool.config +++ b/reltool.config @@ -1,6 +1,5 @@ %% -*- mode: erlang -*- {sys, [ - {lib_dirs, [".."]}, {erts, [{mod_cond, derived}, {app_file, strip}]}, {app_file, strip}, {rel, "catlfish", "0.2.0-dev", @@ -12,6 +11,7 @@ ]}, {boot_rel, "catlfish"}, {excl_archive_filters, ["^include$","^priv$","^\\.git$"]}, - {app, catlfish, [{app_file, all}]} -%% {app, plop, [{mod_cond, app}, {incl_cond, include}, {lib_dir, "../plop"}]} + {app, catlfish, [{app_file, all}, {lib_dir, "."}]}, + {app, plop, [{app_file, all}, {lib_dir, "../plop"}]}, + {app, jiffy, [{app_file, all}, {lib_dir, "../jiffy"}]} ]}. -- cgit v1.1 From eca189ce80ec353035902dc0c361c748e042371a Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Sat, 27 Sep 2014 20:34:18 +0200 Subject: Fix api problems --- src/catlfish.erl | 38 +++++++++++++++++++++++++------------- src/v1.erl | 2 +- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/catlfish.erl b/src/catlfish.erl index 2f95cfb..bd3c106 100644 --- a/src/catlfish.erl +++ b/src/catlfish.erl @@ -58,6 +58,15 @@ serialise_signature_type(certificate_timestamp) -> serialise_signature_type(tree_hash) -> <<1:8>>. +build_mtl(Timestamp, LeafCert) -> + TSE = #timestamped_entry{timestamp = Timestamp, + entry_type = x509_entry, + signed_entry = LeafCert}, + MTL = #mtl{leaf_version = v1, + leaf_type = timestamped_entry, + entry = TSE}, + serialise(MTL). + -spec add_chain(binary(), [binary()]) -> nonempty_string(). add_chain(LeafCert, CertChain) -> EntryHash = crypto:hash(sha256, LeafCert), @@ -117,8 +126,9 @@ entry_and_proof(Index, TreeSize) -> jiffy:encode( case plop:inclusion_and_entry(Index, TreeSize) of {ok, {Entry, Path}} -> - {LeafCertVector, CertChainVector} = unpack_entry(Entry), - {[{leaf_input, base64:encode(LeafCertVector)}, + {Timestamp, LeafCertVector, CertChainVector} = unpack_entry(Entry), + MTL = build_mtl(Timestamp, LeafCertVector), + {[{leaf_input, base64:encode(MTL)}, {extra_data, base64:encode(CertChainVector)}, {audit_path, [base64:encode(X) || X <- Path]}]}; {notfound, Msg} -> @@ -128,25 +138,27 @@ entry_and_proof(Index, TreeSize) -> %% Private functions. unpack_entry(Entry) -> - %% FIXME: Do this with some beatiful binary matching. - LeafCertVectorLen = binary:decode_unsigned(binary_part(Entry, 0, 3)), - LeafCertVector = binary_part(Entry, 3, LeafCertVectorLen), - CertChainVectorPos = 3 + LeafCertVectorLen, - CertChainVector = binary_part( - Entry, CertChainVectorPos, - byte_size(Entry) - CertChainVectorPos), - {LeafCertVector, CertChainVector}. + <> = Entry, + {LeafCertVector, CertChainVector} = decode_tls_vector(LogEntry, 3), + {Timestamp, LeafCertVector, CertChainVector}. -spec x_entries([{non_neg_integer(), binary(), binary()}]) -> list(). x_entries([]) -> []; x_entries([H|T]) -> - {_Index, _Hash, Entry} = H, - {LeafCertVector, CertChainVector} = unpack_entry(Entry), - [{[{leaf_input, LeafCertVector}, {extra_data, CertChainVector}]} | + [_Index, _Hash, Entry] = H, + {Timestamp, LeafCertVector, CertChainVector} = unpack_entry(Entry), + MTL = build_mtl(Timestamp, LeafCertVector), + [{[{leaf_input, base64:encode(MTL)}, {extra_data, base64:encode(CertChainVector)}]} | x_entries(T)]. -spec encode_tls_vector(binary(), non_neg_integer()) -> binary(). encode_tls_vector(Binary, LengthLen) -> Length = byte_size(Binary), <>. + +-spec decode_tls_vector(binary(), non_neg_integer()) -> {binary(), binary()}. +decode_tls_vector(Binary, LengthLen) -> + <> = Binary, + <> = Rest, + {ExtractedBinary, Rest2}. diff --git a/src/v1.erl b/src/v1.erl index b58516d..304b0a8 100644 --- a/src/v1.erl +++ b/src/v1.erl @@ -82,7 +82,7 @@ binary_to_list( jiffy:encode( case plop:inclusion(Hash, TreeSize) of - {ok, {Index, Path}} -> + {ok, Index, Path} -> {[{leaf_index, Index}, {audit_path, [base64:encode(X) || X <- Path]}]}; -- cgit v1.1 From c915ef319d63f73231202443419dcad3aa32b5f4 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Sun, 28 Sep 2014 02:01:15 +0200 Subject: Use raw file storage --- Makefile | 3 ++- catlfish.config | 8 +++++++- src/catlfish.erl | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 5a13905..df00505 100644 --- a/Makefile +++ b/Makefile @@ -7,9 +7,10 @@ release: mkdir rel ./makerelease.erl (cd rel; \ - ln -s ../../plop/Mnesia.nonode@nohost .; \ ln -s ../../plop/test .) cp httpd_props.conf rel cp catlfish.config rel mkdir rel/catlfish + mkdir rel/db + printf "0" > rel/db/treesize cp -r webroot rel/catlfish diff --git a/catlfish.config b/catlfish.config index cda8789..9b80c6e 100644 --- a/catlfish.config +++ b/catlfish.config @@ -9,4 +9,10 @@ {error_logger_mf_maxfiles, 10}]}, {inets, [{services, - [{httpd, [{proplist_file, "httpd_props.conf"}]}]}]}]. + [{httpd, [{proplist_file, "httpd_props.conf"}]}]}]}, + {plop, + [{entry_root_path, "db/certentries/"}, + {index_path, "db/index"}, + {entryhash_root_path, "db/entryhash/"}, + {treesize_path, "db/treesize"}, + {indexforhash_root_path, "db/certindex/"}]}]. diff --git a/src/catlfish.erl b/src/catlfish.erl index bd3c106..d105591 100644 --- a/src/catlfish.erl +++ b/src/catlfish.erl @@ -146,7 +146,7 @@ unpack_entry(Entry) -> x_entries([]) -> []; x_entries([H|T]) -> - [_Index, _Hash, Entry] = H, + {_Index, _Hash, Entry} = H, {Timestamp, LeafCertVector, CertChainVector} = unpack_entry(Entry), MTL = build_mtl(Timestamp, LeafCertVector), [{[{leaf_input, base64:encode(MTL)}, {extra_data, base64:encode(CertChainVector)}]} | -- cgit v1.1 From 732aaa8f2c2d57e8a5ad989fee0ba7b8c3c82479 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Sun, 28 Sep 2014 02:10:27 +0200 Subject: Remove reference to creating database --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 7b6f625..21d86e5 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,6 @@ A compiled https://github.com/davisp/jiffy (for JSON encoding and decoding) in . # Start -Make sure plop has a database. See `plop/README`. Then run catlfish: - $ (cd rel ; bin/erl -config catlfish) # Logs and traces -- cgit v1.1