%%% Copyright (c) 2014, NORDUnet A/S. %%% See LICENSE for licensing information. %%% %%% @doc Server holding log entries in a database and hashes in a %%% Merkle tree. A backend for things like Certificate Transparency %%% (RFC 6962). %%% %%% When you submit data for insertion in the log, it's stored in an %%% append only database with an accompanying Merkle tree. The leaves %%% of the tree hold hashes of submitted data and makes it possible %%% for anyone to verify wether a given piece of data is or is not %%% present in the log. %%% %%% In return you will get a signed timestamp which is a promise that %%% your entry will be present in the log within a certain time period %%% (the MMD). This signed timestamp can later, together with the %%% public key of the log, be used to ensure that your entry is indeed %%% present in the log. %%% TODO %%% - get rid of CT-specific stuff that has creeped in -module(plop). -behaviour(gen_server). %% API. -export([start_link/0, stop/0]). -export([get_logid/0, serialise/1]). -export([add/3, sth/0, get/1, get/2, spt/1, consistency/2, inclusion/2, inclusion_and_entry/2]). -export([generate_timestamp/0]). %% API for tests. -export([testing_get_pubkey/0]). %% gen_server callbacks. -export([init/1, handle_call/3, terminate/2, handle_cast/2, handle_info/2, code_change/3]). -import(stacktrace, [call/2]). -include("plop.hrl"). %%-include("db.hrl"). -include_lib("public_key/include/public_key.hrl"). -include_lib("eunit/include/eunit.hrl"). -record(state, {http_requests, own_requests }). %%%%% moved from plop.hrl, maybe remove -define(PLOPVERSION, 0). -type signature_type() :: certificate_timestamp | tree_hash | test. % uint8 %%%%% %% @doc The parts of an STH which is to be signed. Used as the %% interface to plop:sth/1, for testing. -record(sth_signed, { version = ?PLOPVERSION :: non_neg_integer(), signature_type :: signature_type(), timestamp = now :: 'now' | integer(), tree_size :: integer(), root_hash :: binary() % SHA-256 }). -type sth_signed() :: #sth_signed{}. start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). stop() -> call(?MODULE, stop). add_http_request(Plop, RequestId, Data) -> Plop#state{http_requests = dict:store(RequestId, Data, Plop#state.http_requests)}. add_own_request(Plop, RequestId, Data) -> Plop#state{own_requests = dict:store(RequestId, Data, Plop#state.own_requests)}. remove_http_request(Plop, RequestId) -> Plop#state{http_requests = dict:erase(RequestId, Plop#state.http_requests)}. remove_own_request(Plop, RequestId) -> Plop#state{own_requests = dict:erase(RequestId, Plop#state.own_requests)}. %%%%%%%%%%%%%%%%%%%% init([]) -> _Tree = ht:reset_tree([db:size() - 1]), {ok, #state{http_requests = dict:new(), own_requests = dict:new()}}. handle_cast(_Request, State) -> {noreply, State}. handle_http_reply(State, {storage_sendentry_http, {OwnRequestId}}, StatusCode, Body) -> lager:debug("http_reply: ~p", [Body]), {struct, PropList} = mochijson2:decode(Body), Result = proplists:get_value(<<"result">>, PropList), case dict:fetch(OwnRequestId, State#state.own_requests) of undefined -> {noreply, State}; {storage_sendentry, {From, Completion, RepliesUntilQuorum}} when Result == <<"ok">>, StatusCode == 200 -> case RepliesUntilQuorum - 1 of 0 -> %% reached quorum lager:debug("reached quorum"), gen_server:reply(From, ok), StateWithCompletion = Completion(State), {noreply, remove_own_request(StateWithCompletion, OwnRequestId)}; NewRepliesUntilQuorum -> lager:debug("replies until quorum: ~p", [NewRepliesUntilQuorum]), {noreply, add_own_request(State, OwnRequestId, {storage_sendentry, {From, Completion, NewRepliesUntilQuorum}})} end end. handle_info({http, {RequestId, {StatusLine, _Headers, Body}}}, Plop) -> {_HttpVersion, StatusCode, _ReasonPhrase} = StatusLine, case dict:fetch(RequestId, Plop#state.http_requests) of undefined -> {noreply, Plop}; ignore -> {noreply, Plop}; HttpRequest -> handle_http_reply(remove_http_request(Plop, RequestId), HttpRequest, StatusCode, Body) end; handle_info(_Info, State) -> {noreply, State}. code_change(_OldVsn, State, _Extra) -> {ok, State}. terminate(_Reason, _State) -> io:format("~p terminating~n", [?MODULE]), ok. %%%%%%%%%%%%%%%%%%%% -spec add(binary(), binary(), binary()) -> ok. add(LogEntry, TreeLeafHash, EntryHash) -> call(?MODULE, {add, {LogEntry, TreeLeafHash, EntryHash}}). sth() -> sth([]). -spec get(non_neg_integer(), non_neg_integer()) -> [{non_neg_integer(), binary(), binary()}]. get(Start, End) -> call(?MODULE, {get, {index, Start, End}}). get(Hash) -> call(?MODULE, {get, {hash, Hash}}). spt(Data) -> #signature{algorithm = #sig_and_hash_alg{ hash_alg = sha256, signature_alg = ecdsa}, signature = sign:sign(Data)}. consistency(TreeSizeFirst, TreeSizeSecond) -> call(?MODULE, {consistency, {TreeSizeFirst, TreeSizeSecond}}). -spec inclusion(binary(), non_neg_integer()) -> {ok, {binary(), binary()}} | {notfound, string()}. inclusion(Hash, TreeSize) -> call(?MODULE, {inclusion, {Hash, TreeSize}}). -spec inclusion_and_entry(non_neg_integer(), non_neg_integer()) -> {ok, {binary(), binary()}} | {notfound, string()}. inclusion_and_entry(Index, TreeSize) -> call(?MODULE, {inclusion_and_entry, {Index, TreeSize}}). get_logid() -> sign:get_logid(). testing_get_pubkey() -> sign:get_pubkey(). storage_nodes() -> application:get_env(plop, storage_nodes, []). storage_nodes_quorum() -> {ok, Value} = application:get_env(plop, storage_nodes_quorum), Value. send_storage_sendentry(URLBase, LogEntry, TreeLeafHash) -> Request = mochijson2:encode( {[{plop_version, 1}, {entry, base64:encode(LogEntry)}, {treeleafhash, base64:encode(TreeLeafHash)} ]}), lager:debug("send sendentry to storage node ~p: ~p", [URLBase, Request]), httpc:request(post, {URLBase ++ "sendentry", [], "text/json", list_to_binary(Request)}, [], [{sync, false}]). send_storage_entrycommitted(URLBase, EntryHash, TreeLeafHash) -> Request = mochijson2:encode( {[{plop_version, 1}, {entryhash, base64:encode(EntryHash)}, {treeleafhash, base64:encode(TreeLeafHash)} ]}), httpc:request(post, {URLBase ++ "entrycommitted", [], "text/json", list_to_binary(Request)}, [], [{sync, false}]). store_at_all_nodes(Nodes, {LogEntry, TreeLeafHash, EntryHash}, From, State) -> lager:debug("leafhash ~p", [TreeLeafHash]), OwnRequestId = make_ref(), Completion = fun(CompletionState) -> RequestIds = [send_storage_entrycommitted(URLBase, EntryHash, TreeLeafHash) || URLBase <- Nodes], lists:foldl(fun({ok, RequestId}, StateAcc) -> add_http_request(StateAcc, RequestId, ignore) end, CompletionState, RequestIds) end, PlopWithOwn = add_own_request(State, OwnRequestId, {storage_sendentry, {From, Completion, storage_nodes_quorum()}}), lager:debug("send requests to ~p", [Nodes]), RequestIds = [send_storage_sendentry(URLBase, LogEntry, TreeLeafHash) || URLBase <- Nodes], PlopWithRequests = lists:foldl(fun({ok, RequestId}, PlopAcc) -> add_http_request(PlopAcc, RequestId, {storage_sendentry_http, {OwnRequestId}}) end, PlopWithOwn, RequestIds), PlopWithRequests. fill_in_entry({_Index, LeafHash, notfetched}) -> db:get_by_leaf_hash(LeafHash). %%%%%%%%%%%%%%%%%%%% handle_call(stop, _From, Plop) -> {stop, normal, stopped, Plop}; handle_call({get, {index, Start, End}}, _From, Plop) -> {reply, lists:map(fun (E) -> fill_in_entry(E) end, db:get_by_indices(Start, End, {sorted, false})), Plop}; handle_call({get, {hash, EntryHash}}, _From, Plop) -> {reply, db:get_by_entry_hash(EntryHash), Plop}; handle_call({add, {LogEntry, TreeLeafHash, EntryHash}}, From, Plop) -> lager:debug("add leafhash ~p", [TreeLeafHash]), case storage_nodes() of [] -> exit(internal_merge_not_supported); Nodes -> {noreply, store_at_all_nodes(Nodes, {LogEntry, TreeLeafHash, EntryHash}, From, Plop)} end; handle_call({consistency, {First, Second}}, _From, Plop) -> {reply, ht:consistency(First - 1, Second - 1), Plop}; handle_call({inclusion, {Hash, TreeSize}}, _From, Plop) -> R = case db:get_by_leaf_hash(Hash) of notfound -> {notfound, "Unknown hash"}; % FIXME: include Hash {I, _MTLHash, _Entry} -> {ok, I, ht:path(I, TreeSize - 1)} end, {reply, R, Plop}; handle_call({inclusion_and_entry, {Index, TreeSize}}, _From, Plop) -> R = case db:get_by_index(Index) of notfound -> {notfound, "Unknown index"}; % FIXME: include Index {I, _MTLHash, Entry} -> {ok, Entry, ht:path(I, TreeSize - 1)} end, {reply, R, Plop}. %% @doc Signed Plop Timestamp, conformant to an SCT in RFC6962 3.2 and %% RFC5246 4.7. %% @doc Signed Tree Head as specified in RFC6962 section 3.2. -spec sth(sth_signed() | list()) -> sth(). sth([]) -> sth(#sth_signed{timestamp = now}); sth(#sth_signed{version = Version, timestamp = Timestamp_in}) -> Timestamp = timestamp(Timestamp_in), Treesize = ht:size(), Roothash = ht:root(), BinToSign = serialise(#sth_signed{ version = Version, signature_type = tree_hash, timestamp = Timestamp, tree_size = Treesize, root_hash = Roothash}), Signature = #signature{ algorithm = #sig_and_hash_alg{ hash_alg = sha256, signature_alg = ecdsa}, signature = sign:sign(BinToSign)}, STH = {Treesize, Timestamp, Roothash, Signature}, %%io:format("STH: ~p~nBinToSign: ~p~nSignature: ~p~nTimestamp: ~p~n", %% [STH, BinToSign, Signature, Timestamp]), STH. %%%%%%%%%%%%%%%%%%%% %% Serialisation of data. %% FIXME: Make the type conversion functions return binaries instead %% of integers. Moving knowledge about width of these to one place. -spec signature_type(signature_type()) -> integer(). signature_type(certificate_timestamp) -> 0; signature_type(tree_hash) -> 1; signature_type(test) -> 2. -spec hash_alg_type(hash_alg_type()) -> integer(). hash_alg_type(none) -> 0; hash_alg_type(md5) -> 1; hash_alg_type(sha1) -> 2; hash_alg_type(sha224) -> 3; hash_alg_type(sha256) -> 4; hash_alg_type(sha384) -> 5; hash_alg_type(sha512) -> 6. -spec signature_alg_type(signature_alg_type()) -> integer(). signature_alg_type(anonymous) -> 0; signature_alg_type(rsa) -> 1; signature_alg_type(dsa) -> 2; signature_alg_type(ecdsa) -> 3. %% TODO: Remove. -spec timestamp(now | integer()) -> integer(). timestamp(Timestamp) -> case Timestamp of now -> {NowMegaSec, NowSec, NowMicroSec} = now(), trunc(NowMegaSec * 1.0e9 + NowSec * 1.0e3 + NowMicroSec / 1.0e3); _ -> Timestamp end. -spec generate_timestamp() -> integer(). generate_timestamp() -> {NowMegaSec, NowSec, NowMicroSec} = now(), trunc(NowMegaSec * 1.0e9 + NowSec * 1.0e3 + NowMicroSec / 1.0e3). -spec serialise(sth() | sth_signed() | sig_and_hash_alg() | signature()) -> binary(). serialise(#sth_signed{ % Signed Tree Head. version = Version, signature_type = SigtypeAtom, timestamp = Timestamp, tree_size = Treesize, root_hash = Roothash }) -> Sigtype = signature_type(SigtypeAtom), <>; serialise(#sig_and_hash_alg{ hash_alg = HashAlgType, signature_alg = SignatureAlgType }) -> HashAlg = hash_alg_type(HashAlgType), SignatureAlg = signature_alg_type(SignatureAlgType), <>; serialise(#signature{ algorithm = Algorithm, signature = Signature % DER encoded. }) -> %% Encode a DSS signature according to RFC5246 section 4.7 and %% don't forget that the signature is a vector as specified in %% section 4.3 and has a length field. SigLen = size(Signature), list_to_binary([serialise(Algorithm), <>]). %%%%%%%%%%%%%%%%%%%% %% Internal tests. For more tests see ../test/. %% serialise_test_() -> %% [?_assertEqual( %% <<0:8, 0:8, 0:64, 0:16, "foo">>, %% serialise(#spt_signed{ %% version = 0, %% signature_type = certificate_timestamp, %% timestamp = 0, %% entry_type = x509, %% signed_entry = <<"foo">>}))].