%%% Copyright (c) 2014, NORDUnet A/S. %%% See LICENSE for licensing information. %%% %%% @doc Server holding log entries in a database and hashes in a %%% Merkle tree. A backend for things like Certificate Transparency %%% (RFC 6962). %%% %%% When you submit data for insertion in the log, it's stored in an %%% append only database with an accompanying Merkle tree. The leaves %%% of the tree hold hashes of submitted data and makes it possible %%% for anyone to verify wether a given piece of data is or is not %%% present in the log. %%% %%% In return you will get a signed timestamp which is a promise that %%% your entry will be present in the log within a certain time period %%% (the MMD). This signed timestamp can later, together with the %%% public key of the log, be used to ensure that your entry is indeed %%% present in the log. %%% TODO %%% - get rid of CT-specific stuff that has creeped in -module(plop). -behaviour(gen_server). %% API. -export([start_link/2, stop/0]). -export([get_logid/0, serialise/1]). -export([add/3, sth/0, get/1, get/2, spt/1, consistency/2, inclusion/2, inclusion_and_entry/2]). -export([generate_timestamp/0]). %% API for tests. -export([read_keyfile_rsa/2, read_keyfiles_ec/2]). -export([testing_get_pubkey/0]). %% gen_server callbacks. -export([init/1, handle_call/3, terminate/2, handle_cast/2, handle_info/2, code_change/3]). -include("plop.hrl"). %%-include("db.hrl"). -include_lib("public_key/include/public_key.hrl"). -include_lib("eunit/include/eunit.hrl"). -define(TESTPRIVKEYFILE, "test/eckey.pem"). -define(TESTPUBKEYFILE, "test/eckey-public.pem"). -record(state, {pubkey :: public_key:rsa_public_key(), privkey :: public_key:rsa_private_key(), logid :: binary(), http_requests, own_requests }). %%%%% moved from plop.hrl, maybe remove -define(PLOPVERSION, 0). -type signature_type() :: certificate_timestamp | tree_hash | test. % uint8 %%%%% %% @doc The parts of an STH which is to be signed. Used as the %% interface to plop:sth/1, for testing. -record(sth_signed, { version = ?PLOPVERSION :: non_neg_integer(), signature_type :: signature_type(), timestamp = now :: 'now' | integer(), tree_size :: integer(), root_hash :: binary() % SHA-256 }). -type sth_signed() :: #sth_signed{}. start_link(Keyfile, Passphrase) -> gen_server:start_link({local, ?MODULE}, ?MODULE, [Keyfile, Passphrase], []). stop() -> gen_server:call(?MODULE, stop). add_http_request(Plop, RequestId, Data) -> Plop#state{http_requests = dict:store(RequestId, Data, Plop#state.http_requests)}. add_own_request(Plop, RequestId, Data) -> Plop#state{own_requests = dict:store(RequestId, Data, Plop#state.own_requests)}. remove_http_request(Plop, RequestId) -> Plop#state{http_requests = dict:erase(RequestId, Plop#state.http_requests)}. remove_own_request(Plop, RequestId) -> Plop#state{own_requests = dict:erase(RequestId, Plop#state.own_requests)}. %%%%%%%%%%%%%%%%%%%% init([PrivKeyfile, PubKeyfile]) -> %% Read RSA keypair. %% {Private_key, Public_key} = read_keyfile_rsa(Keyfile, Passphrase), %% LogID = crypto:hash(sha256, %% public_key:der_encode('RSAPublicKey', Public_key)), %% Read EC keypair. {Private_key, Public_key, LogID} = read_keyfiles_ec(PrivKeyfile, PubKeyfile), _Tree = ht:reset_tree([db:size() - 1]), {ok, #state{pubkey = Public_key, privkey = Private_key, logid = LogID, http_requests = dict:new(), own_requests = dict:new()}}. handle_cast(_Request, State) -> {noreply, State}. handle_http_reply(State, {storage_sendentry_http, {OwnRequestId}}, StatusCode, Body) -> lager:debug("http_reply: ~p", [Body]), {PropList} = (catch jiffy:decode(Body)), Result = proplists:get_value(<<"result">>, PropList), case dict:fetch(OwnRequestId, State#state.own_requests) of undefined -> {noreply, State}; {storage_sendentry, {From, Completion, RepliesUntilQuorum}} when Result == <<"ok">>, StatusCode == 200 -> case RepliesUntilQuorum - 1 of 0 -> %% reached quorum gen_server:reply(From, ok), StateWithCompletion = Completion(State), {noreply, remove_own_request(StateWithCompletion, OwnRequestId)}; NewRepliesUntilQuorum -> {noreply, add_own_request(State, OwnRequestId, {storage_sendentry, {From, Completion, NewRepliesUntilQuorum}})} end end. handle_info({http, {RequestId, {StatusLine, _Headers, Body}}}, Plop) -> {_HttpVersion, StatusCode, _ReasonPhrase} = StatusLine, case dict:fetch(RequestId, Plop#state.http_requests) of undefined -> {noreply, Plop}; ignore -> {noreply, Plop}; HttpRequest -> handle_http_reply(remove_http_request(Plop, RequestId), HttpRequest, StatusCode, Body) end; handle_info(_Info, State) -> {noreply, State}. code_change(_OldVsn, State, _Extra) -> {ok, State}. terminate(_Reason, _State) -> io:format("~p terminating~n", [?MODULE]), ok. %%%%%%%%%%%%%%%%%%%% -spec add(binary(), binary(), binary()) -> ok. add(LogEntry, TreeLeafHash, EntryHash) -> gen_server:call(?MODULE, {add, {LogEntry, TreeLeafHash, EntryHash}}). sth() -> gen_server:call(?MODULE, {sth, []}). -spec get(non_neg_integer(), non_neg_integer()) -> [{non_neg_integer(), binary(), binary()}]. get(Start, End) -> gen_server:call(?MODULE, {get, {index, Start, End}}). get(Hash) -> gen_server:call(?MODULE, {get, {hash, Hash}}). spt(Data) -> gen_server:call(?MODULE, {spt, Data}). consistency(TreeSizeFirst, TreeSizeSecond) -> gen_server:call(?MODULE, {consistency, {TreeSizeFirst, TreeSizeSecond}}). -spec inclusion(binary(), non_neg_integer()) -> {ok, {binary(), binary()}} | {notfound, string()}. inclusion(Hash, TreeSize) -> gen_server:call(?MODULE, {inclusion, {Hash, TreeSize}}). -spec inclusion_and_entry(non_neg_integer(), non_neg_integer()) -> {ok, {binary(), binary()}} | {notfound, string()}. inclusion_and_entry(Index, TreeSize) -> gen_server:call(?MODULE, {inclusion_and_entry, {Index, TreeSize}}). get_logid() -> gen_server:call(?MODULE, {get, logid}). testing_get_pubkey() -> gen_server:call(?MODULE, {test, pubkey}). storage_nodes() -> application:get_env(plop, storage_nodes, []). storage_nodes_quorum() -> {ok, Value} = application:get_env(plop, storage_nodes_quorum), Value. send_storage_sendentry(URLBase, LogEntry, TreeLeafHash) -> Request = jiffy:encode( {[{plop_version, 1}, {entry, base64:encode(LogEntry)}, {treeleafhash, base64:encode(TreeLeafHash)} ]}), httpc:request(post, {URLBase ++ "sendentry", [], "text/json", Request}, [], [{sync, false}]). send_storage_entrycommitted(URLBase, EntryHash, TreeLeafHash) -> Request = jiffy:encode( {[{plop_version, 1}, {entryhash, base64:encode(EntryHash)}, {treeleafhash, base64:encode(TreeLeafHash)} ]}), httpc:request(post, {URLBase ++ "entrycommitted", [], "text/json", Request}, [], [{sync, false}]). store_at_all_nodes(Nodes, {LogEntry, TreeLeafHash, EntryHash}, From, State) -> lager:debug("leafhash ~p", [TreeLeafHash]), OwnRequestId = make_ref(), Completion = fun(CompletionState) -> RequestIds = [send_storage_entrycommitted(URLBase, EntryHash, TreeLeafHash) || URLBase <- Nodes], lists:foldl(fun({ok, RequestId}, StateAcc) -> add_http_request(StateAcc, RequestId, ignore) end, CompletionState, RequestIds) end, PlopWithOwn = add_own_request(State, OwnRequestId, {storage_sendentry, {From, Completion, storage_nodes_quorum()}}), lager:debug("send requests to ~p", [Nodes]), RequestIds = [send_storage_sendentry(URLBase, LogEntry, TreeLeafHash) || URLBase <- Nodes], PlopWithRequests = lists:foldl(fun({ok, RequestId}, PlopAcc) -> add_http_request(PlopAcc, RequestId, {storage_sendentry_http, {OwnRequestId}}) end, PlopWithOwn, RequestIds), PlopWithRequests. %%%%%%%%%%%%%%%%%%%% handle_call(stop, _From, Plop) -> {stop, normal, stopped, Plop}; handle_call({get, {index, Start, End}}, _From, Plop) -> {reply, db:get_by_indices(Start, End, {sorted, false}), Plop}; handle_call({get, {hash, EntryHash}}, _From, Plop) -> {reply, db:get_by_entry_hash(EntryHash), Plop}; handle_call({get, logid}, _From, Plop = #state{logid = LogID}) -> {reply, LogID, Plop}; handle_call({add, {LogEntry, TreeLeafHash, EntryHash}}, From, Plop) -> lager:debug("add leafhash ~p", [TreeLeafHash]), case storage_nodes() of [] -> ok = db:add(TreeLeafHash, EntryHash, LogEntry, ht:size()), ok = ht:add(TreeLeafHash), {reply, ok, Plop}; Nodes -> {noreply, store_at_all_nodes(Nodes, {LogEntry, TreeLeafHash, EntryHash}, From, Plop)} end; handle_call({sth, Data}, _From, Plop = #state{privkey = PrivKey}) -> {reply, sth(PrivKey, Data), Plop}; handle_call({spt, Data}, _From, Plop = #state{privkey = PrivKey}) -> {reply, spt(PrivKey, Data), Plop}; handle_call({consistency, {First, Second}}, _From, Plop) -> {reply, ht:consistency(First - 1, Second - 1), Plop}; handle_call({inclusion, {Hash, TreeSize}}, _From, Plop) -> R = case db:get_by_leaf_hash(Hash) of notfound -> {notfound, "Unknown hash"}; % FIXME: include Hash {I, _MTLHash, _Entry} -> {ok, I, ht:path(I, TreeSize - 1)} end, {reply, R, Plop}; handle_call({inclusion_and_entry, {Index, TreeSize}}, _From, Plop) -> R = case db:get_by_index(Index) of notfound -> {notfound, "Unknown index"}; % FIXME: include Index {I, _MTLHash, Entry} -> {ok, Entry, ht:path(I, TreeSize - 1)} end, {reply, R, Plop}; handle_call({test, pubkey}, _From, Plop = #state{pubkey = PK}) -> {reply, PK, Plop}. %% @doc Signed Plop Timestamp, conformant to an SCT in RFC6962 3.2 and %% RFC5246 4.7. -spec spt(public_key:ec_private_key(), binary()) -> signature(). spt(PrivKey, SerialisedData) -> #signature{algorithm = #sig_and_hash_alg{ hash_alg = sha256, signature_alg = ecdsa}, signature = signhash(SerialisedData, PrivKey)}. %% @doc Signed Tree Head as specified in RFC6962 section 3.2. -spec sth(#'ECPrivateKey'{}, sth_signed() | list()) -> sth(). sth(PrivKey, []) -> sth(PrivKey, #sth_signed{timestamp = now}); sth(PrivKey, #sth_signed{version = Version, timestamp = Timestamp_in}) -> Timestamp = timestamp(Timestamp_in), Treesize = ht:size(), Roothash = ht:root(), BinToSign = serialise(#sth_signed{ version = Version, signature_type = tree_hash, timestamp = Timestamp, tree_size = Treesize, root_hash = Roothash}), Signature = #signature{ algorithm = #sig_and_hash_alg{ hash_alg = sha256, signature_alg = ecdsa}, signature = signhash(BinToSign, PrivKey)}, STH = {Treesize, Timestamp, Roothash, Signature}, %%io:format("STH: ~p~nBinToSign: ~p~nSignature: ~p~nTimestamp: ~p~n", %% [STH, BinToSign, Signature, Timestamp]), STH. %% TODO: Merge the keyfile reading functions. %% @doc Read one password protected PEM file with an RSA keypair. read_keyfile_rsa(Filename, Passphrase) -> {ok, PemBin} = file:read_file(Filename), [KeyPem] = public_key:pem_decode(PemBin), % Use first entry. Privatekey = decode_key(KeyPem, Passphrase), {Privatekey, public_key(Privatekey)}. %% @doc Read two PEM files, one with a private EC key and one with the %% corresponding public EC key. read_keyfiles_ec(PrivkeyFile, Pubkeyfile) -> {ok, PemBinPriv} = file:read_file(PrivkeyFile), [OTPPubParamsPem, PrivkeyPem] = public_key:pem_decode(PemBinPriv), Privatekey = decode_key(PrivkeyPem), {_, ParamsBin, ParamsEnc} = OTPPubParamsPem, PubParamsPem = {'EcpkParameters', ParamsBin, ParamsEnc}, Params = public_key:pem_entry_decode(PubParamsPem), {ok, PemBinPub} = file:read_file(Pubkeyfile), [SPKIPem] = public_key:pem_decode(PemBinPub), %% SPKI is missing #'AlgorithmIdentifier' so pem_entry_decode won't do. %% Publickey = public_key:pem_entry_decode(SPKIPem), #'SubjectPublicKeyInfo'{algorithm = AlgoDer} = SPKIPem, SPKI = public_key:der_decode('SubjectPublicKeyInfo', AlgoDer), #'SubjectPublicKeyInfo'{subjectPublicKey = {_, Octets}} = SPKI, Point = #'ECPoint'{point = Octets}, Publickey = {Point, Params}, KeyID = crypto:hash(sha256, AlgoDer), {Privatekey, Publickey, KeyID}. decode_key(Entry) -> public_key:pem_entry_decode(Entry). decode_key(Entry, Passphrase) -> public_key:pem_entry_decode(Entry, Passphrase). public_key(#'RSAPrivateKey'{modulus = Mod, publicExponent = Exp}) -> #'RSAPublicKey'{modulus = Mod, publicExponent = Exp}. %% FIXME: Merge RSA and DC. signhash(D, K) -> signhash_ec(D, K). -spec signhash_ec(iolist() | binary(), public_key:ec_private_key()) -> binary(). signhash_ec(Data, PrivKey) -> public_key:sign(Data, sha256, PrivKey). %% -spec signhash_rsa(iolist() | binary(), public_key:rsa_private_key()) -> binary(). %% signhash_rsa(Data, PrivKey) -> %% %% Was going to just crypto:sign/3 the hash but looking at %% %% digitally_signed() in lib/ssl/src/ssl_handshake.erl it seems %% %% like we should rather use (undocumented) encrypt_private/3. %% %public_key:sign(hash(sha256, BinToSign), sha256, PrivKey) %% public_key:encrypt_private(crypto:hash(sha256, Data), %% PrivKey, %% [{rsa_pad, rsa_pkcs1_padding}]). %%%%%%%%%%%%%%%%%%%% %% Serialisation of data. %% FIXME: Make the type conversion functions return binaries instead %% of integers. Moving knowledge about width of these to one place. -spec signature_type(signature_type()) -> integer(). signature_type(certificate_timestamp) -> 0; signature_type(tree_hash) -> 1; signature_type(test) -> 2. -spec hash_alg_type(hash_alg_type()) -> integer(). hash_alg_type(none) -> 0; hash_alg_type(md5) -> 1; hash_alg_type(sha1) -> 2; hash_alg_type(sha224) -> 3; hash_alg_type(sha256) -> 4; hash_alg_type(sha384) -> 5; hash_alg_type(sha512) -> 6. -spec signature_alg_type(signature_alg_type()) -> integer(). signature_alg_type(anonymous) -> 0; signature_alg_type(rsa) -> 1; signature_alg_type(dsa) -> 2; signature_alg_type(ecdsa) -> 3. %% TODO: Remove. -spec timestamp(now | integer()) -> integer(). timestamp(Timestamp) -> case Timestamp of now -> {NowMegaSec, NowSec, NowMicroSec} = now(), trunc(NowMegaSec * 1.0e9 + NowSec * 1.0e3 + NowMicroSec / 1.0e3); _ -> Timestamp end. -spec generate_timestamp() -> integer(). generate_timestamp() -> {NowMegaSec, NowSec, NowMicroSec} = now(), trunc(NowMegaSec * 1.0e9 + NowSec * 1.0e3 + NowMicroSec / 1.0e3). -spec serialise(sth() | sth_signed() | sig_and_hash_alg() | signature()) -> binary(). serialise(#sth_signed{ % Signed Tree Head. version = Version, signature_type = SigtypeAtom, timestamp = Timestamp, tree_size = Treesize, root_hash = Roothash }) -> Sigtype = signature_type(SigtypeAtom), <>; serialise(#sig_and_hash_alg{ hash_alg = HashAlgType, signature_alg = SignatureAlgType }) -> HashAlg = hash_alg_type(HashAlgType), SignatureAlg = signature_alg_type(SignatureAlgType), <>; serialise(#signature{ algorithm = Algorithm, signature = Signature % DER encoded. }) -> %% Encode a DSS signature according to RFC5246 section 4.7 and %% don't forget that the signature is a vector as specified in %% section 4.3 and has a length field. SigLen = size(Signature), list_to_binary([serialise(Algorithm), <>]). %%%%%%%%%%%%%%%%%%%% %% Internal tests. For more tests see ../test/. %% serialise_test_() -> %% [?_assertEqual( %% <<0:8, 0:8, 0:64, 0:16, "foo">>, %% serialise(#spt_signed{ %% version = 0, %% signature_type = certificate_timestamp, %% timestamp = 0, %% entry_type = x509, %% signed_entry = <<"foo">>}))].