path: root/src/plop.erl

%%% Copyright (c) 2014, NORDUnet A/S.
%%% See LICENSE for licensing information.
%%%
%%% @doc Server holding log entries in a database and hashes in a
%%% Merkle tree. A backend for things like Certificate Transparency
%%% (RFC 6962).
%%%
%%% When you submit data for insertion in the log, it's stored in an
%%% append only database with an accompanying Merkle tree. The leaves
%%% of the tree hold hashes of submitted data and makes it possible
%%% for anyone to verify wether a given piece of data is or is not
%%% present in the log.
%%%
%%% In return you will get a signed timestamp which is a promise that
%%% your entry will be present in the log within a certain time period
%%% (the MMD). This signed timestamp can later, together with the
%%% public key of the log, be used to ensure that your entry is indeed
%%% present in the log.

%%% TODO
%%% - get rid of CT-specific stuff that has creeped in

-module(plop).

%% API.
-export([initsize/0]).
-export([get_logid/0, serialise/1]).
-export([add/3, sth/0, get/1, get/2, spt/1, consistency/2, inclusion/2, inclusion_and_entry/2]).
-export([generate_timestamp/0, save_sth/1, verify_sth/4]).
%% API for tests.
-export([testing_get_pubkey/0]).

-import(stacktrace, [call/2]).
-include("plop.hrl").
%%-include("db.hrl").
-include_lib("public_key/include/public_key.hrl").
-include_lib("eunit/include/eunit.hrl").

%%%%% moved from plop.hrl, maybe remove
-define(PLOPVERSION, 0).
-type signature_type() :: certificate_timestamp | tree_hash | test. % uint8
%%%%%


%% @doc The parts of an STH which is to be signed. Used as the
%% interface to plop:sth/1, for testing.
-record(sth_signed, {
          version = ?PLOPVERSION :: non_neg_integer(),
          signature_type :: signature_type(),
          timestamp = now :: 'now' | integer(),
          tree_size :: integer(),
          root_hash :: binary()                 % SHA-256
         }).
-type sth_signed() :: #sth_signed{}.

handle_http_reply(TreeLeafHash, RepliesUntilQuorum,
                  StatusCode, Body) ->
    lager:debug("leafhash ~s: http_reply: ~p",
                [mochihex:to_hex(TreeLeafHash), Body]),
    {struct, PropList} = mochijson2:decode(Body),
    Result = proplists:get_value(<<"result">>, PropList),
    if Result == <<"ok">>, StatusCode == 200 ->
            case RepliesUntilQuorum - 1 of
                0 ->
                    %% reached quorum
                    lager:debug("leafhash ~s: reached quorum",
                                [mochihex:to_hex(TreeLeafHash)]),
                    {ok};
                NewRepliesUntilQuorum ->
                    lager:debug("leafhash ~s: replies until quorum: ~p",
                                [mochihex:to_hex(TreeLeafHash),
                                 NewRepliesUntilQuorum]),
                    {continue, NewRepliesUntilQuorum}
            end
    end.

%%%%%%%%%%%%%%%%%%%%
-spec add(binary(), binary(), binary()) -> ok.
add(LogEntry, TreeLeafHash, EntryHash) ->
    lager:debug("add leafhash ~s", [mochihex:to_hex(TreeLeafHash)]),
    case storage_nodes() of
        [] ->
            exit(internal_merge_not_supported);
        Nodes ->
            util:spawn_and_wait(fun () ->
                                        store_at_all_nodes(Nodes, {LogEntry, TreeLeafHash, EntryHash})
                                end)
    end.

save_sth(STH) ->
    {ok, STHFile} = application:get_env(plop, sth_path),
    {struct, PropList} = STH,
    Treesize = proplists:get_value(<<"tree_size">>, PropList),
    lager:debug("writing new sth to ~p: ~p", [STHFile, STH]),
    ok = atomic:replacefile(STHFile, mochijson2:encode(STH)),
    ok = db:set_treesize(Treesize).

initsize() ->
    db:create_size_table(),
    Treesize = case sth() of
                   noentry ->
                       0;
                   {struct, PropList} ->
                       proplists:get_value(<<"tree_size">>, PropList)
               end,
    db:set_treesize(Treesize).

sth() ->
    case application:get_env(plop, sth_path) of
        {ok, STHFile} ->
            case atomic:readfile(STHFile) of
                noentry ->
                    noentry;
                Contents ->
                    mochijson2:decode(Contents)
            end;
        undefined ->
            noentry
    end.

-spec get(non_neg_integer(), non_neg_integer()) ->
                 [{non_neg_integer(), binary(), binary()}].
get(Start, End) ->
    EndBound = min(End, db:size() - 1),
    lists:map(fun (E) -> fill_in_entry(E) end,
              db:get_by_indices(Start, EndBound, {sorted, false})).

-spec get(binary()) -> notfound | {notfetched, binary(), binary()}.
get(Hash) ->
    db:get_by_entry_hash(Hash).

spt(Data) ->
    #signature{algorithm = #sig_and_hash_alg{
                              hash_alg = sha256,
                              signature_alg = ecdsa},
               signature = sign:sign_sct(Data)}.

consistency(TreeSizeFirst, TreeSizeSecond) ->
    TreeSize = db:size(),
    if
        TreeSizeFirst >= TreeSizeSecond ->
            [];
        TreeSizeSecond > TreeSize ->
            [];
        true ->
            ht:consistency(TreeSizeFirst - 1, TreeSizeSecond - 1)
    end.

-spec inclusion(binary(), non_neg_integer()) ->
                       {ok, {binary(), binary()}} | {notfound, string()}.
inclusion(Hash, TreeSize) ->
    LastAllowedTreeSize = db:size(),
    if
        TreeSize > LastAllowedTreeSize ->
            {notfound, "Unknown tree size"};
        true ->
            case db:get_by_leaf_hash(Hash) of
                notfound ->
                    {notfound, "Unknown hash"};     % FIXME: include Hash
                {I, _MTLHash, _Entry} ->
                    {ok, I, ht:path(I, TreeSize - 1)}
            end
    end.

-spec inclusion_and_entry(non_neg_integer(), non_neg_integer()) ->
                                 {ok, {binary(), binary()}} |
                                 {notfound, string()}.
inclusion_and_entry(Index, TreeSize) ->
    LastAllowedTreeSize = db:size(),
    LastAllowedEntry = db:size() - 1,
    if
        TreeSize > LastAllowedTreeSize ->
            {notfound, "Unknown tree size"};
        Index > LastAllowedEntry ->
            {notfound, "Unknown index"};
        true ->
            case db:get_by_index(Index) of
                notfound ->
                    {notfound, "Unknown index"};    % FIXME: include Index
                {I, _MTLHash, Entry} ->
                    {ok, Entry, ht:path(I, TreeSize - 1)}
            end
    end.

get_logid() ->
    sign:get_logid().
testing_get_pubkey() ->
    sign:get_pubkey().

storage_nodes() ->
    application:get_env(plop, storage_nodes, []).

storage_nodes_quorum() ->
    {ok, Value} = application:get_env(plop, storage_nodes_quorum),
    Value.

send_http_request(TreeLeafHash, URL, Headers, RequestBody) ->
    ParentPid = self(),
    RequestId = make_ref(),
    spawn(fun () ->
                  case plop_httputil:request("leafhash " ++ mochihex:to_hex(TreeLeafHash), URL, Headers, RequestBody) of
                      {error, Error} ->
                          lager:info("request error: ~p", [Error]),
                          drop;
                      {failure, _StatusLine, _RespHeaders, _Body}  ->
                          lager:debug("auth check failed"),
                          drop;
                      {success, StatusLine, RespHeaders, Body} ->
                          lager:debug("auth check succeeded"),
                          ParentPid ! {http, {RequestId,
                                              {StatusLine, RespHeaders, Body}}};
                      {noauth, StatusLine, RespHeaders, Body} ->
                          lager:debug("no auth"),
                          ParentPid ! {http, {RequestId,
                                              {StatusLine, RespHeaders, Body}}}
                  end
          end),
    RequestId.

send_storage_sendentry(URLBase, LogEntry, TreeLeafHash) ->
    Request = mochijson2:encode(
                {[{plop_version, 1},
                  {entry, base64:encode(LogEntry)},
                  {treeleafhash, base64:encode(TreeLeafHash)}
                 ]}),
    lager:debug("leafhash ~s: send sendentry to storage node ~p",
                [mochihex:to_hex(TreeLeafHash), URLBase]),
    RequestId = send_http_request(TreeLeafHash, URLBase ++ "sendentry", [{"Content-Type", "text/json"}], list_to_binary(Request)),
    {RequestId, URLBase}.

send_storage_entrycommitted(URLBase, EntryHash, TreeLeafHash) ->
    Request = mochijson2:encode(
                {[{plop_version, 1},
                  {entryhash, base64:encode(EntryHash)},
                  {treeleafhash, base64:encode(TreeLeafHash)}
                 ]}),
    send_http_request(TreeLeafHash, URLBase ++ "entrycommitted", [{"Content-Type", "text/json"}], list_to_binary(Request)).

store_loop(TreeLeafHash, Requests, RepliesUntilQuorum) ->
    receive
        {http, {RequestId, {StatusLine, _Headers, Body}}} ->
            {_HttpVersion, StatusCode, _ReasonPhrase} = StatusLine,
            case dict:is_key(RequestId, Requests) of
                false ->
                    lager:info("leafhash ~s: stray storage reply: ~p",
                               [mochihex:to_hex(TreeLeafHash),
                                {StatusLine, Body}]),
                    store_loop(TreeLeafHash, Requests, RepliesUntilQuorum);
                true ->
                    NewRequests = dict:erase(RequestId, Requests),
                    case handle_http_reply(TreeLeafHash, RepliesUntilQuorum,
                                           StatusCode, Body) of
                        {ok} ->
                            ok;
                        {continue, NewRepliesUntilQuorum} ->
                            store_loop(TreeLeafHash, NewRequests,
                                       NewRepliesUntilQuorum)
                    end
            end
    after
        2000 ->
            lager:error("leafhash ~s: storage failed: " ++
                            "~p replies until quorum, nodes left: ~p",
                        [mochihex:to_hex(TreeLeafHash), RepliesUntilQuorum,
                         lists:map(fun({_Key, Value}) ->
                                           Value
                                   end, dict:to_list(Requests))]),
            error
    end.


store_at_all_nodes(Nodes, {LogEntry, TreeLeafHash, EntryHash}) ->
    lager:debug("leafhash ~s: send requests to ~p",
                [mochihex:to_hex(TreeLeafHash), Nodes]),
    Requests = [send_storage_sendentry(URLBase, LogEntry, TreeLeafHash) || URLBase <- Nodes],
    case store_loop(TreeLeafHash, dict:from_list(Requests),
                    storage_nodes_quorum()) of
        ok ->
            lager:debug("leafhash ~s: all requests answered",
                        [mochihex:to_hex(TreeLeafHash)]),
            lists:foreach(fun (URLBase) ->
                                  send_storage_entrycommitted(URLBase, EntryHash,
                                                              TreeLeafHash)
                          end, Nodes),
            ok;
        Any ->
            lager:debug("leafhash ~s: error: ~p",
                        [mochihex:to_hex(TreeLeafHash), Any]),
            Any
    end.

fill_in_entry({_Index, LeafHash, notfetched}) ->
    db:get_by_leaf_hash(LeafHash).

%%%%%%%%%%%%%%%%%%%%

verify_sth(Treesize, Timestamp, Roothash, PackedSignature) ->
    STH = serialise(#sth_signed{
                       version = ?PLOPVERSION,
                       signature_type = tree_hash,
                       timestamp = Timestamp,
                       tree_size = Treesize,
                       root_hash = Roothash}),
    <<_HashAlg:8, _SigAlg:8, _SigLen:16, Signature/binary>> = PackedSignature,
    sign:verify_sth(STH, Signature).

%%%%%%%%%%%%%%%%%%%%
%% Serialisation of data.

%% FIXME: Make the type conversion functions return binaries instead
%% of integers. Moving knowledge about width of these to one place.

-spec signature_type(signature_type()) -> integer().
signature_type(certificate_timestamp) -> 0;
signature_type(tree_hash) -> 1;
signature_type(test) -> 2.

-spec hash_alg_type(hash_alg_type()) -> integer().
hash_alg_type(none) -> 0;
hash_alg_type(md5) -> 1;
hash_alg_type(sha1) -> 2;
hash_alg_type(sha224) -> 3;
hash_alg_type(sha256) -> 4;
hash_alg_type(sha384) -> 5;
hash_alg_type(sha512) -> 6.

-spec signature_alg_type(signature_alg_type()) -> integer().
signature_alg_type(anonymous) -> 0;
signature_alg_type(rsa) -> 1;
signature_alg_type(dsa) -> 2;
signature_alg_type(ecdsa) -> 3.

-spec generate_timestamp() -> integer().
generate_timestamp() ->
    {NowMegaSec, NowSec, NowMicroSec} = now(),
    trunc(NowMegaSec * 1.0e9
          + NowSec * 1.0e3
          + NowMicroSec / 1.0e3).

-spec serialise(sth() | sth_signed() | sig_and_hash_alg() | signature()) ->
                       binary().
serialise(#sth_signed{                          % Signed Tree Head.
             version = Version,
             signature_type = SigtypeAtom,
             timestamp = Timestamp,
             tree_size = Treesize,
             root_hash = Roothash
            }) ->
    Sigtype = signature_type(SigtypeAtom),
    <<Version:8,
      Sigtype:8,
      Timestamp:64,
      Treesize:64,
      Roothash/binary>>;
serialise(#sig_and_hash_alg{
             hash_alg = HashAlgType,
             signature_alg = SignatureAlgType
            }) ->
    HashAlg = hash_alg_type(HashAlgType),
    SignatureAlg = signature_alg_type(SignatureAlgType),
    <<HashAlg:8, SignatureAlg:8>>;
serialise(#signature{
             algorithm = Algorithm,
             signature = Signature              % DER encoded.
             }) ->
    %% Encode a DSS signature according to RFC5246 section 4.7 and
    %% don't forget that the signature is a vector as specified in
    %% section 4.3 and has a length field.
    SigLen = size(Signature),
    list_to_binary([serialise(Algorithm),
                    <<SigLen:16, Signature/binary>>]).

%%%%%%%%%%%%%%%%%%%%
%% Internal tests. For more tests see ../test/.
%% serialise_test_() ->
%%     [?_assertEqual(
%%         <<0:8, 0:8, 0:64, 0:16, "foo">>,
%%         serialise(#spt_signed{
%%                      version = 0,
%%                      signature_type = certificate_timestamp,
%%                      timestamp = 0,
%%                      entry_type = x509,
%%                      signed_entry = <<"foo">>}))].