Require that storage servers sign stored entries

6 files changed, 144 insertions, 43 deletions

diff --git a/src/http_auth.erl b/src/http_auth.erl
index ff8c506..276e1cd 100644
--- a/src/http_auth.erl
+++ b/src/http_auth.erl
@@ -2,7 +2,7 @@
 %%% See LICENSE for licensing information.
 
 -module(http_auth).
--export([verify_auth/4, create_auth/3, init_key_table/0]).
+-export([verify_auth/4, create_auth/3, init_key_table/0, sign_stored/1, verify_stored/3]).
 
 -define(KEY_TABLE, http_auth_keys).
 
@@ -135,6 +135,23 @@ verify_auth(AuthHeader, Method, Path, Data) ->
             failure
     end.
 
+sign_stored(Data) ->
+    {Key, KeyName} = own_key(),
+    Signature = public_key:sign(Data, sha256, Key),
+    {KeyName, Signature}.
+
+verify_stored(KeyName, Data, Signature) ->
+    case lookup_publickey(KeyName) of
+        nokey ->
+            lager:error("key name ~p could not be found", [KeyName]),
+            false;
+        failure ->
+            lager:error("signature ~p with key name ~p and data ~p did not check out", [Signature, KeyName, Data]),
+            false;
+        Key ->
+            public_key:verify(Data, sha256, Signature, Key)
+    end.
+
 create_auth(Method, Path, Data) ->
     case own_key() of
         {Key, KeyName} ->
diff --git a/src/plop.erl b/src/plop.erl
index 8ce3186..46b402b 100644
--- a/src/plop.erl
+++ b/src/plop.erl
@@ -25,9 +25,10 @@
 %% API.
 -export([initsize/0]).
 -export([get_logid/0, serialise/1, signature_type/1]).
--export([add/3, sth/0, get/1, get/2, spt/1, consistency/2, inclusion/2, inclusion_and_entry/2]).
+-export([add/2, commit/3, sth/0, get/1, get/2, spt/2,
+         consistency/2, inclusion/2, inclusion_and_entry/2]).
 -export([generate_timestamp/0, save_sth/1, verify_sth/4]).
--export([get_by_leaf_hash/1, entry_for_leafhash/1]).
+-export([get_by_leaf_hash/1, entry_for_leafhash/1, spt_from_entry/1, get_spt/1, add_spt/2]).
 %% API for tests.
 -export([testing_get_pubkey/0]).
 
@@ -60,18 +61,19 @@ handle_http_reply(TreeLeafHash, RepliesUntilQuorum,
                 [mochihex:to_hex(TreeLeafHash), Body]),
     {struct, PropList} = mochijson2:decode(Body),
     Result = proplists:get_value(<<"result">>, PropList),
+    Signature = proplists:get_value(<<"sig">>, PropList),
     if Result == <<"ok">>, StatusCode == 200 ->
             case RepliesUntilQuorum - 1 of
                 0 ->
                     %% reached quorum
                     lager:debug("leafhash ~s: reached quorum",
                                 [mochihex:to_hex(TreeLeafHash)]),
-                    {ok};
+                    {ok, Signature};
                 NewRepliesUntilQuorum ->
                     lager:debug("leafhash ~s: replies until quorum: ~p",
                                 [mochihex:to_hex(TreeLeafHash),
                                  NewRepliesUntilQuorum]),
-                    {continue, NewRepliesUntilQuorum}
+                    {continue, NewRepliesUntilQuorum, Signature}
             end
     end.
 
@@ -90,20 +92,47 @@ unwrap_entry(WrappedEntry) ->
             error
     end.
 
+spt_data(Entry) ->
+    {ok, {Module, Function}} = application:get_env(plop, spt_data),
+    Module:Function(Entry).
+
+spt_from_entry(LogEntry) ->
+    spt_data(unwrap_entry(LogEntry)).
+
+get_spt(Hash) ->
+    perm:getvalue(sptcache, Hash).
+
+add_spt(LeafHash, SPT) ->
+    ok = perm:addvalue(sptcache, LeafHash, SPT),
+    perm:commit(sptcache),
+    ok.
+
 %%%%%%%%%%%%%%%%%%%%
--spec add(binary(), binary(), binary()) -> ok.
-add(LogEntry, TreeLeafHash, EntryHash) ->
+-spec add(binary(), binary()) -> {ok, list()}.
+add(LogEntry, TreeLeafHash) ->
     lager:debug("add leafhash ~s", [mochihex:to_hex(TreeLeafHash)]),
     WrappedLogEntry = wrap_entry(LogEntry),
     case storage_nodes() of
         [] ->
             exit(internal_merge_not_supported);
         Nodes ->
-            util:spawn_and_wait(fun () ->
-                                        store_at_all_nodes(Nodes, {WrappedLogEntry, TreeLeafHash, EntryHash})
-                                end)
+            util:spawn_and_wait(
+              fun () ->
+                      store_at_all_nodes(Nodes, {WrappedLogEntry, TreeLeafHash})
+              end)
     end.
 
+commit(TreeLeafHash, EntryHash, SPT) ->
+    Nodes = storage_nodes(),
+    util:spawn_and_wait(
+      fun () ->
+              lists:foreach(
+                fun (URLBase) ->
+                        send_storage_entrycommitted(URLBase, EntryHash,
+                                                    TreeLeafHash, SPT)
+                end, Nodes)
+      end).
+
 save_sth(STH) ->
     {ok, STHFile} = application:get_env(plop, sth_path),
     {struct, PropList} = STH,
@@ -151,11 +180,11 @@ get(Hash) ->
             {notfetched, LeafHash, unwrap_entry(Entry)}
     end.
 
-spt(Data) ->
+spt(Data, Signatures) ->
     #signature{algorithm = #sig_and_hash_alg{
                               hash_alg = sha256,
                               signature_alg = ecdsa},
-               signature = sign:sign_sct(Data)}.
+               signature = sign:sign_sct(Data, Signatures)}.
 
 consistency(TreeSizeFirst, TreeSizeSecond) ->
     TreeSize = db:size(),
@@ -250,15 +279,21 @@ send_storage_sendentry(URLBase, LogEntry, TreeLeafHash) ->
     RequestId = send_http_request(TreeLeafHash, URLBase ++ "sendentry", [{"Content-Type", "text/json"}], list_to_binary(Request)),
     {RequestId, URLBase}.
 
-send_storage_entrycommitted(URLBase, EntryHash, TreeLeafHash) ->
+send_storage_entrycommitted(URLBase, EntryHash, TreeLeafHash, SPT) ->
     Request = mochijson2:encode(
                 {[{plop_version, 1},
                   {entryhash, base64:encode(EntryHash)},
-                  {treeleafhash, base64:encode(TreeLeafHash)}
+                  {treeleafhash, base64:encode(TreeLeafHash)},
+                  {timestamp_signature, base64:encode(SPT)}
                  ]}),
     send_http_request(TreeLeafHash, URLBase ++ "entrycommitted", [{"Content-Type", "text/json"}], list_to_binary(Request)).
 
-store_loop(TreeLeafHash, Requests, RepliesUntilQuorum) ->
+add_if_not_undefined(undefined, Signatures) ->
+    Signatures;
+add_if_not_undefined(Signature, Signatures) ->
+    [Signature | Signatures].
+
+store_loop(TreeLeafHash, Requests, RepliesUntilQuorum, Signatures) ->
     receive
         {http, {RequestId, {StatusLine, _Headers, Body}}} ->
             {_HttpVersion, StatusCode, _ReasonPhrase} = StatusLine,
@@ -267,16 +302,16 @@ store_loop(TreeLeafHash, Requests, RepliesUntilQuorum) ->
                     lager:info("leafhash ~s: stray storage reply: ~p",
                                [mochihex:to_hex(TreeLeafHash),
                                 {StatusLine, Body}]),
-                    store_loop(TreeLeafHash, Requests, RepliesUntilQuorum);
+                    store_loop(TreeLeafHash, Requests, RepliesUntilQuorum, Signatures);
                 true ->
                     NewRequests = dict:erase(RequestId, Requests),
                     case handle_http_reply(TreeLeafHash, RepliesUntilQuorum,
                                            StatusCode, Body) of
-                        {ok} ->
-                            ok;
-                        {continue, NewRepliesUntilQuorum} ->
+                        {ok, Signature} ->
+                            {ok, add_if_not_undefined(Signature, Signatures)};
+                        {continue, NewRepliesUntilQuorum, Signature} ->
                             store_loop(TreeLeafHash, NewRequests,
-                                       NewRepliesUntilQuorum)
+                                       NewRepliesUntilQuorum, add_if_not_undefined(Signature, Signatures))
                     end
             end
     after
@@ -291,20 +326,16 @@ store_loop(TreeLeafHash, Requests, RepliesUntilQuorum) ->
     end.
 
 
-store_at_all_nodes(Nodes, {LogEntry, TreeLeafHash, EntryHash}) ->
+store_at_all_nodes(Nodes, {LogEntry, TreeLeafHash}) ->
     lager:debug("leafhash ~s: send requests to ~p",
                 [mochihex:to_hex(TreeLeafHash), Nodes]),
     Requests = [send_storage_sendentry(URLBase, LogEntry, TreeLeafHash) || URLBase <- Nodes],
     case store_loop(TreeLeafHash, dict:from_list(Requests),
-                    storage_nodes_quorum()) of
-        ok ->
-            lager:debug("leafhash ~s: all requests answered",
-                        [mochihex:to_hex(TreeLeafHash)]),
-            lists:foreach(fun (URLBase) ->
-                                  send_storage_entrycommitted(URLBase, EntryHash,
-                                                              TreeLeafHash)
-                          end, Nodes),
-            ok;
+                    storage_nodes_quorum(), []) of
+        {ok, Signatures} ->
+            lager:debug("leafhash ~s: all requests answered: ~p",
+                        [mochihex:to_hex(TreeLeafHash), Signatures]),
+            {ok, Signatures};
         Any ->
             lager:debug("leafhash ~s: error: ~p",
                         [mochihex:to_hex(TreeLeafHash), Any]),
diff --git a/src/plop_sup.erl b/src/plop_sup.erl
index 6bac20e..27f7680 100644
--- a/src/plop_sup.erl
+++ b/src/plop_sup.erl
@@ -43,6 +43,7 @@ init([]) ->
                                     {perm, the_entryhash_db, entryhash_db, entryhash_root_path},
                                     {perm, the_indexforhash_db, indexforhash_db, indexforhash_root_path},
                                     {perm, the_entry_db, entry_db, entry_root_path},
+                                    {perm, the_sptcache, sptcache, sptcache_root_path},
                                     {index, the_index_db, index_db, index_path},
                                     {index, the_newentries_db, newentries_db, newentries_path}
                                    ]),
diff --git a/src/sign.erl b/src/sign.erl
index 6e0e0bd..f7c7194 100644
--- a/src/sign.erl
+++ b/src/sign.erl
@@ -8,7 +8,7 @@
 
 %% API.
 -export([start_link/0, stop/0]).
--export([sign_sct/1, sign_sth/1, get_pubkey/0, get_logid/0, verify_sth/2]).
+-export([sign_sct/2, sign_sth/1, get_pubkey/0, get_logid/0, verify_sth/2]).
 -export([read_keyfile_ec/1, pem_entry_decode/1]).
 %% API for tests.
 -export([read_keyfile_rsa/2]).
@@ -145,20 +145,58 @@ remote_sign_request([URL|RestURLs], Request) ->
             remote_sign_request(RestURLs, Request)
     end.
 
+verify_storage_signature(Data, Signature) ->
+    case string:tokens(Signature, ":") of
+        [Nodename, Sig] ->
+            case http_auth:verify_stored(Nodename, Data, base64:decode(Sig)) of
+                true ->
+                    {ok, Nodename};
+                _ ->
+                    error
+            end;
+        _ ->
+            error
+    end.
+
+verify_storage_signatures(Data, Signatures) ->
+    Nodenames = sets:from_list(verify_storage_signatures(Data, Signatures, [])),
+    AllowedStorageNodes = sets:from_list(plopconfig:get_env(storage_node_names, [])),
+    sets:intersection(Nodenames, AllowedStorageNodes).
+
+verify_storage_signatures(_Data, [], Nodes) ->
+    Nodes;
+verify_storage_signatures(Data, [Signature | Rest], Nodes) ->
+    case verify_storage_signature(Data, Signature) of
+        {ok, Node} ->
+            verify_storage_signatures(Data, Rest, [Node | Nodes]);
+        _ ->
+            []
+    end.
+
 %%%%%%%%%%%%%%%%%%%%
 %% Public API.
 
 sign_sct(Data = <<_Version:8,
                   ?CERTIFICATE_TIMESTAMP:8,
-                  _/binary>>) ->
-    case plopconfig:get_env(signing_nodes) of
-        {ok, URLBases} ->
-            Request = {[{plop_version, 1},
-                        {data, base64:encode(Data)}
-                       ]},
-            remote_sign_request([URLBase ++ "sct" || URLBase <- URLBases], Request);
-        undefined ->
-            call(?MODULE, {sign, Data})
+                  _/binary>>, Signatures) ->
+    Nodenames = verify_storage_signatures(Data, Signatures),
+    lager:debug("sign_sct: signatures ~p ~p", [Signatures, sets:to_list(Nodenames)]),
+    {ok, Quorum} = plopconfig:get_env(storage_sign_quorum),
+    case sets:size(Nodenames) >= Quorum of
+        true ->
+            case plopconfig:get_env(signing_nodes) of
+                {ok, URLBases} ->
+                    Request = {[{plop_version, 1},
+                                {data, base64:encode(Data)},
+                                {signatures, Signatures}
+                               ]},
+                    remote_sign_request([URLBase ++ "sct" || URLBase <- URLBases], Request);
+                undefined ->
+                    call(?MODULE, {sign, Data})
+            end;
+        _ ->
+            lager:error("signatures (~p) less than quorum (~p)", [sets:size(Nodenames), Quorum]),
+            error
     end.
 
 sign_sth(Data = <<_Version:8,
diff --git a/src/signing.erl b/src/signing.erl
index 86ccb88..5bcd3eb 100644
--- a/src/signing.erl
+++ b/src/signing.erl
@@ -15,8 +15,13 @@ request(post, ?APPURL_PLOP_SIGNING, "sct", Input) ->
             html("sendentry: bad input:", E);
         {struct, PropList} ->
             Data = base64:decode(proplists:get_value(<<"data">>, PropList)),
-
-            Result = sign:sign_sct(Data),
+            Signatures = case proplists:get_value(<<"signatures">>, PropList) of
+                             undefined ->
+                                 [];
+                             Sigs ->
+                                 Sigs
+                         end,
+            Result = sign:sign_sct(Data, Signatures),
             success({[{result, base64:encode(Result)}]})
     end;
 request(post, ?APPURL_PLOP_SIGNING, "sth", Input) ->
diff --git a/src/storage.erl b/src/storage.erl
index efe4768..489bf91 100644
--- a/src/storage.erl
+++ b/src/storage.erl
@@ -19,7 +19,10 @@ request(post, ?APPURL_PLOP_STORAGE, "sendentry", Input) ->
 
             ok = db:add_entry_sync(TreeLeafHash, LogEntry),
             ok = storagedb:add(TreeLeafHash),
-            success({[{result, <<"ok">>}]})
+            {KeyName, Sig} = http_auth:sign_stored(plop:spt_from_entry(LogEntry)),
+            success({[{result, <<"ok">>},
+                      {"sig", KeyName ++ ":" ++ base64:encode_to_string(Sig)}
+                     ]})
     end;
 request(post, ?APPURL_PLOP_STORAGE, "entrycommitted", Input) ->
     case (catch mochijson2:decode(Input)) of
@@ -30,6 +33,12 @@ request(post, ?APPURL_PLOP_STORAGE, "entrycommitted", Input) ->
             LeafHash = base64:decode(proplists:get_value(<<"treeleafhash">>, PropList)),
             db:add_entryhash(LeafHash, EntryHash),
             db:commit_entryhash(),
+            case proplists:get_value(<<"timestamp_signature">>, PropList) of
+                undefined ->
+                    none;
+                TimestampSignature ->
+                    plop:add_spt(LeafHash, base64:decode(TimestampSignature))
+            end,
             success({[{result, <<"ok">>}]})
     end;
 request(get, ?APPURL_PLOP_STORAGE, "fetchnewentries", _Input) ->