From 088e4de4f1e2499f6cc0e332ae8cb34b935a6425 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 20 Oct 2014 12:22:50 +0200 Subject: Added HTTP API:s for external merge --- src/frontend.erl | 102 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 src/frontend.erl (limited to 'src/frontend.erl') diff --git a/src/frontend.erl b/src/frontend.erl new file mode 100644 index 0000000..8d0eccd --- /dev/null +++ b/src/frontend.erl @@ -0,0 +1,102 @@ +%%% Copyright (c) 2014, NORDUnet A/S. +%%% See LICENSE for licensing information. + +%%% @doc Frontend node API + +-module(frontend). +%% API (URL) +-export([sendlog/3, missingentries/3, sendentry/3, sendsth/3, currentposition/3]). + +sendentry(SessionID, _Env, Input) -> + R = (catch case (catch jiffy:decode(Input)) of + {error, E} -> + html("sendentry: bad input:", E); + {PropList} -> + LogEntry = base64:decode(proplists:get_value(<<"entry">>, PropList)), + TreeLeafHash = base64:decode(proplists:get_value(<<"treeleafhash">>, PropList)), + + ok = db:add(TreeLeafHash, LogEntry), + binary_to_list( + jiffy:encode( + {[{result, <<"ok">>}]})) + end), + deliver(SessionID, R). + +sendlog(SessionID, _Env, Input) -> + R = (catch case (catch jiffy:decode(Input)) of + {error, E} -> + html("sendentry: bad input:", E); + {PropList} -> + Start = proplists:get_value(<<"start">>, PropList), + Hashes = lists:map(fun (S) -> base64:decode(S) end, proplists:get_value(<<"hashes">>, PropList)), + + Indices = lists:seq(Start, Start + length(Hashes) - 1), + lists:foreach(fun ({Hash, Index}) -> + ok = db:add_index(Hash, Index) + end, lists:zip(Hashes, Indices)), + binary_to_list( + jiffy:encode( + {[{result, <<"ok">>}]})) + end), + deliver(SessionID, R). + +sendsth(SessionID, _Env, Input) -> + R = (catch case (catch jiffy:decode(Input)) of + {error, E} -> + html("sendentry: bad input:", E); + {PropList} -> + Treesize = proplists:get_value(<<"tree_size">>, PropList), + + ok = db:set_treesize(Treesize), + + ht:reset_tree([db:size() - 1]), + + binary_to_list( + jiffy:encode( + {[{result, <<"ok">>}]})) + end), + deliver(SessionID, R). + +currentposition(SessionID, _Env, _Input) -> + Size = db:size(), + R = binary_to_list( + jiffy:encode( + {[{result, <<"ok">>}, {position, Size}]})), + deliver(SessionID, R). + +fetchmissingentries(Index) -> + lists:reverse(fetchmissingentries(Index, [])). + +fetchmissingentries(Index, Acc) -> + case db:leafhash_for_index(Index) of + noentry -> + Acc; + Hash -> + case db:entry_for_leafhash(Hash) of + noentry -> + fetchmissingentries(Index + 1, [Hash | Acc]); + _ -> + fetchmissingentries(Index + 1, Acc) + end + end. + +missingentries(SessionID, _Env, _Input) -> + Size = db:size(), + Missing = fetchmissingentries(Size), + R = binary_to_list( + jiffy:encode( + {[{result, <<"ok">>}, {entries, Missing}]})), + deliver(SessionID, R). + +%% Private functions. +html(Text, Input) -> + io_lib:format( + "Content-Type: text/html\r\n\r\n" ++ + "

~n" ++ + "~s~n" ++ + "~p~n" ++ + "~n", [Text, Input]). + +-spec deliver(any(), string()) -> ok | {error, _Reason}. +deliver(Session, Data) -> + mod_esi:deliver(Session, Data). -- cgit v1.1 From 868a029e39ec8e9aa368da917146d088edee4d2f Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Sat, 25 Oct 2014 01:51:46 +0200 Subject: Move internal HTTP APIs to mochiweb. Stop using jiffy. --- src/frontend.erl | 135 +++++++++++++++++++++++++------------------------------ 1 file changed, 61 insertions(+), 74 deletions(-) (limited to 'src/frontend.erl') diff --git a/src/frontend.erl b/src/frontend.erl index 8d0eccd..a8a8b9e 100644 --- a/src/frontend.erl +++ b/src/frontend.erl @@ -5,64 +5,59 @@ -module(frontend). %% API (URL) --export([sendlog/3, missingentries/3, sendentry/3, sendsth/3, currentposition/3]). - -sendentry(SessionID, _Env, Input) -> - R = (catch case (catch jiffy:decode(Input)) of - {error, E} -> - html("sendentry: bad input:", E); - {PropList} -> - LogEntry = base64:decode(proplists:get_value(<<"entry">>, PropList)), - TreeLeafHash = base64:decode(proplists:get_value(<<"treeleafhash">>, PropList)), - - ok = db:add(TreeLeafHash, LogEntry), - binary_to_list( - jiffy:encode( - {[{result, <<"ok">>}]})) - end), - deliver(SessionID, R). - -sendlog(SessionID, _Env, Input) -> - R = (catch case (catch jiffy:decode(Input)) of - {error, E} -> - html("sendentry: bad input:", E); - {PropList} -> - Start = proplists:get_value(<<"start">>, PropList), - Hashes = lists:map(fun (S) -> base64:decode(S) end, proplists:get_value(<<"hashes">>, PropList)), - - Indices = lists:seq(Start, Start + length(Hashes) - 1), - lists:foreach(fun ({Hash, Index}) -> - ok = db:add_index(Hash, Index) - end, lists:zip(Hashes, Indices)), - binary_to_list( - jiffy:encode( - {[{result, <<"ok">>}]})) - end), - deliver(SessionID, R). - -sendsth(SessionID, _Env, Input) -> - R = (catch case (catch jiffy:decode(Input)) of - {error, E} -> - html("sendentry: bad input:", E); - {PropList} -> - Treesize = proplists:get_value(<<"tree_size">>, PropList), - - ok = db:set_treesize(Treesize), - - ht:reset_tree([db:size() - 1]), - - binary_to_list( - jiffy:encode( - {[{result, <<"ok">>}]})) - end), - deliver(SessionID, R). - -currentposition(SessionID, _Env, _Input) -> +-export([request/3]). + +request(post, "ct/frontend/sendentry", Input) -> + case (catch mochijson2:decode(Input)) of + {error, E} -> + html("sendentry: bad input:", E); + {struct, PropList} -> + LogEntry = base64:decode(proplists:get_value(<<"entry">>, PropList)), + TreeLeafHash = base64:decode(proplists:get_value(<<"treeleafhash">>, PropList)), + + ok = db:add(TreeLeafHash, LogEntry), + success({[{result, <<"ok">>}]}) + end; + +request(post, "ct/frontend/sendlog", Input) -> + case (catch mochijson2:decode(Input)) of + {error, E} -> + html("sendentry: bad input:", E); + {struct, PropList} -> + Start = proplists:get_value(<<"start">>, PropList), + Hashes = lists:map(fun (S) -> base64:decode(S) end, proplists:get_value(<<"hashes">>, PropList)), + + Indices = lists:seq(Start, Start + length(Hashes) - 1), + lists:foreach(fun ({Hash, Index}) -> + ok = db:add_index(Hash, Index) + end, lists:zip(Hashes, Indices)), + success({[{result, <<"ok">>}]}) + end; + +request(post, "ct/frontend/sendsth", Input) -> + case (catch mochijson2:decode(Input)) of + {error, E} -> + html("sendentry: bad input:", E); + {struct, PropList} -> + Treesize = proplists:get_value(<<"tree_size">>, PropList), + + ok = db:set_treesize(Treesize), + + ht:reset_tree([db:size() - 1]), + + success({[{result, <<"ok">>}]}) + end; + +request(get, "ct/frontend/currentposition", _Query) -> + Size = db:size(), + success({[{result, <<"ok">>}, + {position, Size}]}); + +request(get, "ct/frontend/missingentries", _Query) -> Size = db:size(), - R = binary_to_list( - jiffy:encode( - {[{result, <<"ok">>}, {position, Size}]})), - deliver(SessionID, R). + Missing = fetchmissingentries(Size), + success({[{result, <<"ok">>}, + {entries, Missing}]}). fetchmissingentries(Index) -> lists:reverse(fetchmissingentries(Index, [])). @@ -80,23 +75,15 @@ fetchmissingentries(Index, Acc) -> end end. -missingentries(SessionID, _Env, _Input) -> - Size = db:size(), - Missing = fetchmissingentries(Size), - R = binary_to_list( - jiffy:encode( - {[{result, <<"ok">>}, {entries, Missing}]})), - deliver(SessionID, R). %% Private functions. html(Text, Input) -> - io_lib:format( - "Content-Type: text/html\r\n\r\n" ++ - "

~n" ++ - "~s~n" ++ - "~p~n" ++ - "~n", [Text, Input]). - --spec deliver(any(), string()) -> ok | {error, _Reason}. -deliver(Session, Data) -> - mod_esi:deliver(Session, Data). + {400, [{"Content-Type", "text/html"}], + io_lib:format( + "

~n" ++ + "~s~n" ++ + "~p~n" ++ + "~n", [Text, Input])}. + +success(Data) -> + {200, [{"Content-Type", "text/json"}], mochijson2:encode(Data)}. -- cgit v1.1 From 79caa8decbb21228cf3f5cbe32fbf972c40e70dc Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 27 Oct 2014 01:30:15 +0100 Subject: Check that entries are actually present when receiving new STH from merge nodes --- src/frontend.erl | 55 ++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 7 deletions(-) (limited to 'src/frontend.erl') diff --git a/src/frontend.erl b/src/frontend.erl index a8a8b9e..9c69517 100644 --- a/src/frontend.erl +++ b/src/frontend.erl @@ -39,13 +39,33 @@ request(post, "ct/frontend/sendsth", Input) -> {error, E} -> html("sendentry: bad input:", E); {struct, PropList} -> + OldSize = db:size(), Treesize = proplists:get_value(<<"tree_size">>, PropList), - - ok = db:set_treesize(Treesize), - - ht:reset_tree([db:size() - 1]), - - success({[{result, <<"ok">>}]}) + Indexsize = db:indexsize(), + + if + Treesize < OldSize -> + html("Size is older than current size", OldSize); + Treesize == OldSize -> + success({[{result, <<"ok">>}]}); + Treesize > Indexsize -> + html("Has too few entries", Indexsize); + true -> + NewEntries = db:leafhash_for_indices(OldSize, Treesize - 1), + lager:debug("old size: ~p new size: ~p entries: ~p", + [OldSize, Treesize, NewEntries]), + + Errors = check_entries(NewEntries, OldSize, Treesize - 1), + + case Errors of + [] -> + ok = db:set_treesize(Treesize), + ht:reset_tree([db:size() - 1]), + success({[{result, <<"ok">>}]}); + _ -> + html("Database not complete", Errors) + end + end end; request(get, "ct/frontend/currentposition", _Query) -> @@ -56,19 +76,40 @@ request(get, "ct/frontend/currentposition", _Query) -> request(get, "ct/frontend/missingentries", _Query) -> Size = db:size(), Missing = fetchmissingentries(Size), + lager:debug("missingentries: ~p", [Missing]), success({[{result, <<"ok">>}, - {entries, Missing}]}). + {entries, lists:map(fun (Entry) -> base64:encode(Entry) end, + Missing)}]}). +check_entries(Entries, Start, End) -> + lists:foldl(fun ({Hash, Index}, Acc) -> + case check_entry(Hash, Index) of + ok -> + Acc; + Error -> + [Error | Acc] + end + end, [], lists:zip(Entries, lists:seq(Start, End))). + +check_entry(Hash, Index) -> + case db:get_by_leaf_hash(Hash) of + notfound -> + {notfound, Index}; + _ -> + ok + end. fetchmissingentries(Index) -> lists:reverse(fetchmissingentries(Index, [])). fetchmissingentries(Index, Acc) -> + lager:debug("index ~p", [Index]), case db:leafhash_for_index(Index) of noentry -> Acc; Hash -> case db:entry_for_leafhash(Hash) of noentry -> + lager:debug("didn't find hash ~p", [Hash]), fetchmissingentries(Index + 1, [Hash | Acc]); _ -> fetchmissingentries(Index + 1, Acc) -- cgit v1.1