From 92a84310aaabdb44c523ba90aa38aecce2314a63 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 20 Oct 2014 12:22:50 +0200 Subject: Added HTTP API:s for external merge --- src/db.erl | 38 +++++++++++++++++++-- src/frontend.erl | 102 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/storage.erl | 77 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 215 insertions(+), 2 deletions(-) create mode 100644 src/frontend.erl create mode 100644 src/storage.erl diff --git a/src/db.erl b/src/db.erl index 640718c..fcc3f38 100644 --- a/src/db.erl +++ b/src/db.erl @@ -6,8 +6,8 @@ %% API. -export([start_link/0, stop/0]). --export([add/4, size/0]). --export([get_by_index/1, get_by_indices/3, get_by_leaf_hash/1, get_by_entry_hash/1]). +-export([add/4, add/2, add_entryhash/2, add_index/2, set_treesize/1, size/0]). +-export([get_by_index/1, get_by_indices/3, get_by_leaf_hash/1, get_by_entry_hash/1, entry_for_leafhash/1, leafhash_for_index/1]). %% gen_server callbacks. -export([init/1, handle_call/3, terminate/2, handle_cast/2, handle_info/2, code_change/3]). @@ -34,6 +34,22 @@ stop() -> add(LeafHash, EntryHash, Data, Index) -> gen_server:call(?MODULE, {add, {LeafHash, EntryHash, Data, Index}}). +-spec add(binary(), binary()) -> ok. +add(LeafHash, Data) -> + gen_server:call(?MODULE, {add, {LeafHash, Data}}). + +-spec add_entryhash(binary(), binary()) -> ok. +add_entryhash(LeafHash, EntryHash) -> + gen_server:call(?MODULE, {add_entryhash, {LeafHash, EntryHash}}). + +-spec add_index(binary(), non_neg_integer()) -> ok. +add_index(LeafHash, Index) -> + gen_server:call(?MODULE, {add_index, {LeafHash, Index}}). + +-spec set_treesize(non_neg_integer()) -> ok. +set_treesize(Size) -> + gen_server:call(?MODULE, {set_treesize, Size}). + -spec get_by_indices(integer(), integer(), {sorted, true|false}) -> [{non_neg_integer(), binary(), binary()}]. get_by_indices(Start, End, {sorted, Sorted}) -> @@ -145,6 +161,24 @@ handle_call({add, {LeafHash, EntryHash, Data, Index}}, _From, State) -> ok = atomic:replacefile(treesize_path(), integer_to_list(Index+1)), {reply, ok, State}; +handle_call({add, {LeafHash, Data}}, _From, State) -> + ok = perm:ensurefile(entry_root_path(), LeafHash, Data), + {reply, ok, State}; + +handle_call({add_entryhash, {LeafHash, EntryHash}}, _From, State) -> + ok = perm:ensurefile(entryhash_root_path(), EntryHash, LeafHash), + {reply, ok, State}; + +handle_call({add_index, {LeafHash, Index}}, _From, State) -> + ok = perm:ensurefile(indexforhash_root_path(), + LeafHash, integer_to_binary(Index)), + ok = index:add(index_path(), Index, LeafHash), + {reply, ok, State}; + +handle_call({set_treesize, Size}, _From, State) -> + ok = atomic:replacefile(treesize_path(), integer_to_list(Size)), + {reply, ok, State}; + handle_call({get_by_indices, {Start, End, _Sorted}}, _From, State) -> {reply, get_by_indices_helper(Start, End), State}; diff --git a/src/frontend.erl b/src/frontend.erl new file mode 100644 index 0000000..8d0eccd --- /dev/null +++ b/src/frontend.erl @@ -0,0 +1,102 @@ +%%% Copyright (c) 2014, NORDUnet A/S. +%%% See LICENSE for licensing information. + +%%% @doc Frontend node API + +-module(frontend). +%% API (URL) +-export([sendlog/3, missingentries/3, sendentry/3, sendsth/3, currentposition/3]). + +sendentry(SessionID, _Env, Input) -> + R = (catch case (catch jiffy:decode(Input)) of + {error, E} -> + html("sendentry: bad input:", E); + {PropList} -> + LogEntry = base64:decode(proplists:get_value(<<"entry">>, PropList)), + TreeLeafHash = base64:decode(proplists:get_value(<<"treeleafhash">>, PropList)), + + ok = db:add(TreeLeafHash, LogEntry), + binary_to_list( + jiffy:encode( + {[{result, <<"ok">>}]})) + end), + deliver(SessionID, R). + +sendlog(SessionID, _Env, Input) -> + R = (catch case (catch jiffy:decode(Input)) of + {error, E} -> + html("sendentry: bad input:", E); + {PropList} -> + Start = proplists:get_value(<<"start">>, PropList), + Hashes = lists:map(fun (S) -> base64:decode(S) end, proplists:get_value(<<"hashes">>, PropList)), + + Indices = lists:seq(Start, Start + length(Hashes) - 1), + lists:foreach(fun ({Hash, Index}) -> + ok = db:add_index(Hash, Index) + end, lists:zip(Hashes, Indices)), + binary_to_list( + jiffy:encode( + {[{result, <<"ok">>}]})) + end), + deliver(SessionID, R). + +sendsth(SessionID, _Env, Input) -> + R = (catch case (catch jiffy:decode(Input)) of + {error, E} -> + html("sendentry: bad input:", E); + {PropList} -> + Treesize = proplists:get_value(<<"tree_size">>, PropList), + + ok = db:set_treesize(Treesize), + + ht:reset_tree([db:size() - 1]), + + binary_to_list( + jiffy:encode( + {[{result, <<"ok">>}]})) + end), + deliver(SessionID, R). + +currentposition(SessionID, _Env, _Input) -> + Size = db:size(), + R = binary_to_list( + jiffy:encode( + {[{result, <<"ok">>}, {position, Size}]})), + deliver(SessionID, R). + +fetchmissingentries(Index) -> + lists:reverse(fetchmissingentries(Index, [])). + +fetchmissingentries(Index, Acc) -> + case db:leafhash_for_index(Index) of + noentry -> + Acc; + Hash -> + case db:entry_for_leafhash(Hash) of + noentry -> + fetchmissingentries(Index + 1, [Hash | Acc]); + _ -> + fetchmissingentries(Index + 1, Acc) + end + end. + +missingentries(SessionID, _Env, _Input) -> + Size = db:size(), + Missing = fetchmissingentries(Size), + R = binary_to_list( + jiffy:encode( + {[{result, <<"ok">>}, {entries, Missing}]})), + deliver(SessionID, R). + +%% Private functions. +html(Text, Input) -> + io_lib:format( + "Content-Type: text/html\r\n\r\n" ++ + "

~n" ++ + "~s~n" ++ + "~p~n" ++ + "~n", [Text, Input]). + +-spec deliver(any(), string()) -> ok | {error, _Reason}. +deliver(Session, Data) -> + mod_esi:deliver(Session, Data). diff --git a/src/storage.erl b/src/storage.erl new file mode 100644 index 0000000..243cc6c --- /dev/null +++ b/src/storage.erl @@ -0,0 +1,77 @@ +%%% Copyright (c) 2014, NORDUnet A/S. +%%% See LICENSE for licensing information. + +%%% @doc Storage node API + +-module(storage). +%% API (URL) +-export([sendentry/3, entrycommitted/3, fetchnewentries/3]). + +newentries_path() -> + {ok, Value} = application:get_env(plop, newentries_path), + Value. + +sendentry(SessionID, _Env, Input) -> + R = (catch case (catch jiffy:decode(Input)) of + {error, E} -> + html("sendentry: bad input:", E); + {PropList} -> + LogEntry = base64:decode(proplists:get_value(<<"entry">>, PropList)), + TreeLeafHash = base64:decode(proplists:get_value(<<"treeleafhash">>, PropList)), + + ok = db:add(TreeLeafHash, LogEntry), + ok = index:addlast(newentries_path(), TreeLeafHash), + binary_to_list( + jiffy:encode( + {[{result, <<"ok">>}]})) + end), + deliver(SessionID, R). + +entrycommitted(SessionID, _Env, Input) -> + R = (catch case (catch jiffy:decode(Input)) of + {error, E} -> + html("entrycommitted: bad input:", E); + {PropList} -> + EntryHash = base64:decode(proplists:get_value(<<"entryhash">>, PropList)), + LeafHash = base64:decode(proplists:get_value(<<"treeleafhash">>, PropList)), + ok = db:add_entryhash(LeafHash, EntryHash), + binary_to_list( + jiffy:encode( + {[{result, <<"ok">>}]})) + end), + deliver(SessionID, R). + +fetchnewhashes(Index) -> + lists:reverse(fetchnewhashes(Index, [])). + +fetchnewhashes(Index, Acc) -> + case index:get(newentries_path(), Index) of + noentry -> + Acc; + Entry -> + fetchnewhashes(Index + 1, [Entry | Acc]) + end. + +fetchnewentries(SessionID, _Env, _Input) -> + NewHashes = fetchnewhashes(0), + Entries = lists:map(fun(LeafHash) -> + {[{hash, base64:encode(LeafHash)}, + {entry, base64:encode(db:entry_for_leafhash(LeafHash))}]} + end, NewHashes), + R = (catch binary_to_list( + jiffy:encode( + {[{result, <<"ok">>}, {entries, Entries}]}))), + deliver(SessionID, R). + +%% Private functions. +html(Text, Input) -> + io_lib:format( + "Content-Type: text/html\r\n\r\n" ++ + "

~n" ++ + "~s~n" ++ + "~p~n" ++ + "~n", [Text, Input]). + +-spec deliver(any(), string()) -> ok | {error, _Reason}. +deliver(Session, Data) -> + mod_esi:deliver(Session, Data). -- cgit v1.1