From 088e4de4f1e2499f6cc0e332ae8cb34b935a6425 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 20 Oct 2014 12:22:50 +0200 Subject: Added HTTP API:s for external merge --- src/storage.erl | 77 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 src/storage.erl (limited to 'src/storage.erl') diff --git a/src/storage.erl b/src/storage.erl new file mode 100644 index 0000000..243cc6c --- /dev/null +++ b/src/storage.erl @@ -0,0 +1,77 @@ +%%% Copyright (c) 2014, NORDUnet A/S. +%%% See LICENSE for licensing information. + +%%% @doc Storage node API + +-module(storage). +%% API (URL) +-export([sendentry/3, entrycommitted/3, fetchnewentries/3]). + +newentries_path() -> + {ok, Value} = application:get_env(plop, newentries_path), + Value. + +sendentry(SessionID, _Env, Input) -> + R = (catch case (catch jiffy:decode(Input)) of + {error, E} -> + html("sendentry: bad input:", E); + {PropList} -> + LogEntry = base64:decode(proplists:get_value(<<"entry">>, PropList)), + TreeLeafHash = base64:decode(proplists:get_value(<<"treeleafhash">>, PropList)), + + ok = db:add(TreeLeafHash, LogEntry), + ok = index:addlast(newentries_path(), TreeLeafHash), + binary_to_list( + jiffy:encode( + {[{result, <<"ok">>}]})) + end), + deliver(SessionID, R). + +entrycommitted(SessionID, _Env, Input) -> + R = (catch case (catch jiffy:decode(Input)) of + {error, E} -> + html("entrycommitted: bad input:", E); + {PropList} -> + EntryHash = base64:decode(proplists:get_value(<<"entryhash">>, PropList)), + LeafHash = base64:decode(proplists:get_value(<<"treeleafhash">>, PropList)), + ok = db:add_entryhash(LeafHash, EntryHash), + binary_to_list( + jiffy:encode( + {[{result, <<"ok">>}]})) + end), + deliver(SessionID, R). + +fetchnewhashes(Index) -> + lists:reverse(fetchnewhashes(Index, [])). + +fetchnewhashes(Index, Acc) -> + case index:get(newentries_path(), Index) of + noentry -> + Acc; + Entry -> + fetchnewhashes(Index + 1, [Entry | Acc]) + end. + +fetchnewentries(SessionID, _Env, _Input) -> + NewHashes = fetchnewhashes(0), + Entries = lists:map(fun(LeafHash) -> + {[{hash, base64:encode(LeafHash)}, + {entry, base64:encode(db:entry_for_leafhash(LeafHash))}]} + end, NewHashes), + R = (catch binary_to_list( + jiffy:encode( + {[{result, <<"ok">>}, {entries, Entries}]}))), + deliver(SessionID, R). + +%% Private functions. +html(Text, Input) -> + io_lib:format( + "Content-Type: text/html\r\n\r\n" ++ + "

~n" ++ + "~s~n" ++ + "~p~n" ++ + "~n", [Text, Input]). + +-spec deliver(any(), string()) -> ok | {error, _Reason}. +deliver(Session, Data) -> + mod_esi:deliver(Session, Data). -- cgit v1.1 From b968cb1330ecb13f26e35d948c0511882b89ab2a Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Fri, 24 Oct 2014 15:32:58 +0200 Subject: Added lager for logging --- src/storage.erl | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/storage.erl') diff --git a/src/storage.erl b/src/storage.erl index 243cc6c..b966a12 100644 --- a/src/storage.erl +++ b/src/storage.erl @@ -12,6 +12,7 @@ newentries_path() -> Value. sendentry(SessionID, _Env, Input) -> + lager:debug("~p", [Input]), R = (catch case (catch jiffy:decode(Input)) of {error, E} -> html("sendentry: bad input:", E); @@ -25,6 +26,7 @@ sendentry(SessionID, _Env, Input) -> jiffy:encode( {[{result, <<"ok">>}]})) end), + lager:debug("result ~p", [R]), deliver(SessionID, R). entrycommitted(SessionID, _Env, Input) -> -- cgit v1.1 From 868a029e39ec8e9aa368da917146d088edee4d2f Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Sat, 25 Oct 2014 01:51:46 +0200 Subject: Move internal HTTP APIs to mochiweb. Stop using jiffy. --- src/storage.erl | 89 ++++++++++++++++++++++++--------------------------------- 1 file changed, 38 insertions(+), 51 deletions(-) (limited to 'src/storage.erl') diff --git a/src/storage.erl b/src/storage.erl index b966a12..e09acdb 100644 --- a/src/storage.erl +++ b/src/storage.erl @@ -5,43 +5,42 @@ -module(storage). %% API (URL) --export([sendentry/3, entrycommitted/3, fetchnewentries/3]). +-export([request/3]). newentries_path() -> {ok, Value} = application:get_env(plop, newentries_path), Value. -sendentry(SessionID, _Env, Input) -> - lager:debug("~p", [Input]), - R = (catch case (catch jiffy:decode(Input)) of - {error, E} -> - html("sendentry: bad input:", E); - {PropList} -> - LogEntry = base64:decode(proplists:get_value(<<"entry">>, PropList)), - TreeLeafHash = base64:decode(proplists:get_value(<<"treeleafhash">>, PropList)), +request(post, "ct/storage/sendentry", Input) -> + case (catch mochijson2:decode(Input)) of + {error, E} -> + html("sendentry: bad input:", E); + {struct, PropList} -> + LogEntry = base64:decode(proplists:get_value(<<"entry">>, PropList)), + TreeLeafHash = base64:decode(proplists:get_value(<<"treeleafhash">>, PropList)), - ok = db:add(TreeLeafHash, LogEntry), - ok = index:addlast(newentries_path(), TreeLeafHash), - binary_to_list( - jiffy:encode( - {[{result, <<"ok">>}]})) - end), - lager:debug("result ~p", [R]), - deliver(SessionID, R). - -entrycommitted(SessionID, _Env, Input) -> - R = (catch case (catch jiffy:decode(Input)) of - {error, E} -> - html("entrycommitted: bad input:", E); - {PropList} -> - EntryHash = base64:decode(proplists:get_value(<<"entryhash">>, PropList)), - LeafHash = base64:decode(proplists:get_value(<<"treeleafhash">>, PropList)), - ok = db:add_entryhash(LeafHash, EntryHash), - binary_to_list( - jiffy:encode( - {[{result, <<"ok">>}]})) - end), - deliver(SessionID, R). + ok = db:add(TreeLeafHash, LogEntry), + ok = index:addlast(newentries_path(), TreeLeafHash), + success({[{result, <<"ok">>}]}) + end; +request(post, "ct/storage/entrycommitted", Input) -> + case (catch mochijson2:decode(Input)) of + {error, E} -> + html("entrycommitted: bad input:", E); + {struct, PropList} -> + EntryHash = base64:decode(proplists:get_value(<<"entryhash">>, PropList)), + LeafHash = base64:decode(proplists:get_value(<<"treeleafhash">>, PropList)), + ok = db:add_entryhash(LeafHash, EntryHash), + success({[{result, <<"ok">>}]}) + end; +request(get, "ct/storage/fetchnewentries", _Input) -> + NewHashes = fetchnewhashes(0), + Entries = lists:map(fun(LeafHash) -> + {[{hash, base64:encode(LeafHash)}, + {entry, base64:encode(db:entry_for_leafhash(LeafHash))}]} + end, NewHashes), + success({[{result, <<"ok">>}, + {entries, Entries}]}). fetchnewhashes(Index) -> lists:reverse(fetchnewhashes(Index, [])). @@ -54,26 +53,14 @@ fetchnewhashes(Index, Acc) -> fetchnewhashes(Index + 1, [Entry | Acc]) end. -fetchnewentries(SessionID, _Env, _Input) -> - NewHashes = fetchnewhashes(0), - Entries = lists:map(fun(LeafHash) -> - {[{hash, base64:encode(LeafHash)}, - {entry, base64:encode(db:entry_for_leafhash(LeafHash))}]} - end, NewHashes), - R = (catch binary_to_list( - jiffy:encode( - {[{result, <<"ok">>}, {entries, Entries}]}))), - deliver(SessionID, R). - %% Private functions. html(Text, Input) -> - io_lib:format( - "Content-Type: text/html\r\n\r\n" ++ - "

~n" ++ - "~s~n" ++ - "~p~n" ++ - "~n", [Text, Input]). + {400, [{"Content-Type", "text/html"}], + io_lib:format( + "

~n" ++ + "~s~n" ++ + "~p~n" ++ + "~n", [Text, Input])}. --spec deliver(any(), string()) -> ok | {error, _Reason}. -deliver(Session, Data) -> - mod_esi:deliver(Session, Data). +success(Data) -> + {200, [{"Content-Type", "text/json"}], mochijson2:encode(Data)}. -- cgit v1.1 From ebc9d5bac1a69ba25044a73674b8e9ea18217f60 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Sat, 25 Oct 2014 23:56:41 +0200 Subject: Optimize fetchnewentries --- src/storage.erl | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) (limited to 'src/storage.erl') diff --git a/src/storage.erl b/src/storage.erl index e09acdb..df6641a 100644 --- a/src/storage.erl +++ b/src/storage.erl @@ -43,15 +43,8 @@ request(get, "ct/storage/fetchnewentries", _Input) -> {entries, Entries}]}). fetchnewhashes(Index) -> - lists:reverse(fetchnewhashes(Index, [])). - -fetchnewhashes(Index, Acc) -> - case index:get(newentries_path(), Index) of - noentry -> - Acc; - Entry -> - fetchnewhashes(Index + 1, [Entry | Acc]) - end. + Size = index:size(newentries_path()), + index:getrange(newentries_path(), Index, Size - 1). %% Private functions. html(Text, Input) -> -- cgit v1.1 From fb3b9591cc81158824db13818cf6320d5f4a0f7b Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 27 Oct 2014 01:28:32 +0100 Subject: Fix mistake in ebc9d5ba (Optimize fetchnewentries) --- src/storage.erl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'src/storage.erl') diff --git a/src/storage.erl b/src/storage.erl index df6641a..8136308 100644 --- a/src/storage.erl +++ b/src/storage.erl @@ -43,8 +43,12 @@ request(get, "ct/storage/fetchnewentries", _Input) -> {entries, Entries}]}). fetchnewhashes(Index) -> - Size = index:size(newentries_path()), - index:getrange(newentries_path(), Index, Size - 1). + case index:indexsize(newentries_path()) of + 0 -> + []; + Size -> + index:getrange(newentries_path(), Index, Size - 1) + end. %% Private functions. html(Text, Input) -> -- cgit v1.1