From 80c8ef847d996af04ec677a79555d640733641f2 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Sun, 19 Oct 2014 01:37:29 +0200 Subject: db:get_by_leaf_hash(): Return notfound instead of crashing when no entry could be found. db:get_by_entry_hash(): Don't fetch index, isn't used and might not exist. index:add(): Allow writes at exiting indicies. --- src/db.erl | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) (limited to 'src/db.erl') diff --git a/src/db.erl b/src/db.erl index 6315ae5..6fce8a3 100644 --- a/src/db.erl +++ b/src/db.erl @@ -103,7 +103,12 @@ entry_for_leafhash(LeafHash) -> perm:readfile(entry_root_path(), LeafHash). index_for_leafhash(LeafHash) -> - binary_to_integer(perm:readfile(indexforhash_root_path(), LeafHash)). + case perm:readfile(indexforhash_root_path(), LeafHash) of + noentry -> + noentry; + Index -> + binary_to_integer(Index) + end. leafhash_for_index(Index) -> index:get(index_path(), Index). @@ -148,9 +153,17 @@ handle_call({get_by_index, Index}, _From, State) -> {reply, R, State}; handle_call({get_by_leaf_hash, LeafHash}, _From, State) -> - Entry = entry_for_leafhash(LeafHash), - Index = index_for_leafhash(LeafHash), - R = {Index, LeafHash, Entry}, + R = case entry_for_leafhash(LeafHash) of + noentry -> + notfound; + Entry -> + case index_for_leafhash(LeafHash) of + noentry -> + notfound; + Index -> + {Index, LeafHash, Entry} + end + end, {reply, R, State}; handle_call({get_by_entry_hash, EntryHash}, _From, State) -> @@ -159,7 +172,7 @@ handle_call({get_by_entry_hash, EntryHash}, _From, State) -> notfound; LeafHash -> Entry = entry_for_leafhash(LeafHash), - Index = index_for_leafhash(LeafHash), - {Index, LeafHash, Entry} + %% Don't fetch index, isn't used and might not exist + {notfetched, LeafHash, Entry} end, {reply, R, State}. -- cgit v1.1 From 088e4de4f1e2499f6cc0e332ae8cb34b935a6425 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 20 Oct 2014 12:22:50 +0200 Subject: Added HTTP API:s for external merge --- src/db.erl | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) (limited to 'src/db.erl') diff --git a/src/db.erl b/src/db.erl index 6fce8a3..413f4b9 100644 --- a/src/db.erl +++ b/src/db.erl @@ -6,8 +6,8 @@ %% API. -export([start_link/0, stop/0]). --export([add/4, size/0]). --export([get_by_index/1, get_by_indices/3, get_by_leaf_hash/1, get_by_entry_hash/1]). +-export([add/4, add/2, add_entryhash/2, add_index/2, set_treesize/1, size/0]). +-export([get_by_index/1, get_by_indices/3, get_by_leaf_hash/1, get_by_entry_hash/1, entry_for_leafhash/1, leafhash_for_index/1]). %% gen_server callbacks. -export([init/1, handle_call/3, terminate/2, handle_cast/2, handle_info/2, code_change/3]). @@ -34,6 +34,22 @@ stop() -> add(LeafHash, EntryHash, Data, Index) -> gen_server:call(?MODULE, {add, {LeafHash, EntryHash, Data, Index}}). +-spec add(binary(), binary()) -> ok. +add(LeafHash, Data) -> + gen_server:call(?MODULE, {add, {LeafHash, Data}}). + +-spec add_entryhash(binary(), binary()) -> ok. +add_entryhash(LeafHash, EntryHash) -> + gen_server:call(?MODULE, {add_entryhash, {LeafHash, EntryHash}}). + +-spec add_index(binary(), non_neg_integer()) -> ok. +add_index(LeafHash, Index) -> + gen_server:call(?MODULE, {add_index, {LeafHash, Index}}). + +-spec set_treesize(non_neg_integer()) -> ok. +set_treesize(Size) -> + gen_server:call(?MODULE, {set_treesize, Size}). + -spec get_by_indices(integer(), integer(), {sorted, true|false}) -> [{non_neg_integer(), binary(), binary()}]. get_by_indices(Start, End, {sorted, Sorted}) -> @@ -143,6 +159,24 @@ handle_call({add, {LeafHash, EntryHash, Data, Index}}, _From, State) -> ok = atomic:replacefile(treesize_path(), integer_to_list(Index+1)), {reply, ok, State}; +handle_call({add, {LeafHash, Data}}, _From, State) -> + ok = perm:ensurefile(entry_root_path(), LeafHash, Data), + {reply, ok, State}; + +handle_call({add_entryhash, {LeafHash, EntryHash}}, _From, State) -> + ok = perm:ensurefile(entryhash_root_path(), EntryHash, LeafHash), + {reply, ok, State}; + +handle_call({add_index, {LeafHash, Index}}, _From, State) -> + ok = perm:ensurefile(indexforhash_root_path(), + LeafHash, integer_to_binary(Index)), + ok = index:add(index_path(), Index, LeafHash), + {reply, ok, State}; + +handle_call({set_treesize, Size}, _From, State) -> + ok = atomic:replacefile(treesize_path(), integer_to_list(Size)), + {reply, ok, State}; + handle_call({get_by_indices, {Start, End, _Sorted}}, _From, State) -> {reply, get_by_indices_helper(Start, End), State}; -- cgit v1.1 From b968cb1330ecb13f26e35d948c0511882b89ab2a Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Fri, 24 Oct 2014 15:32:58 +0200 Subject: Added lager for logging --- src/db.erl | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/db.erl') diff --git a/src/db.erl b/src/db.erl index 413f4b9..fade7ce 100644 --- a/src/db.erl +++ b/src/db.erl @@ -160,7 +160,9 @@ handle_call({add, {LeafHash, EntryHash, Data, Index}}, _From, State) -> {reply, ok, State}; handle_call({add, {LeafHash, Data}}, _From, State) -> + lager:debug("add leafhash ~p", [LeafHash]), ok = perm:ensurefile(entry_root_path(), LeafHash, Data), + lager:debug("leafhash ~p added", [LeafHash]), {reply, ok, State}; handle_call({add_entryhash, {LeafHash, EntryHash}}, _From, State) -> -- cgit v1.1 From 2483f0cf09ccc4cf73558c7a85bbb51a72d29c3a Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Sat, 25 Oct 2014 15:22:09 +0200 Subject: Optimize db:get_by_indices by not fetching entry and implementing index:getrange --- src/db.erl | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'src/db.erl') diff --git a/src/db.erl b/src/db.erl index fade7ce..f7c2057 100644 --- a/src/db.erl +++ b/src/db.erl @@ -129,6 +129,9 @@ index_for_leafhash(LeafHash) -> leafhash_for_index(Index) -> index:get(index_path(), Index). +leafhash_for_indices(Start, End) -> + index:getrange(index_path(), Start, End). + leafhash_for_entryhash(EntryHash) -> perm:readfile(entryhash_root_path(), EntryHash). @@ -138,11 +141,10 @@ get_by_indices_helper(Start, End) -> EndBound = min(End, size() - 1), case Start =< EndBound of true -> - lists:map(fun (Index) -> - LeafHash = leafhash_for_index(Index), - Entry = entry_for_leafhash(LeafHash), - {Index, LeafHash, Entry} - end, lists:seq(Start, EndBound)); + lists:map(fun ({LeafHash, Index}) -> + {Index, LeafHash, notfetched} + end, lists:zip(leafhash_for_indices(Start, EndBound), + lists:seq(Start, EndBound))); false -> [] end. -- cgit v1.1 From 79caa8decbb21228cf3f5cbe32fbf972c40e70dc Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 27 Oct 2014 01:30:15 +0100 Subject: Check that entries are actually present when receiving new STH from merge nodes --- src/db.erl | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'src/db.erl') diff --git a/src/db.erl b/src/db.erl index f7c2057..2a64935 100644 --- a/src/db.erl +++ b/src/db.erl @@ -7,7 +7,7 @@ %% API. -export([start_link/0, stop/0]). -export([add/4, add/2, add_entryhash/2, add_index/2, set_treesize/1, size/0]). --export([get_by_index/1, get_by_indices/3, get_by_leaf_hash/1, get_by_entry_hash/1, entry_for_leafhash/1, leafhash_for_index/1]). +-export([get_by_index/1, get_by_indices/3, get_by_leaf_hash/1, get_by_entry_hash/1, entry_for_leafhash/1, leafhash_for_index/1, leafhash_for_indices/2, indexsize/0]). %% gen_server callbacks. -export([init/1, handle_call/3, terminate/2, handle_cast/2, handle_info/2, code_change/3]). @@ -18,6 +18,9 @@ size() -> binary_to_integer(atomic:readfile(treesize_path())). +indexsize() -> + index:indexsize(index_path()). + init(_Args) -> {ok, []}. @@ -158,7 +161,7 @@ handle_call({add, {LeafHash, EntryHash, Data, Index}}, _From, State) -> ok = perm:ensurefile(indexforhash_root_path(), LeafHash, integer_to_binary(Index)), ok = index:add(index_path(), Index, LeafHash), - ok = atomic:replacefile(treesize_path(), integer_to_list(Index+1)), + ok = atomic:replacefile(treesize_path(), integer_to_binary(Index+1)), {reply, ok, State}; handle_call({add, {LeafHash, Data}}, _From, State) -> @@ -178,7 +181,7 @@ handle_call({add_index, {LeafHash, Index}}, _From, State) -> {reply, ok, State}; handle_call({set_treesize, Size}, _From, State) -> - ok = atomic:replacefile(treesize_path(), integer_to_list(Size)), + ok = atomic:replacefile(treesize_path(), integer_to_binary(Size)), {reply, ok, State}; handle_call({get_by_indices, {Start, End, _Sorted}}, _From, State) -> -- cgit v1.1