Optimize db:get_by_indices by not fetching entry and implementing index:getrangeoptimize-startup

Conflicts:
	src/index.erl
	src/plop.erl

3 files changed, 38 insertions, 23 deletions

diff --git a/src/db.erl b/src/db.erl
index 6315ae5..a0855b9 100644
--- a/src/db.erl
+++ b/src/db.erl
@@ -108,6 +108,9 @@ index_for_leafhash(LeafHash) ->
 leafhash_for_index(Index) ->
     index:get(index_path(), Index).
 
+leafhash_for_indices(Start, End) ->
+    index:getrange(index_path(), Start, End).
+
 leafhash_for_entryhash(EntryHash) ->
     perm:readfile(entryhash_root_path(), EntryHash).
 
@@ -117,11 +120,10 @@ get_by_indices_helper(Start, End) ->
     EndBound = min(End, size() - 1),
     case Start =< EndBound of
         true ->
-            lists:map(fun (Index) ->
-                              LeafHash = leafhash_for_index(Index),
-                              Entry = entry_for_leafhash(LeafHash),
-                              {Index, LeafHash, Entry}
-                      end, lists:seq(Start, EndBound));
+            lists:map(fun ({LeafHash, Index}) ->
+                              {Index, LeafHash, notfetched}
+                      end, lists:zip(leafhash_for_indices(Start, EndBound),
+                                     lists:seq(Start, EndBound)));
         false ->
             []
     end.
diff --git a/src/index.erl b/src/index.erl
index 7871215..a2b5c4a 100644
--- a/src/index.erl
+++ b/src/index.erl
@@ -11,7 +11,7 @@
 %% TODO: Checksums
 
 -module(index).
--export([get/2, add/3, addlast/2, truncate/2]).
+-export([get/2, getrange/3, add/3, addlast/2, truncate/2]).
 
 -define(ENTRYSIZE, 32).
 -define(ENTRYSIZEINFILE, (?ENTRYSIZE*2+1)).
@@ -54,31 +54,38 @@ truncate(Basepath, Index) ->
 addlast(Basepath, Entry) ->
     add(Basepath, last, Entry).
 
-%% From lib/stdlib/src/lists.erl. For supporting < R17.
--spec droplast(nonempty_list()) -> list().
-droplast([_T]) -> [];
-droplast([H|T]) -> [H|droplast(T)].
+decodedata(Binary) ->
+    lists:reverse(decodedata(Binary, [])).
 
-decodedata(EntryText) when length(EntryText) == ?ENTRYSIZEINFILE ->
-    case [lists:last(EntryText)] of
-        "\n" ->
-            hex:hexstr_to_bin(droplast(EntryText));
-        _ ->
-            util:exit_with_error(badformat, readindex,
-                                 "Index line not ending with linefeed")
-    end.
+decodedata(<<>>, Acc) ->
+    Acc;
+decodedata(<<Entry:?ENTRYSIZE/binary-unit:16, "\n", Rest/binary>>, Acc) ->
+    decodedata(Rest, [mochihex:to_bin(binary_to_list(Entry)) | Acc]);
+decodedata(<<_:?ENTRYSIZE/binary-unit:16, _>>, _Acc) ->
+    util:exit_with_error(badformat, readindex,
+                         "Index line not ending with linefeed").
 
 -spec get(string(), integer()) -> binary().
 get(Basepath, Index) ->
+    case getrange(Basepath, Index, Index) of
+        noentry ->
+            noentry;
+        [Entry] ->
+            Entry
+    end.
+
+-spec getrange(string(), integer(), integer()) -> [binary()].
+getrange(Basepath, Start, End) when Start =< End ->
     case file:open(Basepath, [read, binary]) of
         {ok, File} ->
             {ok, Filesize} = file:position(File, eof),
             if
-                Index * ?ENTRYSIZEINFILE + ?ENTRYSIZEINFILE =< Filesize ->
+                End * ?ENTRYSIZEINFILE + ?ENTRYSIZEINFILE =< Filesize ->
                     {ok, _Position} = file:position(File,
-                                                    Index * ?ENTRYSIZEINFILE),
-                    {ok, EntryText} = file:read(File, ?ENTRYSIZEINFILE),
-                    Entry = decodedata(binary_to_list(EntryText)),
+                                                    Start * ?ENTRYSIZEINFILE),
+                    {ok, EntryText} =
+                        file:read(File, ?ENTRYSIZEINFILE * (End - Start + 1)),
+                    Entry = decodedata(EntryText),
                     file:close(File),
                     Entry;
                 true ->
diff --git a/src/plop.erl b/src/plop.erl
index 0b101be..5244144 100644
--- a/src/plop.erl
+++ b/src/plop.erl
@@ -131,12 +131,18 @@ get_logid() ->
     gen_server:call(?MODULE, {get, logid}).
 testing_get_pubkey() ->
     gen_server:call(?MODULE, {test, pubkey}).
+
+fill_in_entry({_Index, LeafHash, notfetched}) ->
+    db:get_by_leaf_hash(LeafHash).
+
 %%%%%%%%%%%%%%%%%%%%
 handle_call(stop, _From, Plop) ->
     {stop, normal, stopped, Plop};
 
 handle_call({get, {index, Start, End}}, _From, Plop) ->
-    {reply, db:get_by_indices(Start, End, {sorted, false}), Plop};
+    {reply, lists:map(fun (E) -> fill_in_entry(E) end,
+                      db:get_by_indices(Start, End, {sorted, false})),
+     Plop};
 
 handle_call({get, {hash, EntryHash}}, _From, Plop) ->
     {reply, db:get_by_entry_hash(EntryHash), Plop};