Merge remote-tracking branch 'refs/remotes/map/external-merge3' into merging-external-merge

Conflicts:
	src/db.erl
	src/frontend.erl
	src/index.erl
	src/plop.erl
	src/storage.erl
	src/ts.erl

1 files changed, 48 insertions, 7 deletions

diff --git a/src/frontend.erl b/src/frontend.erl
index a8a8b9e..9c69517 100644
--- a/src/frontend.erl
+++ b/src/frontend.erl
@@ -39,13 +39,33 @@ request(post, "ct/frontend/sendsth", Input) ->
         {error, E} ->
             html("sendentry: bad input:", E);
         {struct, PropList} ->
+            OldSize = db:size(),
             Treesize = proplists:get_value(<<"tree_size">>, PropList),
-
-            ok = db:set_treesize(Treesize),
-
-            ht:reset_tree([db:size() - 1]),
-
-            success({[{result, <<"ok">>}]})
+            Indexsize = db:indexsize(),
+
+            if
+                Treesize < OldSize ->
+                    html("Size is older than current size", OldSize);
+                Treesize == OldSize ->
+                    success({[{result, <<"ok">>}]});
+                Treesize > Indexsize ->
+                    html("Has too few entries", Indexsize);
+                true ->
+                    NewEntries = db:leafhash_for_indices(OldSize, Treesize - 1),
+                    lager:debug("old size: ~p new size: ~p entries: ~p",
+                                [OldSize, Treesize, NewEntries]),
+
+                    Errors = check_entries(NewEntries, OldSize, Treesize - 1),
+
+                    case Errors of
+                        [] ->
+                            ok = db:set_treesize(Treesize),
+                            ht:reset_tree([db:size() - 1]),
+                            success({[{result, <<"ok">>}]});
+                        _ ->
+                            html("Database not complete", Errors)
+                    end
+            end
     end;
 
 request(get, "ct/frontend/currentposition", _Query) ->
@@ -56,19 +76,40 @@ request(get, "ct/frontend/currentposition", _Query) ->
 request(get, "ct/frontend/missingentries", _Query) ->
     Size = db:size(),
     Missing = fetchmissingentries(Size),
+    lager:debug("missingentries: ~p", [Missing]),
     success({[{result, <<"ok">>},
-              {entries, Missing}]}).
+              {entries, lists:map(fun (Entry) -> base64:encode(Entry) end,
+                                  Missing)}]}).
+check_entries(Entries, Start, End) ->
+    lists:foldl(fun ({Hash, Index}, Acc) ->
+                        case check_entry(Hash, Index) of
+                            ok ->
+                                Acc;
+                            Error ->
+                                [Error | Acc]
+                        end
+                end, [], lists:zip(Entries, lists:seq(Start, End))).
+
+check_entry(Hash, Index) ->
+    case db:get_by_leaf_hash(Hash) of
+        notfound ->
+            {notfound, Index};
+        _ ->
+            ok
+    end.
 
 fetchmissingentries(Index) ->
     lists:reverse(fetchmissingentries(Index, [])).
 
 fetchmissingentries(Index, Acc) ->
+    lager:debug("index ~p", [Index]),
     case db:leafhash_for_index(Index) of
         noentry ->
             Acc;
         Hash ->
             case db:entry_for_leafhash(Hash) of
                 noentry ->
+                    lager:debug("didn't find hash ~p", [Hash]),
                     fetchmissingentries(Index + 1, [Hash | Acc]);
                 _ ->
                     fetchmissingentries(Index + 1, Acc)