Merge remote-tracking branch 'refs/remotes/map/external-merge3' into merging-external-merge

Conflicts:
	src/v1.erl
	tools/merge.py
	tools/testcase1.py

6 files changed, 146 insertions, 148 deletions

diff --git a/src/catlfish.erl b/src/catlfish.erl
index 73066bb..5d96278 100644
--- a/src/catlfish.erl
+++ b/src/catlfish.erl
@@ -5,6 +5,7 @@
 -export([add_chain/2, entries/2, entry_and_proof/2]).
 -export([known_roots/0, update_known_roots/0]).
 -include_lib("eunit/include/eunit.hrl").
+-include("catlfish.hrl").
 
 -define(PROTOCOL_VERSION, 0).
 
@@ -98,13 +99,11 @@ add_chain(LeafCert, CertChain) ->
         plop:spt(list_to_binary([<<?PROTOCOL_VERSION:8>>,
                                  serialise_signature_type(certificate_timestamp),
                                  serialise(TimestampedEntry)])),
-    binary_to_list(
-      jiffy:encode(
-        {[{sct_version, ?PROTOCOL_VERSION},
-          {id, base64:encode(plop:get_logid())},
-          {timestamp, TimestampedEntry#timestamped_entry.timestamp},
-          {extensions, base64:encode(<<>>)},
-          {signature, base64:encode(plop:serialise(SCT_sig))}]})).
+    {[{sct_version, ?PROTOCOL_VERSION},
+      {id, base64:encode(plop:get_logid())},
+      {timestamp, TimestampedEntry#timestamped_entry.timestamp},
+      {extensions, base64:encode(<<>>)},
+      {signature, base64:encode(plop:serialise(SCT_sig))}]}.
 
 -spec serialise_logentry(integer(), binary(), [binary()]) -> binary().
 serialise_logentry(Timestamp, LeafCert, CertChain) ->
@@ -118,24 +117,21 @@ serialise_logentry(Timestamp, LeafCert, CertChain) ->
 
 -spec entries(non_neg_integer(), non_neg_integer()) -> list().
 entries(Start, End) ->
-    binary_to_list(
-      jiffy:encode({[{entries, x_entries(plop:get(Start, End))}]})).
+    {[{entries, x_entries(plop:get(Start, End))}]}.
 
 -spec entry_and_proof(non_neg_integer(), non_neg_integer()) -> list().
 entry_and_proof(Index, TreeSize) ->
-    binary_to_list(
-      jiffy:encode(
-        case plop:inclusion_and_entry(Index, TreeSize) of
-            {ok, Entry, Path} ->
-                {Timestamp, LeafCertVector, CertChainVector} = unpack_entry(Entry),
-                MTL = build_mtl(Timestamp, LeafCertVector),
-                {[{leaf_input, base64:encode(MTL)},
-                  {extra_data, base64:encode(CertChainVector)},
-                  {audit_path, [base64:encode(X) || X <- Path]}]};
-            {notfound, Msg} ->
-                {[{success, false},
-                  {error_message, list_to_binary(Msg)}]}
-        end)).
+    case plop:inclusion_and_entry(Index, TreeSize) of
+        {ok, Entry, Path} ->
+            {Timestamp, LeafCertVector, CertChainVector} = unpack_entry(Entry),
+            MTL = build_mtl(Timestamp, LeafCertVector),
+            {[{leaf_input, base64:encode(MTL)},
+              {extra_data, base64:encode(CertChainVector)},
+              {audit_path, [base64:encode(X) || X <- Path]}]};
+        {notfound, Msg} ->
+            {[{success, false},
+              {error_message, list_to_binary(Msg)}]}
+    end.
 
 %% Private functions.
 unpack_entry(Entry) ->
@@ -164,7 +160,7 @@ decode_tls_vector(Binary, LengthLen) ->
     <<ExtractedBinary:Length/binary-unit:8, Rest2/binary>> = Rest,
     {ExtractedBinary, Rest2}.
 
--define(ROOTS_TABLE, catlfish_roots).
+-define(ROOTS_CACHE_KEY, roots).
 
 update_known_roots() ->
     case application:get_env(catlfish, known_roots_path) of
@@ -183,22 +179,20 @@ known_roots() ->
 
 -spec known_roots(file:filename(), use_cache|update_tab) -> list().
 known_roots(Directory, CacheUsage) ->
-    case ets:info(?ROOTS_TABLE) of
-        undefined ->
-            read_pemfiles_from_dir(
-              ets:new(?ROOTS_TABLE, [set, protected, named_table]),
-              Directory);
-        _ ->
-            case CacheUsage of
-                use_cache ->
-                    ets:lookup_element(?ROOTS_TABLE, list, 2);
-                update_tab ->
-                    read_pemfiles_from_dir(?ROOTS_TABLE, Directory)
-            end
+    case CacheUsage of
+        use_cache ->
+            case ets:lookup(?CACHE_TABLE, ?ROOTS_CACHE_KEY) of
+                [] ->
+                    read_pemfiles_from_dir(Directory);
+                [{roots, DerList}] ->
+                    DerList
+            end;
+        update_tab ->
+            read_pemfiles_from_dir(Directory)
     end.
 
--spec read_pemfiles_from_dir(ets:tab(), file:filename()) -> list().
-read_pemfiles_from_dir(Tab, Dir) ->
+-spec read_pemfiles_from_dir(file:filename()) -> list().
+read_pemfiles_from_dir(Dir) ->
     DerList =
         case file:list_dir(Dir) of
             {error, enoent} ->
@@ -213,7 +207,7 @@ read_pemfiles_from_dir(Tab, Dir) ->
                           Filenames),
                 ders_from_pemfiles(Dir, Files)
         end,
-    true = ets:insert(Tab, {list, DerList}),
+    true = ets:insert(?CACHE_TABLE, {?ROOTS_CACHE_KEY, DerList}),
     DerList.
 
 ders_from_pemfiles(Dir, Filenames) ->
@@ -256,7 +250,7 @@ read_pemfiles_test_() ->
      fun() -> {known_roots(?PEMFILES_DIR_OK, use_cache),
                known_roots(?PEMFILES_DIR_OK, use_cache)}
      end,
-     fun(_) -> ets:delete(?ROOTS_TABLE) end,
+     fun(_) -> ets:delete(?CACHE_TABLE, ?ROOTS_CACHE_KEY) end,
      fun({L, LCached}) ->
              [?_assertMatch(7, length(L)),
               ?_assertEqual(L, LCached)]
@@ -265,5 +259,5 @@ read_pemfiles_test_() ->
 read_pemfiles_fail_test_() ->
     {setup,
      fun() -> known_roots(?PEMFILES_DIR_NONEXISTENT, use_cache) end,
-     fun(_) -> ets:delete(?ROOTS_TABLE) end,
+     fun(_) -> ets:delete(?CACHE_TABLE, ?ROOTS_CACHE_KEY) end,
      fun(Empty) -> [?_assertMatch([], Empty)] end}.
diff --git a/src/catlfish.hrl b/src/catlfish.hrl
new file mode 100644
index 0000000..46e882b
--- /dev/null
+++ b/src/catlfish.hrl
@@ -0,0 +1,4 @@
+%%% Copyright (c) 2014, NORDUnet A/S.
+%%% See LICENSE for licensing information.
+
+-define(CACHE_TABLE, catlfish_cache).
diff --git a/src/catlfish_app.erl b/src/catlfish_app.erl
index cfb55cd..e24a1bb 100644
--- a/src/catlfish_app.erl
+++ b/src/catlfish_app.erl
@@ -8,11 +8,20 @@
 %% Application callbacks
 -export([start/2, stop/1]).
 
+-include("catlfish.hrl").
+
 %% ===================================================================
 %% Application callbacks
 %% ===================================================================
 
 start(normal, Args) ->
+    case ets:info(?CACHE_TABLE) of
+	undefined ->
+	    ok;
+	_ ->
+	    ets:delete(?CACHE_TABLE)
+    end,
+    ets:new(?CACHE_TABLE, [set, public, named_table]),
     catlfish_sup:start_link(Args).
 
 stop(_State) ->
diff --git a/src/catlfish_sup.erl b/src/catlfish_sup.erl
index abdac44..0b6c306 100644
--- a/src/catlfish_sup.erl
+++ b/src/catlfish_sup.erl
@@ -16,7 +16,7 @@ init([]) ->
          {cacertfile, application:get_env(catlfish, https_cacertfile, none)}],
     Servers =
         lists:map(fun (Config) ->
-                          {IpAddress, Port, Module} = Config,
+                          {ChildName, IpAddress, Port, Module} = Config,
                           {ok, IPv4Address} =
                               inet:parse_ipv4strict_address(IpAddress),
                           WebConfig = [{ip, IPv4Address},
@@ -24,11 +24,12 @@ init([]) ->
                                        {ssl, true},
                                        {ssl_opts, SSLOptions}
                                       ],
-                          {catlfish_web,
+                          {ChildName,
                            {catlfish_web, start, [WebConfig, Module]},
                            permanent, 5000,
                            worker, dynamic}
                   end, application:get_env(catlfish, https_servers, [])),
+    lager:debug("Starting servers ~p", [Servers]),
     {ok,
      {{one_for_one, 3, 10},
       Servers}}.
diff --git a/src/catlfish_web.erl b/src/catlfish_web.erl
index cdc1a39..f3231e4 100644
--- a/src/catlfish_web.erl
+++ b/src/catlfish_web.erl
@@ -2,16 +2,14 @@
 %%% See LICENSE for licensing information.
 
 -module(catlfish_web).
--export([start/2, stop/0, loop/2]).
+-export([start/2, loop/2]).
 
 start(Options, Module) ->
+    lager:debug("Starting catlfish web server: ~p", [Module]),
     Loop = fun (Req) ->
                    ?MODULE:loop(Req, Module)
            end,
-    mochiweb_http:start([{name, ?MODULE}, {loop, Loop} | Options]).
-
-stop() ->
-    mochiweb_http:stop(?MODULE).
+    mochiweb_http:start([{name, Module}, {loop, Loop} | Options]).
 
 loop(Req, Module) ->
     "/" ++ Path = Req:get(path),
@@ -42,7 +40,8 @@ loop(Req, Module) ->
         end
     catch
         Type:What ->
-            lager:error("Crash in ~p for path ~p: ~p ~n~p~n~p~n", [Module, Path, Type, What, erlang:get_stacktrace()]),
+            [CrashFunction | Stack] = erlang:get_stacktrace(),
+            lager:error("Crash in ~p for path ~p: ~p ~p~n~p~n~p~n", [Module, Path, Type, What, CrashFunction, Stack]),
             Req:respond({500, [{"Content-Type", "text/plain"}],
                          "Internal Server Error\n"})
     end.
diff --git a/src/v1.erl b/src/v1.erl
index 707f8ea..d9796fa 100644
--- a/src/v1.erl
+++ b/src/v1.erl
@@ -9,32 +9,31 @@
 
 %% Public functions, i.e. part of URL.
 request(post, "ct/v1/add-chain", Input) ->
-    R = case (catch jiffy:decode(Input)) of
-            {error, E} ->
-                html("add-chain: bad input:", E);
-            {[{<<"chain">>, ChainBase64}]} ->
-                case (catch [base64:decode(X) || X <- ChainBase64]) of
-                    {'EXIT', _} ->
-                        html("add-chain: invalid base64-encoded chain: ",
-                             [ChainBase64]);
-                    [LeafCert | CertChain] ->
-                        Roots = catlfish:known_roots(),
-                        case x509:normalise_chain(Roots, [LeafCert|CertChain]) of
-                            {ok, [Leaf | Chain]} ->
-                                io:format("[info] adding ~p~n",
-                                          [x509:cert_string(LeafCert)]),
-                                success(catlfish:add_chain(Leaf, Chain));
-                            {error, Reason} ->
-                                io:format("[info] rejecting ~p: ~p~n",
-                                          [x509:cert_string(LeafCert), Reason]),
-                                html("add-chain: invalid chain", Reason)
-                        end;
-                    Invalid ->
-                        html("add-chain: chain is not a list: ", [Invalid])
-                end;
-            _ -> html("add-chain: missing input: chain", Input)
-        end,
-    R;
+    case (catch mochijson2:decode(Input)) of
+        {error, E} ->
+            html("add-chain: bad input:", E);
+        {struct, [{<<"chain">>, ChainBase64}]} ->
+            case (catch [base64:decode(X) || X <- ChainBase64]) of
+                {'EXIT', _} ->
+                    html("add-chain: invalid base64-encoded chain: ",
+                         [ChainBase64]);
+                [LeafCert | CertChain] ->
+                    Roots = catlfish:known_roots(),
+                    case x509:normalise_chain(Roots, [LeafCert|CertChain]) of
+                        {ok, [Leaf | Chain]} ->
+                            lager:info("adding ~p",
+                                           [x509:cert_string(LeafCert)]),
+                            success(catlfish:add_chain(Leaf, Chain));
+                        {error, Reason} ->
+                            lager:info("rejecting ~p: ~p",
+                                       [x509:cert_string(LeafCert), Reason]),
+                            html("add-chain: invalid chain", Reason)
+                    end;
+                Invalid ->
+                    html("add-chain: chain is not a list: ", [Invalid])
+            end;
+        _ -> html("add-chain: missing input: chain", Input)
+    end;
 
 request(post, "ct/v1/add-pre-chain", _Input) ->
     niy();
@@ -49,91 +48,83 @@ request(get, "ct/v1/get-sth", _Query) ->
          {sha256_root_hash, base64:encode(Roothash)},
          {tree_head_signature, base64:encode(
                                  plop:serialise(Signature))}],
-    success(jiffy:encode({R}));
+    success({R});
 
 request(get, "ct/v1/get-sth-consistency", Query) ->
-    R = case lists:sort(Query) of
-            [{"first", FirstInput}, {"second", SecondInput}] ->
-                {First, _} = string:to_integer(FirstInput),
-                {Second, _} = string:to_integer(SecondInput),
-                case lists:member(error, [First, Second]) of
-                    true ->
-                        html("get-sth-consistency: bad input:",
-                             [FirstInput, SecondInput]);
-                    false ->
-                        success(
-                          jiffy:encode(
-                            {[{consistency,
-                               [base64:encode(X) ||
-                                   X <- plop:consistency(First, Second)]}]}))
-                end;
-            _ -> html("get-sth-consistency: bad input:", Query)
-        end,
-    R;
+    case lists:sort(Query) of
+        [{"first", FirstInput}, {"second", SecondInput}] ->
+            {First, _} = string:to_integer(FirstInput),
+            {Second, _} = string:to_integer(SecondInput),
+            case lists:member(error, [First, Second]) of
+                true ->
+                    html("get-sth-consistency: bad input:",
+                         [FirstInput, SecondInput]);
+                false ->
+                    success(
+                      {[{consistency,
+                         [base64:encode(X) ||
+                             X <- plop:consistency(First, Second)]}]})
+            end;
+        _ -> html("get-sth-consistency: bad input:", Query)
+    end;
 
 request(get, "ct/v1/get-proof-by-hash", Query) ->
-    R = case lists:sort(Query) of
-            [{"hash", HashInput}, {"tree_size", TreeSizeInput}] ->
-                Hash = case (catch base64:decode(HashInput)) of
-                           {'EXIT', _} -> error;
-                           H -> H
-                       end,
-                {TreeSize, _} = string:to_integer(TreeSizeInput),
-                case lists:member(error, [Hash, TreeSize]) of
-                    true ->
-                        html("get-proof-by-hash: bad input:",
-                             [HashInput, TreeSizeInput]);
-                    false ->
-                        success(
-                          jiffy:encode(
-                            case plop:inclusion(Hash, TreeSize) of
-                                {ok, Index, Path} ->
-                                    {[{leaf_index, Index},
-                                      {audit_path,
-                                       [base64:encode(X) || X <- Path]}]};
-                                {notfound, Msg} ->
-                                    %% FIXME: http status 400
-                                    {[{success, false},
-                                      {error_message, list_to_binary(Msg)}]}
-                            end))
-                end;
-            _ -> html("get-proof-by-hash: bad input:", Query)
-        end,
-    R;
+    case lists:sort(Query) of
+        [{"hash", HashInput}, {"tree_size", TreeSizeInput}] ->
+            Hash = case (catch base64:decode(HashInput)) of
+                       {'EXIT', _} -> error;
+                       H -> H
+                   end,
+            {TreeSize, _} = string:to_integer(TreeSizeInput),
+            case lists:member(error, [Hash, TreeSize]) of
+                true ->
+                    html("get-proof-by-hash: bad input:",
+                         [HashInput, TreeSizeInput]);
+                false ->
+                      case plop:inclusion(Hash, TreeSize) of
+                          {ok, Index, Path} ->
+                              success({[{leaf_index, Index},
+                                        {audit_path,
+                                         [base64:encode(X) || X <- Path]}]});
+                          {notfound, Msg} ->
+                              html("get-proof-by-hash: hash not found", Msg)
+                      end
+            end;
+        _ -> html("get-proof-by-hash: bad input:", Query)
+    end;
 
 request(get, "ct/v1/get-entries", Query) ->
-    R = case lists:sort(Query) of
-            [{"end", EndInput}, {"start", StartInput}] ->
-                {Start, _} = string:to_integer(StartInput),
-                {End, _} = string:to_integer(EndInput),
-                case lists:member(error, [Start, End]) of
-                    true -> html("get-entries: bad input:", [Start, End]);
-                    false -> success(catlfish:entries(Start, min(End, Start + 999)))
-                end;
-            _ -> html("get-entries: bad input:", Query)
-        end,
-    R;
+    case lists:sort(Query) of
+        [{"end", EndInput}, {"start", StartInput}] ->
+            {Start, _} = string:to_integer(StartInput),
+            {End, _} = string:to_integer(EndInput),
+            case lists:member(error, [Start, End]) of
+                true -> html("get-entries: bad input:", [Start, End]);
+                false -> success(
+                           catlfish:entries(Start, min(End, Start + 999)))
+            end;
+        _ -> html("get-entries: bad input:", Query)
+    end;
 
 request(get, "ct/v1/get-entry-and-proof", Query) ->
-    R = case lists:sort(Query) of
-            [{"leaf_index", IndexInput}, {"tree_size", TreeSizeInput}] ->
-                {Index, _} = string:to_integer(IndexInput),
-                {TreeSize, _} = string:to_integer(TreeSizeInput),
-                case lists:member(error, [Index, TreeSize]) of
-                    true ->
-                        html("get-entry-and-proof: not integers: ",
-                             [IndexInput, TreeSizeInput]);
-                    false -> success(catlfish:entry_and_proof(Index, TreeSize))
-                end;
-            _ -> html("get-entry-and-proof: bad input:", Query)
-        end,
-    R;
+    case lists:sort(Query) of
+        [{"leaf_index", IndexInput}, {"tree_size", TreeSizeInput}] ->
+            {Index, _} = string:to_integer(IndexInput),
+            {TreeSize, _} = string:to_integer(TreeSizeInput),
+            case lists:member(error, [Index, TreeSize]) of
+                true ->
+                    html("get-entry-and-proof: not integers: ",
+                         [IndexInput, TreeSizeInput]);
+                false -> success(catlfish:entry_and_proof(Index, TreeSize))
+            end;
+        _ -> html("get-entry-and-proof: bad input:", Query)
+    end;
 
 request(get, "ct/v1/get-roots", _Query) ->
     R = [{certificates,
           [base64:encode(Der) ||
-              Der <- catlfish:known_roots()]}],
-    success(jiffy:encode({R}));
+              Der <- catlfish:update_known_roots()]}],
+    success({R});
 
 request(_Method, _Path, _) ->
     none.
@@ -151,4 +142,4 @@ niy() ->
     html("NIY - Not Implemented Yet|", []).
 
 success(Data) ->
-    {200, [{"Content-Type", "text/json"}], Data}.
+    {200, [{"Content-Type", "text/json"}], mochijson2:encode(Data)}.