Merge pull request #414 from tsloughter/hex_pkg_improvements

Hex pkg improvements

7 files changed, 171 insertions, 79 deletions

diff --git a/src/rebar.hrl b/src/rebar.hrl
index 1f051d7..4540b1a 100644
--- a/src/rebar.hrl
+++ b/src/rebar.hrl
@@ -22,6 +22,7 @@
 -define(DEFAULT_TEST_DEPS_DIR, "test/lib").
 -define(DEFAULT_RELEASE_DIR, "rel").
 -define(DEFAULT_CONFIG_FILE, "rebar.config").
+-define(DEFAULT_CDN, "https://s3.amazonaws.com/s3.hex.pm/tarballs").
 -define(LOCK_FILE, "rebar.lock").
 
 -ifdef(namespaced_types).
diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl
index 20bf46b..0aca308 100644
--- a/src/rebar_fetch.erl
+++ b/src/rebar_fetch.erl
@@ -26,42 +26,34 @@ lock_source(AppDir, Source, State) ->
 -spec download_source(file:filename_all(), rebar_resource:resource(), rebar_state:t()) ->
                              true | {error, any()}.
 download_source(AppDir, Source, State) ->
-    try
-        Resources = rebar_state:resources(State),
-        Module = get_resource_type(Source, Resources),
-        TmpDir = ec_file:insecure_mkdtemp(),
-        AppDir1 = ec_cnv:to_list(AppDir),
-        case Module:download(TmpDir, Source, State) of
-            {ok, _} ->
-                ec_file:mkdir_p(AppDir1),
-                code:del_path(filename:absname(filename:join(AppDir1, "ebin"))),
-                ec_file:remove(filename:absname(AppDir1), [recursive]),
-                ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]),
-                ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)),
-                true;
-            {tarball, File} ->
-                Contents = filename:join(TmpDir, "contents"),
-                ec_file:mkdir_p(AppDir1),
-                ec_file:mkdir_p(Contents),
-                ok = erl_tar:extract(File, [{cwd, TmpDir}]),
-                ok = erl_tar:extract(filename:join(TmpDir, "contents.tar.gz"),
-                                    [{cwd, Contents}, compressed]),
-                code:del_path(filename:absname(filename:join(AppDir1, "ebin"))),
-                ec_file:remove(filename:absname(AppDir1), [recursive]),
-
-                ?DEBUG("Moving contents ~p to ~p", [Contents, filename:absname(AppDir1)]),
-                ok = rebar_file_utils:mv(Contents, filename:absname(AppDir1)),
-
-                ?DEBUG("Removing tmp dir ~p", [TmpDir]),
-                ec_file:remove(TmpDir, [recursive]),
-                true
-        end
+    try download_source_(AppDir, Source, State) of
+        true ->
+            true;
+        Error ->
+            throw(?PRV_ERROR(Error))
     catch
         C:T ->
             ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, erlang:get_stacktrace()]),
             throw(?PRV_ERROR({fetch_fail, Source}))
     end.
 
+download_source_(AppDir, Source, State) ->
+    Resources = rebar_state:resources(State),
+    Module = get_resource_type(Source, Resources),
+    TmpDir = ec_file:insecure_mkdtemp(),
+    AppDir1 = ec_cnv:to_list(AppDir),
+    case Module:download(TmpDir, Source, State) of
+        {ok, _} ->
+            ec_file:mkdir_p(AppDir1),
+            code:del_path(filename:absname(filename:join(AppDir1, "ebin"))),
+            ec_file:remove(filename:absname(AppDir1), [recursive]),
+            ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]),
+            ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)),
+            true;
+        Error ->
+            Error
+    end.
+
 -spec needs_update(file:filename_all(), rebar_resource:resource(), rebar_state:t()) -> boolean() | {error, string()}.
 needs_update(AppDir, Source, State) ->
     Resources = rebar_state:resources(State),
@@ -73,8 +65,18 @@ needs_update(AppDir, Source, State) ->
             true
     end.
 
+format_error({bad_download, CachePath}) ->
+    io_lib:format("Download of package does not match md5sum from server: ~s", [CachePath]);
+format_error({failed_extract, CachePath}) ->
+    io_lib:format("Failed to extract package: ~s", [CachePath]);
+format_error({bad_etag, Source}) ->
+    io_lib:format("MD5 Checksum comparison failed for: ~s", [Source]);
 format_error({fetch_fail, Source}) ->
-    io_lib:format("Failed to fetch and copy dep: ~p", [Source]).
+    io_lib:format("Failed to fetch and copy dep: ~s", [Source]);
+format_error({bad_checksum, File}) ->
+    io_lib:format("Checksum mismatch against tarball in ~s", [File]);
+format_error({bad_registry_checksum, File}) ->
+    io_lib:format("Checksum mismatch against registry in ~s", [File]).
 
 get_resource_type({Type, Location}, Resources) ->
     find_resource_module(Type, Location, Resources);
diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl
index 4ab5f9f..e21f1fd 100644
--- a/src/rebar_packages.erl
+++ b/src/rebar_packages.erl
@@ -2,7 +2,9 @@
 
 -export([get_packages/1
         ,registry/1
+        ,package_dir/1
         ,check_registry/3
+        ,registry_checksum/2
         ,find_highest_matching/3]).
 
 -export_type([package/0]).
@@ -15,8 +17,7 @@
 
 -spec get_packages(rebar_state:t()) -> {rebar_dict(), rebar_digraph()}.
 get_packages(State) ->
-    RebarDir = rebar_dir:global_cache_dir(State),
-    RegistryDir = filename:join(RebarDir, "packages"),
+    RegistryDir = package_dir(State),
     DictFile = filename:join(RegistryDir, "dict"),
     Edges = filename:join(RegistryDir, "edges"),
     Vertices = filename:join(RegistryDir, "vertices"),
@@ -42,8 +43,7 @@ get_packages(State) ->
     end.
 
 registry(State) ->
-    Dir = rebar_dir:global_cache_dir(State),
-    RegistryDir = filename:join(Dir, "packages"),
+    RegistryDir = package_dir(State),
     HexFile = filename:join(RegistryDir, "registry"),
     case ets:file2tab(HexFile) of
         {ok, T} ->
@@ -53,6 +53,17 @@ registry(State) ->
             error
     end.
 
+package_dir(State) ->
+    CacheDir = rebar_dir:global_cache_dir(State),
+    CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN),
+    {ok, {_, _, Host, _, Path, _}} = http_uri:parse(CDN),
+    CDNHostPath = lists:reverse(string:tokens(Host, ".")),
+    CDNPath = tl(filename:split(Path)),
+    PackageDir = filename:join([CacheDir, "hex"] ++ CDNHostPath ++ CDNPath ++ ["packages"]),
+    ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")),
+    PackageDir.
+
+
 check_registry(Pkg, Vsn, State) ->
     case rebar_state:registry(State) of
         {ok, T} ->
@@ -66,6 +77,15 @@ check_registry(Pkg, Vsn, State) ->
             false
     end.
 
+registry_checksum({pkg, Name, Vsn}, State) ->
+    {ok, Registry} = registry(State),
+    case ets:lookup(Registry, {Name, Vsn}) of
+        [{{_, _}, [_, Checksum | _]}] ->
+            Checksum;
+        [] ->
+            none
+    end.
+
 %% Hex supports use of ~> to specify the version required for a dependency.
 %% Since rebar3 requires exact versions to choose from we find the highest
 %% available version of the dep that passes the constraint.
diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl
index 3b44fc8..59ce0dc 100644
--- a/src/rebar_pkg_resource.erl
+++ b/src/rebar_pkg_resource.erl
@@ -23,12 +23,99 @@ needs_update(Dir, {pkg, _Name, Vsn}) ->
             true
     end.
 
-download(Dir, {pkg, Name, Vsn}, State) ->
-    TmpFile = filename:join(Dir, "package.tar.gz"),
-    CDN = rebar_state:get(State, rebar_packages_cdn, "https://s3.amazonaws.com/s3.hex.pm/tarballs"),
-    Url = string:join([CDN, binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>)], "/"),
-    {ok, saved_to_file} = httpc:request(get, {Url, []}, [], [{stream, TmpFile}]),
-    {tarball, TmpFile}.
+download(TmpDir, Pkg={pkg, Name, Vsn}, State) ->
+    CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN),
+    PackageDir = rebar_packages:package_dir(State),
+    Package = binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>),
+    CachePath = filename:join(PackageDir, Package),
+    Url = string:join([CDN, Package], "/"),
+    cached_download(TmpDir, CachePath, Pkg, Url, etag(CachePath), State).
+
+cached_download(TmpDir, CachePath, Pkg, Url, ETag, State) ->
+    case request(Url, ETag) of
+        {ok, cached} ->
+            serve_from_cache(TmpDir, CachePath, Pkg, State);
+        {ok, Body, NewETag} ->
+            serve_from_download(TmpDir, CachePath, Pkg, NewETag, Body, State);
+        error when ETag =/= false ->
+            ?DEBUG("Download ~s error, using ~s from cache", [Url, CachePath]),
+            serve_from_cache(TmpDir, CachePath, Pkg, State);
+        error ->
+            request_failed
+    end.
+
+serve_from_cache(TmpDir, CachePath, Pkg, State) ->
+    {Files, Contents, Version, Meta} = extract(TmpDir, CachePath),
+    case checksums(Pkg, Files, Contents, Version, Meta, State) of
+        {Chk, Chk, Chk} ->
+            ok = erl_tar:extract({binary, Contents}, [{cwd, TmpDir}, compressed]),
+            {ok, true};
+        {_Bin, Chk, Chk} ->
+            ?DEBUG("Checksums: registry: ~p, pkg: ~p", [Chk, _Bin]),
+            {failed_extract, CachePath};
+        {Chk, _Reg, Chk} ->
+            ?DEBUG("Checksums: registry: ~p, pkg: ~p", [_Reg, Chk]),
+            {bad_registry_checksum, CachePath};
+        {_Bin, _Reg, _Tar} ->
+            ?DEBUG("Checksums: registry: ~p, pkg: ~p, meta: ~p", [_Reg, _Bin, _Tar]),
+            {bad_checksum, CachePath}
+    end.
+
+serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State) ->
+    ?DEBUG("Writing ~p to cache at ~s", [Package, CachePath]),
+    file:write_file(CachePath, Binary),
+    case etag(CachePath) of
+        ETag ->
+            serve_from_cache(TmpDir, CachePath, Package, State);
+        FileETag ->
+            ?DEBUG("Downloaded file ~s ETag ~s doesn't match returned ETag ~s", [CachePath, ETag, FileETag]),
+            {bad_download, CachePath}
+    end.
+
+
+extract(TmpDir, CachePath) ->
+    ec_file:mkdir_p(TmpDir),
+    {ok, Files} = erl_tar:extract(CachePath, [memory]),
+    {"contents.tar.gz", Contents} = lists:keyfind("contents.tar.gz", 1, Files),
+    {"VERSION", Version} = lists:keyfind("VERSION", 1, Files),
+    {"metadata.config", Meta} = lists:keyfind("metadata.config", 1, Files),
+    {Files, Contents, Version, Meta}.
+
+checksums(Pkg, Files, Contents, Version, Meta, State) ->
+    Blob = <<Version/binary, Meta/binary, Contents/binary>>,
+    <<X:256/big-unsigned>> = crypto:hash(sha256, Blob),
+    BinChecksum = list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))),
+    RegistryChecksum = rebar_packages:registry_checksum(Pkg, State),
+    {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files),
+    {BinChecksum, RegistryChecksum, TarChecksum}.
 
 make_vsn(_) ->
     {error, "Replacing version of type pkg not supported."}.
+
+request(Url, ETag) ->
+    case httpc:request(get, {Url, [{"if-none-match", ETag} || ETag =/= false]},
+                       [{relaxed, true}],
+                       [{body_format, binary}]) of
+        {ok, {{_Version, 200, _Reason}, Headers, Body}} ->
+            ?DEBUG("Successfully downloaded ~s", [Url]),
+            {"etag", ETag1} = lists:keyfind("etag", 1, Headers),
+            {ok, Body, string:strip(ETag1, both, $")};
+        {ok, {{_Version, 304, _Reason}, _Headers, _Body}} ->
+            ?DEBUG("Cached copy of ~s still valid", [Url]),
+            {ok, cached};
+        {ok, {{_Version, Code, _Reason}, _Headers, _Body}} ->
+            ?DEBUG("Request to ~p failed: status code ~p", [Url, Code]),
+            error;
+        {error, Reason} ->
+            ?DEBUG("Request to ~p failed: ~p", [Url, Reason]),
+            error
+    end.
+
+etag(Path) ->
+    case file:read_file(Path) of
+        {ok, Binary} ->
+            <<X:128/big-unsigned-integer>> = crypto:hash(md5, Binary),
+            string:to_lower(lists:flatten(io_lib:format("~32.16.0b", [X])));
+        {error, _} ->
+            false
+    end.
diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl
index bc0a113..ba49532 100644
--- a/src/rebar_prv_install_deps.erl
+++ b/src/rebar_prv_install_deps.erl
@@ -151,7 +151,7 @@ handle_deps(Profile, State0, Deps, Upgrade, Locks) ->
                                             ,{Packages, Graph}),
 
                 update_pkg_deps(Profile, Packages, PkgDeps1
-                               ,Graph, Upgrade, Seen, State2)
+                               ,Graph, Upgrade, Seen, State2, Locks)
         end,
 
     AllDeps = lists:ukeymerge(2
@@ -184,7 +184,7 @@ find_cycles(Apps) ->
 cull_compile(TopSortedDeps, ProjectApps) ->
     lists:dropwhile(fun not_needs_compile/1, TopSortedDeps -- ProjectApps).
 
-update_pkg_deps(Profile, Packages, PkgDeps, Graph, Upgrade, Seen, State) ->
+update_pkg_deps(Profile, Packages, PkgDeps, Graph, Upgrade, Seen, State, Locks) ->
     case PkgDeps of
         [] -> %% No pkg deps
             {[], State};
@@ -196,13 +196,16 @@ update_pkg_deps(Profile, Packages, PkgDeps, Graph, Upgrade, Seen, State) ->
                 {ok, Solution, []} ->
                     Solution;
                 {ok, Solution, Discarded} ->
-                    [warn_skip_pkg(Pkg, State) || Pkg <- Discarded],
+                    [warn_skip_pkg(Pkg, State) || Pkg <- Discarded, not(pkg_locked(Pkg, Locks))],
                     Solution
             end,
-            update_pkg_deps(Profile, S, Packages, Upgrade, Seen, State)
+            update_pkg_deps(Profile, S, Packages, Upgrade, Seen, State, Locks)
     end.
 
-update_pkg_deps(Profile, Pkgs, Packages, Upgrade, Seen, State) ->
+pkg_locked({Name, _}, Locks) ->
+    false =/= lists:keyfind(Name, 1, Locks).
+
+update_pkg_deps(Profile, Pkgs, Packages, Upgrade, Seen, State, _Locks) ->
     %% Create app_info record for each pkg dep
     DepsDir = profile_dep_dir(State, Profile),
     {Solved, _, State1}
@@ -563,10 +566,10 @@ fetch_app(AppInfo, AppDir, State) ->
     ?INFO("Fetching ~s (~p)", [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)]),
     Source = rebar_app_info:source(AppInfo),
     case rebar_fetch:download_source(AppDir, Source, State) of
-        {error, Reason} ->
-            throw(Reason);
-        Result ->
-            Result
+        true ->
+            true;
+        Error ->
+            throw(Error)
     end.
 
 update_app_info(AppInfo) ->
@@ -587,10 +590,10 @@ maybe_upgrade(AppInfo, AppDir, true, State) ->
         true ->
             ?INFO("Upgrading ~s", [rebar_app_info:name(AppInfo)]),
             case rebar_fetch:download_source(AppDir, Source, State) of
-                {error, Reason} ->
-                    throw(Reason);
-                Result ->
-                    Result
+                true ->
+                    true;
+                Error ->
+                    throw(Error)
             end;
         false ->
             ?INFO("No upgrade needed for ~s", [rebar_app_info:name(AppInfo)]),
@@ -610,7 +613,7 @@ parse_goal(Name, Constraint) ->
 
 warn_skip_deps(AppInfo, State) ->
     Msg = "Skipping ~s (from ~p) as an app of the same name "
-          "has already been fetched~n",
+          "has already been fetched",
     Args = [rebar_app_info:name(AppInfo),
             rebar_app_info:source(AppInfo)],
     case rebar_state:get(State, deps_error_on_conflict, false) of
@@ -620,7 +623,7 @@ warn_skip_deps(AppInfo, State) ->
 
 warn_skip_pkg({Name, Source}, State) ->
     Msg = "Skipping ~s (version ~s from package index) as an app of the same "
-          "name has already been fetched~n",
+          "name has already been fetched",
     Args = [Name, Source],
     case rebar_state:get(State, deps_error_on_conflict, false) of
         false -> ?WARN(Msg, Args);
diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl
index 942b386..dfb719a 100644
--- a/src/rebar_prv_update.erl
+++ b/src/rebar_prv_update.erl
@@ -35,8 +35,7 @@ init(State) ->
 do(State) ->
     ?INFO("Updating package index...", []),
     try
-        Dir = rebar_dir:global_cache_dir(State),
-        RegistryDir = filename:join(Dir, "packages"),
+        RegistryDir = rebar_packages:package_dir(State),
         filelib:ensure_dir(filename:join(RegistryDir, "dummy")),
         HexFile = filename:join(RegistryDir, "registry"),
         TmpDir = ec_file:insecure_mkdtemp(),
@@ -64,8 +63,7 @@ format_error(package_index_write) ->
     "Failed to write package index.".
 
 write_registry(Dict, {digraph, Edges, Vertices, Neighbors, _}, State) ->
-    Dir = rebar_dir:global_cache_dir(State),
-    RegistryDir = filename:join(Dir, "packages"),
+    RegistryDir = rebar_packages:package_dir(State),
     filelib:ensure_dir(filename:join(RegistryDir, "dummy")),
     ets:tab2file(Edges, filename:join(RegistryDir, "edges")),
     ets:tab2file(Vertices, filename:join(RegistryDir, "vertices")),
diff --git a/src/rebar_resource.erl b/src/rebar_resource.erl
index 7c58135..cdce7a8 100644
--- a/src/rebar_resource.erl
+++ b/src/rebar_resource.erl
@@ -14,23 +14,6 @@
 -type location() :: string().
 -type ref() :: any().
 
--ifdef(no_callback_support).
-
-%% In the case where R14 or lower is being used to compile the system
-%% we need to export a behaviour info
--export([behaviour_info/1]).
-
--spec behaviour_info(atom()) -> [{atom(), arity()}] | undefined.
-behaviour_info(callbacks) ->
-    [{lock, 2},
-     {download, 3},
-     {needs_update,2},
-     {make_vsn, 1}];
-behaviour_info(_) ->
-    undefined.
-
--else.
-
 -callback lock(file:filename_all(), tuple()) ->
     rebar_resource:resource().
 -callback download(file:filename_all(), tuple(), rebar_state:t()) ->
@@ -39,5 +22,3 @@ behaviour_info(_) ->
     boolean().
 -callback make_vsn(file:filename_all()) ->
     {plain, string()} | {error, string()}.
-
--endif.