Proper custom pkg index support, some tests

- The rebar package index files have been moved off the default path and
  will require a new `rebar3 update`
- Caching of downloaded packages automatically takes place in a path
  relative to the CDN used
- The cache path is not shared with hex as we now write and modify data
  in there arbitrarily
- Basic tests plus the working set for more of them is included

4 files changed, 25 insertions, 26 deletions

diff --git a/src/rebar.hrl b/src/rebar.hrl
index 1f051d7..4540b1a 100644
--- a/src/rebar.hrl
+++ b/src/rebar.hrl
@@ -22,6 +22,7 @@
 -define(DEFAULT_TEST_DEPS_DIR, "test/lib").
 -define(DEFAULT_RELEASE_DIR, "rel").
 -define(DEFAULT_CONFIG_FILE, "rebar.config").
+-define(DEFAULT_CDN, "https://s3.amazonaws.com/s3.hex.pm/tarballs").
 -define(LOCK_FILE, "rebar.lock").
 
 -ifdef(namespaced_types).
diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl
index a344328..e21f1fd 100644
--- a/src/rebar_packages.erl
+++ b/src/rebar_packages.erl
@@ -2,6 +2,7 @@
 
 -export([get_packages/1
         ,registry/1
+        ,package_dir/1
         ,check_registry/3
         ,registry_checksum/2
         ,find_highest_matching/3]).
@@ -16,8 +17,7 @@
 
 -spec get_packages(rebar_state:t()) -> {rebar_dict(), rebar_digraph()}.
 get_packages(State) ->
-    RebarDir = rebar_dir:global_cache_dir(State),
-    RegistryDir = filename:join(RebarDir, "packages"),
+    RegistryDir = package_dir(State),
     DictFile = filename:join(RegistryDir, "dict"),
     Edges = filename:join(RegistryDir, "edges"),
     Vertices = filename:join(RegistryDir, "vertices"),
@@ -43,8 +43,7 @@ get_packages(State) ->
     end.
 
 registry(State) ->
-    Dir = rebar_dir:global_cache_dir(State),
-    RegistryDir = filename:join(Dir, "packages"),
+    RegistryDir = package_dir(State),
     HexFile = filename:join(RegistryDir, "registry"),
     case ets:file2tab(HexFile) of
         {ok, T} ->
@@ -54,6 +53,17 @@ registry(State) ->
             error
     end.
 
+package_dir(State) ->
+    CacheDir = rebar_dir:global_cache_dir(State),
+    CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN),
+    {ok, {_, _, Host, _, Path, _}} = http_uri:parse(CDN),
+    CDNHostPath = lists:reverse(string:tokens(Host, ".")),
+    CDNPath = tl(filename:split(Path)),
+    PackageDir = filename:join([CacheDir, "hex"] ++ CDNHostPath ++ CDNPath ++ ["packages"]),
+    ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")),
+    PackageDir.
+
+
 check_registry(Pkg, Vsn, State) ->
     case rebar_state:registry(State) of
         {ok, T} ->
diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl
index f6bb29b..59ce0dc 100644
--- a/src/rebar_pkg_resource.erl
+++ b/src/rebar_pkg_resource.erl
@@ -11,8 +11,6 @@
 
 -include("rebar.hrl").
 
--define(DEFAULT_CDN, "https://s3.amazonaws.com/s3.hex.pm/tarballs").
-
 lock(_AppDir, Source) ->
     Source.
 
@@ -27,7 +25,7 @@ needs_update(Dir, {pkg, _Name, Vsn}) ->
 
 download(TmpDir, Pkg={pkg, Name, Vsn}, State) ->
     CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN),
-    PackageDir = hex_package_dir(CDN, State),
+    PackageDir = rebar_packages:package_dir(State),
     Package = binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>),
     CachePath = filename:join(PackageDir, Package),
     Url = string:join([CDN, Package], "/"),
@@ -53,10 +51,13 @@ serve_from_cache(TmpDir, CachePath, Pkg, State) ->
             ok = erl_tar:extract({binary, Contents}, [{cwd, TmpDir}, compressed]),
             {ok, true};
         {_Bin, Chk, Chk} ->
+            ?DEBUG("Checksums: registry: ~p, pkg: ~p", [Chk, _Bin]),
             {failed_extract, CachePath};
         {Chk, _Reg, Chk} ->
+            ?DEBUG("Checksums: registry: ~p, pkg: ~p", [_Reg, Chk]),
             {bad_registry_checksum, CachePath};
         {_Bin, _Reg, _Tar} ->
+            ?DEBUG("Checksums: registry: ~p, pkg: ~p, meta: ~p", [_Reg, _Bin, _Tar]),
             {bad_checksum, CachePath}
     end.
 
@@ -67,7 +68,7 @@ serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State) ->
         ETag ->
             serve_from_cache(TmpDir, CachePath, Package, State);
         FileETag ->
-            ?DEBUG("Download ETag ~s doesn't match returned ETag ~s", [ETag, FileETag]),
+            ?DEBUG("Downloaded file ~s ETag ~s doesn't match returned ETag ~s", [CachePath, ETag, FileETag]),
             {bad_download, CachePath}
     end.
 
@@ -84,28 +85,17 @@ checksums(Pkg, Files, Contents, Version, Meta, State) ->
     Blob = <<Version/binary, Meta/binary, Contents/binary>>,
     <<X:256/big-unsigned>> = crypto:hash(sha256, Blob),
     BinChecksum = list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))),
-    RegistryChecksum = rebar_packages:registry_sum(Pkg, State),
+    RegistryChecksum = rebar_packages:registry_checksum(Pkg, State),
     {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files),
     {BinChecksum, RegistryChecksum, TarChecksum}.
 
 make_vsn(_) ->
     {error, "Replacing version of type pkg not supported."}.
 
-%% Use the shared hex package directory unless a non-default package repo is used
-hex_package_dir(?DEFAULT_CDN, _) ->
-    filename:join([rebar_dir:home_dir(), ".hex", "packages"]);
-hex_package_dir(CDN, State) ->
-    CacheDir = rebar_dir:global_cache_dir(State),
-    {ok, {_, _, Host, _, _, _}} = http_uri:parse(CDN),
-    CDNPath = filename:join(lists:reverse(string:tokens(Host, "."))),
-    PackageDir = filename:join([CacheDir, "hex", CDNPath, "packages"]),
-    ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")),
-    PackageDir.
-
 request(Url, ETag) ->
     case httpc:request(get, {Url, [{"if-none-match", ETag} || ETag =/= false]},
-                      [{relaxed, true}],
-                      [{body_format, binary}]) of
+                       [{relaxed, true}],
+                       [{body_format, binary}]) of
         {ok, {{_Version, 200, _Reason}, Headers, Body}} ->
             ?DEBUG("Successfully downloaded ~s", [Url]),
             {"etag", ETag1} = lists:keyfind("etag", 1, Headers),
diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl
index 942b386..dfb719a 100644
--- a/src/rebar_prv_update.erl
+++ b/src/rebar_prv_update.erl
@@ -35,8 +35,7 @@ init(State) ->
 do(State) ->
     ?INFO("Updating package index...", []),
     try
-        Dir = rebar_dir:global_cache_dir(State),
-        RegistryDir = filename:join(Dir, "packages"),
+        RegistryDir = rebar_packages:package_dir(State),
         filelib:ensure_dir(filename:join(RegistryDir, "dummy")),
         HexFile = filename:join(RegistryDir, "registry"),
         TmpDir = ec_file:insecure_mkdtemp(),
@@ -64,8 +63,7 @@ format_error(package_index_write) ->
     "Failed to write package index.".
 
 write_registry(Dict, {digraph, Edges, Vertices, Neighbors, _}, State) ->
-    Dir = rebar_dir:global_cache_dir(State),
-    RegistryDir = filename:join(Dir, "packages"),
+    RegistryDir = rebar_packages:package_dir(State),
     filelib:ensure_dir(filename:join(RegistryDir, "dummy")),
     ets:tab2file(Edges, filename:join(RegistryDir, "edges")),
     ets:tab2file(Vertices, filename:join(RegistryDir, "vertices")),