replace package management with hex.pm

14 files changed, 326 insertions, 157 deletions

diff --git a/Makefile b/Makefile
index 2e02370..f36c177 100644
--- a/Makefile
+++ b/Makefile
@@ -6,7 +6,7 @@ RETEST=$(PWD)/deps/retest/retest
 DEPS_PLT=$(CURDIR)/.depsolver_plt
 
 all:
-	./bootstrap/bootstrap
+	@rebar get-deps compile escriptize
 
 clean:
 	@rm -rf rebar3 ebin/*.beam inttest/rt.work rt.work .eunit
diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl
index dca5aa8..9d779be 100644
--- a/src/rebar_app_info.erl
+++ b/src/rebar_app_info.erl
@@ -79,7 +79,7 @@ new(AppName, Vsn) ->
 new(AppName, Vsn, Dir) ->
     {ok, #app_info_t{name=ec_cnv:to_binary(AppName),
                      original_vsn=Vsn,
-                     dir=Dir}}.
+                     dir=ec_cnv:to_list(Dir)}}.
 
 %% @doc build a complete version of the app info with all fields set.
 -spec new(atom() | binary() | string(), binary() | string(), file:name(), list()) ->
@@ -87,7 +87,7 @@ new(AppName, Vsn, Dir) ->
 new(AppName, Vsn, Dir, Deps) ->
     {ok, #app_info_t{name=ec_cnv:to_binary(AppName),
                      original_vsn=Vsn,
-                     dir=Dir,
+                     dir=ec_cnv:to_list(Dir),
                      deps=Deps}}.
 
 %% @doc discover a complete version of the app info with all fields set.
@@ -200,7 +200,7 @@ dir(#app_info_t{dir=Dir}) ->
 
 -spec dir(t(), file:name()) -> t().
 dir(AppInfo=#app_info_t{}, Dir) ->
-    AppInfo#app_info_t{dir=Dir}.
+    AppInfo#app_info_t{dir=ec_cnv:to_list(Dir)}.
 
 -spec ebin_dir(t()) -> file:name().
 ebin_dir(#app_info_t{dir=Dir}) ->
diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl
index b096705..cb342a8 100644
--- a/src/rebar_erlc_compiler.erl
+++ b/src/rebar_erlc_compiler.erl
@@ -144,17 +144,13 @@ doterl_compile(Config, Dir, MoreSources, ErlOpts) ->
                                 filename:join(Dir, X)
                         end, rebar_dir:src_dirs(proplists:append_values(src_dirs, ErlOpts))),
     AllErlFiles = gather_src(SrcDirs, []) ++ MoreSources,
-    %% NOTE: If and when erl_first_files is not inherited anymore
-    %% (rebar_state:get instead of rebar_state:get_list), consider
-    %% logging a warning message for any file listed in erl_first_files which
-    %% wasn't found via gather_src.
-    
+
     %% Issue: rebar/rebar3#140 (fix matching based on same path + order of
     %% erl_first_files)
     ErlFirstFiles = get_erl_first_files(ErlFirstFilesConf, AllErlFiles),
     RestErls = [ File || File <- AllErlFiles,
                          not lists:member(File, ErlFirstFiles) ],
-   
+
     %% Make sure that ebin/ exists and is on the path
     ok = filelib:ensure_dir(filename:join([Dir, "ebin", "dummy.beam"])),
     CurrPath = code:get_path(),
@@ -634,5 +630,3 @@ log_files(Prefix, Files) ->
         _ ->
             ?DEBUG("~s:~n~p", [Prefix, Files])
     end.
-
-
diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl
index 088a49a..4dfd790 100644
--- a/src/rebar_fetch.erl
+++ b/src/rebar_fetch.erl
@@ -8,12 +8,13 @@
 -module(rebar_fetch).
 
 -export([lock_source/2,
-         download_source/2,
+         download_source/3,
          needs_update/2]).
 
 -export([format_error/1]).
 
 -include("rebar.hrl").
+-include_lib("providers/include/providers.hrl").
 
 %% map short versions of resources to module names
 -define(RESOURCES, [{git, rebar_git_resource}, {pkg, rebar_pkg_resource}]).
@@ -24,13 +25,14 @@ lock_source(AppDir, Source) ->
     Module = get_resource_type(Source),
     Module:lock(AppDir, Source).
 
--spec download_source(file:filename_all(), rebar_resource:resource()) -> true | {error, any()}.
-download_source(AppDir, Source) ->
+-spec download_source(file:filename_all(), rebar_resource:resource(), rebar_state:t()) ->
+                             true | {error, any()}.
+download_source(AppDir, Source, State) ->
     try
         Module = get_resource_type(Source),
         TmpDir = ec_file:insecure_mkdtemp(),
         AppDir1 = ec_cnv:to_list(AppDir),
-        case Module:download(TmpDir, Source) of
+        case Module:download(TmpDir, Source, State) of
             {ok, _} ->
                 ec_file:mkdir_p(AppDir1),
                 code:del_path(filename:absname(filename:join(AppDir1, "ebin"))),
@@ -38,20 +40,21 @@ download_source(AppDir, Source) ->
                 ok = ec_file:copy(TmpDir, filename:absname(AppDir1), [recursive]),
                 true;
             {tarball, File} ->
+                Contents = filename:join(TmpDir, "contents"),
                 ec_file:mkdir_p(AppDir1),
-                ok = erl_tar:extract(File, [{cwd, TmpDir}
-                                           ,compressed]),
-                BaseName = filename:basename(AppDir1),
-                [FromDir] = filelib:wildcard(filename:join(TmpDir, BaseName++"-*")),
+                ec_file:mkdir_p(Contents),
+                ok = erl_tar:extract(File, [{cwd, TmpDir}]),
+                ok = erl_tar:extract(filename:join(TmpDir, "contents.tar.gz"),
+                                    [{cwd, Contents}, compressed]),
                 code:del_path(filename:absname(filename:join(AppDir1, "ebin"))),
                 ec_file:remove(filename:absname(AppDir1), [recursive]),
-                ok = ec_file:copy(FromDir, filename:absname(AppDir1), [recursive]),
-                true = code:add_patha(filename:join(AppDir1, "ebin")),
+                ok = ec_file:copy(Contents, filename:absname(AppDir1), [recursive]),
                 true
         end
     catch
-        _:_ ->
-            {error, {rebar_fetch, {fetch_fail, Source}}}
+        C:T ->
+            ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, erlang:get_stacktrace()]),
+            throw(?PRV_ERROR({fetch_fail, Source}))
     end.
 
 -spec needs_update(file:filename_all(), rebar_resource:resource()) -> boolean() | {error, string()}.
diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl
index bd0a978..785c362 100644
--- a/src/rebar_git_resource.erl
+++ b/src/rebar_git_resource.erl
@@ -5,7 +5,7 @@
 -behaviour(rebar_resource).
 
 -export([lock/2
-        ,download/2
+        ,download/3
         ,needs_update/2
         ,make_vsn/1]).
 
@@ -75,28 +75,28 @@ parse_git_url("https://" ++ HostPath) ->
     [Host | Path] = string:tokens(HostPath, "/"),
     {Host, filename:rootname(filename:join(Path), ".git")}.
 
-download(Dir, {git, Url}) ->
+download(Dir, {git, Url}, State) ->
     ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []),
-    download(Dir, {git, Url, {branch, "master"}});
-download(Dir, {git, Url, ""}) ->
+    download(Dir, {git, Url, {branch, "master"}}, State);
+download(Dir, {git, Url, ""}, State) ->
     ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []),
-    download(Dir, {git, Url, {branch, "master"}});
-download(Dir, {git, Url, {branch, Branch}}) ->
+    download(Dir, {git, Url, {branch, "master"}}, State);
+download(Dir, {git, Url, {branch, Branch}}, _State) ->
     ok = filelib:ensure_dir(Dir),
     rebar_utils:sh(?FMT("git clone ~s ~s -b ~s --single-branch",
                        [Url, filename:basename(Dir), Branch]),
                    [{cd, filename:dirname(Dir)}]);
-download(Dir, {git, Url, {tag, Tag}}) ->
+download(Dir, {git, Url, {tag, Tag}}, _State) ->
     ok = filelib:ensure_dir(Dir),
     rebar_utils:sh(?FMT("git clone ~s ~s -b ~s --single-branch",
                         [Url, filename:basename(Dir), Tag]),
                    [{cd, filename:dirname(Dir)}]);
-download(Dir, {git, Url, {ref, Ref}}) ->
+download(Dir, {git, Url, {ref, Ref}}, _State) ->
     ok = filelib:ensure_dir(Dir),
     rebar_utils:sh(?FMT("git clone -n ~s ~s", [Url, filename:basename(Dir)]),
                    [{cd, filename:dirname(Dir)}]),
     rebar_utils:sh(?FMT("git checkout -q ~s", [Ref]), [{cd, Dir}]);
-download(Dir, {git, Url, Rev}) ->
+download(Dir, {git, Url, Rev}, _State) ->
     ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []),
     ok = filelib:ensure_dir(Dir),
     rebar_utils:sh(?FMT("git clone -n ~s ~s", [Url, filename:basename(Dir)]),
diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl
index 30d11c9..5400544 100644
--- a/src/rebar_packages.erl
+++ b/src/rebar_packages.erl
@@ -13,13 +13,21 @@
 -spec get_packages(rebar_state:t()) -> {rebar_dict(), rebar_digraph()}.
 get_packages(State) ->
     RebarDir = rebar_dir:global_config_dir(State),
-    PackagesFile = filename:join(RebarDir, "packages"),
-    case ec_file:exists(PackagesFile) of
+    RegistryDir = filename:join(RebarDir, "packages"),
+    DictFile = filename:join(RegistryDir, "dict"),
+    Edges = filename:join(RegistryDir, "edges"),
+    Vertices = filename:join(RegistryDir, "vertices"),
+    Neighbors = filename:join(RegistryDir, "neighbors"),
+
+    case lists:all(fun(X) -> filelib:is_file(X) end, [DictFile, Edges, Vertices, Neighbors]) of
         true ->
             try
-                {ok, Binary} = file:read_file(PackagesFile),
-                {Dict, Graph} = binary_to_term(Binary),
-                {Dict, rebar_digraph:restore_graph(Graph)}
+                {ok, DictBinary} = file:read_file(DictFile),
+                Dict = binary_to_term(DictBinary),
+                {ok, EdgesTab} = ets:file2tab(Edges),
+                {ok, VerticesTab} = ets:file2tab(Vertices),
+                {ok, NeighborsTab} = ets:file2tab(Neighbors),
+                {Dict, {digraph, EdgesTab, VerticesTab, NeighborsTab, true}}
             catch
                 _:_ ->
                     ?ERROR("Bad packages index, try to fix with `rebar3 update`", []),
diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl
index 5cd6fc8..0508e2c 100644
--- a/src/rebar_pkg_resource.erl
+++ b/src/rebar_pkg_resource.erl
@@ -5,7 +5,7 @@
 -behaviour(rebar_resource).
 
 -export([lock/2
-        ,download/2
+        ,download/3
         ,needs_update/2
         ,make_vsn/1]).
 
@@ -23,9 +23,11 @@ needs_update(Dir, {pkg, _Name, Vsn, _Url}) ->
             true
     end.
 
-download(Dir, {pkg, _Name, _Vsn, Url}) ->
+download(Dir, {pkg, Name, Vsn}, State) ->
     TmpFile = filename:join(Dir, "package.tar.gz"),
-    {ok, saved_to_file} = httpc:request(get, {binary_to_list(Url), []}, [], [{stream, TmpFile}]),
+    CDN = rebar_state:get(State, rebar_packages_cdn, "https://s3.amazonaws.com/s3.hex.pm/tarballs"),
+    Url = string:join([CDN, binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>)], "/"),
+    {ok, saved_to_file} = httpc:request(get, {Url, []}, [], [{stream, TmpFile}]),
     {tarball, TmpFile}.
 
 make_vsn(_) ->
diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl
index dab4c24..61fbd70 100644
--- a/src/rebar_prv_install_deps.erl
+++ b/src/rebar_prv_install_deps.erl
@@ -75,19 +75,19 @@ do(State) ->
         Profiles = rebar_state:current_profiles(State),
         ProjectApps = rebar_state:project_apps(State),
 
-        {SrcApps, State1} =
-            lists:foldl(fun(Profile, {SrcAppsAcc, StateAcc}) ->
+        {Apps, State1} =
+            lists:foldl(fun(Profile, {AppsAcc, StateAcc}) ->
                                 Locks = rebar_state:get(StateAcc, {locks, Profile}, []),
-                                {ok, NewSrcApps, NewState} =
+                                {ok, NewApps, NewState} =
                                     handle_deps(Profile
                                                ,StateAcc
                                                ,rebar_state:get(StateAcc, {deps, Profile}, [])
                                                ,Locks),
-                                {NewSrcApps++SrcAppsAcc, NewState}
+                                {NewApps++AppsAcc, NewState}
                         end, {[], State}, lists:reverse(Profiles)),
 
-        Source = ProjectApps ++ SrcApps,
-        case find_cycles(Source ++ rebar_state:all_deps(State1)) of
+        Source = ProjectApps ++ Apps,
+        case find_cycles(Source) of
             {cycles, Cycles} ->
                 ?PRV_ERROR({cycles, Cycles});
             {error, Error} ->
@@ -117,7 +117,9 @@ find_cycles(Apps) ->
 
 -spec format_error(any()) -> iolist().
 format_error({parse_dep, Dep}) ->
-    io_lib:format("Failed parsing dep ~p~n", [Dep]);
+    io_lib:format("Failed parsing dep ~p", [Dep]);
+format_error({missing_package, Package, Version}) ->
+    io_lib:format("Package not found in registry: ~s-~s", [Package, Version]);
 format_error({cycles, Cycles}) ->
     Prints = [["applications: ",
                [io_lib:format("~s ", [Dep]) || Dep <- Cycle],
@@ -178,7 +180,7 @@ handle_deps(Profile, State, Deps, Upgrade, Locks) ->
     %% Sort all apps to build order
     State3 = rebar_state:all_deps(State2, AllDeps),
 
-    {ok, SrcApps, State3}.
+    {ok, AllDeps, State3}.
 
 %% ===================================================================
 %% Internal functions
@@ -193,7 +195,7 @@ update_pkg_deps(Profile, Pkgs, Packages, Upgrade, Seen, State) ->
                                                       ,Packages
                                                       ,Pkg),
                               {SeenAcc1, StateAcc1} = maybe_lock(Profile, AppInfo, SeenAcc, StateAcc, 0),
-                              case maybe_fetch(AppInfo, Upgrade, SeenAcc1) of
+                              case maybe_fetch(AppInfo, Upgrade, SeenAcc1, State) of
                                   true ->
                                       {[AppInfo | Acc], SeenAcc1, StateAcc1};
                                   false ->
@@ -227,15 +229,14 @@ maybe_lock(Profile, AppInfo, Seen, State, Level) ->
 package_to_app(DepsDir, Packages, {Name, Vsn}) ->
     case dict:find({Name, Vsn}, Packages) of
         error ->
-            {error, missing_package};
+            throw(?PRV_ERROR({missing_package, Name, Vsn}));
         {ok, P} ->
             PkgDeps = [{PkgName, PkgVsn}
                        || {PkgName,PkgVsn} <- proplists:get_value(<<"deps">>, P, [])],
-            Link = proplists:get_value(<<"link">>, P, ""),
             {ok, AppInfo} = rebar_app_info:new(Name, Vsn),
             AppInfo1 = rebar_app_info:deps(AppInfo, PkgDeps),
             AppInfo2 = rebar_app_info:dir(AppInfo1, rebar_dir:deps_dir(DepsDir, Name)),
-            rebar_app_info:source(AppInfo2, {pkg, Name, Vsn, Link})
+            rebar_app_info:source(AppInfo2, {pkg, Name, Vsn})
     end.
 
 -spec update_src_deps(atom(), non_neg_integer(), list(), list(), list(), rebar_state:t(), boolean(), sets:set(binary()), list()) -> {rebar_state:t(), list(), list(), sets:set(binary())}.
@@ -282,7 +283,7 @@ update_src_deps(Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen,
                                                               ,StateAcc1
                                                               ,LocksAcc);
                                              _ ->
-                                                 maybe_fetch(AppInfo, false, SeenAcc),
+                                                 maybe_fetch(AppInfo, false, SeenAcc, StateAcc),
                                                  handle_dep(AppInfo
                                                            ,SrcDepsAcc
                                                            ,PkgDepsAcc
@@ -306,7 +307,7 @@ handle_upgrade(AppInfo, SrcDeps, PkgDeps, SrcApps, Level, State, Locks) ->
     Name = rebar_app_info:name(AppInfo),
     case lists:keyfind(Name, 1, Locks) of
         false ->
-            case maybe_fetch(AppInfo, true, sets:new()) of
+            case maybe_fetch(AppInfo, true, sets:new(), State) of
                 true ->
                     handle_dep(AppInfo
                               ,SrcDeps
@@ -357,8 +358,8 @@ handle_dep(State, DepsDir, AppInfo, Locks, Level) ->
     {AppInfo2, SrcDeps, PkgDeps, Locks++NewLocks}.
 
 -spec maybe_fetch(rebar_app_info:t(), boolean() | {true, binary(), integer()},
-                  sets:set(binary())) -> boolean().
-maybe_fetch(AppInfo, Upgrade, Seen) ->
+                  sets:set(binary()), rebar_state:t()) -> boolean().
+maybe_fetch(AppInfo, Upgrade, Seen, State) ->
     AppDir = ec_cnv:to_list(rebar_app_info:dir(AppInfo)),
     Apps = rebar_app_discover:find_apps(["_checkouts"], all),
     case rebar_app_utils:find(rebar_app_info:name(AppInfo), Apps) of
@@ -368,13 +369,13 @@ maybe_fetch(AppInfo, Upgrade, Seen) ->
         error ->
             case not app_exists(AppDir) of
                 true ->
-                    fetch_app(AppInfo, AppDir);
+                    fetch_app(AppInfo, AppDir, State);
                 false ->
                     case sets:is_element(rebar_app_info:name(AppInfo), Seen) of
                         true ->
                             false;
                         false ->
-                            maybe_upgrade(AppInfo, AppDir, Upgrade)
+                            maybe_upgrade(AppInfo, AppDir, Upgrade, State)
                     end
             end
     end.
@@ -457,25 +458,25 @@ app_exists(AppDir) ->
             end
     end.
 
-fetch_app(AppInfo, AppDir) ->
+fetch_app(AppInfo, AppDir, State) ->
     ?INFO("Fetching ~s (~p)", [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)]),
     Source = rebar_app_info:source(AppInfo),
-    case rebar_fetch:download_source(AppDir, Source) of
+    case rebar_fetch:download_source(AppDir, Source, State) of
         {error, Reason} ->
             throw(Reason);
         Result ->
             Result
     end.
 
-maybe_upgrade(AppInfo, AppDir, false) ->
+maybe_upgrade(AppInfo, AppDir, false, _State) ->
     Source = rebar_app_info:source(AppInfo),
     rebar_fetch:needs_update(AppDir, Source);
-maybe_upgrade(AppInfo, AppDir, true) ->
+maybe_upgrade(AppInfo, AppDir, true, State) ->
     Source = rebar_app_info:source(AppInfo),
     case rebar_fetch:needs_update(AppDir, Source) of
         true ->
             ?INFO("Updating ~s", [rebar_app_info:name(AppInfo)]),
-            case rebar_fetch:download_source(AppDir, Source) of
+            case rebar_fetch:download_source(AppDir, Source, State) of
                 {error, Reason} ->
                     throw(Reason);
                 Result ->
diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl
index 524a684..655e5a3 100644
--- a/src/rebar_prv_update.erl
+++ b/src/rebar_prv_update.erl
@@ -34,18 +34,22 @@ init(State) ->
 do(State) ->
     ?INFO("Updating package index...", []),
     try
-        Url = url(State),
+        Url = rebar_state:get(State, rebar_packages_cdn, "https://s3.amazonaws.com/s3.hex.pm/registry.ets.gz"),
         TmpDir = ec_file:insecure_mkdtemp(),
-        TmpFile = filename:join(TmpDir, "packages"),
-        Home = rebar_dir:home_dir(),
-        PackagesFile = filename:join([Home, ?CONFIG_DIR, "packages"]),
-        filelib:ensure_dir(PackagesFile),
-        {ok, _RequestId} = httpc:request(get, {Url, [{"Accept", "application/erlang"}]},
-                                        [], [{stream, TmpFile}, {sync, true}]),
-        ok = ec_file:copy(TmpFile, PackagesFile)
+        TmpFile = filename:join(TmpDir, "packages.gz"),
+        HexFile = filename:join(TmpDir, "packages"),
+        {ok, _RequestId} = httpc:request(get, {Url, []},
+                                         [], [{stream, TmpFile}, {sync, true}]),
+        {ok, Data} = file:read_file(TmpFile),
+        Unzipped = zlib:gunzip(Data),
+        ok = file:write_file(HexFile, Unzipped),
+        {Dict, Graph} = hex_to_graph(HexFile),
+        write_registry(Dict, Graph, State),
+        ok
     catch
-        _:_ ->
-            {error, {?MODULE, package_index_write}}
+        E:C ->
+            io:format("E C ~p ~p~n", [E, C]),
+            throw({error, {?MODULE, package_index_write}})
     end,
 
     {ok, State}.
@@ -54,9 +58,39 @@ do(State) ->
 format_error(package_index_write) ->
     "Failed to write package index.".
 
-url(State) ->
-    ErtsVsn = erlang:system_info(version),
+write_registry(Dict, {digraph, Edges, Vertices, Neighbors, _}, State) ->
+    Dir = rebar_dir:global_config_dir(State),
+    RegistryDir = filename:join(Dir, "packages"),
+    filelib:ensure_dir(filename:join(RegistryDir, "dummy")),
+    ets:tab2file(Edges, filename:join(RegistryDir, "edges")),
+    ets:tab2file(Vertices, filename:join(RegistryDir, "vertices")),
+    ets:tab2file(Neighbors, filename:join(RegistryDir, "neighbors")),
+    file:write_file(filename:join(RegistryDir, "dict"), term_to_binary(Dict)).
 
-    Qs = [io_lib:format("~s=~s", [X, Y]) || {X, Y} <- [{"erts", ErtsVsn}]],
-    Url = rebar_state:get(State, rebar_packages_url, "http://packages.rebar3.org/packages"),
-    Url ++ "?" ++ string:join(Qs, "&").
+hex_to_graph(Filename) ->
+    {ok, T} = ets:file2tab(Filename),
+    Graph = digraph:new(),
+    ets:foldl(fun({Pkg, [Versions]}, ok) when is_binary(Pkg), is_list(Versions) ->
+                      lists:foreach(fun(Version) ->
+                                            digraph:add_vertex(Graph, {Pkg, Version}, 1)
+                                    end, Versions);
+                 (_, ok) ->
+                      ok
+              end, ok, T),
+
+    Dict1 = ets:foldl(fun({{Pkg, PkgVsn}, [Deps | _]}, Dict) ->
+                              DepsList = lists:foldl(fun([Dep, DepVsn, _, _], DepsListAcc) ->
+                                                             case DepVsn of
+                                                                 <<"~> ", Vsn/binary>> ->
+                                                                     digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, Vsn}),
+                                                                     [{Dep, DepVsn} | DepsListAcc];
+                                                                 Vsn ->
+                                                                     digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, Vsn}),
+                                                                     [{Dep, Vsn} | DepsListAcc]
+                                                             end
+                                                     end, [], Deps),
+                              dict:store({Pkg, PkgVsn}, DepsList, Dict);
+                         (_, Dict) ->
+                              Dict
+                      end, dict:new(), T),
+    {Dict1, Graph}.
diff --git a/src/rebar_resource.erl b/src/rebar_resource.erl
index 2bc28e2..7c58135 100644
--- a/src/rebar_resource.erl
+++ b/src/rebar_resource.erl
@@ -23,7 +23,7 @@
 -spec behaviour_info(atom()) -> [{atom(), arity()}] | undefined.
 behaviour_info(callbacks) ->
     [{lock, 2},
-     {download, 2},
+     {download, 3},
      {needs_update,2},
      {make_vsn, 1}];
 behaviour_info(_) ->
@@ -33,7 +33,7 @@ behaviour_info(_) ->
 
 -callback lock(file:filename_all(), tuple()) ->
     rebar_resource:resource().
--callback download(file:filename_all(), tuple()) ->
+-callback download(file:filename_all(), tuple(), rebar_state:t()) ->
     {tarball, file:filename_all()} | {ok, any()} | {error, any()}.
 -callback needs_update(file:filename_all(), tuple()) ->
     boolean().
diff --git a/test/mock_git_resource.erl b/test/mock_git_resource.erl
index d2f0207..6940f54 100644
--- a/test/mock_git_resource.erl
+++ b/test/mock_git_resource.erl
@@ -103,7 +103,7 @@ mock_download(Opts) ->
     Overrides = proplists:get_value(override_vsn, Opts, []),
     meck:expect(
         ?MOD, download,
-        fun (Dir, Git) ->
+        fun (Dir, Git, _) ->
             filelib:ensure_dir(Dir),
             {git, Url, {_, Vsn}} = normalize_git(Git, Overrides, Default),
             App = app(Url),
diff --git a/test/mock_pkg_resource.erl b/test/mock_pkg_resource.erl
index ab2c0d3..c7276b6 100644
--- a/test/mock_pkg_resource.erl
+++ b/test/mock_pkg_resource.erl
@@ -21,7 +21,7 @@ mock() -> mock([]).
             | {not_in_index, [{App, Vsn}]}
             | {pkgdeps, [{{App,Vsn}, [Dep]}]},
     App :: string(),
-    Dep :: {App, string(), {pkg, App, Vsn, Url::string()}},
+    Dep :: {App, string(), {pkg, App, Vsn}},
     Vsn :: string().
 mock(Opts) ->
     meck:new(?MOD, [no_link]),
@@ -50,7 +50,7 @@ mock_update(Opts) ->
     ToUpdate = proplists:get_value(update, Opts, []),
     meck:expect(
         ?MOD, needs_update,
-        fun(_Dir, {pkg, App, _Vsn, _Url}) ->
+        fun(_Dir, {pkg, App, _Vsn}) ->
             lists:member(App, ToUpdate)
         end).
 
@@ -65,7 +65,7 @@ mock_vsn(_Opts) ->
 %% @doc For each app to download, create a dummy app on disk instead.
 %% The configuration for this one (passed in from `mock/1') includes:
 %%
-%% - Specify a version with `{pkg, _, Vsn, _}'
+%% - Specify a version with `{pkg, _, Vsn}'
 %% - Dependencies for each application must be passed of the form:
 %%   `{pkgdeps, [{"app1", [{app2, ".*", {pkg, ...}}]}]}' -- basically
 %%   the `pkgdeps' option takes a key/value list of terms to output directly
@@ -74,7 +74,7 @@ mock_download(Opts) ->
     Deps = proplists:get_value(pkgdeps, Opts, []),
     meck:expect(
         ?MOD, download,
-        fun (Dir, {pkg, AppBin, Vsn, _Url}) ->
+        fun (Dir, {pkg, AppBin, Vsn}, _) ->
             App = binary_to_list(AppBin),
             filelib:ensure_dir(Dir),
             AppDeps = proplists:get_value({App,Vsn}, Deps, []),
@@ -83,11 +83,15 @@ mock_download(Opts) ->
                 [element(1,D) || D  <- AppDeps]
             ),
             rebar_test_utils:create_config(Dir, [{deps, AppDeps}]),
-            Tarball = filename:join([Dir, "package.tar.gz"]),
+            Tarball = filename:join([Dir, App++"-"++binary_to_list(Vsn)++".tar"]),
+            Contents = filename:join([Dir, "contents.tar.gz"]),
             Files = all_files(rebar_app_info:dir(AppInfo)),
-            ok = erl_tar:create(Tarball,
+            ok = erl_tar:create(Contents,
                                 archive_names(Dir, App, Vsn, Files),
                                 [compressed]),
+            ok = erl_tar:create(Tarball,
+                                [{"contents.tar.gz", Contents}],
+                                []),
             [file:delete(F) || F <- Files],
             {tarball, Tarball}
         end).
@@ -117,8 +121,7 @@ all_files(Dir) ->
     filelib:wildcard(filename:join([Dir, "**"])).
 
 archive_names(Dir, App, Vsn, Files) ->
-    ArchName = App++"-"++binary_to_list(Vsn),
-    [{ArchName ++ (F -- Dir), F} || F <- Files].
+    [{(F -- Dir) -- "/", F} || F <- Files].
 
 find_parts(Apps, Skip) -> find_parts(Apps, Skip, dict:new()).
 
diff --git a/test/rebar_deps_SUITE.erl b/test/rebar_deps_SUITE.erl
index 4aaa7e3..633e9a2 100644
--- a/test/rebar_deps_SUITE.erl
+++ b/test/rebar_deps_SUITE.erl
@@ -1,14 +1,16 @@
-%% TODO: add tests for:
-%% - only part of deps fetched
-%% - only part of deps locked
-%% - output only shown once
-%% - modification asterisk on locked file
 -module(rebar_deps_SUITE).
 -compile(export_all).
 -include_lib("common_test/include/ct.hrl").
 -include_lib("eunit/include/eunit.hrl").
 
-all() -> [default_nodep, default_lock].
+all() -> [{group, git}, {group, pkg}].
+
+groups() ->
+    [{all, [], [flat, pick_highest_left, pick_highest_right,
+                pick_smallest1, pick_smallest2,
+                circular1, circular2, circular_skip]},
+     {git, [], [{group, all}]},
+     {pkg, [], [{group, all}]}].
 
 init_per_suite(Config) ->
     application:start(meck),
@@ -17,79 +19,202 @@ init_per_suite(Config) ->
 end_per_suite(_Config) ->
     application:stop(meck).
 
+init_per_group(git, Config) ->
+    [{deps_type, git} | Config];
+init_per_group(pkg, Config) ->
+    [{deps_type, pkg} | Config];
+init_per_group(_, Config) ->
+    Config.
+
+end_per_group(_, Config) ->
+    Config.
+
 init_per_testcase(Case, Config) ->
-    meck:new(io, [no_link, passthrough, unstick]),
-    setup_project(Case, Config).
+    {Deps, Warnings, Expect} = deps(Case),
+    Expected = case Expect of
+        {ok, List} -> {ok, format_expected_deps(List)};
+        {error, Reason} -> {error, Reason}
+    end,
+    DepsType = ?config(deps_type, Config),
+    mock_warnings(),
+    [{expect, Expected},
+     {warnings, Warnings}
+    | setup_project(Case, Config, expand_deps(DepsType, Deps))].
 
 end_per_testcase(_, Config) ->
     meck:unload(),
     Config.
 
-config_content() ->
-    [{deps, [
-        {src_a, ".*", {git, "https://example.org/ferd/src_a.git", {branch, "master"}}},
-        {src_b, {git, "https://example.org/ferd/src_b.git", {branch, "master"}}},
-        {pkg_a, "1.0.0"}
-     ]},
-     {profiles,
-      [{test,
-        [{deps, [
-            {tdep, {git, "git://example.org/ferd/tdep.git", {tag, "0.8.2"}}}
-        ]}]
-      }]}
-    ].
-
-setup_project(Case, Config0) ->
+format_expected_deps(Deps) ->
+    [case Dep of
+        {N,V} -> {dep, N, V};
+        N -> {dep, N}
+     end || Dep <- Deps].
+
+%% format:
+%% {Spec,
+%%  [Warning],
+%%  {ok, Result} | {error, Reason}}
+%%
+%% Spec is a list of levelled dependencies of two possible forms:
+%% - {"Name", Spec}
+%% - {"Name", "Vsn", Spec}
+%%
+%% Warnings are going to match on mocked ?WARN(...)
+%% calls to be evaluated. An empty list means we do not care about
+%% warnings, not that no warnings will be printed. This means
+%% the list of warning isn't interpreted to be exhaustive, and more
+%% warnings may be generated than are listed.
+deps(flat) ->
+    {[{"B", []},
+      {"C", []}],
+     [],
+     {ok, ["B", "C"]}};
+deps(pick_highest_left) ->
+    {[{"B", [{"C", "2", []}]},
+      {"C", "1", []}],
+     [{"C","2"}],
+     {ok, ["B", {"C", "1"}]}};
+deps(pick_highest_right) ->
+    {[{"B", "1", []},
+      {"C", [{"B", "2", []}]}],
+     [{"B","2"}],
+     {ok, [{"B","1"}, "C"]}};
+deps(pick_smallest1) ->
+    {[{"B", [{"D", "1", []}]},
+      {"C", [{"D", "2", []}]}],
+     [{"D","2"}],
+     %% we pick D1 because B < C
+     {ok, ["B","C",{"D","1"}]}};
+deps(pick_smallest2) ->
+    {[{"C", [{"D", "2", []}]},
+      {"B", [{"D", "1", []}]}],
+     [{"D","2"}],
+     %% we pick D1 because B < C
+     {ok, ["B","C",{"D","1"}]}};
+deps(circular1) ->
+    {[{"B", [{"A", []}]}, % A is the top-level app
+      {"C", []}],
+     [],
+     {error, {rebar_prv_install_deps, {cycles, [[<<"A">>,<<"B">>]]}}}};
+deps(circular2) ->
+    {[{"B", [{"C", [{"B", []}]}]},
+      {"C", []}],
+     [],
+     {error, {rebar_prv_install_deps, {cycles, [[<<"B">>,<<"C">>]]}}}};
+deps(circular_skip) ->
+    %% Never spot the circular dep due to being to low in the deps tree
+    %% in source deps
+    {[{"B", [{"C", "2", [{"B", []}]}]},
+      {"C", "1", [{"D",[]}]}],
+     [{"C","2"}],
+     {ok, ["B", {"C","1"}, "D"]}}.
+
+expand_deps(_, []) -> [];
+expand_deps(git, [{Name, Deps} | Rest]) ->
+    Dep = {Name, ".*", {git, "https://example.org/user/"++Name++".git", "master"}},
+    [{Dep, expand_deps(git, Deps)} | expand_deps(git, Rest)];
+expand_deps(git, [{Name, Vsn, Deps} | Rest]) ->
+    Dep = {Name, Vsn, {git, "https://example.org/user/"++Name++".git", {tag, Vsn}}},
+    [{Dep, expand_deps(git, Deps)} | expand_deps(git, Rest)];
+expand_deps(pkg, [{Name, Deps} | Rest]) ->
+    Dep = {pkg, Name, "0.0.0"},
+    [{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)];
+expand_deps(pkg, [{Name, Vsn, Deps} | Rest]) ->
+    Dep = {pkg, Name, Vsn},
+    [{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)].
+
+setup_project(Case, Config0, Deps) ->
+    DepsType = ?config(deps_type, Config0),
     Config = rebar_test_utils:init_rebar_state(
             Config0,
-            atom_to_list(Case)++"_"
+            atom_to_list(Case)++"_"++atom_to_list(DepsType)++"_"
     ),
     AppDir = ?config(apps, Config),
     rebar_test_utils:create_app(AppDir, "A", "0.0.0", [kernel, stdlib]),
-    TopDeps = proplists:get_value(deps, config_content()),
-    StringDeps = [erlang:setelement(1, Dep, atom_to_list(element(1,Dep)))
-                  || Dep <- TopDeps],
+    TopDeps = top_level_deps(Deps),
     RebarConf = rebar_test_utils:create_config(AppDir, [{deps, TopDeps}]),
-    mock_git_resource:mock([{deps, lists:filter(fun src_dep/1, StringDeps)}]),
-    mock_pkg_resource:mock([{pkgdeps,
-      [{{ec_cnv:to_binary(N),
-         ec_cnv:to_binary(V)},[]}
-       || {N,V} <- lists:filter(fun pkg_dep/1, StringDeps)]}]),
+    case DepsType of
+        git ->
+            mock_git_resource:mock([{deps, flat_deps(Deps)}]);
+        pkg ->
+            mock_pkg_resource:mock([{pkgdeps, flat_pkgdeps(Deps)}])
+    end,
     [{rebarconfig, RebarConf} | Config].
 
-src_dep(Dep) ->
-    case element(1, Dep) of
-        "src_"++_ -> true;
-        _ -> false
-    end.
 
-pkg_dep(Dep) ->
-    case element(1, Dep) of
-        "pkg_"++_ -> true;
-        _ -> false
-    end.
+flat_deps([]) -> [];
+flat_deps([{{Name,_Vsn,Ref}, Deps} | Rest]) ->
+    [{{Name,vsn_from_ref(Ref)}, top_level_deps(Deps)}]
+    ++
+    flat_deps(Deps)
+    ++
+    flat_deps(Rest).
 
-default_nodep(Config) ->
-    {ok, RebarConfig} = file:consult(?config(rebarconfig, Config)),
-    rebar_test_utils:run_and_check(
-        Config, RebarConfig, ["deps"], {ok, []}
-    ),
-    History = meck:history(io),
-    Strings = [io_lib:format(Str, Args) || {_, {io, format, [Str, Args]}, _} <- History],
-    {match, _} = re:run(Strings, "src_a\\* \\(git source\\)"),
-    {match, _} = re:run(Strings, "src_b\\* \\(git source\\)"),
-    {match, _} = re:run(Strings, "pkg_a\\* \\(package 1.0.0\\)").
+vsn_from_ref({git, _, {_, Vsn}}) -> Vsn;
+vsn_from_ref({git, _, Vsn}) -> Vsn.
 
-default_lock(Config) ->
+flat_pkgdeps([]) -> [];
+flat_pkgdeps([{{pkg, Name, Vsn}, Deps} | Rest]) ->
+    [{{iolist_to_binary(Name),iolist_to_binary(Vsn)}, top_level_deps(Deps)}]
+    ++
+    flat_pkgdeps(Deps)
+    ++
+    flat_pkgdeps(Rest).
+
+top_level_deps([]) -> [];
+top_level_deps([{{pkg, Name, Vsn}, _} | Deps]) ->
+    [{list_to_atom(Name), Vsn} | top_level_deps(Deps)];
+top_level_deps([{{Name, Vsn, Ref}, _} | Deps]) ->
+    [{list_to_atom(Name), Vsn, Ref} | top_level_deps(Deps)].
+
+app_vsn([]) -> [];
+app_vsn([{Source, Deps} | Rest]) ->
+    {Name, Vsn} = case Source of
+        {pkg, N, V} -> {N,V};
+        {N,V,_Ref} -> {N,V}
+    end,
+    [{Name, Vsn}] ++ app_vsn(Deps) ++ app_vsn(Rest).
+
+mock_warnings() ->
+    %% just let it do its thing, we check warnings through
+    %% the call log.
+    meck:new(rebar_log, [no_link, passthrough]).
+
+%%% TESTS %%%
+flat(Config) -> run(Config).
+pick_highest_left(Config) -> run(Config).
+pick_highest_right(Config) -> run(Config).
+pick_smallest1(Config) -> run(Config).
+pick_smallest2(Config) -> run(Config).
+circular1(Config) -> run(Config).
+circular2(Config) -> run(Config).
+circular_skip(Config) -> run(Config).
+
+run(Config) ->
     {ok, RebarConfig} = file:consult(?config(rebarconfig, Config)),
     rebar_test_utils:run_and_check(
-        Config, RebarConfig, ["lock"], {ok, []}
+        Config, RebarConfig, ["install_deps"], ?config(expect, Config)
     ),
-    rebar_test_utils:run_and_check(
-        Config, RebarConfig, ["deps"], {ok, []}
-    ),
-    History = meck:history(io),
-    Strings = [io_lib:format(Str, Args) || {_, {io, format, [Str, Args]}, _} <- History],
-    {match, _} = re:run(Strings, "src_a\\ \\(locked git source\\)"),
-    {match, _} = re:run(Strings, "src_b\\ \\(locked git source\\)"),
-    {match, _} = re:run(Strings, "pkg_a\\ \\(locked package 1.0.0\\)").
+    check_warnings(warning_calls(), ?config(warnings, Config), ?config(deps_type, Config)).
+
+warning_calls() ->
+    History = meck:history(rebar_log),
+    [{Str, Args} || {_, {rebar_log, log, [warn, Str, Args]}, _} <- History].
+
+check_warnings(_, [], _) ->
+    ok;
+check_warnings(Warns, [{Name, Vsn} | Rest], Type) ->
+    ct:pal("Checking for warning ~p in ~p", [{Name,Vsn},Warns]),
+    ?assert(in_warnings(Type, Warns, Name, Vsn)),
+    check_warnings(Warns, Rest, Type).
+
+in_warnings(git, Warns, NameRaw, VsnRaw) ->
+    Name = iolist_to_binary(NameRaw),
+    1 =< length([1 || {_, [AppName, {git, _, {_, Vsn}}]} <- Warns,
+                      AppName =:= Name, Vsn =:= VsnRaw]);
+in_warnings(pkg, Warns, NameRaw, VsnRaw) ->
+    Name = iolist_to_binary(NameRaw),
+    Vsn = iolist_to_binary(VsnRaw),
+    1 =< length([1 || {_, [AppName, AppVsn]} <- Warns,
+                      AppName =:= Name, AppVsn =:= Vsn]).
diff --git a/test/rebar_upgrade_SUITE.erl b/test/rebar_upgrade_SUITE.erl
index 39b9687..b4daef9 100644
--- a/test/rebar_upgrade_SUITE.erl
+++ b/test/rebar_upgrade_SUITE.erl
@@ -359,7 +359,7 @@ upgrades(delete_d) ->
 top_level_deps([]) -> [];
 top_level_deps([{{Name, Vsn, Ref}, _} | Deps]) ->
     [{list_to_atom(Name), Vsn, Ref} | top_level_deps(Deps)];
-top_level_deps([{{pkg, Name, Vsn, _URL}, _} | Deps]) ->
+top_level_deps([{{pkg, Name, Vsn}, _} | Deps]) ->
     [{list_to_atom(Name), Vsn} | top_level_deps(Deps)].
 
 mock_deps(git, Deps, Upgrades) ->
@@ -381,7 +381,7 @@ vsn_from_ref({git, _, {_, Vsn}}) -> Vsn;
 vsn_from_ref({git, _, Vsn}) -> Vsn.
 
 flat_pkgdeps([]) -> [];
-flat_pkgdeps([{{pkg, Name, Vsn, _Url}, Deps} | Rest]) ->
+flat_pkgdeps([{{pkg, Name, Vsn}, Deps} | Rest]) ->
     [{{iolist_to_binary(Name),iolist_to_binary(Vsn)}, top_level_deps(Deps)}]
     ++
     flat_pkgdeps(Deps)
@@ -396,10 +396,10 @@ expand_deps(git, [{Name, Vsn, Deps} | Rest]) ->
     Dep = {Name, Vsn, {git, "https://example.org/user/"++Name++".git", {tag, Vsn}}},
     [{Dep, expand_deps(git, Deps)} | expand_deps(git, Rest)];
 expand_deps(pkg, [{Name, Deps} | Rest]) ->
-    Dep = {pkg, Name, "0.0.0", "https://example.org/user/"++Name++".tar.gz"},
+    Dep = {pkg, Name, "0.0.0"},
     [{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)];
 expand_deps(pkg, [{Name, Vsn, Deps} | Rest]) ->
-    Dep = {pkg, Name, Vsn, "https://example.org/user/"++Name++".tar.gz"},
+    Dep = {pkg, Name, Vsn},
     [{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)].
 
 normalize_unlocks({App, Locks}) ->
@@ -472,4 +472,3 @@ run(Config) ->
     rebar_test_utils:run_and_check(
         Config, NewRebarConfig, ["upgrade", App], Expectation
     ).
-