improved dep handling and add package list task

9 files changed, 399 insertions, 296 deletions

diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl
index 72a4a97..6831fea 100644
--- a/src/rebar_app_info.erl
+++ b/src/rebar_app_info.erl
@@ -20,7 +20,7 @@
 
 -export_type([t/0]).
 
--record(app_info_t, {name :: atom(),
+-record(app_info_t, {name :: binary(),
                      app_file_src :: file:name() | undefined,
                      app_file :: file:name(),
                      config :: rebar_config:config() | undefined,
@@ -44,11 +44,10 @@ new() ->
     {ok, #app_info_t{}}.
 
 %% @doc build a complete version of the app info with all fields set.
--spec new(atom(), string(), file:name()) ->
+-spec new(atom() | binary() | string(), string(), file:name()) ->
                  {ok, t()}.
-new(AppName, Vsn, Dir)
-  when erlang:is_atom(AppName) ->
-    {ok, #app_info_t{name=AppName,
+new(AppName, Vsn, Dir) ->
+    {ok, #app_info_t{name=ec_cnv:to_binary(AppName),
                      original_vsn=Vsn,
                      dir=Dir}}.
 
@@ -56,10 +55,9 @@ new(AppName, Vsn, Dir)
 name(#app_info_t{name=Name}) ->
     Name.
 
--spec name(t(), atom()) -> t().
-name(AppInfo=#app_info_t{}, AppName)
-  when erlang:is_atom(AppName) ->
-    AppInfo#app_info_t{name=AppName}.
+-spec name(t(), atom() | binary() | string()) -> t().
+name(AppInfo=#app_info_t{}, AppName) ->
+    AppInfo#app_info_t{name=ec_cnv:to_binary(AppName)}.
 
 -spec config(t()) -> rebar_config:confg().
 config(#app_info_t{config=Config}) ->
diff --git a/src/rebar_base_compiler.erl b/src/rebar_base_compiler.erl
index e8fb26f..446fa89 100644
--- a/src/rebar_base_compiler.erl
+++ b/src/rebar_base_compiler.erl
@@ -135,10 +135,10 @@ compile_each([], _Config, _CompileFn) ->
 compile_each([Source | Rest], Config, CompileFn) ->
     case compile(Source, Config, CompileFn) of
         ok ->
-            ?INFO("~sCompiled ~s\n", [rebar_utils:indent(1), filename:basename(Source)]);
+            ?DEBUG("~sCompiled ~s\n", [rebar_utils:indent(1), filename:basename(Source)]);
         {ok, Warnings} ->
             report(Warnings),
-            ?INFO("~sCompiled ~s\n", [rebar_utils:indent(1), filename:basename(Source)]);
+            ?DEBUG("~sCompiled ~s\n", [rebar_utils:indent(1), filename:basename(Source)]);
         skipped ->
             ?DEBUG("~sSkipped ~s\n", [rebar_utils:indent(1), filename:basename(Source)]);
         Error ->
@@ -173,11 +173,11 @@ compile_queue(Config, Pids, Targets) ->
 
         {compiled, Source, Warnings} ->
             report(Warnings),
-            ?INFO("~sCompiled ~s\n", [rebar_utils:indent(1), filename:basename(Source)]),
+            ?DEBUG("~sCompiled ~s\n", [rebar_utils:indent(1), filename:basename(Source)]),
             compile_queue(Config, Pids, Targets);
 
         {compiled, Source} ->
-            ?INFO("~sCompiled ~s\n", [rebar_utils:indent(1), filename:basename(Source)]),
+            ?DEBUG("~sCompiled ~s\n", [rebar_utils:indent(1), filename:basename(Source)]),
             compile_queue(Config, Pids, Targets);
         {skipped, Source} ->
             ?DEBUG("~sSkipped ~s~n", [rebar_utils:indent(1), filename:basename(Source)]),
diff --git a/src/rebar_core.erl b/src/rebar_core.erl
index 675b272..8732074 100644
--- a/src/rebar_core.erl
+++ b/src/rebar_core.erl
@@ -39,7 +39,7 @@ process_command(State, Command) ->
     LibDirs = rebar_state:get(State, lib_dirs, ?DEFAULT_LIB_DIRS),
     DepsDir = rebar_state:get(State, deps_dir, ?DEFAULT_DEPS_DIRS),
     _UpdatedCodePaths = update_code_path([DepsDir | LibDirs]),
-    rebar_prv_deps:setup_env(State),
+    rebar_prv_install_deps:setup_env(State),
 
     TargetProviders = rebar_provider:get_target_providers(Command, State),
 
diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl
index 024e032..09e52a9 100644
--- a/src/rebar_fetch.erl
+++ b/src/rebar_fetch.erl
@@ -19,13 +19,6 @@
 
 -include("rebar.hrl").
 
--type dep_name() :: atom().
--type dep_vsn() :: ec_semver:any_version().
--type dep_source() :: {atom(), string(), ref()}.
--type ref() :: string() | {atom(), string()}.
-
--type dep() :: { dep_name(), dep_vsn(), dep_source() }.
-
 -record(p4_settings, {
           client=undefined,
           transport="tcp4:perforce:1666",
@@ -50,12 +43,18 @@ init_p4_settings(Basename) ->
 current_ref(AppDir, {git, _, _}) ->
     string:strip(os:cmd("git --git-dir='" ++ AppDir ++ "/.git' rev-parse --verify HEAD"), both, $\n).
 
-download_source(AppDir, {p4, Url}) ->
-    download_source(AppDir, {p4, Url, "#head"});
-download_source(AppDir, {p4, Url, Rev}) ->
-    download_source(AppDir, {p4, Url, Rev, init_p4_settings(filename:basename(AppDir))});
-download_source(AppDir, {p4, Url, _Rev, Settings}) ->
-    ok = filelib:ensure_dir(AppDir),
+download_source(AppDir, Source) ->
+    ec_file:mkdir_p(AppDir),
+    TmpDir = ec_file:insecure_mkdtemp(),
+    download_source_tmp(TmpDir, Source),
+    ok = ec_file:copy(TmpDir, binary_to_list(filename:absname(AppDir)), [recursive]).
+
+download_source_tmp(TmpDir, {p4, Url}) ->
+    download_source_tmp(TmpDir, {p4, Url, "#head"});
+download_source_tmp(TmpDir, {p4, Url, Rev}) ->
+    download_source_tmp(TmpDir, {p4, Url, Rev, init_p4_settings(filename:basename(TmpDir))});
+download_source_tmp(TmpDir, {p4, Url, _Rev, Settings}) ->
+    ok = filelib:ensure_dir(TmpDir),
     rebar_utils:sh_send("p4 client -i",
                    ?FMT("Client: ~s~n"
                         ++"Description: generated by Rebar~n"
@@ -63,68 +62,67 @@ download_source(AppDir, {p4, Url, _Rev, Settings}) ->
                         ++"View:~n"
                         ++"  ~s/...  //~s/...~n",
                         [Settings#p4_settings.client,
-                         AppDir,
+                         TmpDir,
                          Url,
                          Settings#p4_settings.client]),
                        []),
     rebar_utils:sh(?FMT("p4 -c ~s sync -f", [Settings#p4_settings.client]), []);
-download_source(AppDir, {hg, Url, Rev}) ->
-    ok = filelib:ensure_dir(AppDir),
-    rebar_utils:sh(?FMT("hg clone -U ~s ~s", [Url, filename:basename(AppDir)]),
-                   [{cd, filename:dirname(AppDir)}]),
-    rebar_utils:sh(?FMT("hg update ~s", [Rev]), [{cd, AppDir}]);
-download_source(AppDir, {git, Url}) ->
-    download_source(AppDir, {git, Url, {branch, "HEAD"}});
-download_source(AppDir, {git, Url, ""}) ->
-    download_source(AppDir, {git, Url, {branch, "HEAD"}});
-download_source(AppDir, {git, Url, {branch, Branch}}) ->
-    ok = filelib:ensure_dir(AppDir),
-    rebar_utils:sh(?FMT("git clone -n ~s ~s", [Url, filename:basename(AppDir)]),
-                   [{cd, filename:dirname(AppDir)}]),
-    rebar_utils:sh(?FMT("git checkout -q origin/~s", [Branch]), [{cd, AppDir}]);
-download_source(AppDir, {git, Url, {tag, Tag}}) ->
-    ok = filelib:ensure_dir(AppDir),
-    rebar_utils:sh(?FMT("git clone -n ~s ~s", [Url, filename:basename(AppDir)]),
-                   [{cd, filename:dirname(AppDir)}]),
-    rebar_utils:sh(?FMT("git checkout -q ~s", [Tag]), [{cd, AppDir}]);
-download_source(AppDir, {git, Url, Rev}) ->
-    ok = filelib:ensure_dir(AppDir),
-    rebar_utils:sh(?FMT("git clone -n ~s ~s", [Url, filename:basename(AppDir)]),
-                   [{cd, filename:dirname(AppDir)}]),
-    rebar_utils:sh(?FMT("git checkout -q ~s", [Rev]), [{cd, AppDir}]);
-download_source(AppDir, {bzr, Url, Rev}) ->
-    ok = filelib:ensure_dir(AppDir),
+download_source_tmp(TmpDir, {hg, Url, Rev}) ->
+    ok = filelib:ensure_dir(TmpDir),
+    rebar_utils:sh(?FMT("hg clone -U ~s ~s", [Url, filename:basename(TmpDir)]),
+                   [{cd, filename:dirname(TmpDir)}]),
+    rebar_utils:sh(?FMT("hg update ~s", [Rev]), [{cd, TmpDir}]);
+download_source_tmp(TmpDir, {git, Url}) ->
+    download_source_tmp(TmpDir, {git, Url, {branch, "HEAD"}});
+download_source_tmp(TmpDir, {git, Url, ""}) ->
+    download_source_tmp(TmpDir, {git, Url, {branch, "HEAD"}});
+download_source_tmp(TmpDir, {git, Url, {branch, Branch}}) ->
+    ok = filelib:ensure_dir(TmpDir),
+    rebar_utils:sh(?FMT("git clone -n ~s ~s", [Url, filename:basename(TmpDir)]),
+                   [{cd, filename:dirname(TmpDir)}]),
+    rebar_utils:sh(?FMT("git checkout -q origin/~s", [Branch]), [{cd, TmpDir}]);
+download_source_tmp(TmpDir, {git, Url, {tag, Tag}}) ->
+    ok = filelib:ensure_dir(TmpDir),
+    rebar_utils:sh(?FMT("git clone -n ~s ~s", [Url, filename:basename(TmpDir)]),
+                   [{cd, filename:dirname(TmpDir)}]),
+    rebar_utils:sh(?FMT("git checkout -q ~s", [Tag]), [{cd, TmpDir}]);
+download_source_tmp(TmpDir, {git, Url, Rev}) ->
+    ok = filelib:ensure_dir(TmpDir),
+    rebar_utils:sh(?FMT("git clone -n ~s ~s", [Url, filename:basename(TmpDir)]),
+                   [{cd, filename:dirname(TmpDir)}]),
+    rebar_utils:sh(?FMT("git checkout -q ~s", [Rev]), [{cd, TmpDir}]);
+download_source_tmp(TmpDir, {bzr, Url, Rev}) ->
+    ok = filelib:ensure_dir(TmpDir),
     rebar_utils:sh(?FMT("bzr branch -r ~s ~s ~s",
-                        [Rev, Url, filename:basename(AppDir)]),
-                   [{cd, filename:dirname(AppDir)}]);
-download_source(AppDir, {svn, Url, Rev}) ->
-    ok = filelib:ensure_dir(AppDir),
+                        [Rev, Url, filename:basename(TmpDir)]),
+                   [{cd, filename:dirname(TmpDir)}]);
+download_source_tmp(TmpDir, {svn, Url, Rev}) ->
+    ok = filelib:ensure_dir(TmpDir),
     rebar_utils:sh(?FMT("svn checkout -r ~s ~s ~s",
-                        [Rev, Url, filename:basename(AppDir)]),
-                   [{cd, filename:dirname(AppDir)}]);
-download_source(AppDir, {rsync, Url}) ->
-    ok = filelib:ensure_dir(AppDir),
-    rebar_utils:sh(?FMT("rsync -az --delete ~s/ ~s", [Url, AppDir]), []);
-download_source(AppDir, {fossil, Url}) ->
-    download_source(AppDir, {fossil, Url, ""});
-download_source(AppDir, {fossil, Url, Version}) ->
-    Repository = filename:join(AppDir, filename:basename(AppDir) ++ ".fossil"),
+                        [Rev, Url, filename:basename(TmpDir)]),
+                   [{cd, filename:dirname(TmpDir)}]);
+download_source_tmp(TmpDir, {rsync, Url}) ->
+    ok = filelib:ensure_dir(TmpDir),
+    rebar_utils:sh(?FMT("rsync -az --delete ~s/ ~s", [Url, TmpDir]), []);
+download_source_tmp(TmpDir, {fossil, Url}) ->
+    download_source_tmp(TmpDir, {fossil, Url, ""});
+download_source_tmp(TmpDir, {fossil, Url, Version}) ->
+    Repository = filename:join(TmpDir, filename:basename(TmpDir) ++ ".fossil"),
     ok = filelib:ensure_dir(Repository),
-    ok = file:set_cwd(AppDir),
+    ok = file:set_cwd(TmpDir),
     rebar_utils:sh(?FMT("fossil clone ~s ~s", [Url, Repository]),
-                   [{cd, AppDir}]),
+                   [{cd, TmpDir}]),
     rebar_utils:sh(?FMT("fossil open ~s ~s --nested", [Repository, Version]),
                    []);
-download_source(AppDir, {AppName, AppVersion, Url}) when is_binary(AppName)
-                                                       , is_binary(AppVersion) ->
-    TmpDir = ec_file:insecure_mkdtemp(),
+download_source_tmp(TmpDir, {AppName, AppVersion, Url}) when is_binary(AppName)
+                                                           , is_binary(AppVersion) ->
     TmpFile = binary_to_list(filename:join(TmpDir, <<AppName/binary, "-", AppVersion/binary>>)),
     {ok, saved_to_file} = httpc:request(get, {binary_to_list(Url), []}, [], [{stream, TmpFile}]),
-    ok = erl_tar:extract(TmpFile, [{cwd, filename:dirname(AppDir)}, compressed]),
+    ok = erl_tar:extract(TmpFile, [{cwd, filename:dirname(TmpDir)}, compressed]),
     ok.
 
 update_source1(AppDir, Args) when element(1, Args) =:= p4 ->
-    download_source(AppDir, Args);
+    download_source_tmp(AppDir, Args);
 update_source1(AppDir, {git, Url}) ->
     update_source1(AppDir, {git, Url, {branch, "HEAD"}});
 update_source1(AppDir, {git, Url, ""}) ->
diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl
new file mode 100644
index 0000000..103d3a3
--- /dev/null
+++ b/src/rebar_packages.erl
@@ -0,0 +1,23 @@
+-module(rebar_packages).
+
+-export([get_packages/1]).
+
+-include("rebar.hrl").
+
+-spec get_packages(rebar_state:t()) -> {list(), rlx_depsolver:t()}.
+get_packages(State) ->
+    RebarDir = rebar_state:get(State, global_rebar_dir, filename:join(os:getenv("HOME"), ".rebar")),
+    PackagesFile = filename:join(RebarDir, "packages"),
+    case ec_file:exists(PackagesFile) of
+        true ->
+            try
+                {ok, Binary} = file:read_file(PackagesFile),
+                binary_to_term(Binary)
+            catch
+                _:_ ->
+                    ?ERROR("Bad packages index, try to fix with `rebar update`~n", []),
+                    {[], rlx_depsolver:new()}
+            end;
+        false ->
+            {[], rlx_depsolver:new()}
+    end.
diff --git a/src/rebar_prv_app_builder.erl b/src/rebar_prv_app_builder.erl
index f9ba1c4..688345a 100644
--- a/src/rebar_prv_app_builder.erl
+++ b/src/rebar_prv_app_builder.erl
@@ -9,7 +9,7 @@
 -include("rebar.hrl").
 
 -define(PROVIDER, compile).
--define(DEPS, [deps]).
+-define(DEPS, [install_deps]).
 
 %% ===================================================================
 %% Public API
@@ -32,7 +32,7 @@ do(State) ->
     Apps = rebar_state:apps_to_build(State),
 
     lists:foreach(fun(AppInfo) ->
-                          ?INFO("Compiling ~p ~s~n", [rebar_app_info:name(AppInfo)
+                          ?INFO("Compiling ~s ~s~n", [rebar_app_info:name(AppInfo)
                                                      ,rebar_app_info:original_vsn(AppInfo)]),
                           _AppInfo1 = build(State, AppInfo)
                   end, Apps),
diff --git a/src/rebar_prv_deps.erl b/src/rebar_prv_deps.erl
index 0e5cb31..834e7dc 100644
--- a/src/rebar_prv_deps.erl
+++ b/src/rebar_prv_deps.erl
@@ -1,29 +1,3 @@
-%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
-%% ex: ts=4 sw=4 et
-%% -------------------------------------------------------------------
-%%
-%% rebar: Erlang Build Tools
-%%
-%% Copyright (c) 2009 Dave Smith (dizzyd@dizzyd.com)
-%%
-%% Permission is hereby granted, free of charge, to any person obtaining a copy
-%% of this software and associated documentation files (the "Software"), to deal
-%% in the Software without restriction, including without limitation the rights
-%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-%% copies of the Software, and to permit persons to whom the Software is
-%% furnished to do so, subject to the following conditions:
-%%
-%% The above copyright notice and this permission notice shall be included in
-%% all copies or substantial portions of the Software.
-%%
-%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-%% THE SOFTWARE.
-%% -------------------------------------------------------------------
 -module(rebar_prv_deps).
 
 -behaviour(rebar_provider).
@@ -33,22 +7,8 @@
 
 -include("rebar.hrl").
 
--export([setup_env/1]).
-
-%% for internal use only
--export([get_deps_dir/1]).
--export([get_deps_dir/2]).
-
 -define(PROVIDER, deps).
--define(DEPS, [app_discovery]).
-
--record(dep, {name :: binary(),
-              vsn :: binary(),
-              source :: binary()}).
-
-%% ===================================================================
-%% Public API
-%% ===================================================================
+-define(DEPS, []).
 
 -spec init(rebar_state:t()) -> {ok, rebar_state:t()}.
 init(State) ->
@@ -57,190 +17,14 @@ init(State) ->
                                                        bare = false,
                                                        deps = ?DEPS,
                                                        example = "rebar deps",
-                                                       short_desc = "Install dependencies",
-                                                       desc = info("Install dependencies"),
+                                                       short_desc = "List dependencies",
+                                                       desc = info("List dependencies"),
                                                        opts = []}),
     {ok, State1}.
 
 -spec do(rebar_state:t()) -> {ok, rebar_state:t()}.
 do(State) ->
-    %% Read in package index and dep graph
-    {Packages, Graph} = get_packages(State),
-
-    case rebar_state:get(State, deps, []) of
-        [] ->
-            {ok, State};
-        Deps ->
-            %% Split source deps form binary deps, needed to keep backwards compatibility
-            case parse_deps(Deps) of
-                {SrcDeps, []} ->
-                    {State1, SrcDeps1} = update_src_deps(State, SrcDeps),
-                    {ok, rebar_state:set(State1, deps, SrcDeps1)};
-                {SrcDeps, Goals} ->
-                    {State1, _SrcDeps1} = update_src_deps(State, SrcDeps),
-                    {ok, Solved} = rlx_depsolver:solve(Graph, Goals),
-                    M = lists:map(fun({Name, Vsn}) ->
-                                          FmtVsn = to_binary(rlx_depsolver:format_version(Vsn)),
-                                          {ok, P} = dict:find({Name, FmtVsn}, Packages),
-                                          Link = proplists:get_value(<<"link">>, P),
-                                          {Name, Vsn, {Name
-                                                      ,FmtVsn
-                                                      ,Link}}
-                                  end, Solved),
-
-                    {State2, Deps1} = update_deps(State1, M),
-                    State3 = rebar_state:set(State2, deps, Deps1),
-                    {ok, rebar_state:set(State3, goals, Goals)}
-            end
-    end.
-
-update_deps(State, Deps) ->
-    DepsDir = get_deps_dir(State),
-
-    %% Find available apps to fulfill dependencies
-    %% Should only have to do this once, not every iteration
-    UnbuiltApps = rebar_app_discover:find_unbuilt_apps([DepsDir]),
-    FoundApps = rebar_app_discover:find_apps([DepsDir]),
-
-    download_missing_deps(State, DepsDir, FoundApps, UnbuiltApps, Deps).
-
-update_src_deps(State, Deps) ->
-    DepsDir = get_deps_dir(State),
-
-    %% Find available apps to fulfill dependencies
-    %% Should only have to do this once, not every iteration
-    UnbuiltApps = rebar_app_discover:find_unbuilt_apps([DepsDir]),
-    FoundApps = rebar_app_discover:find_apps([DepsDir]),
-
-    %% Resolve deps and their dependencies
-    Deps1 = handle_src_deps(Deps, UnbuiltApps++FoundApps),
-    {State1, Missing} = download_missing_deps(State, DepsDir, FoundApps, UnbuiltApps, Deps1),
-    case dict:is_empty(Missing) of
-        true ->
-            {State1, Deps1};
-        false ->
-            update_src_deps(State1, Missing)
-    end.
-
-handle_src_deps(Deps, Found) ->
-    lists:foldl(fun(X, DepsAcc) ->
-                        C = rebar_config:consult(rebar_app_info:dir(X)),
-                        S = rebar_state:new(rebar_state:new(), C, rebar_app_info:dir(X)),
-                        {ParsedDeps, _Goals} = parse_deps(rebar_state:get(S, deps, [])),
-                        dict:merge(fun(_K, V1, _V2) -> V1 end, DepsAcc, ParsedDeps)
-                end, Deps, Found).
-
-download_missing_deps(State, DepsDir, Found, Unbuilt, Deps) ->
-    Missing =
-        dict:filter(fun(Key, _) ->
-                            not lists:any(fun(F) ->
-                                                  Key =:= to_binary(rebar_app_info:name(F))
-                                          end, Found++Unbuilt)
-                    end, Deps),
-    dict:map(fun(_Key, #dep{name=Name, source=Source}) ->
-                     TargetDir = get_deps_dir(DepsDir, Name),
-                     case filelib:is_dir(TargetDir) of
-                         true ->
-                             ok;
-                         false ->
-                             ?INFO("Fetching ~s ~s~n", [Name
-                                                       ,element(2, Source)]),
-                             rebar_fetch:download_source(TargetDir, Source),
-                             case rebar_app_discover:find_unbuilt_apps([TargetDir]) of
-                                 [AppSrc] ->
-                                     C = rebar_config:consult(rebar_app_info:dir(AppSrc)),
-                                     S = rebar_state:new(rebar_state:new()
-                                                        ,C
-                                                        ,rebar_app_info:dir(AppSrc)),
-                                     rebar_prv_app_builder:build(S, AppSrc);
-                                 [] ->
-                                     []
-                             end
-                     end
-             end, Missing),
-
-    {State, Missing}.
-
-%% set REBAR_DEPS_DIR and ERL_LIBS environment variables
-setup_env(State) ->
-    DepsDir = get_deps_dir(State),
-    %% include rebar's DepsDir in ERL_LIBS
-    Separator = case os:type() of
-                    {win32, nt} ->
-                        ";";
-                    _ ->
-                        ":"
-                end,
-    ERL_LIBS = case os:getenv("ERL_LIBS") of
-                   false ->
-                       {"ERL_LIBS", DepsDir};
-                   PrevValue ->
-                       {"ERL_LIBS", DepsDir ++ Separator ++ PrevValue}
-               end,
-    [{"REBAR_DEPS_DIR", DepsDir}, ERL_LIBS].
-
-
-get_deps_dir(State) ->
-    BaseDir = rebar_state:get(State, base_dir, ""),
-    get_deps_dir(BaseDir, "deps").
-
-get_deps_dir(DepsDir, App) ->
-    filename:join(DepsDir, App).
-
-%% ===================================================================
-%% Internal functions
-%% ===================================================================
-
-new({Name, Vsn, Source})->
-    #dep{name=to_binary(Name), vsn=to_binary(Vsn), source=Source};
-new(Name) ->
-    #dep{name=to_binary(Name)}.
-
--spec name(record(dep)) -> binary().
-name(#dep{name=Name}) ->
-    Name.
-
--spec vsn(record(dep)) -> binary().
-vsn(#dep{vsn=Vsn}) ->
-    Vsn.
-
--spec source(record(dep)) -> tuple().
-source(#dep{source=Source}) ->
-    Source.
-
-to_binary(X) when is_binary(X) ->
-    X;
-to_binary(X) when is_atom(X) ->
-    atom_to_binary(X, utf8);
-to_binary(X) when is_list(X) ->
-    iolist_to_binary(X).
-
-parse_deps(Deps) ->
-    lists:foldl(fun({Name, Vsn}, {SrcDepsAcc, GoalsAcc}) ->
-                        {SrcDepsAcc, [{to_binary(Name), to_binary(Vsn)} | GoalsAcc]};
-                   (Name, {SrcDepsAcc, GoalsAcc}) when is_atom(Name) ->
-                        {SrcDepsAcc, [to_binary(Name) | GoalsAcc]};
-                   (SrcDep, {SrcDepsAcc, GoalsAcc}) ->
-                        Dep = new(SrcDep),
-                        {dict:store(name(Dep), Dep, SrcDepsAcc), GoalsAcc}
-                end, {dict:new(), []}, Deps).
-
-get_packages(State) ->
-    RebarDir = rebar_state:get(State, global_rebar_dir, filename:join(os:getenv("HOME"), ".rebar")),
-    PackagesFile = filename:join(RebarDir, "packages"),
-    case ec_file:exists(PackagesFile) of
-        true ->
-            try
-                {ok, Binary} = file:read_file(PackagesFile),
-                binary_to_term(Binary)
-            catch
-                _:_ ->
-                    ?ERROR("Bad packages index, try to fix with `rebar update`~n", []),
-                    {[], rlx_depsolver:new()}
-            end;
-        false ->
-            {[], rlx_depsolver:new()}
-    end.
+    {ok, State}.
 
 info(Description) ->
     io_lib:format("~s.~n"
diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl
new file mode 100644
index 0000000..245dcb2
--- /dev/null
+++ b/src/rebar_prv_install_deps.erl
@@ -0,0 +1,242 @@
+%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 et
+%% -------------------------------------------------------------------
+%%
+%% rebar: Erlang Build Tools
+%%
+%% Copyright (c) 2009 Dave Smith (dizzyd@dizzyd.com)
+%%
+%% Permission is hereby granted, free of charge, to any person obtaining a copy
+%% of this software and associated documentation files (the "Software"), to deal
+%% in the Software without restriction, including without limitation the rights
+%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+%% copies of the Software, and to permit persons to whom the Software is
+%% furnished to do so, subject to the following conditions:
+%%
+%% The above copyright notice and this permission notice shall be included in
+%% all copies or substantial portions of the Software.
+%%
+%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+%% THE SOFTWARE.
+%% -------------------------------------------------------------------
+-module(rebar_prv_install_deps).
+
+-behaviour(rebar_provider).
+
+-export([init/1,
+         do/1]).
+
+-include("rebar.hrl").
+
+-export([setup_env/1]).
+
+%% for internal use only
+-export([get_deps_dir/1]).
+-export([get_deps_dir/2]).
+
+-define(PROVIDER, install_deps).
+-define(DEPS, [app_discovery]).
+
+-record(dep, {name :: binary(),
+              vsn :: binary(),
+              source :: binary()}).
+
+%% ===================================================================
+%% Public API
+%% ===================================================================
+
+-spec init(rebar_state:t()) -> {ok, rebar_state:t()}.
+init(State) ->
+    State1 = rebar_state:add_provider(State, #provider{name = ?PROVIDER,
+                                                       provider_impl = ?MODULE,
+                                                       bare = false,
+                                                       deps = ?DEPS,
+                                                       example = "rebar deps",
+                                                       short_desc = "Install dependencies",
+                                                       desc = info("Install dependencies"),
+                                                       opts = []}),
+    {ok, State1}.
+
+-spec do(rebar_state:t()) -> {ok, rebar_state:t()}.
+do(State) ->
+    %% Read in package index and dep graph
+    {Packages, Graph} = rebar_packages:get_packages(State),
+
+    case rebar_state:get(State, deps, []) of
+        [] ->
+            {ok, State};
+        Deps ->
+            %% Split source deps form binary deps, needed to keep backwards compatibility
+            {SrcDeps, Goals} = parse_deps(Deps),
+            case update_src_deps(State, SrcDeps, Goals) of
+                {State1, SrcDeps1, []} ->
+                    {ok, rebar_state:set(State1, deps, SrcDeps1)};
+                {State1, SrcDeps1, Goals1} ->
+                    {ok, Solved} = rlx_depsolver:solve(Graph, Goals1),
+                    M = lists:map(fun({Name, Vsn}) ->
+                                          FmtVsn = ec_cnv:to_binary(rlx_depsolver:format_version(Vsn)),
+                                          {ok, P} = dict:find({Name, FmtVsn}, Packages),
+                                          Link = proplists:get_value(<<"link">>, P),
+                                          #dep{name=Name,
+                                              vsn=FmtVsn,
+                                              source={Name
+                                                     ,FmtVsn
+                                                     ,Link}}
+                                  end, Solved),
+                    {State2, Deps1} = update_deps(State1, M),
+                    State3 = rebar_state:set(State2, deps, SrcDeps1++Deps1),
+                    {ok, rebar_state:set(State3, goals, Goals1)}
+            end
+    end.
+
+%% set REBAR_DEPS_DIR and ERL_LIBS environment variables
+setup_env(State) ->
+    DepsDir = get_deps_dir(State),
+    %% include rebar's DepsDir in ERL_LIBS
+    Separator = case os:type() of
+                    {win32, nt} ->
+                        ";";
+                    _ ->
+                        ":"
+                end,
+    ERL_LIBS = case os:getenv("ERL_LIBS") of
+                   false ->
+                       {"ERL_LIBS", DepsDir};
+                   PrevValue ->
+                       {"ERL_LIBS", DepsDir ++ Separator ++ PrevValue}
+               end,
+    [{"REBAR_DEPS_DIR", DepsDir}, ERL_LIBS].
+
+
+get_deps_dir(State) ->
+    BaseDir = rebar_state:get(State, base_dir, ""),
+    get_deps_dir(BaseDir, "deps").
+
+get_deps_dir(DepsDir, App) ->
+    filename:join(DepsDir, App).
+
+%% ===================================================================
+%% Internal functions
+%% ===================================================================
+
+new({Name, Vsn, Source})->
+    #dep{name=ec_cnv:to_binary(Name), vsn=ec_cnv:to_binary(Vsn), source=Source};
+new(Name) ->
+    #dep{name=ec_cnv:to_binary(Name)}.
+
+%% Fetch missing binary deps
+update_deps(State, Deps) ->
+    DepsDir = get_deps_dir(State),
+
+    %% Find available apps to fulfill dependencies
+    %% Should only have to do this once, not every iteration
+    UnbuiltApps = rebar_app_discover:find_unbuilt_apps([DepsDir]),
+    FoundApps = rebar_app_discover:find_apps([DepsDir]),
+
+    download_missing_deps(State, DepsDir, FoundApps, UnbuiltApps, Deps).
+
+
+%% Find source deps to build and download
+update_src_deps(State, Deps, Goals) ->
+    DepsDir = get_deps_dir(State),
+
+    %% Find available apps to fulfill dependencies
+    %% Should only have to do this once, not every iteration
+    UnbuiltApps = rebar_app_discover:find_unbuilt_apps([DepsDir]),
+    FoundApps = rebar_app_discover:find_apps([DepsDir]),
+
+    %% Resolve deps and their dependencies
+    {Deps1, NewGoals} = handle_src_deps(Deps, UnbuiltApps++FoundApps, Goals),
+    case download_missing_deps(State, DepsDir, FoundApps, UnbuiltApps, Deps1) of
+        {State1, []} ->
+            {State1, Deps1, NewGoals};
+        {State1, Missing} ->
+            update_src_deps(State1, Missing, NewGoals)
+    end.
+
+%% Collect deps of new deps
+handle_src_deps(Deps, Found, Goals) ->
+    lists:foldl(fun(X, {DepsAcc, GoalsAcc}) ->
+                        C = rebar_config:consult(rebar_app_info:dir(X)),
+                        S = rebar_state:new(rebar_state:new(), C, rebar_app_info:dir(X)),
+                        {ParsedDeps, NewGoals} = parse_deps(rebar_state:get(S, deps, [])),
+                        %lists:keymerge(2, DepsAcc, ParsedDeps)
+                        {ParsedDeps++DepsAcc, NewGoals++GoalsAcc}
+                end, {Deps, Goals}, Found).
+
+%% Fetch missing deps from source
+download_missing_deps(State, DepsDir, Found, Unbuilt, Deps) ->
+    Missing =
+        lists:filter(fun(#dep{name=Name}) ->
+                            not lists:any(fun(F) ->
+                                                  Name =:= rebar_app_info:name(F)
+                                          end, Found++Unbuilt)
+                    end, Deps),
+    lists:foreach(fun(#dep{name=Name, source=Source}) ->
+                          TargetDir = get_deps_dir(DepsDir, Name),
+                          ?INFO("Fetching ~s ~s~n", [Name
+                                                    ,element(2, Source)]),
+                          rebar_fetch:download_source(TargetDir, Source),
+                          case rebar_app_discover:find_unbuilt_apps([TargetDir]) of
+                              [AppSrc] ->
+                                  C = rebar_config:consult(rebar_app_info:dir(AppSrc)),
+                                  S = rebar_state:new(rebar_state:new()
+                                                     ,C
+                                                     ,rebar_app_info:dir(AppSrc)),
+                                  rebar_prv_app_builder:build(S, AppSrc);
+                              [] ->
+                                  []
+                          end
+              end, Missing),
+
+    {State, Missing}.
+
+parse_deps(Deps) ->
+    lists:foldl(fun({Name, Vsn}, {SrcDepsAcc, GoalsAcc}) ->
+                        {SrcDepsAcc, [{ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)} | GoalsAcc]};
+                   (Name, {SrcDepsAcc, GoalsAcc}) when is_atom(Name) ->
+                        {SrcDepsAcc, [ec_cnv:to_binary(Name) | GoalsAcc]};
+                   (SrcDep, {SrcDepsAcc, GoalsAcc}) ->
+                        Dep = new(SrcDep),
+                        {[Dep | SrcDepsAcc], GoalsAcc}
+                end, {[], []}, Deps).
+
+info(Description) ->
+    io_lib:format("~s.~n"
+                 "~n"
+                 "Valid rebar.config options:~n"
+                 "  ~p~n"
+                 "  ~p~n"
+                 "Valid command line options:~n"
+                 "  deps_dir=\"deps\" (override default or rebar.config deps_dir)~n",
+                 [
+                 Description,
+                 {deps_dir, "deps"},
+                 {deps,
+                  [app_name,
+                   {rebar, "1.0.*"},
+                   {rebar, ".*",
+                    {git, "git://github.com/rebar/rebar.git"}},
+                   {rebar, ".*",
+                    {git, "git://github.com/rebar/rebar.git", "Rev"}},
+                   {rebar, "1.0.*",
+                    {git, "git://github.com/rebar/rebar.git", {branch, "master"}}},
+                   {rebar, "1.0.0",
+                    {git, "git://github.com/rebar/rebar.git", {tag, "1.0.0"}}},
+                   {rebar, "",
+                    {git, "git://github.com/rebar/rebar.git", {branch, "master"}},
+                    [raw]},
+                   {app_name, ".*", {hg, "https://www.example.org/url"}},
+                   {app_name, ".*", {rsync, "Url"}},
+                   {app_name, ".*", {svn, "https://www.example.org/url"}},
+                   {app_name, ".*", {svn, "svn://svn.example.org/url"}},
+                   {app_name, ".*", {bzr, "https://www.example.org/url", "Rev"}},
+                   {app_name, ".*", {fossil, "https://www.example.org/url"}},
+                   {app_name, ".*", {fossil, "https://www.example.org/url", "Vsn"}},
+                   {app_name, ".*", {p4, "//depot/subdir/app_dir"}}]}
+                 ]).
diff --git a/src/rebar_prv_packages.erl b/src/rebar_prv_packages.erl
new file mode 100644
index 0000000..8b4a963
--- /dev/null
+++ b/src/rebar_prv_packages.erl
@@ -0,0 +1,58 @@
+-module(rebar_prv_packages).
+
+-behaviour(rebar_provider).
+
+-export([init/1,
+         do/1]).
+
+-include("rebar.hrl").
+
+-define(PROVIDER, pkgs).
+-define(DEPS, []).
+
+-spec init(rebar_state:t()) -> {ok, rebar_state:t()}.
+init(State) ->
+    State1 = rebar_state:add_provider(State, #provider{name = ?PROVIDER,
+                                                       provider_impl = ?MODULE,
+                                                       bare = false,
+                                                       deps = ?DEPS,
+                                                       example = "rebar pkgs",
+                                                       short_desc = "List available packages",
+                                                       desc = info("List available packages"),
+                                                       opts = []}),
+    {ok, State1}.
+
+-spec do(rebar_state:t()) -> {ok, rebar_state:t()}.
+do(State) ->
+    {Packages, _Graph} = rebar_packages:get_packages(State),
+    print_packages(Packages),
+    {ok, State}.
+
+print_packages(Packages) ->
+    Keys = lists:keysort(1, dict:fetch_keys(Packages)),
+    Pkgs = merge(Keys),
+    lists:foreach(fun({Name, Vsns}) ->
+                          VsnStr = join(Vsns, <<", ">>),
+                          io:format("~s:~n    Versions: ~s~n~n", [Name, VsnStr])
+                  end, Pkgs).
+
+-spec merge([{binary(), binary()}]) -> [{binary(), [binary()]}].
+merge(List) ->
+    merge([], List).
+
+merge(List, []) ->
+    List;
+merge([{Key, Values} | T], [{Key, Value} | Rest]) ->
+    merge([{Key, [Value | Values]} | T], Rest);
+merge(List, [{Key, Value} | Rest]) ->
+    merge([{Key, [Value]} | List], Rest).
+
+-spec join([binary()], binary()) -> binary().
+join([Bin], _Sep) ->
+    <<Bin/binary>>;
+join([Bin | T], Sep) ->
+    <<Bin/binary, Sep/binary, (join(T, Sep))/binary>>.
+
+
+info(Description) ->
+    io_lib:format("~s.~n", [Description]).