Merge pull request #718 from tsloughter/ferd-add-mixed-deps-tests

Ferd add mixed deps tests

6 files changed, 180 insertions, 85 deletions

diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl
index 3b34539..e81a323 100644
--- a/src/rebar_app_discover.erl
+++ b/src/rebar_app_discover.erl
@@ -16,7 +16,25 @@ do(State, LibDirs) ->
     Apps = find_apps(Dirs, all),
     ProjectDeps = rebar_state:deps_names(State),
     DepsDir = rebar_dir:deps_dir(State),
+    CurrentProfiles = rebar_state:current_profiles(State),
+
+    %% There may be a top level src which is an app and there may not
+    %% Find it here if there is, otherwise define the deps parent as root
+    TopLevelApp = define_root_app(Apps, State),
+
+    %% Handle top level deps
+    State1 = lists:foldl(fun(Profile, StateAcc) ->
+                                 ProfileDeps = rebar_state:get(StateAcc, {deps, Profile}, []),
+                                 ProfileDeps2 = rebar_utils:tup_dedup(rebar_utils:tup_sort(ProfileDeps)),
+                                 ParsedDeps = parse_profile_deps(Profile
+                                                                ,TopLevelApp
+                                                                ,ProfileDeps2
+                                                                ,StateAcc
+                                                                ,StateAcc),
+                                 rebar_state:set(StateAcc, {parsed_deps, Profile}, ParsedDeps)
+                         end, State, lists:reverse(CurrentProfiles)),
 
+    %% Handle sub project apps deps
     %% Sort apps so we get the same merged deps config everytime
     SortedApps = rebar_utils:sort_deps(Apps),
     lists:foldl(fun(AppInfo, StateAcc) ->
@@ -33,7 +51,19 @@ do(State, LibDirs) ->
                                 ?INFO("Ignoring ~s", [Name]),
                                 StateAcc
                         end
-                end, State, SortedApps).
+                end, State1, SortedApps).
+
+define_root_app(Apps, State) ->
+    RootDir = rebar_dir:root_dir(State),
+    case ec_lists:find(fun(X) ->
+                               ec_file:real_dir_path(rebar_app_info:dir(X)) =:=
+                                   ec_file:real_dir_path(RootDir)
+                       end, Apps) of
+        {ok, App} ->
+            rebar_app_info:name(App);
+        error ->
+            root
+    end.
 
 format_error({module_list, File}) ->
     io_lib:format("Error reading module list from ~p~n", [File]);
@@ -51,24 +81,46 @@ merge_deps(AppInfo, State) ->
                  rebar_state:apply_profiles(
                    rebar_state:new(reset_hooks(rebar_state:opts(State, Default)), C,
                                   rebar_app_info:dir(AppInfo)), CurrentProfiles), Name),
+    AppState1 = rebar_state:overrides(AppState, rebar_state:get(AppState, overrides, [])),
 
-    rebar_utils:check_min_otp_version(rebar_state:get(AppState, minimum_otp_vsn, undefined)),
-    rebar_utils:check_blacklisted_otp_versions(rebar_state:get(AppState, blacklisted_otp_vsns, [])),
+    rebar_utils:check_min_otp_version(rebar_state:get(AppState1, minimum_otp_vsn, undefined)),
+    rebar_utils:check_blacklisted_otp_versions(rebar_state:get(AppState1, blacklisted_otp_vsns, [])),
 
-    AppState1 = rebar_state:set(AppState, artifacts, []),
-    AppInfo1 = rebar_app_info:state(AppInfo, AppState1),
+    AppState2 = rebar_state:set(AppState1, artifacts, []),
+    AppInfo1 = rebar_app_info:state(AppInfo, AppState2),
 
     State1 = lists:foldl(fun(Profile, StateAcc) ->
-                                 AppProfDeps = rebar_state:get(AppState, {deps, Profile}, []),
-                                 TopLevelProfDeps = rebar_state:get(StateAcc, {deps, Profile}, []),
-                                 ProfDeps2 = rebar_utils:tup_dedup(rebar_utils:tup_umerge(
-                                                     rebar_utils:tup_sort(TopLevelProfDeps)
-                                                     ,rebar_utils:tup_sort(AppProfDeps))),
-                                 rebar_state:set(StateAcc, {deps, Profile}, ProfDeps2)
+                                 handle_profile(Profile, Name, AppState1, StateAcc)
                          end, State, lists:reverse(CurrentProfiles)),
 
     {AppInfo1, State1}.
 
+handle_profile(Profile, Name, AppState, State) ->
+    {TopSrc, TopPkg} = rebar_state:get(State, {parsed_deps, Profile}, {[], []}),
+    TopLevelProfileDeps = rebar_state:get(State, {deps, Profile}, []),
+    AppProfileDeps = rebar_state:get(AppState, {deps, Profile}, []),
+    AppProfileDeps2 = rebar_utils:tup_dedup(rebar_utils:tup_sort(AppProfileDeps)),
+    ProfileDeps2 = rebar_utils:tup_dedup(rebar_utils:tup_umerge(
+                                           rebar_utils:tup_sort(TopLevelProfileDeps)
+                                           ,rebar_utils:tup_sort(AppProfileDeps2))),
+    State1 = rebar_state:set(State, {deps, Profile}, ProfileDeps2),
+
+    %% Only deps not also specified in the top level config need
+    %% to be included in the parsed deps
+    NewDeps = ProfileDeps2 -- TopLevelProfileDeps,
+    {ParsedSrc, ParsedPkg} = parse_profile_deps(Profile, Name, NewDeps, AppState, State1),
+    rebar_state:set(State1, {parsed_deps, Profile}, {TopSrc++ParsedSrc, TopPkg++ParsedPkg}).
+
+parse_profile_deps(Profile, Name, Deps, AppState, State) ->
+    DepsDir = rebar_prv_install_deps:profile_dep_dir(State, Profile),
+    Locks = rebar_state:get(State, {locks, Profile}, []),
+    rebar_prv_install_deps:parse_deps(Name
+                                     ,DepsDir
+                                     ,Deps
+                                     ,AppState
+                                     ,Locks
+                                     ,1).
+
 project_app_config(AppInfo, State) ->
     C = rebar_config:consult(rebar_app_info:dir(AppInfo)),
     Dir = rebar_app_info:dir(AppInfo),
diff --git a/src/rebar_digraph.erl b/src/rebar_digraph.erl
index d21ff77..4ece286 100644
--- a/src/rebar_digraph.erl
+++ b/src/rebar_digraph.erl
@@ -3,6 +3,7 @@
 -export([compile_order/1
         ,restore_graph/1
         ,cull_deps/2
+        ,cull_deps/3
         ,subgraph/2
         ,format_error/1]).
 
@@ -72,8 +73,12 @@ restore_graph({Vs, Es}) ->
 %% The first dep while traversing the graph is chosen and any conflicting
 %% dep encountered later on is ignored.
 cull_deps(Graph, Vertices) ->
-    {Solution, Levels} = build_initial_dicts(Vertices),
-    cull_deps(Graph, Vertices, Levels, Solution, []).
+    cull_deps(Graph, Vertices, sets:new()).
+
+cull_deps(Graph, Vertices, Seen) ->
+    Vertices1 = lists:keysort(2, Vertices),
+    {Solution, Levels, Discarded} = {dict:new(), dict:new(), []},
+    cull_deps(Graph, Vertices1, Levels, Solution, Seen, Discarded).
 
 format_error(no_solution) ->
     io_lib:format("No solution for packages found.", []).
@@ -82,79 +87,106 @@ format_error(no_solution) ->
 %% Internal Functions
 %%====================================================================
 
-cull_deps(_Graph, [], Levels, Solution, Discarded) ->
+cull_deps(_Graph, [], Levels, Solution, _, Discarded) ->
     {_, Vertices} = lists:unzip(dict:to_list(Solution)),
-    LvlVertices = [{Profile, {Parent,App,Vsn,dict:fetch(App,Levels)}} || {Profile, {Parent,App,Vsn}} <- Vertices],
+    LvlVertices = [{Profile, {Parent, App, Vsn, dict:fetch(App, Levels)}}
+                  || {Profile, {Parent,App,Vsn}} <- Vertices],
     {ok, LvlVertices, Discarded};
-cull_deps(Graph, [{Profile, Level, Vs} | Vertices], Levels, Solution, Discarded) ->
+cull_deps(Graph, [{Profile, Level, Vs} | Vertices], Levels, Solution, Seen, Discarded) ->
     {NV, NS, LS, DS} =
-        lists:foldl(fun({_, Name, Vsn}, {Acc, SolutionAcc, LevelsAcc, DiscardedAcc}) ->
-                            OutNeighbors = lists:keysort(1, digraph:out_neighbours(Graph, {Name,Vsn})),
-                            handle_neighbors(Profile, Level, Name
-                                            ,OutNeighbors, Acc, SolutionAcc
-                                            ,LevelsAcc, DiscardedAcc)
-
-                    end, {[], Solution, Levels, Discarded}, lists:keysort(2, Vs)),
-
-    cull_deps(Graph, Vertices++NV, LS, NS, DS).
+        lists:foldl(fun({Parent, Name, Vsn}, {Acc, SolutionAcc, LevelsAcc, DiscardedAcc}) ->
+                            {SolutionAcc1, LevelsAcc1, DiscardedAcc1} =
+                                maybe_add_to_solution(Profile, Level, Name, {Name, Vsn}, Parent
+                                                     ,SolutionAcc
+                                                     ,LevelsAcc, Seen, DiscardedAcc),
+                            OutNeighbors = digraph:out_neighbours(Graph, {Name,Vsn}),
+                            {NewVertices, DiscardedAcc2} = handle_neighbors(Profile, Level, Name
+                                                          ,OutNeighbors, Acc, SolutionAcc1
+                                                                           ,Seen, DiscardedAcc1),
+                            {NewVertices, SolutionAcc1, LevelsAcc1, DiscardedAcc2}
+                    end, {[], Solution, Levels, Discarded}, Vs),
+    NewVertices = combine_profile_levels(Vertices, NV),
+    cull_deps(Graph, NewVertices, LS, NS, Seen, DS).
+
+%% Combine lists of deps that have the same profile and level
+combine_profile_levels(Vertices, NewVertices) ->
+    V = lists:foldl(fun({Profile, Level, Vs}, Acc) ->
+                            case ec_lists:find(fun({P, L, _}) ->
+                                                       P =:= Profile andalso L =:= Level
+                                               end, Acc) of
+                            {ok, {_, _, OldVs}=Old} ->
+                                lists:delete(Old, Acc)++[{Profile, Level, lists:keysort(1, OldVs++Vs)}];
+                            error ->
+                                Acc++[{Profile, Level, Vs}]
+                        end
+                end, Vertices, NewVertices),
+    lists:keysort(2, V).
 
 %% For each outgoing edge of a dep check if it should be added to the solution
 %% and add it to the list of vertices to do the same for
 handle_neighbors(Profile, Level, Parent, OutNeighbors, Vertices
-                ,Solution, Levels, Discarded) ->
-        case lists:foldl(fun({Name, _}=N, {NewVertices, Solution1, Levels1, Discarded1}) ->
-                            maybe_add_to_solution(Profile, Level, Name, N, Parent
-                                                 ,NewVertices, Solution1
-                                                 ,Levels1, Discarded1)
-                    end, {[], Solution, Levels, Discarded}, OutNeighbors) of
-            {[], SolutionAcc2, LevelsAcc2, DiscardedAcc2} ->
-                {Vertices, SolutionAcc2, LevelsAcc2, DiscardedAcc2};
-            {NewVertices1, SolutionAcc2, LevelsAcc2, DiscardedAcc2} ->
-                {Vertices++[{Profile, Level+1, NewVertices1}]
-                ,SolutionAcc2, LevelsAcc2, DiscardedAcc2}
+                ,Solution, Seen, Discarded) ->
+    case lists:foldl(fun({Name, Vsn}=Value, {NewVertices, Discarded1}) ->
+                             case dict:find(Name, Solution) of
+                                 {ok, {Profile, {Parent, Name, Vsn}}} -> % already seen
+                                     {NewVertices,
+                                      Discarded1};
+                                 {ok, _} -> % conflict resolution!
+                                     %% Warn on different version
+                                     {NewVertices,
+                                      [Value|Discarded1]};
+                                 error ->
+                                     %% We check Seen separately because we don't care
+                                     %% to warn if the exact same version of a package
+                                     %% was already part of the solution but we do
+                                     %% if it was simply seen in source deps
+                                     case sets:is_element(Name, Seen) of
+                                         true ->
+                                             {NewVertices,
+                                              [Value|Discarded1]};
+                                         false ->
+                                             {[{Parent, Name, Vsn} | NewVertices],
+                                              Discarded1}
+                                     end
+                             end
+                    end, {[], Discarded}, OutNeighbors) of
+            {[], DiscardedAcc2} ->
+                {Vertices, DiscardedAcc2};
+            {NewVertices1, DiscardedAcc2} ->
+                {Vertices++[{Profile, Level+1, NewVertices1}] ,DiscardedAcc2}
         end.
 
 maybe_add_to_solution(Profile, Level, Key, {Name, Vsn}=Value, Parent
-                     ,Vertices ,Solution, Levels, Discarded) ->
+                     ,Solution, Levels, Seen, Discarded) ->
     case dict:find(Key, Solution) of
         {ok, {Profile, {Parent, Name, Vsn}}} -> % already seen
-            {Vertices,
-             Solution,
+            {Solution,
              Levels,
              Discarded};
         {ok, _} -> % conflict resolution!
-            {Vertices,
-             Solution,
+            %% Warn on different version
+            {Solution,
              Levels,
              [Value|Discarded]};
         error ->
-            {[{Parent, Name, Vsn} | Vertices],
-             dict:store(Key, {Profile, {Parent, Name, Vsn}}, Solution),
-             dict:store(Key, Level+1, Levels),
-             Discarded}
+            %% We check Seen separately because we don't care to warn if the exact
+            %% same version of a package was already part of the solution but we do
+            %% if it was simply seen in source deps
+            case sets:is_element(Name, Seen) of
+                true ->
+                    {Solution,
+                     Levels,
+                     [Value|Discarded]};
+                false ->
+                    {dict:store(Key, {Profile, {Parent, Name, Vsn}}, Solution),
+                     dict:store(Key, Level, Levels),
+                     Discarded}
+            end
     end.
 
 subgraph(Graph, Vertices) ->
     digraph_utils:subgraph(Graph, Vertices).
 
-maybe_add_to_dict(Key, Value, Dict) ->
-    case dict:is_key(Key, Dict) of
-        true ->
-            Dict;
-        false ->
-            dict:store(Key, Value, Dict)
-    end.
-
-%% Track the profile (so we know where to install it), name/vsn of each dep
-%% and the level it is from (for the lock file)
-build_initial_dicts(Vertices) ->
-    lists:foldl(fun({Profile, Level, Vs}, {Solution, Levels}) ->
-                        lists:foldl(fun({Parent, Key, Vsn}, {SAcc, LAcc}) ->
-                                            {maybe_add_to_dict(Key, {Profile, {Parent,Key,Vsn}}, SAcc),
-                                             maybe_add_to_dict(Key, Level, LAcc)}
-                                    end, {Solution, Levels}, Vs)
-                end, {dict:new(), dict:new()}, Vertices).
-
 -spec names_to_apps([atom()], [rebar_app_info:t()]) -> [rebar_app_info:t()].
 names_to_apps(Names, Apps) ->
     [element(2, App) || App <- [find_app_by_name(Name, Apps) || Name <- Names], App =/= error].
diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl
index 2b024cf..1165631 100644
--- a/src/rebar_prv_common_test.erl
+++ b/src/rebar_prv_common_test.erl
@@ -361,10 +361,10 @@ remove_links(Path) ->
     end.
 
 delete_dir_link(Path) ->
-	case os:type() of
-		{unix, _} -> file:delete(Path);
-		{win32, _} -> file:del_dir(Path)
-	end.
+    case os:type() of
+        {unix, _} -> file:delete(Path);
+        {win32, _} -> file:del_dir(Path)
+    end.
 
 dir_entries(Path) ->
     {ok, SubDirs} = file:list_dir(Path),
diff --git a/src/rebar_prv_deps_tree.erl b/src/rebar_prv_deps_tree.erl
index 6c69aba..d429c52 100644
--- a/src/rebar_prv_deps_tree.erl
+++ b/src/rebar_prv_deps_tree.erl
@@ -29,7 +29,7 @@ init(State) ->
 do(State) ->
     {Args, _} = rebar_state:command_parsed_args(State),
     Verbose = proplists:get_value(verbose, Args, false),
-    print_deps_tree(rebar_state:all_deps(State), Verbose),
+    print_deps_tree(rebar_state:all_deps(State), Verbose, State),
     {ok, State}.
 
 -spec format_error(any()) -> iolist().
@@ -38,7 +38,7 @@ format_error(Reason) ->
 
 %% Internal functions
 
-print_deps_tree(SrcDeps, Verbose) ->
+print_deps_tree(SrcDeps, Verbose, State) ->
     D = lists:foldl(fun(App, Dict) ->
                             Name = rebar_app_info:name(App),
                             Vsn = rebar_app_info:original_vsn(App),
@@ -46,11 +46,14 @@ print_deps_tree(SrcDeps, Verbose) ->
                             Parent = rebar_app_info:parent(App),
                             dict:append_list(Parent, [{Name, Vsn, Source}], Dict)
                     end, dict:new(), SrcDeps),
+    ProjectAppNames = [{rebar_app_info:name(App)
+                       ,rebar_app_info:original_vsn(App)
+                       ,project} || App <- rebar_state:project_apps(State)],
     case dict:find(root, D) of
         {ok, Children} ->
-            print_children(-1, lists:keysort(1, Children), D, Verbose);
+            print_children(-1, lists:keysort(1, Children++ProjectAppNames), D, Verbose);
         error ->
-            none
+            print_children(-1, lists:keysort(1, ProjectAppNames), D, Verbose)
     end.
 
 print_children(_, [], _, _) ->
@@ -68,6 +71,8 @@ print_children(Indent, [{Name, Vsn, Source} | Rest], Dict, Verbose) ->
             print_children(Indent, Rest, Dict, Verbose)
     end.
 
+type(project, _) ->
+    "project app";
 type(Source, Verbose) when is_tuple(Source) ->
     case {element(1, Source), Verbose} of
         {pkg, _} ->
diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl
index e521b25..11a4250 100644
--- a/src/rebar_prv_install_deps.erl
+++ b/src/rebar_prv_install_deps.erl
@@ -37,6 +37,8 @@
 
 -export([handle_deps_as_profile/4,
          parse_deps/5,
+         parse_deps/6,
+         profile_dep_dir/2,
          find_cycles/1,
          cull_compile/2]).
 
@@ -155,10 +157,8 @@ deps_per_profile(Profiles, Upgrade, State) ->
     handle_profile_pkg_level(PkgDeps1, AllApps, Seen, Upgrade, Locks, State1).
 
 parse_profile_deps(State, Profile, Level) ->
-    DepsDir = profile_dep_dir(State, Profile),
     Locks = rebar_state:get(State, {locks, Profile}, []),
-    Deps = rebar_state:get(State, {deps, Profile}, []),
-    {SrcDeps, PkgDeps} = parse_deps(DepsDir, Deps, State, Locks, Level),
+    {SrcDeps, PkgDeps} = rebar_state:get(State, {parsed_deps, Profile}, {[], []}),
     {{Profile, SrcDeps, Locks, Level}, {Profile, Level, PkgDeps}}.
 
 %% Level-order traversal of all dependencies, across profiles.
@@ -190,9 +190,12 @@ handle_profile_pkg_level(PkgDeps, AllApps, Seen, Upgrade, Locks, State) ->
     State1 = rebar_state:packages(rebar_state:registry(State, Registry)
                                  ,{Packages, Graph}),
 
-    S = case rebar_digraph:cull_deps(Graph, lists:keysort(2, PkgDeps)) of
-            {ok, [], _} ->
+    S = case rebar_digraph:cull_deps(Graph, lists:keysort(2, PkgDeps), Seen) of
+            {ok, [], []} ->
                 throw({rebar_digraph, no_solution});
+            {ok, [], Discarded} ->
+                [warn_skip_pkg(Pkg, State) || Pkg <- Discarded, not(pkg_locked(Pkg, Locks))],
+                [];
             {ok, Solution, []} ->
                 Solution;
             {ok, Solution, Discarded} ->
@@ -376,9 +379,9 @@ handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) ->
     Profiles = rebar_state:current_profiles(State),
     Name = rebar_app_info:name(AppInfo),
 
-    C = rebar_config:consult(rebar_app_info:dir(AppInfo)),
-
+    %% Deps may be under a sub project app, find it and use its state if so
     S = rebar_app_info:state(AppInfo),
+    C = rebar_config:consult(rebar_app_info:dir(AppInfo)),
     S1 = rebar_state:new(S, C, rebar_app_info:dir(AppInfo)),
     S2 = rebar_state:apply_overrides(S1, Name),
 
diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl
index 3a371ca..f49eafe 100644
--- a/src/rebar_prv_upgrade.erl
+++ b/src/rebar_prv_upgrade.erl
@@ -53,18 +53,21 @@ do(State) ->
         {Locks0, _Unlocks0} ->
             Deps0 = top_level_deps(Deps, Locks),
             State1 = rebar_state:set(State, {deps, default}, Deps0),
-            State2 = rebar_state:set(State1, {locks, default}, Locks0),
-            State3 = rebar_state:set(State2, upgrade, true),
-            UpdatedLocks = [L || L <- rebar_state:lock(State3),
+            DepsDir = rebar_prv_install_deps:profile_dep_dir(State, default),
+            D = rebar_prv_install_deps:parse_deps(root, DepsDir, Deps0, State1, Locks0, 0),
+            State2 = rebar_state:set(State1, {parsed_deps, default}, D),
+            State3 = rebar_state:set(State2, {locks, default}, Locks0),
+            State4 = rebar_state:set(State3, upgrade, true),
+            UpdatedLocks = [L || L <- rebar_state:lock(State4),
                                  lists:keymember(rebar_app_info:name(L), 1, Locks0)],
-            Res = rebar_prv_install_deps:do(rebar_state:lock(State3, UpdatedLocks)),
+            Res = rebar_prv_install_deps:do(rebar_state:lock(State4, UpdatedLocks)),
             case Res of
-                {ok, State4} ->
+                {ok, State5} ->
                     rebar_utils:info_useless(
                       [element(1,Lock) || Lock <- Locks],
-                      [rebar_app_info:name(App) || App <- rebar_state:lock(State4)]
+                      [rebar_app_info:name(App) || App <- rebar_state:lock(State5)]
                      ),
-                    rebar_prv_lock:do(State4);
+                    rebar_prv_lock:do(State5);
                 _ ->
                     Res
             end