From cfea721d98469c58953436d27a7663fdd19dfd3f Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 2 Jul 2015 12:51:46 +0000 Subject: wip: deps install refactor --- src/rebar_plugins.erl | 2 +- src/rebar_prv_install_deps.erl | 168 +++++++++++++++++--------------------- src/rebar_prv_plugins_upgrade.erl | 5 +- 3 files changed, 77 insertions(+), 98 deletions(-) (limited to 'src') diff --git a/src/rebar_plugins.erl b/src/rebar_plugins.erl index 7e12324..0f3cafc 100644 --- a/src/rebar_plugins.erl +++ b/src/rebar_plugins.erl @@ -63,7 +63,7 @@ handle_plugins(Profile, Plugins, State, Upgrade) -> handle_plugin(Profile, Plugin, State, Upgrade) -> try - {ok, Apps, State2} = rebar_prv_install_deps:handle_deps(Profile, State, [Plugin], Upgrade), + {Apps, State2} = rebar_prv_install_deps:handle_profile_deps(Profile, State, [Plugin], Upgrade), {no_cycle, Sorted} = rebar_prv_install_deps:find_cycles(Apps), ToBuild = rebar_prv_install_deps:cull_compile(Sorted, []), diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index 768d41a..fa7cc44 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -35,10 +35,7 @@ -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). --export([handle_deps/3, - handle_deps/4, - handle_deps/5, - +-export([handle_profile_deps/4, find_cycles/1, cull_compile/2]). @@ -76,8 +73,9 @@ do(State) -> Profiles = rebar_state:current_profiles(State), ProjectApps = rebar_state:project_apps(State), - {Apps, State1} = - lists:foldl(fun deps_per_profile/2, {[], State}, lists:reverse(Profiles)), + Upgrade = rebar_state:get(State, upgrade, false), + {Apps, State1} = deps_per_profile(Profiles, Upgrade, State), + %lists:foldl(fun deps_per_profile/2, {[], State}, lists:reverse(Profiles)), State2 = rebar_state:update_all_deps(State1, Apps), CodePaths = [rebar_app_info:ebin_dir(A) || A <- Apps], @@ -119,63 +117,65 @@ format_error({cycles, Cycles}) -> format_error(Reason) -> io_lib:format("~p", [Reason]). --spec handle_deps(atom(), rebar_state:t(), list()) -> - {ok, [rebar_app_info:t()], rebar_state:t()} | {error, string()}. -handle_deps(Profile, State, Deps) -> - handle_deps(Profile, State, Deps, false, []). - --spec handle_deps(atom(), rebar_state:t(), list(), list() | boolean()) -> - {ok, [rebar_app_info:t()], rebar_state:t()} | {error, string()}. -handle_deps(Profile, State, Deps, Upgrade) when is_boolean(Upgrade) -> - handle_deps(Profile, State, Deps, Upgrade, []); -handle_deps(Profile, State, Deps, Locks) when is_list(Locks) -> - Upgrade = rebar_state:get(State, upgrade, false), - handle_deps(Profile, State, Deps, Upgrade, Locks). - --spec handle_deps(atom(), rebar_state:t(), list(), boolean() | {true, binary(), integer()}, list()) - -> {ok, [rebar_app_info:t()], rebar_state:t()} | {error, string()}. -handle_deps(_Profile, State, [], _, _) -> - {ok, [], State}; -handle_deps(Profile, State0, Deps, Upgrade, Locks) -> - %% Split source deps from pkg deps, needed to keep backwards compatibility - DepsDir = profile_dep_dir(State0, Profile), - - {SrcDeps, PkgDeps} = parse_deps(DepsDir, Deps, State0, Locks, 0), - - %% Fetch transitive src deps - {State1, SrcApps, PkgDeps1, Seen} = update_src_deps(Profile, 0, SrcDeps, PkgDeps, [] - ,State0, Upgrade, sets:new(), Locks), - - {Solved, State4} = - case PkgDeps1 of - [] -> - {[], State1}; - _ -> - %% Read in package index and dep graph - {Packages, Graph} = rebar_state:packages(State1), - Registry = rebar_packages:registry(State1), - State2 = rebar_state:packages(rebar_state:registry(State1, Registry) - ,{Packages, Graph}), - - update_pkg_deps(Profile, Packages, PkgDeps1 - ,Graph, Upgrade, Seen, State2, Locks) - end, - - AllDeps = lists:ukeymerge(2 - ,lists:ukeysort(2, SrcApps) - ,lists:ukeysort(2, Solved)), - - {ok, AllDeps, State4}. - %% =================================================================== %% Internal functions %% =================================================================== -deps_per_profile(Profile, {Apps, State}) -> - Locks = rebar_state:get(State, {locks, Profile}, []), - ProfileDeps = rebar_state:get(State, {deps, Profile}, []), - {ok, NewApps, NewState} = handle_deps(Profile, State, ProfileDeps, Locks), - {NewApps++Apps, NewState}. +handle_profile_deps(Profile, State, Deps, Upgrade) -> + Locks = [], + Level = 0, + DepsDir = profile_dep_dir(State, Profile), + + {SrcDeps, PkgDeps} = parse_deps(DepsDir, Deps, State, Locks, Level), + AllProfileDeps = [{Profile, SrcDeps, Locks, Level}], + {AllApps, PkgDeps1, Seen, State1} = handle_profile_level(AllProfileDeps, [{Profile, Locks, PkgDeps}], Locks, sets:new(), Upgrade, State), + + handle_profile_pkg_level(PkgDeps1, AllApps, Seen, Upgrade, State1). + +deps_per_profile(Profiles, Upgrade, State) -> + Level = 0, + {AllProfileDeps, PkgDeps} = lists:foldl(fun(Profile, {ProfileDepsAcc, PkgDepsAcc}) -> + DepsDir = profile_dep_dir(State, Profile), + Locks = rebar_state:get(State, {locks, Profile}, []), + Deps = rebar_state:get(State, {deps, Profile}, []), + {SrcDeps, PkgDeps} = parse_deps(DepsDir, Deps, State, Locks, Level), + {[{Profile, SrcDeps, Locks, Level} | ProfileDepsAcc] + ,[{Profile, Locks, PkgDeps} | PkgDepsAcc]} + end, {[],[]}, Profiles), + {AllApps, PkgDeps1, Seen, State1} = handle_profile_level(AllProfileDeps, PkgDeps, [], sets:new(), Upgrade, State), + + handle_profile_pkg_level(PkgDeps1, AllApps, Seen, Upgrade, State1). + +handle_profile_level([], PkgDeps, SrcApps, Seen, _Upgrade, State) -> + {SrcApps, PkgDeps, Seen, State}; +handle_profile_level([{Profile, SrcDeps, Locks, Level} | Rest], PkgDeps, SrcApps, Seen, Upgrade, State) -> + case update_src_deps(Profile, Level, SrcDeps, [], SrcApps + ,State, Upgrade, Seen, Locks) of + {[], PkgDeps1, SrcApps1, State1, Seen1, Locks1} -> + handle_profile_level(Rest, [{Profile, Locks1, PkgDeps1} | PkgDeps], SrcApps1++SrcApps, sets:union(Seen, Seen1), Upgrade, State1); + {SrcDeps1, PkgDeps1, SrcApps1, State1, Seen1, Locks1} -> + handle_profile_level(Rest++[{Profile, SrcDeps1, Locks1, Level+1}], [{Profile, Locks1, PkgDeps1} | PkgDeps], SrcApps1++SrcApps, sets:union(Seen, Seen1), Upgrade, State1) + end. + +handle_profile_pkg_level(PkgDeps, AllApps, Seen, Upgrade, State) -> + %% Read in package index and dep graph + {Packages, Graph} = rebar_state:packages(State), + Registry = rebar_packages:registry(State), + State1 = rebar_state:packages(rebar_state:registry(State, Registry) + ,{Packages, Graph}), + + lists:foldl(fun({_Profile, _, []}, {AllAcc, StateAcc}) -> + {AllAcc, StateAcc}; + ({Profile1, Locks, PkgDeps2}, {AllAcc, StateAcc}) -> + {Solved, StateAcc2} = update_pkg_deps(Profile1, Packages, PkgDeps2 + ,Graph, Upgrade, Seen, StateAcc, Locks), + + AllDeps = lists:ukeymerge(2 + ,lists:ukeysort(2, AllAcc) + ,lists:ukeysort(2, Solved)), + + {AllDeps, StateAcc2} + end, {AllApps, State1}, PkgDeps). find_cycles(Apps) -> case rebar_digraph:compile_order(Apps) of @@ -279,19 +279,15 @@ package_to_app(DepsDir, Packages, {Name, Vsn, Level}, IsLock, State) -> -spec update_src_deps(atom(), non_neg_integer(), list(), list(), list(), rebar_state:t(), boolean(), sets:set(binary()), list()) -> {rebar_state:t(), list(), list(), sets:set(binary())}. update_src_deps(Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, Locks) -> - case lists:foldl( - fun(AppInfo, {SrcDepsAcc, PkgDepsAcc, SrcAppsAcc, StateAcc, SeenAcc, LocksAcc}) -> - update_src_dep(AppInfo, Profile, Level, - SrcDepsAcc, PkgDepsAcc, SrcAppsAcc, StateAcc, - Upgrade, SeenAcc, Locks, LocksAcc) - end, - {[], PkgDeps, SrcApps, State, Seen, Locks}, - rebar_utils:sort_deps(SrcDeps)) of - {[], NewPkgDeps, NewSrcApps, State1, Seen1, _NewLocks} -> - {State1, NewSrcApps, NewPkgDeps, Seen1}; - {NewSrcDeps, NewPkgDeps, NewSrcApps, State1, Seen1, NewLocks} -> - update_src_deps(Profile, Level+1, NewSrcDeps, NewPkgDeps, NewSrcApps, State1, Upgrade, Seen1, NewLocks) - end. + lists:foldl( + fun(AppInfo, {SrcDepsAcc, PkgDepsAcc, SrcAppsAcc, StateAcc, SeenAcc, LocksAcc}) -> + update_src_dep(AppInfo, Profile, Level, + SrcDepsAcc, PkgDepsAcc, SrcAppsAcc, StateAcc, + Upgrade, SeenAcc, Locks, LocksAcc) + end, + {[], PkgDeps, SrcApps, State, Seen, Locks}, + rebar_utils:sort_deps(SrcDeps)). + update_src_dep(AppInfo, Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, BaseLocks, Locks) -> %% If not seen, add to list of locks to write out @@ -414,41 +410,27 @@ maybe_fetch(AppInfo, Profile, Upgrade, Seen, State) -> false -> case rebar_app_discover:find_app(AppDir, all) of false -> - case already_in_default(AppInfo, State) of - false -> - case fetch_app(AppInfo, AppDir, State) of - true -> - maybe_symlink_default(State, Profile, AppDir, AppInfo), - {true, update_app_info(AppDir, AppInfo)}; - Other -> - {Other, AppInfo} - end; - {true, FoundApp} -> - %% Preserve the state we created with overrides - AppState = rebar_app_info:state(AppInfo), - FoundApp1 = rebar_app_info:state(FoundApp, AppState), - symlink_dep(rebar_app_info:dir(FoundApp1), AppDir), - {true, FoundApp1} + case fetch_app(AppInfo, AppDir, State) of + true -> + maybe_symlink_default(State, Profile, AppDir, AppInfo), + {true, update_app_info(AppDir, AppInfo)}; + Other -> + {Other, AppInfo} end; {true, AppInfo1} -> %% Preserve the state we created with overrides AppState = rebar_app_info:state(AppInfo), AppInfo2 = rebar_app_info:state(AppInfo1, AppState), - maybe_symlink_default(State, Profile, AppDir, AppInfo2), case sets:is_element(rebar_app_info:name(AppInfo), Seen) of true -> {false, AppInfo2}; false -> + maybe_symlink_default(State, Profile, AppDir, AppInfo2), {maybe_upgrade(AppInfo, AppDir, Upgrade, State), AppInfo2} end end end. -already_in_default(AppInfo, State) -> - Name = ec_cnv:to_list(rebar_app_info:name(AppInfo)), - DefaultAppDir = filename:join([rebar_state:get(State, base_dir, []), "default", "lib", Name]), - rebar_app_discover:find_app(DefaultAppDir, all). - needs_symlinking(State, Profile) -> case {rebar_state:current_profiles(State), Profile} of {[default], default} -> diff --git a/src/rebar_prv_plugins_upgrade.erl b/src/rebar_prv_plugins_upgrade.erl index 5ccd054..7830b24 100644 --- a/src/rebar_prv_plugins_upgrade.erl +++ b/src/rebar_prv_plugins_upgrade.erl @@ -62,10 +62,7 @@ upgrade(Plugin, State) -> ?PRV_ERROR({not_found, Plugin}); {ok, P, Profile} -> State1 = rebar_state:set(State, deps_dir, ?DEFAULT_PLUGINS_DIR), - {ok, Apps, _State2} = rebar_prv_install_deps:handle_deps(Profile - ,State1 - ,[P] - ,true), + {Apps, _State2} = rebar_prv_install_deps:handle_profile_deps(Profile, State1, [P], true), {no_cycle, Sorted} = rebar_prv_install_deps:find_cycles(Apps), ToBuild = rebar_prv_install_deps:cull_compile(Sorted, []), -- cgit v1.1 From 7840a93e859e939f5dae92d026d5a008435bc56f Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 2 Jul 2015 14:57:55 +0000 Subject: Minor refactors --- src/rebar_plugins.erl | 2 +- src/rebar_prv_install_deps.erl | 60 ++++++++++++++++++++++++--------------- src/rebar_prv_plugins_upgrade.erl | 2 +- 3 files changed, 39 insertions(+), 25 deletions(-) (limited to 'src') diff --git a/src/rebar_plugins.erl b/src/rebar_plugins.erl index 0f3cafc..3e855de 100644 --- a/src/rebar_plugins.erl +++ b/src/rebar_plugins.erl @@ -63,7 +63,7 @@ handle_plugins(Profile, Plugins, State, Upgrade) -> handle_plugin(Profile, Plugin, State, Upgrade) -> try - {Apps, State2} = rebar_prv_install_deps:handle_profile_deps(Profile, State, [Plugin], Upgrade), + {Apps, State2} = rebar_prv_install_deps:handle_deps_as_profile(Profile, State, [Plugin], Upgrade), {no_cycle, Sorted} = rebar_prv_install_deps:find_cycles(Apps), ToBuild = rebar_prv_install_deps:cull_compile(Sorted, []), diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index fa7cc44..7160704 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -35,7 +35,7 @@ -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). --export([handle_profile_deps/4, +-export([handle_deps_as_profile/4, find_cycles/1, cull_compile/2]). @@ -117,45 +117,57 @@ format_error({cycles, Cycles}) -> format_error(Reason) -> io_lib:format("~p", [Reason]). -%% =================================================================== -%% Internal functions -%% =================================================================== - -handle_profile_deps(Profile, State, Deps, Upgrade) -> +%% Allows other providers to install deps in a given profile +%% manually, outside of what is provided by rebar3's deps tuple. +handle_deps_as_profile(Profile, State, Deps, Upgrade) -> Locks = [], Level = 0, DepsDir = profile_dep_dir(State, Profile), {SrcDeps, PkgDeps} = parse_deps(DepsDir, Deps, State, Locks, Level), - AllProfileDeps = [{Profile, SrcDeps, Locks, Level}], - {AllApps, PkgDeps1, Seen, State1} = handle_profile_level(AllProfileDeps, [{Profile, Locks, PkgDeps}], Locks, sets:new(), Upgrade, State), + AllSrcProfileDeps = [{Profile, SrcDeps, Locks, Level}], + AllPkgProfileDeps = [{Profile, Locks, PkgDeps}], + {AllApps, PkgDeps1, Seen, State1} = handle_profile_level(AllSrcProfileDeps, AllPkgProfileDeps, Locks, sets:new(), Upgrade, State), handle_profile_pkg_level(PkgDeps1, AllApps, Seen, Upgrade, State1). + +%% =================================================================== +%% Internal functions +%% =================================================================== + +%% finds all the deps in `{deps, ...}` for each profile provided. deps_per_profile(Profiles, Upgrade, State) -> Level = 0, - {AllProfileDeps, PkgDeps} = lists:foldl(fun(Profile, {ProfileDepsAcc, PkgDepsAcc}) -> - DepsDir = profile_dep_dir(State, Profile), - Locks = rebar_state:get(State, {locks, Profile}, []), - Deps = rebar_state:get(State, {deps, Profile}, []), - {SrcDeps, PkgDeps} = parse_deps(DepsDir, Deps, State, Locks, Level), - {[{Profile, SrcDeps, Locks, Level} | ProfileDepsAcc] - ,[{Profile, Locks, PkgDeps} | PkgDepsAcc]} - end, {[],[]}, Profiles), + {AllProfileDeps, PkgDeps} = lists:foldl(fun(Profile, {SrcAcc, PkgAcc}) -> + {Src, Pkg} = parse_profile_deps(State, Profile, Level), + {[Src | SrcAcc], [Pkg | PkgAcc]} + end, {[], []}, Profiles), {AllApps, PkgDeps1, Seen, State1} = handle_profile_level(AllProfileDeps, PkgDeps, [], sets:new(), Upgrade, State), handle_profile_pkg_level(PkgDeps1, AllApps, Seen, Upgrade, State1). +parse_profile_deps(State, Profile, Level) -> + DepsDir = profile_dep_dir(State, Profile), + Locks = rebar_state:get(State, {locks, Profile}, []), + Deps = rebar_state:get(State, {deps, Profile}, []), + {SrcDeps, PkgDeps} = parse_deps(DepsDir, Deps, State, Locks, Level), + {{Profile, SrcDeps, Locks, Level}, {Profile, Locks, PkgDeps}}. + +%% Level-order traversal of all dependencies, across profiles. +%% If profiles x,y,z are present, then the traversal will go: +%% x0, y0, z0, x1, y1, z1, ..., xN, yN, zN. handle_profile_level([], PkgDeps, SrcApps, Seen, _Upgrade, State) -> {SrcApps, PkgDeps, Seen, State}; handle_profile_level([{Profile, SrcDeps, Locks, Level} | Rest], PkgDeps, SrcApps, Seen, Upgrade, State) -> - case update_src_deps(Profile, Level, SrcDeps, [], SrcApps - ,State, Upgrade, Seen, Locks) of - {[], PkgDeps1, SrcApps1, State1, Seen1, Locks1} -> - handle_profile_level(Rest, [{Profile, Locks1, PkgDeps1} | PkgDeps], SrcApps1++SrcApps, sets:union(Seen, Seen1), Upgrade, State1); - {SrcDeps1, PkgDeps1, SrcApps1, State1, Seen1, Locks1} -> - handle_profile_level(Rest++[{Profile, SrcDeps1, Locks1, Level+1}], [{Profile, Locks1, PkgDeps1} | PkgDeps], SrcApps1++SrcApps, sets:union(Seen, Seen1), Upgrade, State1) - end. + {SrcDeps1, PkgDeps1, SrcApps1, State1, Seen1, Locks1} = + update_src_deps(Profile, Level, SrcDeps, [], SrcApps + ,State, Upgrade, Seen, Locks), + SrcDeps2 = case SrcDeps1 of + [] -> Rest; + _ -> Rest ++ [{Profile, SrcDeps1, Locks1, Level+1}] + end, + handle_profile_level(SrcDeps2, [{Profile, Locks1, PkgDeps1} | PkgDeps], SrcApps1++SrcApps, sets:union(Seen, Seen1), Upgrade, State1). handle_profile_pkg_level(PkgDeps, AllApps, Seen, Upgrade, State) -> %% Read in package index and dep graph @@ -486,6 +498,7 @@ parse_deps(DepsDir, Deps, State, Locks, Level) -> end, {[], []}, Deps). parse_dep({Name, Vsn}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_list(Vsn) -> + %% Versioned Package dependency CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), case rebar_app_info:discover(CheckoutsDir) of {ok, _App} -> @@ -496,6 +509,7 @@ parse_dep({Name, Vsn}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is ,ec_cnv:to_binary(Vsn)) | PkgDepsAcc]} end; parse_dep(Name, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_atom(Name) -> + %% Unversioned package dependency {PkgName, PkgVsn} = get_package(ec_cnv:to_binary(Name), State), CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), case rebar_app_info:discover(CheckoutsDir) of diff --git a/src/rebar_prv_plugins_upgrade.erl b/src/rebar_prv_plugins_upgrade.erl index 7830b24..02c185f 100644 --- a/src/rebar_prv_plugins_upgrade.erl +++ b/src/rebar_prv_plugins_upgrade.erl @@ -62,7 +62,7 @@ upgrade(Plugin, State) -> ?PRV_ERROR({not_found, Plugin}); {ok, P, Profile} -> State1 = rebar_state:set(State, deps_dir, ?DEFAULT_PLUGINS_DIR), - {Apps, _State2} = rebar_prv_install_deps:handle_profile_deps(Profile, State1, [P], true), + {Apps, _State2} = rebar_prv_install_deps:handle_deps_as_profile(Profile, State1, [P], true), {no_cycle, Sorted} = rebar_prv_install_deps:find_cycles(Apps), ToBuild = rebar_prv_install_deps:cull_compile(Sorted, []), -- cgit v1.1