From 31a24ad4ffb6a121819087da2cb9b58db5bd287d Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Wed, 19 Aug 2015 22:41:33 -0500 Subject: this patch treats pkg and src deps as equals, so level decides winner Instead fetching and resolving src deps (which could depend on pkg deps) and then pkg deps this patch combines the two into a single set of iterations by level. The only difference between src and pkg deps in this new install_deps is how their deps list is found -- from the config or lock file for src deps and from the neighbors of the vertex for pkg. --- src/rebar_app_discover.erl | 18 +- src/rebar_app_info.erl | 13 +- src/rebar_app_utils.erl | 129 +++++++++++++- src/rebar_digraph.erl | 110 ------------ src/rebar_packages.erl | 26 ++- src/rebar_prv_install_deps.erl | 395 +++++++++-------------------------------- src/rebar_prv_packages.erl | 2 +- src/rebar_prv_upgrade.erl | 3 +- src/rebar_state.erl | 17 +- 9 files changed, 267 insertions(+), 446 deletions(-) (limited to 'src') diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index e81a323..8b1e58e 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -96,7 +96,7 @@ merge_deps(AppInfo, State) -> {AppInfo1, State1}. handle_profile(Profile, Name, AppState, State) -> - {TopSrc, TopPkg} = rebar_state:get(State, {parsed_deps, Profile}, {[], []}), + TopParsedDeps = rebar_state:get(State, {parsed_deps, Profile}, {[], []}), TopLevelProfileDeps = rebar_state:get(State, {deps, Profile}, []), AppProfileDeps = rebar_state:get(AppState, {deps, Profile}, []), AppProfileDeps2 = rebar_utils:tup_dedup(rebar_utils:tup_sort(AppProfileDeps)), @@ -108,18 +108,18 @@ handle_profile(Profile, Name, AppState, State) -> %% Only deps not also specified in the top level config need %% to be included in the parsed deps NewDeps = ProfileDeps2 -- TopLevelProfileDeps, - {ParsedSrc, ParsedPkg} = parse_profile_deps(Profile, Name, NewDeps, AppState, State1), - rebar_state:set(State1, {parsed_deps, Profile}, {TopSrc++ParsedSrc, TopPkg++ParsedPkg}). + ParsedDeps = parse_profile_deps(Profile, Name, NewDeps, AppState, State1), + rebar_state:set(State1, {parsed_deps, Profile}, TopParsedDeps++ParsedDeps). parse_profile_deps(Profile, Name, Deps, AppState, State) -> DepsDir = rebar_prv_install_deps:profile_dep_dir(State, Profile), Locks = rebar_state:get(State, {locks, Profile}, []), - rebar_prv_install_deps:parse_deps(Name - ,DepsDir - ,Deps - ,AppState - ,Locks - ,1). + rebar_app_utils:parse_deps(Name + ,DepsDir + ,Deps + ,AppState + ,Locks + ,1). project_app_config(AppInfo, State) -> C = rebar_config:consult(rebar_app_info:dir(AppInfo)), diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index 7e31f6d..bb99584 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -34,6 +34,8 @@ dir/2, out_dir/1, out_dir/2, + resource_type/1, + resource_type/2, source/1, source/2, state/1, @@ -64,6 +66,7 @@ dep_level=0 :: integer(), dir :: file:name(), out_dir :: file:name(), + resource_type :: pkg | src, source :: string() | tuple() | undefined, state :: rebar_state:t() | undefined, is_lock=false :: boolean(), @@ -274,6 +277,14 @@ out_dir(AppInfo=#app_info_t{}, OutDir) -> ebin_dir(#app_info_t{out_dir=OutDir}) -> ec_cnv:to_list(filename:join(OutDir, "ebin")). +-spec resource_type(t(), pkg | src) -> t(). +resource_type(AppInfo=#app_info_t{}, Type) -> + AppInfo#app_info_t{resource_type=Type}. + +-spec resource_type(t()) -> pkg | src. +resource_type(#app_info_t{resource_type=ResourceType}) -> + ResourceType. + -spec source(t(), string() | tuple()) -> t(). source(AppInfo=#app_info_t{}, Source) -> AppInfo#app_info_t{source=Source}. @@ -316,7 +327,7 @@ is_checkout(#app_info_t{is_checkout=IsCheckout}) -> -spec valid(t()) -> boolean(). valid(AppInfo=#app_info_t{valid=undefined, state=State}) -> - case rebar_app_utils:validate_application_info(AppInfo) + case rebar_app_utils:validate_application_info(AppInfo) =:= true andalso rebar_state:has_all_artifacts(State) =:= true of true -> true; diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index e9745c3..17b435c 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -32,6 +32,9 @@ app_src_to_app/2, validate_application_info/1, validate_application_info/2, + parse_deps/5, + parse_deps/6, + dep_to_app/7, format_error/1]). -include("rebar.hrl"). @@ -87,6 +90,109 @@ validate_application_info(AppInfo, AppDetail) -> end end. +-spec parse_deps(binary(), list(), rebar_state:t(), list(), integer()) -> {[rebar_app_info:t()], [tuple()]}. +parse_deps(DepsDir, Deps, State, Locks, Level) -> + parse_deps(root, DepsDir, Deps, State, Locks, Level). + +parse_deps(Parent, DepsDir, Deps, State, Locks, Level) -> + [parse_dep(Dep, Parent, DepsDir, State, Locks, Level) || Dep <- Deps]. + +parse_dep(Dep, Parent, DepsDir, State, Locks, Level) -> + Name = case Dep of + Dep when is_tuple(Dep) -> + element(1, Dep); + Dep -> + Dep + end, + case lists:keyfind(ec_cnv:to_binary(Name), 1, Locks) of + false -> + parse_dep(Parent, Dep, DepsDir, false, State); + LockedDep -> + LockedLevel = element(3, LockedDep), + case LockedLevel > Level of + true -> + parse_dep(Parent, Dep, DepsDir, false, State); + false -> + parse_dep(Parent, LockedDep, DepsDir, true, State) + end + end. + +parse_dep(Parent, {Name, Vsn}, DepsDir, IsLock, State) when is_list(Vsn); is_binary(Vsn) -> + %% Versioned Package dependency + CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), + case rebar_app_info:discover(CheckoutsDir) of + {ok, _App} -> + dep_to_app(root, DepsDir, Name, [], [], IsLock, State); + not_found -> + {PkgName, PkgVsn} = parse_goal(ec_cnv:to_binary(Name) + ,ec_cnv:to_binary(Vsn)), + Source = {pkg, PkgName, PkgVsn}, + rebar_app_info:resource_type(dep_to_app(Parent, DepsDir, PkgName, PkgVsn, Source, IsLock, State), pkg) + end; +parse_dep(Parent, Name, DepsDir, IsLock, State) when is_atom(Name); is_binary(Name) -> + %% Unversioned package dependency + {PkgName, PkgVsn} = get_package(ec_cnv:to_binary(Name), State), + CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), + case rebar_app_info:discover(CheckoutsDir) of + {ok, _App} -> + dep_to_app(root, DepsDir, Name, [], [], IsLock, State); + not_found -> + Source = {pkg, PkgName, PkgVsn}, + rebar_app_info:resource_type(dep_to_app(Parent, DepsDir, PkgName, PkgVsn, Source, IsLock, State), pkg) + end; +parse_dep(Parent, {Name, Source}, DepsDir, IsLock, State) when is_tuple(Source) -> + dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); +parse_dep(Parent, {Name, _Vsn, Source}, DepsDir, IsLock, State) when is_tuple(Source) -> + dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); +parse_dep(Parent, {Name, _Vsn, Source, Opts}, DepsDir, IsLock, State) when is_tuple(Source) -> + ?WARN("Dependency option list ~p in ~p is not supported and will be ignored", [Opts, Name]), + dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); +parse_dep(Parent, {_Name, {pkg, Name, Vsn}, Level}, DepsDir, IsLock, State) when is_integer(Level) -> + CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), + case rebar_app_info:discover(CheckoutsDir) of + {ok, _App} -> + dep_to_app(root, DepsDir, Name, [], [], IsLock, State); + not_found -> + Source = {pkg, Name, Vsn}, + rebar_app_info:resource_type(dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State), pkg) + end; +parse_dep(Parent, {Name, Source, Level}, DepsDir, IsLock, State) when is_tuple(Source) + , is_integer(Level) -> + dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); +parse_dep(_, Dep, _, _, _) -> + throw(?PRV_ERROR({parse_dep, Dep})). + +dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> + CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), + BaseDir = rebar_state:get(State, base_dir, []), + {ok, Dep} = case rebar_app_info:discover(CheckoutsDir) of + {ok, App} -> + {ok, rebar_app_info:is_checkout(App, true)}; + not_found -> + Dir = ec_cnv:to_list(filename:join(DepsDir, Name)), + case rebar_app_info:discover(Dir) of + {ok, App} -> + {ok, App}; + not_found -> + rebar_app_info:new(Name, Vsn, + ec_cnv:to_list(filename:join(DepsDir, Name))) + end + end, + C = rebar_config:consult(rebar_app_info:dir(Dep)), + S = rebar_state:new(rebar_state:new(), C, rebar_app_info:dir(Dep)), + Overrides = rebar_state:get(State, overrides, []), + ParentOverrides = rebar_state:overrides(State), + S1 = rebar_state:set(rebar_state:overrides(S, ParentOverrides++Overrides), base_dir, BaseDir), + Dep1 = rebar_app_info:state(Dep, S1), + AppInfo = rebar_app_info:is_lock(rebar_app_info:source(Dep1, Source), IsLock), + ResourceType = case Source of + {pkg, _, _} -> + pkg; + _ -> + src + end, + rebar_app_info:resource_type(rebar_app_info:parent(AppInfo, Parent), ResourceType). + format_error(Error) -> io_lib:format("~p", [Error]). @@ -94,11 +200,30 @@ format_error(Error) -> %% Internal functions %% =================================================================== +-spec parse_goal(binary(), binary()) -> {binary(), binary()} | {binary(), binary(), binary()}. +parse_goal(Name, Constraint) -> + case re:run(Constraint, "([^\\d]*)(\\d.*)", [{capture, [1,2], binary}]) of + {match, [<<>>, Vsn]} -> + {Name, Vsn}; + {match, [Op, Vsn]} -> + {Name, Vsn, binary_to_atom(Op, utf8)}; + nomatch -> + throw(?PRV_ERROR({bad_constraint, Name, Constraint})) + end. + +get_package(Dep, State) -> + case rebar_state:registry(State) of + {ok, T} -> + {ok, HighestDepVsn} = rebar_packages:find_highest_matching(Dep, "0", T), + {Dep, HighestDepVsn}; + error -> + throw(?PRV_ERROR({load_registry_fail, Dep})) + end. + -spec has_all_beams(file:filename_all(), [module()]) -> true | ?PRV_ERROR({missing_module, module()}). has_all_beams(EbinDir, [Module | ModuleList]) -> - BeamFile = filename:join([EbinDir, - ec_cnv:to_list(Module) ++ ".beam"]), + BeamFile = filename:join([EbinDir, ec_cnv:to_list(Module) ++ ".beam"]), case filelib:is_file(BeamFile) of true -> has_all_beams(EbinDir, ModuleList); diff --git a/src/rebar_digraph.erl b/src/rebar_digraph.erl index ff0a1d2..363253a 100644 --- a/src/rebar_digraph.erl +++ b/src/rebar_digraph.erl @@ -2,8 +2,6 @@ -export([compile_order/1 ,restore_graph/1 - ,cull_deps/2 - ,cull_deps/3 ,subgraph/2 ,format_error/1]). @@ -69,17 +67,6 @@ restore_graph({Vs, Es}) -> end, Es), Graph. -%% Pick packages to fullfill dependencies -%% The first dep while traversing the graph is chosen and any conflicting -%% dep encountered later on is ignored. -cull_deps(Graph, Vertices) -> - cull_deps(Graph, Vertices, sets:new()). - -cull_deps(Graph, Vertices, Seen) -> - Vertices1 = lists:keysort(2, Vertices), - {Solution, Levels, Discarded} = {dict:new(), dict:new(), sets:new()}, - cull_deps(Graph, Vertices1, Levels, Solution, Seen, Discarded). - format_error(no_solution) -> io_lib:format("No solution for packages found.", []). @@ -87,103 +74,6 @@ format_error(no_solution) -> %% Internal Functions %%==================================================================== -cull_deps(_Graph, [], Levels, Solution, _, Discarded) -> - {_, Vertices} = lists:unzip(dict:to_list(Solution)), - LvlVertices = [{Profile, {Parent, App, Vsn, dict:fetch(App, Levels)}} - || {Profile, {Parent,App,Vsn}} <- Vertices], - {ok, LvlVertices, sets:to_list(Discarded)}; -cull_deps(Graph, [{Profile, Level, Vs} | Vertices], Levels, Solution, Seen, Discarded) -> - {NV, NS, LS, DS} = - lists:foldl(fun({Parent, Name, Vsn}, {Acc, SolutionAcc, LevelsAcc, DiscardedAcc}) -> - {SolutionAcc1, LevelsAcc1, DiscardedAcc1} = - maybe_add_to_solution(Profile, Level, Name, {Name, Vsn}, Parent - ,SolutionAcc - ,LevelsAcc, Seen, DiscardedAcc), - OutNeighbors = digraph:out_neighbours(Graph, {Name,Vsn}), - {NewVertices, DiscardedAcc2} = handle_neighbors(Profile, Level, Name - ,OutNeighbors, Acc, SolutionAcc1 - ,Seen, DiscardedAcc1), - {NewVertices, SolutionAcc1, LevelsAcc1, DiscardedAcc2} - end, {[], Solution, Levels, Discarded}, Vs), - NewVertices = combine_profile_levels(Vertices, NV), - cull_deps(Graph, NewVertices, LS, NS, Seen, DS). - -%% Combine lists of deps that have the same profile and level -combine_profile_levels(Vertices, NewVertices) -> - V = lists:foldl(fun({Profile, Level, Vs}, Acc) -> - case ec_lists:find(fun({P, L, _}) -> - P =:= Profile andalso L =:= Level - end, Acc) of - {ok, {_, _, OldVs}=Old} -> - lists:delete(Old, Acc)++[{Profile, Level, lists:keysort(1, OldVs++Vs)}]; - error -> - Acc++[{Profile, Level, Vs}] - end - end, Vertices, NewVertices), - lists:keysort(2, V). - -%% For each outgoing edge of a dep check if it should be added to the solution -%% and add it to the list of vertices to do the same for -handle_neighbors(Profile, Level, Parent, OutNeighbors, Vertices - ,Solution, Seen, Discarded) -> - case lists:foldl(fun({Name, Vsn}=Value, {NewVertices, Discarded1}) -> - case dict:find(Name, Solution) of - {ok, {Profile, {Parent, Name, Vsn}}} -> % already seen - {NewVertices, - Discarded1}; - {ok, _} -> % conflict resolution! - %% Warn on different version - {NewVertices, - sets:add_element(Value, Discarded1)}; - error -> - %% We check Seen separately because we don't care - %% to warn if the exact same version of a package - %% was already part of the solution but we do - %% if it was simply seen in source deps - case sets:is_element(Name, Seen) of - true -> - {NewVertices, - sets:add_element(Value, Discarded1)}; - false -> - {[{Parent, Name, Vsn} | NewVertices], - Discarded1} - end - end - end, {[], Discarded}, OutNeighbors) of - {[], DiscardedAcc2} -> - {Vertices, DiscardedAcc2}; - {NewVertices1, DiscardedAcc2} -> - {Vertices++[{Profile, Level+1, NewVertices1}] ,DiscardedAcc2} - end. - -maybe_add_to_solution(Profile, Level, Key, {Name, Vsn}=Value, Parent - ,Solution, Levels, Seen, Discarded) -> - case dict:find(Key, Solution) of - {ok, {Profile, {Parent, Name, Vsn}}} -> % already seen - {Solution, - Levels, - Discarded}; - {ok, _} -> % conflict resolution! - %% Warn on different version - {Solution, - Levels, - sets:add_element(Value, Discarded)}; - error -> - %% We check Seen separately because we don't care to warn if the exact - %% same version of a package was already part of the solution but we do - %% if it was simply seen in source deps - case sets:is_element(Name, Seen) of - true -> - {Solution, - Levels, - sets:add_element(Value, Discarded)}; - false -> - {dict:store(Key, {Profile, {Parent, Name, Vsn}}, Solution), - dict:store(Key, Level, Levels), - Discarded} - end - end. - subgraph(Graph, Vertices) -> digraph_utils:subgraph(Graph, Vertices). diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index ca3b676..5f3d1c9 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -1,6 +1,7 @@ -module(rebar_packages). -export([packages/1 + ,packages_graph/1 ,registry/1 ,package_dir/1 ,check_registry/3 @@ -15,32 +16,43 @@ -type vsn() :: binary(). -type package() :: pkg_name() | {pkg_name(), vsn()}. --spec packages(rebar_state:t()) -> {rebar_dict(), rebar_digraph()}. +-spec packages(rebar_state:t()) -> rebar_dict(). %% DON'T USE IT! Use rebar_state:packages(State) instead. packages(State) -> RegistryDir = package_dir(State), DictFile = filename:join(RegistryDir, "dict"), + + try + {ok, DictBinary} = file:read_file(DictFile), + binary_to_term(DictBinary) + catch + _:_ -> + ?ERROR("Bad packages index, try to fix with `rebar3 update`", []), + dict:new() + end. + +-spec packages_graph(rebar_state:t()) -> rebar_digraph(). +packages_graph(State) -> + RegistryDir = package_dir(State), Edges = filename:join(RegistryDir, "edges"), Vertices = filename:join(RegistryDir, "vertices"), Neighbors = filename:join(RegistryDir, "neighbors"), - case lists:all(fun(X) -> filelib:is_file(X) end, [DictFile, Edges, Vertices, Neighbors]) of + case lists:all(fun(X) -> filelib:is_file(X) end, [Edges, Vertices, Neighbors]) of true -> try - {ok, DictBinary} = file:read_file(DictFile), - Dict = binary_to_term(DictBinary), {ok, EdgesTab} = ets:file2tab(Edges), {ok, VerticesTab} = ets:file2tab(Vertices), {ok, NeighborsTab} = ets:file2tab(Neighbors), - {Dict, {digraph, EdgesTab, VerticesTab, NeighborsTab, true}} + {digraph, EdgesTab, VerticesTab, NeighborsTab, true} catch _:_ -> ?ERROR("Bad packages index, try to fix with `rebar3 update`", []), - {dict:new(), digraph:new()} + digraph:new() end; false -> ?ERROR("Bad packages index, try to fix with `rebar3 update`", []), - {dict:new(), digraph:new()} + digraph:new() end. -spec registry(rebar_state:t()) -> {ok, ets:tid()} | {error, any()}. diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index 3a5a7cd..0883fab 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -36,8 +36,6 @@ -include_lib("providers/include/providers.hrl"). -export([handle_deps_as_profile/4, - parse_deps/5, - parse_deps/6, profile_dep_dir/2, find_cycles/1, cull_compile/2]). @@ -125,17 +123,12 @@ handle_deps_as_profile(Profile, State, Deps, Upgrade) -> Locks = [], Level = 0, DepsDir = profile_dep_dir(State, Profile), - {SrcDeps, PkgDeps} = parse_deps(DepsDir, Deps, State, Locks, Level), - AllSrcProfileDeps = [{Profile, SrcDeps, Locks, Level}], - AllPkgProfileDeps = case PkgDeps of - [] -> - []; - _ -> - [{Profile, Level, PkgDeps}] - end, - {AllApps, PkgDeps1, Seen, State1} = handle_profile_level(AllSrcProfileDeps, AllPkgProfileDeps, Locks, sets:new(), Upgrade, State), - - handle_profile_pkg_level(PkgDeps1, AllApps, Seen, Upgrade, [], State1). + Deps1 = rebar_app_utils:parse_deps(DepsDir, Deps, State, Locks, Level), + ProfileLevelDeps = [{Profile, Deps1, Level}], + Graph = rebar_state:packages_graph(State), + Registry = rebar_packages:registry(State), + State1 = rebar_state:packages_graph(rebar_state:registry(State, Registry), Graph), + handle_profile_level(ProfileLevelDeps, [], sets:new(), Upgrade, Locks, State1, Graph). %% =================================================================== %% Internal functions @@ -144,71 +137,33 @@ handle_deps_as_profile(Profile, State, Deps, Upgrade) -> %% finds all the deps in `{deps, ...}` for each profile provided. deps_per_profile(Profiles, Upgrade, State) -> Level = 0, - {AllProfileDeps, PkgDeps} = lists:foldl(fun(Profile, {SrcAcc, PkgAcc}) -> - case parse_profile_deps(State, Profile, Level) of - {Src, {_, _, []}} -> - {[Src | SrcAcc], PkgAcc}; - {Src, Pkg} -> - {[Src | SrcAcc], [Pkg | PkgAcc]} - end - end, {[], []}, Profiles), - {AllApps, PkgDeps1, Seen, State1} = handle_profile_level(AllProfileDeps, PkgDeps, [], sets:new(), Upgrade, State), Locks = rebar_state:get(State, {locks, default}, []), - handle_profile_pkg_level(PkgDeps1, AllApps, Seen, Upgrade, Locks, State1). - -parse_profile_deps(State, Profile, Level) -> - Locks = rebar_state:get(State, {locks, Profile}, []), - {SrcDeps, PkgDeps} = rebar_state:get(State, {parsed_deps, Profile}, {[], []}), - {{Profile, SrcDeps, Locks, Level}, {Profile, Level, PkgDeps}}. + Deps = lists:foldl(fun(Profile, DepAcc) -> + [parsed_profile_deps(State, Profile, Level) | DepAcc] + end, [], Profiles), + Graph = rebar_state:packages_graph(State), + Registry = rebar_packages:registry(State), + State1 = rebar_state:packages_graph(rebar_state:registry(State, Registry), Graph), + handle_profile_level(Deps, [], sets:new(), Upgrade, Locks, State1, Graph). + +parsed_profile_deps(State, Profile, Level) -> + ParsedDeps = rebar_state:get(State, {parsed_deps, Profile}, []), + {Profile, ParsedDeps, Level}. %% Level-order traversal of all dependencies, across profiles. %% If profiles x,y,z are present, then the traversal will go: %% x0, y0, z0, x1, y1, z1, ..., xN, yN, zN. -handle_profile_level([], PkgDeps, SrcApps, Seen, _Upgrade, State) -> - {SrcApps, PkgDeps, Seen, State}; -handle_profile_level([{Profile, SrcDeps, Locks, Level} | Rest], PkgDeps, SrcApps, Seen, Upgrade, State) -> - {SrcDeps1, PkgDeps1, SrcApps1, State1, Seen1, Locks1} = - update_src_deps(Profile, Level, SrcDeps, [], SrcApps - ,State, Upgrade, Seen, Locks), - SrcDeps2 = case SrcDeps1 of +handle_profile_level([], Apps, _Seen, _Upgrade, _Locks, State, _Graph) -> + {Apps, State}; +handle_profile_level([{Profile, Deps, Level} | Rest], Apps, Seen, Upgrade, Locks, State, Graph) -> + {Deps1, Apps1, State1, Seen1} = + update_deps(Profile, Level, Deps, Apps + ,State, Upgrade, Seen, Locks, Graph), + Deps2 = case Deps1 of [] -> Rest; - _ -> Rest ++ [{Profile, SrcDeps1, Locks1, Level+1}] + _ -> Rest ++ [{Profile, Deps1, Level+1}] end, - handle_profile_level(SrcDeps2, case PkgDeps1 of - [] -> - PkgDeps; - _ -> - [{Profile, Level+1, PkgDeps1} | PkgDeps] - end, SrcApps1, sets:union(Seen, Seen1), Upgrade, State1). - -handle_profile_pkg_level([], AllApps, _Seen, _Upgrade, _Locks, State) -> - {AllApps, State}; -handle_profile_pkg_level(PkgDeps, AllApps, Seen, Upgrade, Locks, State) -> - %% Read in package index and dep graph - {Packages, Graph} = rebar_state:packages(State), - Registry = rebar_state:registry(State), - State1 = rebar_state:packages(rebar_state:registry(State, Registry) - ,{Packages, Graph}), - - S = case rebar_digraph:cull_deps(Graph, lists:keysort(2, PkgDeps), Seen) of - {ok, [], []} -> - throw({rebar_digraph, no_solution}); - {ok, [], Discarded} -> - [warn_skip_pkg(Pkg, State) || Upgrade =:= false, - Pkg <- Discarded, - not(pkg_locked(Pkg, Locks))], - []; - {ok, Solution, []} -> - Solution; - {ok, Solution, Discarded} -> - [warn_skip_pkg(Pkg, State) || Upgrade =:= false, - Pkg <- Discarded, - not(pkg_locked(Pkg, Locks))], - Solution - end, - - {PkgApps, State2} = update_pkg_deps(S, Packages, Upgrade, Seen, State1, Locks), - {AllApps++PkgApps, State2}. + handle_profile_level(Deps2, Apps1, sets:union(Seen, Seen1), Upgrade, Locks, State1, Graph). find_cycles(Apps) -> case rebar_digraph:compile_order(Apps) of @@ -220,38 +175,6 @@ find_cycles(Apps) -> cull_compile(TopSortedDeps, ProjectApps) -> lists:dropwhile(fun not_needs_compile/1, TopSortedDeps -- ProjectApps). -pkg_locked({_, Name, _, _}, Locks) -> - pkg_locked(Name, Locks); -pkg_locked({Name, _}, Locks) -> - pkg_locked(Name, Locks); -pkg_locked(Name, Locks) -> - false =/= lists:keyfind(Name, 1, Locks). - -update_pkg_deps(Pkgs, Packages, Upgrade, Seen, State, Locks) -> - {Solved, _, State1} - = lists:foldl(fun({Profile, Pkg}, {Acc, SeenAcc, StateAcc}) -> - DepsDir = profile_dep_dir(State, Profile), - handle_pkg_dep(Profile, Pkg, Packages, Upgrade, DepsDir, Acc, SeenAcc, Locks, StateAcc) - end, {[], Seen, State}, Pkgs), - {Solved, State1}. - -handle_pkg_dep(Profile, Pkg, Packages, Upgrade, DepsDir, Fetched, Seen, Locks, State) -> - IsLock = pkg_locked(Pkg, Locks), - AppInfo = package_to_app(DepsDir, Packages, Pkg, IsLock, State), - case sets:is_element(rebar_app_info:name(AppInfo), Seen) of - true -> - {Fetched, Seen, State}; - false -> - Deps = rebar_app_info:deps(AppInfo), - Level = rebar_app_info:dep_level(AppInfo), - {NewSeen, NewState} = maybe_lock(Profile, AppInfo, Seen, State, Level), - {_, AppInfo1} = maybe_fetch(AppInfo, Profile, Upgrade, Seen, NewState), - {AppInfo2, _, _, _, _} = - handle_dep(NewState, Profile, DepsDir, AppInfo1, Locks, Level), - AppInfo3 = rebar_app_info:deps(AppInfo2, Deps), - {[AppInfo3 | Fetched], NewSeen, NewState} - end. - maybe_lock(Profile, AppInfo, Seen, State, Level) -> Name = rebar_app_info:name(AppInfo), case rebar_app_info:is_checkout(AppInfo) of @@ -278,49 +201,28 @@ maybe_lock(Profile, AppInfo, Seen, State, Level) -> {sets:add_element(Name, Seen), State} end. -package_to_app(DepsDir, Packages, {Parent, Name, Vsn, Level}, IsLock, State) -> - case dict:find({Name, Vsn}, Packages) of - error -> - case rebar_packages:check_registry(Name, Vsn, State) of - true -> - throw(?PRV_ERROR({not_rebar_package, Name, Vsn})); - false -> - throw(?PRV_ERROR({missing_package, Name, Vsn})) - end; - {ok, PkgDeps} -> - Source = {pkg, Name, Vsn}, - AppInfo = new_dep(root, DepsDir, Name, Vsn, Source, IsLock, State), - AppInfo1 = rebar_app_info:dep_level(rebar_app_info:deps(AppInfo, PkgDeps), Level), - BaseDir = rebar_state:get(State, base_dir, []), - AppState1 = rebar_state:set(rebar_app_info:state(AppInfo1), base_dir, BaseDir), - rebar_app_info:parent(rebar_app_info:state(AppInfo1, AppState1), Parent) - end. - --spec update_src_deps(atom(), non_neg_integer(), list(), list(), list(), rebar_state:t(), boolean(), sets:set(binary()), list()) -> {list(), list(), list(), rebar_state:t(), sets:set(binary()), list()}. -update_src_deps(Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, Locks) -> +update_deps(Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks, Graph) -> lists:foldl( - fun(AppInfo, {SrcDepsAcc, PkgDepsAcc, SrcAppsAcc, StateAcc, SeenAcc, LocksAcc}) -> - update_src_dep(AppInfo, Profile, Level, - SrcDepsAcc, PkgDepsAcc, SrcAppsAcc, StateAcc, - Upgrade, SeenAcc, Locks, LocksAcc) + fun(AppInfo, {DepsAcc, AppsAcc, StateAcc, SeenAcc}) -> + update_dep(AppInfo, Profile, Level, + DepsAcc, AppsAcc, StateAcc, + Upgrade, SeenAcc, Locks, Graph) end, - {[], PkgDeps, SrcApps, State, Seen, Locks}, - rebar_utils:sort_deps(SrcDeps)). + {[], Apps, State, Seen}, + rebar_utils:sort_deps(Deps)). - -update_src_dep(AppInfo, Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, BaseLocks, Locks) -> +update_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks, Graph) -> %% If not seen, add to list of locks to write out Name = rebar_app_info:name(AppInfo), case sets:is_element(Name, Seen) of true -> - update_seen_src_dep(AppInfo, Profile, Level, - SrcDeps, PkgDeps, SrcApps, - State, Upgrade, Seen, BaseLocks, Locks); + update_seen_dep(AppInfo, Profile, Level, + Deps, Apps, + State, Upgrade, Seen, Locks); false -> - update_unseen_src_dep(AppInfo, Profile, Level, - SrcDeps, PkgDeps, SrcApps, - State, Upgrade, Seen, Locks) - + update_unseen_dep(AppInfo, Profile, Level, + Deps, Apps, + State, Upgrade, Seen, Locks, Graph) end. profile_dep_dir(State, Profile) -> @@ -329,41 +231,31 @@ profile_dep_dir(State, Profile) -> _ -> rebar_dir:deps_dir(State) end. -update_seen_src_dep(AppInfo, _Profile, _Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, BaseLocks, Locks) -> +update_seen_dep(AppInfo, _Profile, _Level, Deps, Apps, State, Upgrade, Seen, Locks) -> Name = rebar_app_info:name(AppInfo), %% If seen from lock file or user requested an upgrade %% don't print warning about skipping - case lists:keymember(Name, 1, BaseLocks) of + case lists:keymember(Name, 1, Locks) of false when Upgrade -> ok; false when not Upgrade -> warn_skip_deps(AppInfo, State); true -> ok end, - {SrcDeps, PkgDeps, SrcApps, State, Seen, Locks}. + {Deps, Apps, State, Seen}. -update_unseen_src_dep(AppInfo, Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, Locks) -> +update_unseen_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks, Graph) -> {NewSeen, State1} = maybe_lock(Profile, AppInfo, Seen, State, Level), {_, AppInfo1} = maybe_fetch(AppInfo, Profile, Upgrade, Seen, State1), - {NewSrcDeps, NewPkgDeps, NewSrcApps, State2, NewLocks} = - handle_dep(AppInfo1, Profile, SrcDeps, PkgDeps, SrcApps, - Level, State1, Locks), - {NewSrcDeps, NewPkgDeps, NewSrcApps, State2, NewSeen, NewLocks}. - -handle_dep(AppInfo, Profile, SrcDeps, PkgDeps, SrcApps, Level, State, Locks) -> DepsDir = profile_dep_dir(State, Profile), - {AppInfo1, NewSrcDeps, NewPkgDeps, NewLocks, State1} = - handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level), - AppInfo2 = rebar_app_info:dep_level(AppInfo1, Level), - {NewSrcDeps ++ SrcDeps - ,NewPkgDeps++PkgDeps - ,[AppInfo2 | SrcApps] - ,State1 - ,NewLocks}. - --spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer()) -> - {rebar_app_info:t(), [rebar_app_info:t()], [pkg_dep()], [integer()], rebar_state:t()}. -handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) -> + {AppInfo2, NewDeps, State2} = + handle_dep(State1, Profile, DepsDir, AppInfo1, Locks, Level, Graph), + AppInfo3 = rebar_app_info:dep_level(AppInfo2, Level), + {NewDeps ++ Deps, [AppInfo3 | Apps], State2, NewSeen}. + +-spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer(), rebar_dict()) -> {rebar_app_info:t(), [rebar_app_info:t()], [pkg_dep()], [integer()], rebar_state:t()}. +handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level, Graph) -> Profiles = rebar_state:current_profiles(State), Name = rebar_app_info:name(AppInfo), + Vsn = rebar_app_info:original_vsn(AppInfo), %% Deps may be under a sub project app, find it and use its state if so S0 = rebar_app_info:state(AppInfo), @@ -375,23 +267,26 @@ handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) -> S3 = rebar_state:apply_profiles(S2, Profiles), Plugins = rebar_state:get(S3, plugins, []), S4 = rebar_state:set(S3, {plugins, Profile}, Plugins), - AppInfo1 = rebar_app_info:state(AppInfo, S4), rebar_utils:check_min_otp_version(rebar_state:get(S4, minimum_otp_vsn, undefined)), rebar_utils:check_blacklisted_otp_versions(rebar_state:get(S4, blacklisted_otp_vsns, [])), %% Dep may have plugins to install. Find and install here. S5 = rebar_plugins:install(S4), - AppInfo2 = rebar_app_info:state(AppInfo1, S5), + AppInfo1 = rebar_app_info:state(AppInfo, S5), %% Upgrade lock level to be the level the dep will have in this dep tree - Deps = rebar_state:get(S5, {deps, default}, []), - NewLocks = Locks++[{DepName, Source, LockLevel+Level} || - {DepName, Source, LockLevel} <- rebar_state:get(S5, {locks, default}, [])], - AppInfo3 = rebar_app_info:deps(AppInfo2, rebar_state:deps_names(Deps)), - {SrcDeps, PkgDeps} = parse_deps(rebar_app_info:name(AppInfo3), DepsDir, Deps - ,S5, NewLocks, Level+1), - {AppInfo3, SrcDeps, PkgDeps, NewLocks, State}. + case rebar_app_info:resource_type(AppInfo1) of + pkg -> + NewDeps = digraph:out_neighbours(Graph, {ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}), + NewDeps1 = rebar_app_utils:parse_deps(Name, DepsDir, NewDeps, S5, Locks, Level+1), + {rebar_app_info:deps(AppInfo1, NewDeps), NewDeps1, State}; + _ -> + Deps = rebar_state:get(S5, {deps, default}, []), + AppInfo2 = rebar_app_info:deps(AppInfo1, rebar_state:deps_names(Deps)), + Deps1 = rebar_app_utils:parse_deps(Name, DepsDir, Deps, S5, Locks, Level+1), + {AppInfo2, Deps1, State} + end. -spec maybe_fetch(rebar_app_info:t(), atom(), boolean(), sets:set(binary()), rebar_state:t()) -> {boolean(), rebar_app_info:t()}. @@ -406,20 +301,20 @@ maybe_fetch(AppInfo, Profile, Upgrade, Seen, State) -> false -> true = fetch_app(AppInfo, AppDir, State), maybe_symlink_default(State, Profile, AppDir, AppInfo), - {true, update_app_info(AppDir, AppInfo)}; + {true, rebar_app_info:valid(update_app_info(AppDir, AppInfo), false)}; {true, AppInfo1} -> %% Preserve the state we created with overrides + AppInfo2 = copy_app_info(AppInfo, AppInfo1), AppState = rebar_app_info:state(AppInfo), - Parent = rebar_app_info:parent(AppInfo), - Source = rebar_app_info:source(AppInfo), - AppInfo2 = rebar_app_info:parent(rebar_app_info:state(AppInfo1, AppState), Parent), - AppInfo3 = rebar_app_info:source(AppInfo2, Source), - case sets:is_element(rebar_app_info:name(AppInfo), Seen) of + AppInfo3 = rebar_app_info:state(AppInfo2, AppState), + case sets:is_element(rebar_app_info:name(AppInfo3), Seen) of true -> {false, AppInfo3}; false -> maybe_symlink_default(State, Profile, AppDir, AppInfo3), - {maybe_upgrade(AppInfo, AppDir, Upgrade, State), AppInfo3} + MaybeUpgrade = maybe_upgrade(AppInfo, AppDir, Upgrade, State), + AppInfo4 = update_app_info(AppDir, AppInfo3), + {MaybeUpgrade, AppInfo4} end end end. @@ -467,106 +362,6 @@ make_relative_to_root(State, Path) when is_list(Path) -> Root = rebar_dir:root_dir(State), rebar_dir:make_relative_path(Path, Root). --spec parse_deps(binary(), list(), rebar_state:t(), list(), integer()) -> {[rebar_app_info:t()], [tuple()]}. -parse_deps(DepsDir, Deps, State, Locks, Level) -> - parse_deps(root, DepsDir, Deps, State, Locks, Level). - -parse_deps(Parent, DepsDir, Deps, State, Locks, Level) -> - lists:foldl(fun(Dep, Acc) -> - Name = case Dep of - Dep when is_tuple(Dep) -> - element(1, Dep); - Dep -> - Dep - end, - case lists:keyfind(ec_cnv:to_binary(Name), 1, Locks) of - false -> - parse_dep(Parent, Dep, Acc, DepsDir, false, State); - LockedDep -> - LockedLevel = element(3, LockedDep), - case LockedLevel > Level of - true -> - parse_dep(Parent, Dep, Acc, DepsDir, false, State); - false -> - parse_dep(Parent, LockedDep, Acc, DepsDir, true, State) - end - end - end, {[], []}, Deps). - -parse_dep(Parent, {Name, Vsn}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_list(Vsn) -> - %% Versioned Package dependency - CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), - case rebar_app_info:discover(CheckoutsDir) of - {ok, _App} -> - Dep = new_dep(root, DepsDir, Name, [], [], IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; - not_found -> - {SrcDepsAcc, [parse_goal(Parent - ,ec_cnv:to_binary(Name) - ,ec_cnv:to_binary(Vsn)) | PkgDepsAcc]} - end; -parse_dep(Parent, Name, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_atom(Name); is_binary(Name) -> - %% Unversioned package dependency - {PkgName, PkgVsn} = get_package(ec_cnv:to_binary(Name), State), - CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), - case rebar_app_info:discover(CheckoutsDir) of - {ok, _App} -> - Dep = new_dep(root, DepsDir, Name, [], [], IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; - not_found -> - {SrcDepsAcc, [{Parent, PkgName, PkgVsn} | PkgDepsAcc]} - end; -parse_dep(Parent, {Name, Source}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source) -> - Dep = new_dep(Parent, DepsDir, Name, [], Source, IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; -parse_dep(Parent, {Name, _Vsn, Source}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source) -> - Dep = new_dep(Parent, DepsDir, Name, [], Source, IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; -parse_dep(Parent, {Name, _Vsn, Source, Opts}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source) -> - ?WARN("Dependency option list ~p in ~p is not supported and will be ignored", [Opts, Name]), - Dep = new_dep(Parent, DepsDir, Name, [], Source, IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; -parse_dep(Parent, {_Name, {pkg, Name, Vsn}, Level}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_integer(Level) -> - CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), - case rebar_app_info:discover(CheckoutsDir) of - {ok, _App} -> - Dep = new_dep(root, DepsDir, Name, [], [], IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; - not_found -> - {SrcDepsAcc, [{Parent, Name, Vsn} | PkgDepsAcc]} - end; -parse_dep(Parent, {Name, Source, Level}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source) - , is_integer(Level) -> - Dep = new_dep(Parent, DepsDir, Name, [], Source, IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; -parse_dep(_, Dep, _, _, _, _) -> - throw(?PRV_ERROR({parse_dep, Dep})). - - -new_dep(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> - CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), - {ok, Dep} = case rebar_app_info:discover(CheckoutsDir) of - {ok, App} -> - {ok, rebar_app_info:is_checkout(App, true)}; - not_found -> - Dir = ec_cnv:to_list(filename:join(DepsDir, Name)), - case rebar_app_info:discover(Dir) of - {ok, App} -> - {ok, App}; - not_found -> - rebar_app_info:new(Name, Vsn, - ec_cnv:to_list(filename:join(DepsDir, Name))) - end - end, - C = rebar_config:consult(rebar_app_info:dir(Dep)), - S = rebar_state:new(rebar_state:new(), C, rebar_app_info:dir(Dep)), - Overrides = rebar_state:get(State, overrides, []), - ParentOverrides = rebar_state:overrides(State), - Dep1 = rebar_app_info:state(Dep, - rebar_state:overrides(S, ParentOverrides++Overrides)), - AppInfo = rebar_app_info:is_lock(rebar_app_info:source(Dep1, Source), IsLock), - rebar_app_info:parent(AppInfo, Parent). - fetch_app(AppInfo, AppDir, State) -> ?INFO("Fetching ~s (~p)", [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)]), Source = rebar_app_info:source(AppInfo), @@ -577,15 +372,24 @@ fetch_app(AppInfo, AppDir, State) -> %% be read in an parsed. update_app_info(AppDir, AppInfo) -> {ok, Found} = rebar_app_info:discover(AppDir), - Parent = rebar_app_info:parent(AppInfo), - Source = rebar_app_info:source(AppInfo), AppDetails = rebar_app_info:app_details(Found), + Vsn = rebar_app_info:original_vsn(Found), Applications = proplists:get_value(applications, AppDetails, []), IncludedApplications = proplists:get_value(included_applications, AppDetails, []), - AppInfo1 = rebar_app_info:applications( + AppInfo1 = rebar_app_info:original_vsn(rebar_app_info:applications( rebar_app_info:app_details(AppInfo, AppDetails), - IncludedApplications++Applications), - rebar_app_info:source(rebar_app_info:parent(rebar_app_info:valid(AppInfo1, false), Parent), Source). + IncludedApplications++Applications), Vsn), + AppInfo2 = copy_app_info(AppInfo, AppInfo1), + rebar_app_info:valid(AppInfo2, undefined). + +copy_app_info(OldAppInfo, NewAppInfo) -> + ResourceType = rebar_app_info:resource_type(OldAppInfo), + Parent = rebar_app_info:parent(OldAppInfo), + Source = rebar_app_info:source(OldAppInfo), + + rebar_app_info:resource_type( + rebar_app_info:source( + rebar_app_info:parent(NewAppInfo, Parent), Source), ResourceType). maybe_upgrade(AppInfo, AppDir, Upgrade, State) -> Source = rebar_app_info:source(AppInfo), @@ -608,17 +412,6 @@ maybe_upgrade(AppInfo, AppDir, Upgrade, State) -> false end. --spec parse_goal(binary() | root, binary(), binary()) -> {binary(), binary()} | {binary(), binary(), binary()}. -parse_goal(Parent, Name, Constraint) -> - case re:run(Constraint, "([^\\d]*)(\\d.*)", [{capture, [1,2], binary}]) of - {match, [<<>>, Vsn]} -> - {Parent, Name, Vsn}; - {match, [Op, Vsn]} -> - {Parent, Name, Vsn, binary_to_atom(Op, utf8)}; - nomatch -> - throw(?PRV_ERROR({bad_constraint, Name, Constraint})) - end. - warn_skip_deps(AppInfo, State) -> Msg = "Skipping ~s (from ~p) as an app of the same name " "has already been fetched", @@ -629,25 +422,7 @@ warn_skip_deps(AppInfo, State) -> true -> ?ERROR(Msg, Args), ?FAIL end. -warn_skip_pkg({Name, Source}, State) -> - Msg = "Skipping ~s (version ~s from package index) as an app of the same " - "name has already been fetched", - Args = [Name, Source], - case rebar_state:get(State, deps_error_on_conflict, false) of - false -> ?WARN(Msg, Args); - true -> ?ERROR(Msg, Args), ?FAIL - end. - not_needs_compile(App) -> not(rebar_app_info:is_checkout(App)) andalso rebar_app_info:valid(App) andalso rebar_state:has_all_artifacts(rebar_app_info:state(App)) =:= true. - -get_package(Dep, State) -> - case rebar_state:registry(State) of - {ok, T} -> - {ok, HighestDepVsn} = rebar_packages:find_highest_matching(Dep, "0", T), - {Dep, HighestDepVsn}; - error -> - throw(?PRV_ERROR({load_registry_fail, Dep})) - end. diff --git a/src/rebar_prv_packages.erl b/src/rebar_prv_packages.erl index 82ed2f7..44cde4e 100644 --- a/src/rebar_prv_packages.erl +++ b/src/rebar_prv_packages.erl @@ -27,7 +27,7 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - {Dict, _} = rebar_state:packages(State), + Dict = rebar_packages:packages(State), print_packages(Dict), {ok, State}. diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl index f49eafe..7385cfe 100644 --- a/src/rebar_prv_upgrade.erl +++ b/src/rebar_prv_upgrade.erl @@ -54,7 +54,7 @@ do(State) -> Deps0 = top_level_deps(Deps, Locks), State1 = rebar_state:set(State, {deps, default}, Deps0), DepsDir = rebar_prv_install_deps:profile_dep_dir(State, default), - D = rebar_prv_install_deps:parse_deps(root, DepsDir, Deps0, State1, Locks0, 0), + D = rebar_app_utils:parse_deps(root, DepsDir, Deps0, State1, Locks0, 0), State2 = rebar_state:set(State1, {parsed_deps, default}, D), State3 = rebar_state:set(State2, {locks, default}, Locks0), State4 = rebar_state:set(State3, upgrade, true), @@ -83,7 +83,6 @@ format_error({transitive_dependency, Name}) -> format_error(Reason) -> io_lib:format("~p", [Reason]). - parse_names(Bin, Locks) -> case lists:usort(re:split(Bin, <<" *, *">>, [trim])) of %% Nothing submitted, use *all* apps diff --git a/src/rebar_state.erl b/src/rebar_state.erl index 7616151..54e7f13 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -38,7 +38,8 @@ overrides/1, overrides/2, apply_overrides/2, - packages/1, packages/2, + packages_graph/1, packages_graph/2, + packages/1, registry/1, registry/2, resources/1, resources/2, add_resource/2, @@ -65,7 +66,8 @@ all_plugin_deps = [] :: [rebar_app_info:t()], all_deps = [] :: [rebar_app_info:t()], - packages = undefined :: {rebar_dict(), rebar_digraph()} | undefined, + packages = undefined :: rebar_dict(), + packages_graph = undefined :: rebar_digraph() | undefined, registry = undefined :: {ok, ets:tid()} | error | undefined, overrides = [], resources = [], @@ -96,7 +98,9 @@ new(Config) when is_list(Config) -> load_package_registry(Config0) -> Registry = rebar_packages:registry(Config0), Packages = rebar_packages:packages(Config0), + PackagesGraph = rebar_packages:packages_graph(Config0), Config0#state_t{registry = Registry, + packages_graph = PackagesGraph, packages = Packages}. -spec new(t() | atom(), list()) -> t(). @@ -451,8 +455,13 @@ packages(#state_t{packages=undefined}) -> packages(#state_t{packages=Packages}) -> Packages. -packages(State, Packages) -> - State#state_t{packages=Packages}. +packages_graph(#state_t{packages_graph=undefined}) -> + throw(packages_usage_error); +packages_graph(#state_t{packages_graph=PackagesGraph}) -> + PackagesGraph. + +packages_graph(State, PackagesGraph) -> + State#state_t{packages_graph=PackagesGraph}. registry(#state_t{registry=undefined}) -> throw(registry_usage_error); -- cgit v1.1 From 6242c8155499df2fe141772830565b041959c7b9 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 21 Aug 2015 08:56:49 -0500 Subject: include rebar3 in supported packages buildtool --- src/rebar_prv_update.erl | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index 55c70e2..e73c769 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -84,6 +84,7 @@ write_registry(Dict, {digraph, Edges, Vertices, Neighbors, _}, State) -> is_supported(<<"make">>) -> true; is_supported(<<"rebar">>) -> true; +is_supported(<<"rebar3">>) -> true; is_supported(_) -> false. hex_to_graph(Filename) -> -- cgit v1.1 From cf5390f01876ff8d9e70cff521740ab0dd805929 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 21 Aug 2015 12:18:30 -0500 Subject: replace use of dict of packages and registry with single ets table --- src/rebar.hrl | 6 ++ src/rebar3.erl | 13 +++- src/rebar_app_utils.erl | 9 +-- src/rebar_packages.erl | 168 ++++++++++++++++++++--------------------- src/rebar_plugins.erl | 10 ++- src/rebar_prv_install_deps.erl | 39 ++++------ src/rebar_prv_packages.erl | 12 ++- src/rebar_prv_update.erl | 95 +++++++++++------------ src/rebar_state.erl | 52 ++----------- 9 files changed, 178 insertions(+), 226 deletions(-) (limited to 'src') diff --git a/src/rebar.hrl b/src/rebar.hrl index 8a702df..961b8ea 100644 --- a/src/rebar.hrl +++ b/src/rebar.hrl @@ -23,8 +23,14 @@ -define(DEFAULT_RELEASE_DIR, "rel"). -define(DEFAULT_CONFIG_FILE, "rebar.config"). -define(DEFAULT_CDN, "https://s3.amazonaws.com/s3.hex.pm/tarballs"). +-define(DEFAULT_HEX_REGISTRY, "https://s3.amazonaws.com/s3.hex.pm/registry.ets.gz"). -define(LOCK_FILE, "rebar.lock"). +-define(PACKAGE_INDEX_VERSION, 3). +-define(PACKAGE_TABLE, package_index). +-define(INDEX_FILE, "packages.idx"). +-define(REGISTRY_FILE, "registry"). + -ifdef(namespaced_types). -type rebar_dict() :: dict:dict(). -else. diff --git a/src/rebar3.erl b/src/rebar3.erl index 8004443..ab3e0eb 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -105,6 +105,7 @@ run_aux(State, RawArgs) -> {ok, Providers} = application:get_env(rebar, providers), %% Providers can modify profiles stored in opts, so set default after initializing providers State4 = rebar_state:create_logic_providers(Providers, State3), + rebar_packages:packages(State4), State5 = rebar_plugins:project_apps_install(State4), State6 = rebar_state:default(State5, rebar_state:opts(State5)), @@ -280,9 +281,15 @@ state_from_global_config(Config, GlobalConfigFile) -> %% We don't want to worry about global plugin install state effecting later %% usage. So we throw away the global profile state used for plugin install. GlobalConfigThrowAway = rebar_state:current_profiles(GlobalConfig, [global]), - GlobalState = rebar_plugins:handle_plugins(global, - rebar_state:get(GlobalConfigThrowAway, plugins, []), - GlobalConfigThrowAway), + GlobalState = case rebar_state:get(GlobalConfigThrowAway, plugins, []) of + [] -> + GlobalConfigThrowAway; + GlobalPluginsToInstall -> + rebar_packages:packages(GlobalConfigThrowAway), + rebar_plugins:handle_plugins(global, + GlobalPluginsToInstall, + GlobalConfigThrowAway) + end, GlobalPlugins = rebar_state:providers(GlobalState), GlobalConfig2 = rebar_state:set(GlobalConfig, plugins, []), GlobalConfig3 = rebar_state:set(GlobalConfig2, {plugins, global}, rebar_state:get(GlobalConfigThrowAway, plugins, [])), diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index 17b435c..b14c50a 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -212,13 +212,8 @@ parse_goal(Name, Constraint) -> end. get_package(Dep, State) -> - case rebar_state:registry(State) of - {ok, T} -> - {ok, HighestDepVsn} = rebar_packages:find_highest_matching(Dep, "0", T), - {Dep, HighestDepVsn}; - error -> - throw(?PRV_ERROR({load_registry_fail, Dep})) - end. + {ok, HighestDepVsn} = rebar_packages:find_highest_matching(Dep, "0", ?PACKAGE_TABLE, State), + {Dep, HighestDepVsn}. -spec has_all_beams(file:filename_all(), [module()]) -> true | ?PRV_ERROR({missing_module, module()}). diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 5f3d1c9..f6993c5 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -1,71 +1,60 @@ -module(rebar_packages). -export([packages/1 - ,packages_graph/1 - ,registry/1 + ,close_packages/0 + ,load_and_verify_version/1 + ,deps/3 ,package_dir/1 - ,check_registry/3 ,registry_checksum/2 - ,find_highest_matching/3]). + ,find_highest_matching/4 + ,format_error/1]). -export_type([package/0]). -include("rebar.hrl"). +-include_lib("providers/include/providers.hrl"). -type pkg_name() :: binary() | atom(). -type vsn() :: binary(). -type package() :: pkg_name() | {pkg_name(), vsn()}. --spec packages(rebar_state:t()) -> rebar_dict(). -%% DON'T USE IT! Use rebar_state:packages(State) instead. +-spec packages(rebar_state:t()) -> ets:tid(). packages(State) -> - RegistryDir = package_dir(State), - DictFile = filename:join(RegistryDir, "dict"), - - try - {ok, DictBinary} = file:read_file(DictFile), - binary_to_term(DictBinary) - catch - _:_ -> + catch ets:delete(?PACKAGE_TABLE), + case load_and_verify_version(State) of + true -> + ok; + false -> + ?DEBUG("Error loading package index.", []), ?ERROR("Bad packages index, try to fix with `rebar3 update`", []), - dict:new() + ets:new(?PACKAGE_TABLE, [named_table, public]) end. --spec packages_graph(rebar_state:t()) -> rebar_digraph(). -packages_graph(State) -> - RegistryDir = package_dir(State), - Edges = filename:join(RegistryDir, "edges"), - Vertices = filename:join(RegistryDir, "vertices"), - Neighbors = filename:join(RegistryDir, "neighbors"), +close_packages() -> + catch ets:delete(?PACKAGE_TABLE). - case lists:all(fun(X) -> filelib:is_file(X) end, [Edges, Vertices, Neighbors]) of - true -> - try - {ok, EdgesTab} = ets:file2tab(Edges), - {ok, VerticesTab} = ets:file2tab(Vertices), - {ok, NeighborsTab} = ets:file2tab(Neighbors), - {digraph, EdgesTab, VerticesTab, NeighborsTab, true} - catch - _:_ -> - ?ERROR("Bad packages index, try to fix with `rebar3 update`", []), - digraph:new() +load_and_verify_version(State) -> + RegistryDir = package_dir(State), + case ets:file2tab(filename:join(RegistryDir, ?INDEX_FILE)) of + {ok, _} -> + case ets:lookup_element(?PACKAGE_TABLE, package_index_version, 2) of + ?PACKAGE_INDEX_VERSION -> + true; + _ -> + (catch ets:delete(?PACKAGE_TABLE)), + rebar_prv_update:hex_to_index(State) end; - false -> - ?ERROR("Bad packages index, try to fix with `rebar3 update`", []), - digraph:new() + _ -> + rebar_prv_update:hex_to_index(State) end. --spec registry(rebar_state:t()) -> {ok, ets:tid()} | {error, any()}. -%% DON'T USE IT! Use rebar_state:registry(State) instead. -registry(State) -> - RegistryDir = package_dir(State), - HexFile = filename:join(RegistryDir, "registry"), - case ets:file2tab(HexFile) of - {ok, T} -> - {ok, T}; - {error, Reason} -> - ?DEBUG("Error loading registry: ~p", [Reason]), - error +deps(Name, Vsn, State) -> + try + verify_table(State), + ets:lookup_element(?PACKAGE_TABLE, {ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}, 2) + catch + _:_ -> + throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)})) end. package_dir(State) -> @@ -78,27 +67,13 @@ package_dir(State) -> ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")), PackageDir. - -check_registry(Pkg, Vsn, State) -> - case rebar_state:registry(State) of - {ok, T} -> - case ets:lookup(T, Pkg) of - [{Pkg, [Vsns]}] -> - lists:member(Vsn, Vsns); - _ -> - false - end; - error -> - false - end. - registry_checksum({pkg, Name, Vsn}, State) -> - {ok, Registry} = rebar_state:registry(State), - case ets:lookup(Registry, {Name, Vsn}) of - [{{_, _}, [_, Checksum | _]}] -> - Checksum; - [] -> - none + try + verify_table(State), + ets:lookup_element(?PACKAGE_TABLE, {Name, Vsn}, 3) + catch + _:_ -> + throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)})) end. %% Hex supports use of ~> to specify the version required for a dependency. @@ -116,28 +91,45 @@ registry_checksum({pkg, Name, Vsn}, State) -> %% `~> 2.1.3-dev` | `>= 2.1.3-dev and < 2.2.0` %% `~> 2.0` | `>= 2.0.0 and < 3.0.0` %% `~> 2.1` | `>= 2.1.0 and < 3.0.0` -find_highest_matching(Dep, Constraint, T) -> - case ets:lookup(T, Dep) of - [{Dep, [[Vsn]]}] -> - case ec_semver:pes(Vsn, Constraint) of - true -> - {ok, Vsn}; - false -> - ?WARN("Only existing version of ~s is ~s which does not match constraint ~~> ~s. " - "Using anyway, but it is not guarenteed to work.", [Dep, Vsn, Constraint]), - {ok, Vsn} - end; - [{Dep, [[HeadVsn | VsnTail]]}] -> - {ok, lists:foldl(fun(Version, Highest) -> - case ec_semver:pes(Version, Constraint) andalso - ec_semver:gt(Version, Highest) of - true -> - Version; - false -> - Highest - end - end, HeadVsn, VsnTail)}; +find_highest_matching(Dep, Constraint, Table, State) -> + verify_table(State), + case ets:lookup_element(Table, Dep, 2) of + [[HeadVsn | VsnTail]] -> + {ok, handle_vsns(Constraint, HeadVsn, VsnTail)}; + [[Vsn]] -> + handle_single_vsn(Dep, Vsn, Constraint); + [Vsn] -> + handle_single_vsn(Dep, Vsn, Constraint); + [HeadVsn | VsnTail] -> + {ok, handle_vsns(Constraint, HeadVsn, VsnTail)}; [] -> - ?WARN("Missing registry entry for package ~s", [Dep]), + ?WARN("Missing registry entry for package ~s. Try to fix with `rebar3 update`", [Dep]), none end. + +handle_vsns(Constraint, HeadVsn, VsnTail) -> + lists:foldl(fun(Version, Highest) -> + case ec_semver:pes(Version, Constraint) andalso + ec_semver:gt(Version, Highest) of + true -> + Version; + false -> + Highest + end + end, HeadVsn, VsnTail). + +handle_single_vsn(Dep, Vsn, Constraint) -> + case ec_semver:pes(Vsn, Constraint) of + true -> + {ok, Vsn}; + false -> + ?WARN("Only existing version of ~s is ~s which does not match constraint ~~> ~s. " + "Using anyway, but it is not guarenteed to work.", [Dep, Vsn, Constraint]), + {ok, Vsn} + end. + +format_error({missing_package, Package, Version}) -> + io_lib:format("Package not found in registry: ~s-~s. Try to fix with `rebar3 update`", [Package, Version]). + +verify_table(State) -> + ets:info(?PACKAGE_TABLE, named_table) =:= true orelse ?MODULE:load_and_verify_version(State). diff --git a/src/rebar_plugins.erl b/src/rebar_plugins.erl index bda3fb7..742dc82 100644 --- a/src/rebar_plugins.erl +++ b/src/rebar_plugins.erl @@ -16,10 +16,12 @@ -spec project_apps_install(rebar_state:t()) -> rebar_state:t(). project_apps_install(State) -> + rebar_packages:packages(State), + Profiles = rebar_state:current_profiles(State), ProjectApps = rebar_state:project_apps(State), - lists:foldl(fun(Profile, StateAcc) -> + State1 = lists:foldl(fun(Profile, StateAcc) -> Plugins = rebar_state:get(State, {plugins, Profile}, []), StateAcc1 = handle_plugins(Profile, Plugins, StateAcc), @@ -30,7 +32,11 @@ project_apps_install(State) -> Plugins2 = rebar_state:get(S, {plugins, Profile}, []), handle_plugins(Profile, Plugins2, StateAcc2) end, StateAcc1, ProjectApps) - end, State, Profiles). + end, State, Profiles), + + rebar_packages:close_packages(), + + State1. -spec install(rebar_state:t()) -> rebar_state:t(). install(State) -> diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index 0883fab..0b0f607 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -125,10 +125,7 @@ handle_deps_as_profile(Profile, State, Deps, Upgrade) -> DepsDir = profile_dep_dir(State, Profile), Deps1 = rebar_app_utils:parse_deps(DepsDir, Deps, State, Locks, Level), ProfileLevelDeps = [{Profile, Deps1, Level}], - Graph = rebar_state:packages_graph(State), - Registry = rebar_packages:registry(State), - State1 = rebar_state:packages_graph(rebar_state:registry(State, Registry), Graph), - handle_profile_level(ProfileLevelDeps, [], sets:new(), Upgrade, Locks, State1, Graph). + handle_profile_level(ProfileLevelDeps, [], sets:new(), Upgrade, Locks, State). %% =================================================================== %% Internal functions @@ -141,10 +138,7 @@ deps_per_profile(Profiles, Upgrade, State) -> Deps = lists:foldl(fun(Profile, DepAcc) -> [parsed_profile_deps(State, Profile, Level) | DepAcc] end, [], Profiles), - Graph = rebar_state:packages_graph(State), - Registry = rebar_packages:registry(State), - State1 = rebar_state:packages_graph(rebar_state:registry(State, Registry), Graph), - handle_profile_level(Deps, [], sets:new(), Upgrade, Locks, State1, Graph). + handle_profile_level(Deps, [], sets:new(), Upgrade, Locks, State). parsed_profile_deps(State, Profile, Level) -> ParsedDeps = rebar_state:get(State, {parsed_deps, Profile}, []), @@ -153,17 +147,17 @@ parsed_profile_deps(State, Profile, Level) -> %% Level-order traversal of all dependencies, across profiles. %% If profiles x,y,z are present, then the traversal will go: %% x0, y0, z0, x1, y1, z1, ..., xN, yN, zN. -handle_profile_level([], Apps, _Seen, _Upgrade, _Locks, State, _Graph) -> +handle_profile_level([], Apps, _Seen, _Upgrade, _Locks, State) -> {Apps, State}; -handle_profile_level([{Profile, Deps, Level} | Rest], Apps, Seen, Upgrade, Locks, State, Graph) -> +handle_profile_level([{Profile, Deps, Level} | Rest], Apps, Seen, Upgrade, Locks, State) -> {Deps1, Apps1, State1, Seen1} = update_deps(Profile, Level, Deps, Apps - ,State, Upgrade, Seen, Locks, Graph), + ,State, Upgrade, Seen, Locks), Deps2 = case Deps1 of [] -> Rest; _ -> Rest ++ [{Profile, Deps1, Level+1}] end, - handle_profile_level(Deps2, Apps1, sets:union(Seen, Seen1), Upgrade, Locks, State1, Graph). + handle_profile_level(Deps2, Apps1, sets:union(Seen, Seen1), Upgrade, Locks, State1). find_cycles(Apps) -> case rebar_digraph:compile_order(Apps) of @@ -201,17 +195,17 @@ maybe_lock(Profile, AppInfo, Seen, State, Level) -> {sets:add_element(Name, Seen), State} end. -update_deps(Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks, Graph) -> +update_deps(Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) -> lists:foldl( fun(AppInfo, {DepsAcc, AppsAcc, StateAcc, SeenAcc}) -> update_dep(AppInfo, Profile, Level, DepsAcc, AppsAcc, StateAcc, - Upgrade, SeenAcc, Locks, Graph) + Upgrade, SeenAcc, Locks) end, {[], Apps, State, Seen}, rebar_utils:sort_deps(Deps)). -update_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks, Graph) -> +update_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) -> %% If not seen, add to list of locks to write out Name = rebar_app_info:name(AppInfo), case sets:is_element(Name, Seen) of @@ -222,7 +216,7 @@ update_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks, Gra false -> update_unseen_dep(AppInfo, Profile, Level, Deps, Apps, - State, Upgrade, Seen, Locks, Graph) + State, Upgrade, Seen, Locks) end. profile_dep_dir(State, Profile) -> @@ -242,24 +236,23 @@ update_seen_dep(AppInfo, _Profile, _Level, Deps, Apps, State, Upgrade, Seen, Loc end, {Deps, Apps, State, Seen}. -update_unseen_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks, Graph) -> +update_unseen_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) -> {NewSeen, State1} = maybe_lock(Profile, AppInfo, Seen, State, Level), {_, AppInfo1} = maybe_fetch(AppInfo, Profile, Upgrade, Seen, State1), DepsDir = profile_dep_dir(State, Profile), {AppInfo2, NewDeps, State2} = - handle_dep(State1, Profile, DepsDir, AppInfo1, Locks, Level, Graph), + handle_dep(State1, Profile, DepsDir, AppInfo1, Locks, Level), AppInfo3 = rebar_app_info:dep_level(AppInfo2, Level), {NewDeps ++ Deps, [AppInfo3 | Apps], State2, NewSeen}. --spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer(), rebar_dict()) -> {rebar_app_info:t(), [rebar_app_info:t()], [pkg_dep()], [integer()], rebar_state:t()}. -handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level, Graph) -> +-spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer()) -> {rebar_app_info:t(), [rebar_app_info:t()], [pkg_dep()], [integer()], rebar_state:t()}. +handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) -> Profiles = rebar_state:current_profiles(State), Name = rebar_app_info:name(AppInfo), Vsn = rebar_app_info:original_vsn(AppInfo), %% Deps may be under a sub project app, find it and use its state if so - S0 = rebar_app_info:state(AppInfo), - S = rebar_state:registry(S0, rebar_state:registry(State)), + S = rebar_app_info:state(AppInfo), C = rebar_config:consult(rebar_app_info:dir(AppInfo)), S1 = rebar_state:new(S, C, rebar_app_info:dir(AppInfo)), S2 = rebar_state:apply_overrides(S1, Name), @@ -278,7 +271,7 @@ handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level, Graph) -> %% Upgrade lock level to be the level the dep will have in this dep tree case rebar_app_info:resource_type(AppInfo1) of pkg -> - NewDeps = digraph:out_neighbours(Graph, {ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}), + NewDeps = rebar_packages:deps(Name, Vsn, S5), NewDeps1 = rebar_app_utils:parse_deps(Name, DepsDir, NewDeps, S5, Locks, Level+1), {rebar_app_info:deps(AppInfo1, NewDeps), NewDeps1, State}; _ -> diff --git a/src/rebar_prv_packages.erl b/src/rebar_prv_packages.erl index 44cde4e..f5d9e38 100644 --- a/src/rebar_prv_packages.erl +++ b/src/rebar_prv_packages.erl @@ -27,19 +27,17 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - Dict = rebar_packages:packages(State), - print_packages(Dict), + print_packages(), {ok, State}. -spec format_error(any()) -> iolist(). format_error(load_registry_fail) -> "Failed to load package regsitry. Try running 'rebar3 update' to fix". -print_packages(Dict) -> - Pkgs = lists:keysort(1, dict:fetch_keys(Dict)), - SortedPkgs = lists:foldl(fun({Pkg, Vsn}, Acc) -> - orddict:append_list(Pkg, [Vsn], Acc) - end, orddict:new(), Pkgs), +print_packages() -> + SortedPkgs = ets:foldl(fun({{Pkg, Vsn}, _}, Acc) -> + orddict:append_list(Pkg, [Vsn], Acc) + end, orddict:new(), ?PACKAGE_TABLE), orddict:map(fun(Name, Vsns) -> SortedVsns = lists:sort(fun(A, B) -> diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index e73c769..4b3a155 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -9,21 +9,14 @@ do/1, format_error/1]). +-export([hex_to_index/1]). + -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). -define(PROVIDER, update). -define(DEPS, []). -%% Ignore warning of digraph opaque type when running dialyzer --dialyzer({no_opaque, do/1}). --dialyzer({no_opaque, write_registry/3}). - -%% Ignoring the opaque type warning won't stop dialyzer from warning of -%% no return for functions that had the opaque type warnings --dialyzer({no_return, do/1}). --dialyzer({no_return, write_registry/3}). - %% =================================================================== %% Public API %% =================================================================== @@ -50,16 +43,15 @@ do(State) -> TmpDir = ec_file:insecure_mkdtemp(), TmpFile = filename:join(TmpDir, "packages.gz"), - Url = rebar_state:get(State, rebar_packages_cdn, "https://s3.amazonaws.com/s3.hex.pm/registry.ets.gz"), + Url = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_HEX_REGISTRY), {ok, _RequestId} = httpc:request(get, {Url, []}, [], [{stream, TmpFile}, {sync, true}], rebar), {ok, Data} = file:read_file(TmpFile), Unzipped = zlib:gunzip(Data), ok = file:write_file(HexFile, Unzipped), - {Dict, Graph} = hex_to_graph(HexFile), - write_registry(Dict, Graph, State), - true = digraph:delete(Graph), + + hex_to_index(State), ok catch _E:C -> @@ -73,57 +65,58 @@ do(State) -> format_error(package_index_write) -> "Failed to write package index.". --spec write_registry(rebar_dict(), {digraph, ets:tid(), ets:tid(), ets:tid(), any()}, rebar_state:t()) -> ok | {error, atom()}. -write_registry(Dict, {digraph, Edges, Vertices, Neighbors, _}, State) -> - RegistryDir = rebar_packages:package_dir(State), - filelib:ensure_dir(filename:join(RegistryDir, "dummy")), - ets:tab2file(Edges, filename:join(RegistryDir, "edges")), - ets:tab2file(Vertices, filename:join(RegistryDir, "vertices")), - ets:tab2file(Neighbors, filename:join(RegistryDir, "neighbors")), - file:write_file(filename:join(RegistryDir, "dict"), term_to_binary(Dict)). - is_supported(<<"make">>) -> true; is_supported(<<"rebar">>) -> true; is_supported(<<"rebar3">>) -> true; is_supported(_) -> false. -hex_to_graph(Filename) -> - {ok, T} = ets:file2tab(Filename), - Graph = digraph:new(), - ets:foldl(fun({Pkg, [Versions]}, ok) when is_binary(Pkg), is_list(Versions) -> - lists:foreach(fun(Version) -> - digraph:add_vertex(Graph, {Pkg, Version}, 1) - end, Versions); - (_, ok) -> - ok - end, ok, T), - - Dict1 = ets:foldl(fun({{Pkg, PkgVsn}, [Deps, _, BuildTools | _]}, Dict) when is_list(BuildTools) -> - case lists:any(fun is_supported/1, BuildTools) of - true -> - DepsList = update_graph(Pkg, PkgVsn, Deps, T, Graph), - dict:store({Pkg, PkgVsn}, DepsList, Dict); - false -> - Dict - end; - (_, Dict) -> - Dict - end, dict:new(), T), - {Dict1, Graph}. - -update_graph(Pkg, PkgVsn, Deps, HexRegistry, Graph) -> +hex_to_index(State) -> + RegistryDir = rebar_packages:package_dir(State), + HexFile = filename:join(RegistryDir, "registry"), + try ets:file2tab(HexFile) of + {ok, Registry} -> + try + (catch ets:delete(?PACKAGE_TABLE)), + ets:new(?PACKAGE_TABLE, [named_table, public]), + ets:foldl(fun({{Pkg, PkgVsn}, [Deps, Checksum, BuildTools | _]}, _) when is_list(BuildTools) -> + case lists:any(fun is_supported/1, BuildTools) of + true -> + DepsList = update_deps_list(Deps, Registry, State), + ets:insert(?PACKAGE_TABLE, {{Pkg, PkgVsn}, DepsList, Checksum}); + false -> + true + end; + ({Pkg, [Vsns]}, _) when is_binary(Pkg) -> + ets:insert(?PACKAGE_TABLE, {Pkg, Vsns}); + (_, _) -> + true + end, true, Registry), + + ets:insert(?PACKAGE_TABLE, {package_index_version, ?PACKAGE_INDEX_VERSION}), + ets:tab2file(?PACKAGE_TABLE, filename:join(RegistryDir, "packages.idx")), + true + after + catch ets:delete(Registry) + end; + {error, Reason} -> + ?DEBUG("Error loading package registry: ~p", [Reason]), + false + catch + _:_ -> + fail + end. + +update_deps_list(Deps, HexRegistry, State) -> lists:foldl(fun([Dep, DepVsn, false, _AppName | _], DepsListAcc) -> case DepVsn of <<"~> ", Vsn/binary>> -> - case rebar_packages:find_highest_matching(Dep, Vsn, HexRegistry) of + case rebar_packages:find_highest_matching(Dep, Vsn, HexRegistry, State) of {ok, HighestDepVsn} -> - digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, HighestDepVsn}), - [{Dep, DepVsn} | DepsListAcc]; + [{Dep, HighestDepVsn} | DepsListAcc]; none -> DepsListAcc end; Vsn -> - digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, Vsn}), [{Dep, Vsn} | DepsListAcc] end; ([_Dep, _DepVsn, true, _AppName | _], DepsListAcc) -> diff --git a/src/rebar_state.erl b/src/rebar_state.erl index 54e7f13..213eb2f 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -38,10 +38,6 @@ overrides/1, overrides/2, apply_overrides/2, - packages_graph/1, packages_graph/2, - packages/1, - registry/1, registry/2, - resources/1, resources/2, add_resource/2, providers/1, providers/2, add_provider/2]). @@ -66,9 +62,6 @@ all_plugin_deps = [] :: [rebar_app_info:t()], all_deps = [] :: [rebar_app_info:t()], - packages = undefined :: rebar_dict(), - packages_graph = undefined :: rebar_digraph() | undefined, - registry = undefined :: {ok, ets:tid()} | error | undefined, overrides = [], resources = [], providers = []}). @@ -90,18 +83,9 @@ new(Config) when is_list(Config) -> Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins} | Config], true = rebar_config:verify_config_format(Terms), Opts = dict:from_list(Terms), - load_package_registry( - BaseState#state_t { dir = rebar_dir:get_cwd(), - default = Opts, - opts = Opts }). - -load_package_registry(Config0) -> - Registry = rebar_packages:registry(Config0), - Packages = rebar_packages:packages(Config0), - PackagesGraph = rebar_packages:packages_graph(Config0), - Config0#state_t{registry = Registry, - packages_graph = PackagesGraph, - packages = Packages}. + BaseState#state_t { dir = rebar_dir:get_cwd(), + default = Opts, + opts = Opts }. -spec new(t() | atom(), list()) -> t(). new(Profile, Config) when is_atom(Profile) @@ -113,11 +97,10 @@ new(Profile, Config) when is_atom(Profile) Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins} | Config], true = rebar_config:verify_config_format(Terms), Opts = dict:from_list(Terms), - load_package_registry( - BaseState#state_t { dir = rebar_dir:get_cwd(), - current_profiles = [Profile], - default = Opts, - opts = Opts }); + BaseState#state_t { dir = rebar_dir:get_cwd(), + current_profiles = [Profile], + default = Opts, + opts = Opts }; new(ParentState=#state_t{}, Config) -> %% Load terms from rebar.config, if it exists Dir = rebar_dir:get_cwd(), @@ -450,27 +433,6 @@ namespace(#state_t{namespace=Namespace}) -> namespace(State=#state_t{}, Namespace) -> State#state_t{namespace=Namespace}. -packages(#state_t{packages=undefined}) -> - throw(packages_usage_error); -packages(#state_t{packages=Packages}) -> - Packages. - -packages_graph(#state_t{packages_graph=undefined}) -> - throw(packages_usage_error); -packages_graph(#state_t{packages_graph=PackagesGraph}) -> - PackagesGraph. - -packages_graph(State, PackagesGraph) -> - State#state_t{packages_graph=PackagesGraph}. - -registry(#state_t{registry=undefined}) -> - throw(registry_usage_error); -registry(#state_t{registry=Registry}) -> - Registry. - -registry(State, Registry) -> - State#state_t{registry=Registry}. - -spec resources(t()) -> [{rebar_resource:type(), module()}]. resources(#state_t{resources=Resources}) -> Resources. -- cgit v1.1 From e336d7aaa124bb63c92f4470429d06f5de8e089a Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 21 Aug 2015 19:48:26 -0500 Subject: remove unneeded packages ets table load/unload in plugins --- src/rebar_plugins.erl | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) (limited to 'src') diff --git a/src/rebar_plugins.erl b/src/rebar_plugins.erl index 742dc82..6c2daef 100644 --- a/src/rebar_plugins.erl +++ b/src/rebar_plugins.erl @@ -16,27 +16,21 @@ -spec project_apps_install(rebar_state:t()) -> rebar_state:t(). project_apps_install(State) -> - rebar_packages:packages(State), - Profiles = rebar_state:current_profiles(State), ProjectApps = rebar_state:project_apps(State), - State1 = lists:foldl(fun(Profile, StateAcc) -> + lists:foldl(fun(Profile, StateAcc) -> Plugins = rebar_state:get(State, {plugins, Profile}, []), StateAcc1 = handle_plugins(Profile, Plugins, StateAcc), lists:foldl(fun(App, StateAcc2) -> - AppDir = rebar_app_info:dir(App), - C = rebar_config:consult(AppDir), - S = rebar_state:new(rebar_state:new(), C, AppDir), - Plugins2 = rebar_state:get(S, {plugins, Profile}, []), - handle_plugins(Profile, Plugins2, StateAcc2) - end, StateAcc1, ProjectApps) - end, State, Profiles), - - rebar_packages:close_packages(), - - State1. + AppDir = rebar_app_info:dir(App), + C = rebar_config:consult(AppDir), + S = rebar_state:new(rebar_state:new(), C, AppDir), + Plugins2 = rebar_state:get(S, {plugins, Profile}, []), + handle_plugins(Profile, Plugins2, StateAcc2) + end, StateAcc1, ProjectApps) + end, State, Profiles). -spec install(rebar_state:t()) -> rebar_state:t(). install(State) -> -- cgit v1.1