diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/rebar.hrl | 6 | ||||
| -rw-r--r-- | src/rebar3.erl | 13 | ||||
| -rw-r--r-- | src/rebar_app_discover.erl | 18 | ||||
| -rw-r--r-- | src/rebar_app_info.erl | 13 | ||||
| -rw-r--r-- | src/rebar_app_utils.erl | 124 | ||||
| -rw-r--r-- | src/rebar_digraph.erl | 110 | ||||
| -rw-r--r-- | src/rebar_packages.erl | 158 | ||||
| -rw-r--r-- | src/rebar_plugins.erl | 12 | ||||
| -rw-r--r-- | src/rebar_prv_install_deps.erl | 388 | ||||
| -rw-r--r-- | src/rebar_prv_packages.erl | 12 | ||||
| -rw-r--r-- | src/rebar_prv_update.erl | 96 | ||||
| -rw-r--r-- | src/rebar_prv_upgrade.erl | 3 | ||||
| -rw-r--r-- | src/rebar_state.erl | 43 |
13 files changed, 382 insertions, 614 deletions
diff --git a/src/rebar.hrl b/src/rebar.hrl index 8a702df..961b8ea 100644 --- a/src/rebar.hrl +++ b/src/rebar.hrl @@ -23,8 +23,14 @@ -define(DEFAULT_RELEASE_DIR, "rel"). -define(DEFAULT_CONFIG_FILE, "rebar.config"). -define(DEFAULT_CDN, "https://s3.amazonaws.com/s3.hex.pm/tarballs"). +-define(DEFAULT_HEX_REGISTRY, "https://s3.amazonaws.com/s3.hex.pm/registry.ets.gz"). -define(LOCK_FILE, "rebar.lock"). +-define(PACKAGE_INDEX_VERSION, 3). +-define(PACKAGE_TABLE, package_index). +-define(INDEX_FILE, "packages.idx"). +-define(REGISTRY_FILE, "registry"). + -ifdef(namespaced_types). -type rebar_dict() :: dict:dict(). -else. diff --git a/src/rebar3.erl b/src/rebar3.erl index 8004443..ab3e0eb 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -105,6 +105,7 @@ run_aux(State, RawArgs) -> {ok, Providers} = application:get_env(rebar, providers), %% Providers can modify profiles stored in opts, so set default after initializing providers State4 = rebar_state:create_logic_providers(Providers, State3), + rebar_packages:packages(State4), State5 = rebar_plugins:project_apps_install(State4), State6 = rebar_state:default(State5, rebar_state:opts(State5)), @@ -280,9 +281,15 @@ state_from_global_config(Config, GlobalConfigFile) -> %% We don't want to worry about global plugin install state effecting later %% usage. So we throw away the global profile state used for plugin install. GlobalConfigThrowAway = rebar_state:current_profiles(GlobalConfig, [global]), - GlobalState = rebar_plugins:handle_plugins(global, - rebar_state:get(GlobalConfigThrowAway, plugins, []), - GlobalConfigThrowAway), + GlobalState = case rebar_state:get(GlobalConfigThrowAway, plugins, []) of + [] -> + GlobalConfigThrowAway; + GlobalPluginsToInstall -> + rebar_packages:packages(GlobalConfigThrowAway), + rebar_plugins:handle_plugins(global, + GlobalPluginsToInstall, + GlobalConfigThrowAway) + end, GlobalPlugins = rebar_state:providers(GlobalState), GlobalConfig2 = rebar_state:set(GlobalConfig, plugins, []), GlobalConfig3 = rebar_state:set(GlobalConfig2, {plugins, global}, rebar_state:get(GlobalConfigThrowAway, plugins, [])), diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index e81a323..8b1e58e 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -96,7 +96,7 @@ merge_deps(AppInfo, State) -> {AppInfo1, State1}. handle_profile(Profile, Name, AppState, State) -> - {TopSrc, TopPkg} = rebar_state:get(State, {parsed_deps, Profile}, {[], []}), + TopParsedDeps = rebar_state:get(State, {parsed_deps, Profile}, {[], []}), TopLevelProfileDeps = rebar_state:get(State, {deps, Profile}, []), AppProfileDeps = rebar_state:get(AppState, {deps, Profile}, []), AppProfileDeps2 = rebar_utils:tup_dedup(rebar_utils:tup_sort(AppProfileDeps)), @@ -108,18 +108,18 @@ handle_profile(Profile, Name, AppState, State) -> %% Only deps not also specified in the top level config need %% to be included in the parsed deps NewDeps = ProfileDeps2 -- TopLevelProfileDeps, - {ParsedSrc, ParsedPkg} = parse_profile_deps(Profile, Name, NewDeps, AppState, State1), - rebar_state:set(State1, {parsed_deps, Profile}, {TopSrc++ParsedSrc, TopPkg++ParsedPkg}). + ParsedDeps = parse_profile_deps(Profile, Name, NewDeps, AppState, State1), + rebar_state:set(State1, {parsed_deps, Profile}, TopParsedDeps++ParsedDeps). parse_profile_deps(Profile, Name, Deps, AppState, State) -> DepsDir = rebar_prv_install_deps:profile_dep_dir(State, Profile), Locks = rebar_state:get(State, {locks, Profile}, []), - rebar_prv_install_deps:parse_deps(Name - ,DepsDir - ,Deps - ,AppState - ,Locks - ,1). + rebar_app_utils:parse_deps(Name + ,DepsDir + ,Deps + ,AppState + ,Locks + ,1). project_app_config(AppInfo, State) -> C = rebar_config:consult(rebar_app_info:dir(AppInfo)), diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index 7e31f6d..bb99584 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -34,6 +34,8 @@ dir/2, out_dir/1, out_dir/2, + resource_type/1, + resource_type/2, source/1, source/2, state/1, @@ -64,6 +66,7 @@ dep_level=0 :: integer(), dir :: file:name(), out_dir :: file:name(), + resource_type :: pkg | src, source :: string() | tuple() | undefined, state :: rebar_state:t() | undefined, is_lock=false :: boolean(), @@ -274,6 +277,14 @@ out_dir(AppInfo=#app_info_t{}, OutDir) -> ebin_dir(#app_info_t{out_dir=OutDir}) -> ec_cnv:to_list(filename:join(OutDir, "ebin")). +-spec resource_type(t(), pkg | src) -> t(). +resource_type(AppInfo=#app_info_t{}, Type) -> + AppInfo#app_info_t{resource_type=Type}. + +-spec resource_type(t()) -> pkg | src. +resource_type(#app_info_t{resource_type=ResourceType}) -> + ResourceType. + -spec source(t(), string() | tuple()) -> t(). source(AppInfo=#app_info_t{}, Source) -> AppInfo#app_info_t{source=Source}. @@ -316,7 +327,7 @@ is_checkout(#app_info_t{is_checkout=IsCheckout}) -> -spec valid(t()) -> boolean(). valid(AppInfo=#app_info_t{valid=undefined, state=State}) -> - case rebar_app_utils:validate_application_info(AppInfo) + case rebar_app_utils:validate_application_info(AppInfo) =:= true andalso rebar_state:has_all_artifacts(State) =:= true of true -> true; diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index e9745c3..b14c50a 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -32,6 +32,9 @@ app_src_to_app/2, validate_application_info/1, validate_application_info/2, + parse_deps/5, + parse_deps/6, + dep_to_app/7, format_error/1]). -include("rebar.hrl"). @@ -87,6 +90,109 @@ validate_application_info(AppInfo, AppDetail) -> end end. +-spec parse_deps(binary(), list(), rebar_state:t(), list(), integer()) -> {[rebar_app_info:t()], [tuple()]}. +parse_deps(DepsDir, Deps, State, Locks, Level) -> + parse_deps(root, DepsDir, Deps, State, Locks, Level). + +parse_deps(Parent, DepsDir, Deps, State, Locks, Level) -> + [parse_dep(Dep, Parent, DepsDir, State, Locks, Level) || Dep <- Deps]. + +parse_dep(Dep, Parent, DepsDir, State, Locks, Level) -> + Name = case Dep of + Dep when is_tuple(Dep) -> + element(1, Dep); + Dep -> + Dep + end, + case lists:keyfind(ec_cnv:to_binary(Name), 1, Locks) of + false -> + parse_dep(Parent, Dep, DepsDir, false, State); + LockedDep -> + LockedLevel = element(3, LockedDep), + case LockedLevel > Level of + true -> + parse_dep(Parent, Dep, DepsDir, false, State); + false -> + parse_dep(Parent, LockedDep, DepsDir, true, State) + end + end. + +parse_dep(Parent, {Name, Vsn}, DepsDir, IsLock, State) when is_list(Vsn); is_binary(Vsn) -> + %% Versioned Package dependency + CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), + case rebar_app_info:discover(CheckoutsDir) of + {ok, _App} -> + dep_to_app(root, DepsDir, Name, [], [], IsLock, State); + not_found -> + {PkgName, PkgVsn} = parse_goal(ec_cnv:to_binary(Name) + ,ec_cnv:to_binary(Vsn)), + Source = {pkg, PkgName, PkgVsn}, + rebar_app_info:resource_type(dep_to_app(Parent, DepsDir, PkgName, PkgVsn, Source, IsLock, State), pkg) + end; +parse_dep(Parent, Name, DepsDir, IsLock, State) when is_atom(Name); is_binary(Name) -> + %% Unversioned package dependency + {PkgName, PkgVsn} = get_package(ec_cnv:to_binary(Name), State), + CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), + case rebar_app_info:discover(CheckoutsDir) of + {ok, _App} -> + dep_to_app(root, DepsDir, Name, [], [], IsLock, State); + not_found -> + Source = {pkg, PkgName, PkgVsn}, + rebar_app_info:resource_type(dep_to_app(Parent, DepsDir, PkgName, PkgVsn, Source, IsLock, State), pkg) + end; +parse_dep(Parent, {Name, Source}, DepsDir, IsLock, State) when is_tuple(Source) -> + dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); +parse_dep(Parent, {Name, _Vsn, Source}, DepsDir, IsLock, State) when is_tuple(Source) -> + dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); +parse_dep(Parent, {Name, _Vsn, Source, Opts}, DepsDir, IsLock, State) when is_tuple(Source) -> + ?WARN("Dependency option list ~p in ~p is not supported and will be ignored", [Opts, Name]), + dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); +parse_dep(Parent, {_Name, {pkg, Name, Vsn}, Level}, DepsDir, IsLock, State) when is_integer(Level) -> + CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), + case rebar_app_info:discover(CheckoutsDir) of + {ok, _App} -> + dep_to_app(root, DepsDir, Name, [], [], IsLock, State); + not_found -> + Source = {pkg, Name, Vsn}, + rebar_app_info:resource_type(dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State), pkg) + end; +parse_dep(Parent, {Name, Source, Level}, DepsDir, IsLock, State) when is_tuple(Source) + , is_integer(Level) -> + dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); +parse_dep(_, Dep, _, _, _) -> + throw(?PRV_ERROR({parse_dep, Dep})). + +dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> + CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), + BaseDir = rebar_state:get(State, base_dir, []), + {ok, Dep} = case rebar_app_info:discover(CheckoutsDir) of + {ok, App} -> + {ok, rebar_app_info:is_checkout(App, true)}; + not_found -> + Dir = ec_cnv:to_list(filename:join(DepsDir, Name)), + case rebar_app_info:discover(Dir) of + {ok, App} -> + {ok, App}; + not_found -> + rebar_app_info:new(Name, Vsn, + ec_cnv:to_list(filename:join(DepsDir, Name))) + end + end, + C = rebar_config:consult(rebar_app_info:dir(Dep)), + S = rebar_state:new(rebar_state:new(), C, rebar_app_info:dir(Dep)), + Overrides = rebar_state:get(State, overrides, []), + ParentOverrides = rebar_state:overrides(State), + S1 = rebar_state:set(rebar_state:overrides(S, ParentOverrides++Overrides), base_dir, BaseDir), + Dep1 = rebar_app_info:state(Dep, S1), + AppInfo = rebar_app_info:is_lock(rebar_app_info:source(Dep1, Source), IsLock), + ResourceType = case Source of + {pkg, _, _} -> + pkg; + _ -> + src + end, + rebar_app_info:resource_type(rebar_app_info:parent(AppInfo, Parent), ResourceType). + format_error(Error) -> io_lib:format("~p", [Error]). @@ -94,11 +200,25 @@ format_error(Error) -> %% Internal functions %% =================================================================== +-spec parse_goal(binary(), binary()) -> {binary(), binary()} | {binary(), binary(), binary()}. +parse_goal(Name, Constraint) -> + case re:run(Constraint, "([^\\d]*)(\\d.*)", [{capture, [1,2], binary}]) of + {match, [<<>>, Vsn]} -> + {Name, Vsn}; + {match, [Op, Vsn]} -> + {Name, Vsn, binary_to_atom(Op, utf8)}; + nomatch -> + throw(?PRV_ERROR({bad_constraint, Name, Constraint})) + end. + +get_package(Dep, State) -> + {ok, HighestDepVsn} = rebar_packages:find_highest_matching(Dep, "0", ?PACKAGE_TABLE, State), + {Dep, HighestDepVsn}. + -spec has_all_beams(file:filename_all(), [module()]) -> true | ?PRV_ERROR({missing_module, module()}). has_all_beams(EbinDir, [Module | ModuleList]) -> - BeamFile = filename:join([EbinDir, - ec_cnv:to_list(Module) ++ ".beam"]), + BeamFile = filename:join([EbinDir, ec_cnv:to_list(Module) ++ ".beam"]), case filelib:is_file(BeamFile) of true -> has_all_beams(EbinDir, ModuleList); diff --git a/src/rebar_digraph.erl b/src/rebar_digraph.erl index ff0a1d2..363253a 100644 --- a/src/rebar_digraph.erl +++ b/src/rebar_digraph.erl @@ -2,8 +2,6 @@ -export([compile_order/1 ,restore_graph/1 - ,cull_deps/2 - ,cull_deps/3 ,subgraph/2 ,format_error/1]). @@ -69,17 +67,6 @@ restore_graph({Vs, Es}) -> end, Es), Graph. -%% Pick packages to fullfill dependencies -%% The first dep while traversing the graph is chosen and any conflicting -%% dep encountered later on is ignored. -cull_deps(Graph, Vertices) -> - cull_deps(Graph, Vertices, sets:new()). - -cull_deps(Graph, Vertices, Seen) -> - Vertices1 = lists:keysort(2, Vertices), - {Solution, Levels, Discarded} = {dict:new(), dict:new(), sets:new()}, - cull_deps(Graph, Vertices1, Levels, Solution, Seen, Discarded). - format_error(no_solution) -> io_lib:format("No solution for packages found.", []). @@ -87,103 +74,6 @@ format_error(no_solution) -> %% Internal Functions %%==================================================================== -cull_deps(_Graph, [], Levels, Solution, _, Discarded) -> - {_, Vertices} = lists:unzip(dict:to_list(Solution)), - LvlVertices = [{Profile, {Parent, App, Vsn, dict:fetch(App, Levels)}} - || {Profile, {Parent,App,Vsn}} <- Vertices], - {ok, LvlVertices, sets:to_list(Discarded)}; -cull_deps(Graph, [{Profile, Level, Vs} | Vertices], Levels, Solution, Seen, Discarded) -> - {NV, NS, LS, DS} = - lists:foldl(fun({Parent, Name, Vsn}, {Acc, SolutionAcc, LevelsAcc, DiscardedAcc}) -> - {SolutionAcc1, LevelsAcc1, DiscardedAcc1} = - maybe_add_to_solution(Profile, Level, Name, {Name, Vsn}, Parent - ,SolutionAcc - ,LevelsAcc, Seen, DiscardedAcc), - OutNeighbors = digraph:out_neighbours(Graph, {Name,Vsn}), - {NewVertices, DiscardedAcc2} = handle_neighbors(Profile, Level, Name - ,OutNeighbors, Acc, SolutionAcc1 - ,Seen, DiscardedAcc1), - {NewVertices, SolutionAcc1, LevelsAcc1, DiscardedAcc2} - end, {[], Solution, Levels, Discarded}, Vs), - NewVertices = combine_profile_levels(Vertices, NV), - cull_deps(Graph, NewVertices, LS, NS, Seen, DS). - -%% Combine lists of deps that have the same profile and level -combine_profile_levels(Vertices, NewVertices) -> - V = lists:foldl(fun({Profile, Level, Vs}, Acc) -> - case ec_lists:find(fun({P, L, _}) -> - P =:= Profile andalso L =:= Level - end, Acc) of - {ok, {_, _, OldVs}=Old} -> - lists:delete(Old, Acc)++[{Profile, Level, lists:keysort(1, OldVs++Vs)}]; - error -> - Acc++[{Profile, Level, Vs}] - end - end, Vertices, NewVertices), - lists:keysort(2, V). - -%% For each outgoing edge of a dep check if it should be added to the solution -%% and add it to the list of vertices to do the same for -handle_neighbors(Profile, Level, Parent, OutNeighbors, Vertices - ,Solution, Seen, Discarded) -> - case lists:foldl(fun({Name, Vsn}=Value, {NewVertices, Discarded1}) -> - case dict:find(Name, Solution) of - {ok, {Profile, {Parent, Name, Vsn}}} -> % already seen - {NewVertices, - Discarded1}; - {ok, _} -> % conflict resolution! - %% Warn on different version - {NewVertices, - sets:add_element(Value, Discarded1)}; - error -> - %% We check Seen separately because we don't care - %% to warn if the exact same version of a package - %% was already part of the solution but we do - %% if it was simply seen in source deps - case sets:is_element(Name, Seen) of - true -> - {NewVertices, - sets:add_element(Value, Discarded1)}; - false -> - {[{Parent, Name, Vsn} | NewVertices], - Discarded1} - end - end - end, {[], Discarded}, OutNeighbors) of - {[], DiscardedAcc2} -> - {Vertices, DiscardedAcc2}; - {NewVertices1, DiscardedAcc2} -> - {Vertices++[{Profile, Level+1, NewVertices1}] ,DiscardedAcc2} - end. - -maybe_add_to_solution(Profile, Level, Key, {Name, Vsn}=Value, Parent - ,Solution, Levels, Seen, Discarded) -> - case dict:find(Key, Solution) of - {ok, {Profile, {Parent, Name, Vsn}}} -> % already seen - {Solution, - Levels, - Discarded}; - {ok, _} -> % conflict resolution! - %% Warn on different version - {Solution, - Levels, - sets:add_element(Value, Discarded)}; - error -> - %% We check Seen separately because we don't care to warn if the exact - %% same version of a package was already part of the solution but we do - %% if it was simply seen in source deps - case sets:is_element(Name, Seen) of - true -> - {Solution, - Levels, - sets:add_element(Value, Discarded)}; - false -> - {dict:store(Key, {Profile, {Parent, Name, Vsn}}, Solution), - dict:store(Key, Level, Levels), - Discarded} - end - end. - subgraph(Graph, Vertices) -> digraph_utils:subgraph(Graph, Vertices). diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index ca3b676..f6993c5 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -1,59 +1,60 @@ -module(rebar_packages). -export([packages/1 - ,registry/1 + ,close_packages/0 + ,load_and_verify_version/1 + ,deps/3 ,package_dir/1 - ,check_registry/3 ,registry_checksum/2 - ,find_highest_matching/3]). + ,find_highest_matching/4 + ,format_error/1]). -export_type([package/0]). -include("rebar.hrl"). +-include_lib("providers/include/providers.hrl"). -type pkg_name() :: binary() | atom(). -type vsn() :: binary(). -type package() :: pkg_name() | {pkg_name(), vsn()}. --spec packages(rebar_state:t()) -> {rebar_dict(), rebar_digraph()}. -%% DON'T USE IT! Use rebar_state:packages(State) instead. +-spec packages(rebar_state:t()) -> ets:tid(). packages(State) -> - RegistryDir = package_dir(State), - DictFile = filename:join(RegistryDir, "dict"), - Edges = filename:join(RegistryDir, "edges"), - Vertices = filename:join(RegistryDir, "vertices"), - Neighbors = filename:join(RegistryDir, "neighbors"), - - case lists:all(fun(X) -> filelib:is_file(X) end, [DictFile, Edges, Vertices, Neighbors]) of + catch ets:delete(?PACKAGE_TABLE), + case load_and_verify_version(State) of true -> - try - {ok, DictBinary} = file:read_file(DictFile), - Dict = binary_to_term(DictBinary), - {ok, EdgesTab} = ets:file2tab(Edges), - {ok, VerticesTab} = ets:file2tab(Vertices), - {ok, NeighborsTab} = ets:file2tab(Neighbors), - {Dict, {digraph, EdgesTab, VerticesTab, NeighborsTab, true}} - catch - _:_ -> - ?ERROR("Bad packages index, try to fix with `rebar3 update`", []), - {dict:new(), digraph:new()} - end; + ok; false -> + ?DEBUG("Error loading package index.", []), ?ERROR("Bad packages index, try to fix with `rebar3 update`", []), - {dict:new(), digraph:new()} + ets:new(?PACKAGE_TABLE, [named_table, public]) end. --spec registry(rebar_state:t()) -> {ok, ets:tid()} | {error, any()}. -%% DON'T USE IT! Use rebar_state:registry(State) instead. -registry(State) -> +close_packages() -> + catch ets:delete(?PACKAGE_TABLE). + +load_and_verify_version(State) -> RegistryDir = package_dir(State), - HexFile = filename:join(RegistryDir, "registry"), - case ets:file2tab(HexFile) of - {ok, T} -> - {ok, T}; - {error, Reason} -> - ?DEBUG("Error loading registry: ~p", [Reason]), - error + case ets:file2tab(filename:join(RegistryDir, ?INDEX_FILE)) of + {ok, _} -> + case ets:lookup_element(?PACKAGE_TABLE, package_index_version, 2) of + ?PACKAGE_INDEX_VERSION -> + true; + _ -> + (catch ets:delete(?PACKAGE_TABLE)), + rebar_prv_update:hex_to_index(State) + end; + _ -> + rebar_prv_update:hex_to_index(State) + end. + +deps(Name, Vsn, State) -> + try + verify_table(State), + ets:lookup_element(?PACKAGE_TABLE, {ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}, 2) + catch + _:_ -> + throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)})) end. package_dir(State) -> @@ -66,27 +67,13 @@ package_dir(State) -> ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")), PackageDir. - -check_registry(Pkg, Vsn, State) -> - case rebar_state:registry(State) of - {ok, T} -> - case ets:lookup(T, Pkg) of - [{Pkg, [Vsns]}] -> - lists:member(Vsn, Vsns); - _ -> - false - end; - error -> - false - end. - registry_checksum({pkg, Name, Vsn}, State) -> - {ok, Registry} = rebar_state:registry(State), - case ets:lookup(Registry, {Name, Vsn}) of - [{{_, _}, [_, Checksum | _]}] -> - Checksum; - [] -> - none + try + verify_table(State), + ets:lookup_element(?PACKAGE_TABLE, {Name, Vsn}, 3) + catch + _:_ -> + throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)})) end. %% Hex supports use of ~> to specify the version required for a dependency. @@ -104,28 +91,45 @@ registry_checksum({pkg, Name, Vsn}, State) -> %% `~> 2.1.3-dev` | `>= 2.1.3-dev and < 2.2.0` %% `~> 2.0` | `>= 2.0.0 and < 3.0.0` %% `~> 2.1` | `>= 2.1.0 and < 3.0.0` -find_highest_matching(Dep, Constraint, T) -> - case ets:lookup(T, Dep) of - [{Dep, [[Vsn]]}] -> - case ec_semver:pes(Vsn, Constraint) of - true -> - {ok, Vsn}; - false -> - ?WARN("Only existing version of ~s is ~s which does not match constraint ~~> ~s. " - "Using anyway, but it is not guarenteed to work.", [Dep, Vsn, Constraint]), - {ok, Vsn} - end; - [{Dep, [[HeadVsn | VsnTail]]}] -> - {ok, lists:foldl(fun(Version, Highest) -> - case ec_semver:pes(Version, Constraint) andalso - ec_semver:gt(Version, Highest) of - true -> - Version; - false -> - Highest - end - end, HeadVsn, VsnTail)}; +find_highest_matching(Dep, Constraint, Table, State) -> + verify_table(State), + case ets:lookup_element(Table, Dep, 2) of + [[HeadVsn | VsnTail]] -> + {ok, handle_vsns(Constraint, HeadVsn, VsnTail)}; + [[Vsn]] -> + handle_single_vsn(Dep, Vsn, Constraint); + [Vsn] -> + handle_single_vsn(Dep, Vsn, Constraint); + [HeadVsn | VsnTail] -> + {ok, handle_vsns(Constraint, HeadVsn, VsnTail)}; [] -> - ?WARN("Missing registry entry for package ~s", [Dep]), + ?WARN("Missing registry entry for package ~s. Try to fix with `rebar3 update`", [Dep]), none end. + +handle_vsns(Constraint, HeadVsn, VsnTail) -> + lists:foldl(fun(Version, Highest) -> + case ec_semver:pes(Version, Constraint) andalso + ec_semver:gt(Version, Highest) of + true -> + Version; + false -> + Highest + end + end, HeadVsn, VsnTail). + +handle_single_vsn(Dep, Vsn, Constraint) -> + case ec_semver:pes(Vsn, Constraint) of + true -> + {ok, Vsn}; + false -> + ?WARN("Only existing version of ~s is ~s which does not match constraint ~~> ~s. " + "Using anyway, but it is not guarenteed to work.", [Dep, Vsn, Constraint]), + {ok, Vsn} + end. + +format_error({missing_package, Package, Version}) -> + io_lib:format("Package not found in registry: ~s-~s. Try to fix with `rebar3 update`", [Package, Version]). + +verify_table(State) -> + ets:info(?PACKAGE_TABLE, named_table) =:= true orelse ?MODULE:load_and_verify_version(State). diff --git a/src/rebar_plugins.erl b/src/rebar_plugins.erl index bda3fb7..6c2daef 100644 --- a/src/rebar_plugins.erl +++ b/src/rebar_plugins.erl @@ -24,12 +24,12 @@ project_apps_install(State) -> StateAcc1 = handle_plugins(Profile, Plugins, StateAcc), lists:foldl(fun(App, StateAcc2) -> - AppDir = rebar_app_info:dir(App), - C = rebar_config:consult(AppDir), - S = rebar_state:new(rebar_state:new(), C, AppDir), - Plugins2 = rebar_state:get(S, {plugins, Profile}, []), - handle_plugins(Profile, Plugins2, StateAcc2) - end, StateAcc1, ProjectApps) + AppDir = rebar_app_info:dir(App), + C = rebar_config:consult(AppDir), + S = rebar_state:new(rebar_state:new(), C, AppDir), + Plugins2 = rebar_state:get(S, {plugins, Profile}, []), + handle_plugins(Profile, Plugins2, StateAcc2) + end, StateAcc1, ProjectApps) end, State, Profiles). -spec install(rebar_state:t()) -> rebar_state:t(). diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index 3a5a7cd..0b0f607 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -36,8 +36,6 @@ -include_lib("providers/include/providers.hrl"). -export([handle_deps_as_profile/4, - parse_deps/5, - parse_deps/6, profile_dep_dir/2, find_cycles/1, cull_compile/2]). @@ -125,17 +123,9 @@ handle_deps_as_profile(Profile, State, Deps, Upgrade) -> Locks = [], Level = 0, DepsDir = profile_dep_dir(State, Profile), - {SrcDeps, PkgDeps} = parse_deps(DepsDir, Deps, State, Locks, Level), - AllSrcProfileDeps = [{Profile, SrcDeps, Locks, Level}], - AllPkgProfileDeps = case PkgDeps of - [] -> - []; - _ -> - [{Profile, Level, PkgDeps}] - end, - {AllApps, PkgDeps1, Seen, State1} = handle_profile_level(AllSrcProfileDeps, AllPkgProfileDeps, Locks, sets:new(), Upgrade, State), - - handle_profile_pkg_level(PkgDeps1, AllApps, Seen, Upgrade, [], State1). + Deps1 = rebar_app_utils:parse_deps(DepsDir, Deps, State, Locks, Level), + ProfileLevelDeps = [{Profile, Deps1, Level}], + handle_profile_level(ProfileLevelDeps, [], sets:new(), Upgrade, Locks, State). %% =================================================================== %% Internal functions @@ -144,71 +134,30 @@ handle_deps_as_profile(Profile, State, Deps, Upgrade) -> %% finds all the deps in `{deps, ...}` for each profile provided. deps_per_profile(Profiles, Upgrade, State) -> Level = 0, - {AllProfileDeps, PkgDeps} = lists:foldl(fun(Profile, {SrcAcc, PkgAcc}) -> - case parse_profile_deps(State, Profile, Level) of - {Src, {_, _, []}} -> - {[Src | SrcAcc], PkgAcc}; - {Src, Pkg} -> - {[Src | SrcAcc], [Pkg | PkgAcc]} - end - end, {[], []}, Profiles), - {AllApps, PkgDeps1, Seen, State1} = handle_profile_level(AllProfileDeps, PkgDeps, [], sets:new(), Upgrade, State), Locks = rebar_state:get(State, {locks, default}, []), - handle_profile_pkg_level(PkgDeps1, AllApps, Seen, Upgrade, Locks, State1). + Deps = lists:foldl(fun(Profile, DepAcc) -> + [parsed_profile_deps(State, Profile, Level) | DepAcc] + end, [], Profiles), + handle_profile_level(Deps, [], sets:new(), Upgrade, Locks, State). -parse_profile_deps(State, Profile, Level) -> - Locks = rebar_state:get(State, {locks, Profile}, []), - {SrcDeps, PkgDeps} = rebar_state:get(State, {parsed_deps, Profile}, {[], []}), - {{Profile, SrcDeps, Locks, Level}, {Profile, Level, PkgDeps}}. +parsed_profile_deps(State, Profile, Level) -> + ParsedDeps = rebar_state:get(State, {parsed_deps, Profile}, []), + {Profile, ParsedDeps, Level}. %% Level-order traversal of all dependencies, across profiles. %% If profiles x,y,z are present, then the traversal will go: %% x0, y0, z0, x1, y1, z1, ..., xN, yN, zN. -handle_profile_level([], PkgDeps, SrcApps, Seen, _Upgrade, State) -> - {SrcApps, PkgDeps, Seen, State}; -handle_profile_level([{Profile, SrcDeps, Locks, Level} | Rest], PkgDeps, SrcApps, Seen, Upgrade, State) -> - {SrcDeps1, PkgDeps1, SrcApps1, State1, Seen1, Locks1} = - update_src_deps(Profile, Level, SrcDeps, [], SrcApps - ,State, Upgrade, Seen, Locks), - SrcDeps2 = case SrcDeps1 of +handle_profile_level([], Apps, _Seen, _Upgrade, _Locks, State) -> + {Apps, State}; +handle_profile_level([{Profile, Deps, Level} | Rest], Apps, Seen, Upgrade, Locks, State) -> + {Deps1, Apps1, State1, Seen1} = + update_deps(Profile, Level, Deps, Apps + ,State, Upgrade, Seen, Locks), + Deps2 = case Deps1 of [] -> Rest; - _ -> Rest ++ [{Profile, SrcDeps1, Locks1, Level+1}] + _ -> Rest ++ [{Profile, Deps1, Level+1}] end, - handle_profile_level(SrcDeps2, case PkgDeps1 of - [] -> - PkgDeps; - _ -> - [{Profile, Level+1, PkgDeps1} | PkgDeps] - end, SrcApps1, sets:union(Seen, Seen1), Upgrade, State1). - -handle_profile_pkg_level([], AllApps, _Seen, _Upgrade, _Locks, State) -> - {AllApps, State}; -handle_profile_pkg_level(PkgDeps, AllApps, Seen, Upgrade, Locks, State) -> - %% Read in package index and dep graph - {Packages, Graph} = rebar_state:packages(State), - Registry = rebar_state:registry(State), - State1 = rebar_state:packages(rebar_state:registry(State, Registry) - ,{Packages, Graph}), - - S = case rebar_digraph:cull_deps(Graph, lists:keysort(2, PkgDeps), Seen) of - {ok, [], []} -> - throw({rebar_digraph, no_solution}); - {ok, [], Discarded} -> - [warn_skip_pkg(Pkg, State) || Upgrade =:= false, - Pkg <- Discarded, - not(pkg_locked(Pkg, Locks))], - []; - {ok, Solution, []} -> - Solution; - {ok, Solution, Discarded} -> - [warn_skip_pkg(Pkg, State) || Upgrade =:= false, - Pkg <- Discarded, - not(pkg_locked(Pkg, Locks))], - Solution - end, - - {PkgApps, State2} = update_pkg_deps(S, Packages, Upgrade, Seen, State1, Locks), - {AllApps++PkgApps, State2}. + handle_profile_level(Deps2, Apps1, sets:union(Seen, Seen1), Upgrade, Locks, State1). find_cycles(Apps) -> case rebar_digraph:compile_order(Apps) of @@ -220,38 +169,6 @@ find_cycles(Apps) -> cull_compile(TopSortedDeps, ProjectApps) -> lists:dropwhile(fun not_needs_compile/1, TopSortedDeps -- ProjectApps). -pkg_locked({_, Name, _, _}, Locks) -> - pkg_locked(Name, Locks); -pkg_locked({Name, _}, Locks) -> - pkg_locked(Name, Locks); -pkg_locked(Name, Locks) -> - false =/= lists:keyfind(Name, 1, Locks). - -update_pkg_deps(Pkgs, Packages, Upgrade, Seen, State, Locks) -> - {Solved, _, State1} - = lists:foldl(fun({Profile, Pkg}, {Acc, SeenAcc, StateAcc}) -> - DepsDir = profile_dep_dir(State, Profile), - handle_pkg_dep(Profile, Pkg, Packages, Upgrade, DepsDir, Acc, SeenAcc, Locks, StateAcc) - end, {[], Seen, State}, Pkgs), - {Solved, State1}. - -handle_pkg_dep(Profile, Pkg, Packages, Upgrade, DepsDir, Fetched, Seen, Locks, State) -> - IsLock = pkg_locked(Pkg, Locks), - AppInfo = package_to_app(DepsDir, Packages, Pkg, IsLock, State), - case sets:is_element(rebar_app_info:name(AppInfo), Seen) of - true -> - {Fetched, Seen, State}; - false -> - Deps = rebar_app_info:deps(AppInfo), - Level = rebar_app_info:dep_level(AppInfo), - {NewSeen, NewState} = maybe_lock(Profile, AppInfo, Seen, State, Level), - {_, AppInfo1} = maybe_fetch(AppInfo, Profile, Upgrade, Seen, NewState), - {AppInfo2, _, _, _, _} = - handle_dep(NewState, Profile, DepsDir, AppInfo1, Locks, Level), - AppInfo3 = rebar_app_info:deps(AppInfo2, Deps), - {[AppInfo3 | Fetched], NewSeen, NewState} - end. - maybe_lock(Profile, AppInfo, Seen, State, Level) -> Name = rebar_app_info:name(AppInfo), case rebar_app_info:is_checkout(AppInfo) of @@ -278,49 +195,28 @@ maybe_lock(Profile, AppInfo, Seen, State, Level) -> {sets:add_element(Name, Seen), State} end. -package_to_app(DepsDir, Packages, {Parent, Name, Vsn, Level}, IsLock, State) -> - case dict:find({Name, Vsn}, Packages) of - error -> - case rebar_packages:check_registry(Name, Vsn, State) of - true -> - throw(?PRV_ERROR({not_rebar_package, Name, Vsn})); - false -> - throw(?PRV_ERROR({missing_package, Name, Vsn})) - end; - {ok, PkgDeps} -> - Source = {pkg, Name, Vsn}, - AppInfo = new_dep(root, DepsDir, Name, Vsn, Source, IsLock, State), - AppInfo1 = rebar_app_info:dep_level(rebar_app_info:deps(AppInfo, PkgDeps), Level), - BaseDir = rebar_state:get(State, base_dir, []), - AppState1 = rebar_state:set(rebar_app_info:state(AppInfo1), base_dir, BaseDir), - rebar_app_info:parent(rebar_app_info:state(AppInfo1, AppState1), Parent) - end. - --spec update_src_deps(atom(), non_neg_integer(), list(), list(), list(), rebar_state:t(), boolean(), sets:set(binary()), list()) -> {list(), list(), list(), rebar_state:t(), sets:set(binary()), list()}. -update_src_deps(Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, Locks) -> +update_deps(Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) -> lists:foldl( - fun(AppInfo, {SrcDepsAcc, PkgDepsAcc, SrcAppsAcc, StateAcc, SeenAcc, LocksAcc}) -> - update_src_dep(AppInfo, Profile, Level, - SrcDepsAcc, PkgDepsAcc, SrcAppsAcc, StateAcc, - Upgrade, SeenAcc, Locks, LocksAcc) + fun(AppInfo, {DepsAcc, AppsAcc, StateAcc, SeenAcc}) -> + update_dep(AppInfo, Profile, Level, + DepsAcc, AppsAcc, StateAcc, + Upgrade, SeenAcc, Locks) end, - {[], PkgDeps, SrcApps, State, Seen, Locks}, - rebar_utils:sort_deps(SrcDeps)). + {[], Apps, State, Seen}, + rebar_utils:sort_deps(Deps)). - -update_src_dep(AppInfo, Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, BaseLocks, Locks) -> +update_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) -> %% If not seen, add to list of locks to write out Name = rebar_app_info:name(AppInfo), case sets:is_element(Name, Seen) of true -> - update_seen_src_dep(AppInfo, Profile, Level, - SrcDeps, PkgDeps, SrcApps, - State, Upgrade, Seen, BaseLocks, Locks); + update_seen_dep(AppInfo, Profile, Level, + Deps, Apps, + State, Upgrade, Seen, Locks); false -> - update_unseen_src_dep(AppInfo, Profile, Level, - SrcDeps, PkgDeps, SrcApps, - State, Upgrade, Seen, Locks) - + update_unseen_dep(AppInfo, Profile, Level, + Deps, Apps, + State, Upgrade, Seen, Locks) end. profile_dep_dir(State, Profile) -> @@ -329,45 +225,34 @@ profile_dep_dir(State, Profile) -> _ -> rebar_dir:deps_dir(State) end. -update_seen_src_dep(AppInfo, _Profile, _Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, BaseLocks, Locks) -> +update_seen_dep(AppInfo, _Profile, _Level, Deps, Apps, State, Upgrade, Seen, Locks) -> Name = rebar_app_info:name(AppInfo), %% If seen from lock file or user requested an upgrade %% don't print warning about skipping - case lists:keymember(Name, 1, BaseLocks) of + case lists:keymember(Name, 1, Locks) of false when Upgrade -> ok; false when not Upgrade -> warn_skip_deps(AppInfo, State); true -> ok end, - {SrcDeps, PkgDeps, SrcApps, State, Seen, Locks}. + {Deps, Apps, State, Seen}. -update_unseen_src_dep(AppInfo, Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, Locks) -> +update_unseen_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) -> {NewSeen, State1} = maybe_lock(Profile, AppInfo, Seen, State, Level), {_, AppInfo1} = maybe_fetch(AppInfo, Profile, Upgrade, Seen, State1), - {NewSrcDeps, NewPkgDeps, NewSrcApps, State2, NewLocks} = - handle_dep(AppInfo1, Profile, SrcDeps, PkgDeps, SrcApps, - Level, State1, Locks), - {NewSrcDeps, NewPkgDeps, NewSrcApps, State2, NewSeen, NewLocks}. - -handle_dep(AppInfo, Profile, SrcDeps, PkgDeps, SrcApps, Level, State, Locks) -> DepsDir = profile_dep_dir(State, Profile), - {AppInfo1, NewSrcDeps, NewPkgDeps, NewLocks, State1} = - handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level), - AppInfo2 = rebar_app_info:dep_level(AppInfo1, Level), - {NewSrcDeps ++ SrcDeps - ,NewPkgDeps++PkgDeps - ,[AppInfo2 | SrcApps] - ,State1 - ,NewLocks}. - --spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer()) -> - {rebar_app_info:t(), [rebar_app_info:t()], [pkg_dep()], [integer()], rebar_state:t()}. + {AppInfo2, NewDeps, State2} = + handle_dep(State1, Profile, DepsDir, AppInfo1, Locks, Level), + AppInfo3 = rebar_app_info:dep_level(AppInfo2, Level), + {NewDeps ++ Deps, [AppInfo3 | Apps], State2, NewSeen}. + +-spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer()) -> {rebar_app_info:t(), [rebar_app_info:t()], [pkg_dep()], [integer()], rebar_state:t()}. handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) -> Profiles = rebar_state:current_profiles(State), Name = rebar_app_info:name(AppInfo), + Vsn = rebar_app_info:original_vsn(AppInfo), %% Deps may be under a sub project app, find it and use its state if so - S0 = rebar_app_info:state(AppInfo), - S = rebar_state:registry(S0, rebar_state:registry(State)), + S = rebar_app_info:state(AppInfo), C = rebar_config:consult(rebar_app_info:dir(AppInfo)), S1 = rebar_state:new(S, C, rebar_app_info:dir(AppInfo)), S2 = rebar_state:apply_overrides(S1, Name), @@ -375,23 +260,26 @@ handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) -> S3 = rebar_state:apply_profiles(S2, Profiles), Plugins = rebar_state:get(S3, plugins, []), S4 = rebar_state:set(S3, {plugins, Profile}, Plugins), - AppInfo1 = rebar_app_info:state(AppInfo, S4), rebar_utils:check_min_otp_version(rebar_state:get(S4, minimum_otp_vsn, undefined)), rebar_utils:check_blacklisted_otp_versions(rebar_state:get(S4, blacklisted_otp_vsns, [])), %% Dep may have plugins to install. Find and install here. S5 = rebar_plugins:install(S4), - AppInfo2 = rebar_app_info:state(AppInfo1, S5), + AppInfo1 = rebar_app_info:state(AppInfo, S5), %% Upgrade lock level to be the level the dep will have in this dep tree - Deps = rebar_state:get(S5, {deps, default}, []), - NewLocks = Locks++[{DepName, Source, LockLevel+Level} || - {DepName, Source, LockLevel} <- rebar_state:get(S5, {locks, default}, [])], - AppInfo3 = rebar_app_info:deps(AppInfo2, rebar_state:deps_names(Deps)), - {SrcDeps, PkgDeps} = parse_deps(rebar_app_info:name(AppInfo3), DepsDir, Deps - ,S5, NewLocks, Level+1), - {AppInfo3, SrcDeps, PkgDeps, NewLocks, State}. + case rebar_app_info:resource_type(AppInfo1) of + pkg -> + NewDeps = rebar_packages:deps(Name, Vsn, S5), + NewDeps1 = rebar_app_utils:parse_deps(Name, DepsDir, NewDeps, S5, Locks, Level+1), + {rebar_app_info:deps(AppInfo1, NewDeps), NewDeps1, State}; + _ -> + Deps = rebar_state:get(S5, {deps, default}, []), + AppInfo2 = rebar_app_info:deps(AppInfo1, rebar_state:deps_names(Deps)), + Deps1 = rebar_app_utils:parse_deps(Name, DepsDir, Deps, S5, Locks, Level+1), + {AppInfo2, Deps1, State} + end. -spec maybe_fetch(rebar_app_info:t(), atom(), boolean(), sets:set(binary()), rebar_state:t()) -> {boolean(), rebar_app_info:t()}. @@ -406,20 +294,20 @@ maybe_fetch(AppInfo, Profile, Upgrade, Seen, State) -> false -> true = fetch_app(AppInfo, AppDir, State), maybe_symlink_default(State, Profile, AppDir, AppInfo), - {true, update_app_info(AppDir, AppInfo)}; + {true, rebar_app_info:valid(update_app_info(AppDir, AppInfo), false)}; {true, AppInfo1} -> %% Preserve the state we created with overrides + AppInfo2 = copy_app_info(AppInfo, AppInfo1), AppState = rebar_app_info:state(AppInfo), - Parent = rebar_app_info:parent(AppInfo), - Source = rebar_app_info:source(AppInfo), - AppInfo2 = rebar_app_info:parent(rebar_app_info:state(AppInfo1, AppState), Parent), - AppInfo3 = rebar_app_info:source(AppInfo2, Source), - case sets:is_element(rebar_app_info:name(AppInfo), Seen) of + AppInfo3 = rebar_app_info:state(AppInfo2, AppState), + case sets:is_element(rebar_app_info:name(AppInfo3), Seen) of true -> {false, AppInfo3}; false -> maybe_symlink_default(State, Profile, AppDir, AppInfo3), - {maybe_upgrade(AppInfo, AppDir, Upgrade, State), AppInfo3} + MaybeUpgrade = maybe_upgrade(AppInfo, AppDir, Upgrade, State), + AppInfo4 = update_app_info(AppDir, AppInfo3), + {MaybeUpgrade, AppInfo4} end end end. @@ -467,106 +355,6 @@ make_relative_to_root(State, Path) when is_list(Path) -> Root = rebar_dir:root_dir(State), rebar_dir:make_relative_path(Path, Root). --spec parse_deps(binary(), list(), rebar_state:t(), list(), integer()) -> {[rebar_app_info:t()], [tuple()]}. -parse_deps(DepsDir, Deps, State, Locks, Level) -> - parse_deps(root, DepsDir, Deps, State, Locks, Level). - -parse_deps(Parent, DepsDir, Deps, State, Locks, Level) -> - lists:foldl(fun(Dep, Acc) -> - Name = case Dep of - Dep when is_tuple(Dep) -> - element(1, Dep); - Dep -> - Dep - end, - case lists:keyfind(ec_cnv:to_binary(Name), 1, Locks) of - false -> - parse_dep(Parent, Dep, Acc, DepsDir, false, State); - LockedDep -> - LockedLevel = element(3, LockedDep), - case LockedLevel > Level of - true -> - parse_dep(Parent, Dep, Acc, DepsDir, false, State); - false -> - parse_dep(Parent, LockedDep, Acc, DepsDir, true, State) - end - end - end, {[], []}, Deps). - -parse_dep(Parent, {Name, Vsn}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_list(Vsn) -> - %% Versioned Package dependency - CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), - case rebar_app_info:discover(CheckoutsDir) of - {ok, _App} -> - Dep = new_dep(root, DepsDir, Name, [], [], IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; - not_found -> - {SrcDepsAcc, [parse_goal(Parent - ,ec_cnv:to_binary(Name) - ,ec_cnv:to_binary(Vsn)) | PkgDepsAcc]} - end; -parse_dep(Parent, Name, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_atom(Name); is_binary(Name) -> - %% Unversioned package dependency - {PkgName, PkgVsn} = get_package(ec_cnv:to_binary(Name), State), - CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), - case rebar_app_info:discover(CheckoutsDir) of - {ok, _App} -> - Dep = new_dep(root, DepsDir, Name, [], [], IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; - not_found -> - {SrcDepsAcc, [{Parent, PkgName, PkgVsn} | PkgDepsAcc]} - end; -parse_dep(Parent, {Name, Source}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source) -> - Dep = new_dep(Parent, DepsDir, Name, [], Source, IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; -parse_dep(Parent, {Name, _Vsn, Source}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source) -> - Dep = new_dep(Parent, DepsDir, Name, [], Source, IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; -parse_dep(Parent, {Name, _Vsn, Source, Opts}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source) -> - ?WARN("Dependency option list ~p in ~p is not supported and will be ignored", [Opts, Name]), - Dep = new_dep(Parent, DepsDir, Name, [], Source, IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; -parse_dep(Parent, {_Name, {pkg, Name, Vsn}, Level}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_integer(Level) -> - CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), - case rebar_app_info:discover(CheckoutsDir) of - {ok, _App} -> - Dep = new_dep(root, DepsDir, Name, [], [], IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; - not_found -> - {SrcDepsAcc, [{Parent, Name, Vsn} | PkgDepsAcc]} - end; -parse_dep(Parent, {Name, Source, Level}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source) - , is_integer(Level) -> - Dep = new_dep(Parent, DepsDir, Name, [], Source, IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; -parse_dep(_, Dep, _, _, _, _) -> - throw(?PRV_ERROR({parse_dep, Dep})). - - -new_dep(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> - CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), - {ok, Dep} = case rebar_app_info:discover(CheckoutsDir) of - {ok, App} -> - {ok, rebar_app_info:is_checkout(App, true)}; - not_found -> - Dir = ec_cnv:to_list(filename:join(DepsDir, Name)), - case rebar_app_info:discover(Dir) of - {ok, App} -> - {ok, App}; - not_found -> - rebar_app_info:new(Name, Vsn, - ec_cnv:to_list(filename:join(DepsDir, Name))) - end - end, - C = rebar_config:consult(rebar_app_info:dir(Dep)), - S = rebar_state:new(rebar_state:new(), C, rebar_app_info:dir(Dep)), - Overrides = rebar_state:get(State, overrides, []), - ParentOverrides = rebar_state:overrides(State), - Dep1 = rebar_app_info:state(Dep, - rebar_state:overrides(S, ParentOverrides++Overrides)), - AppInfo = rebar_app_info:is_lock(rebar_app_info:source(Dep1, Source), IsLock), - rebar_app_info:parent(AppInfo, Parent). - fetch_app(AppInfo, AppDir, State) -> ?INFO("Fetching ~s (~p)", [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)]), Source = rebar_app_info:source(AppInfo), @@ -577,15 +365,24 @@ fetch_app(AppInfo, AppDir, State) -> %% be read in an parsed. update_app_info(AppDir, AppInfo) -> {ok, Found} = rebar_app_info:discover(AppDir), - Parent = rebar_app_info:parent(AppInfo), - Source = rebar_app_info:source(AppInfo), AppDetails = rebar_app_info:app_details(Found), + Vsn = rebar_app_info:original_vsn(Found), Applications = proplists:get_value(applications, AppDetails, []), IncludedApplications = proplists:get_value(included_applications, AppDetails, []), - AppInfo1 = rebar_app_info:applications( + AppInfo1 = rebar_app_info:original_vsn(rebar_app_info:applications( rebar_app_info:app_details(AppInfo, AppDetails), - IncludedApplications++Applications), - rebar_app_info:source(rebar_app_info:parent(rebar_app_info:valid(AppInfo1, false), Parent), Source). + IncludedApplications++Applications), Vsn), + AppInfo2 = copy_app_info(AppInfo, AppInfo1), + rebar_app_info:valid(AppInfo2, undefined). + +copy_app_info(OldAppInfo, NewAppInfo) -> + ResourceType = rebar_app_info:resource_type(OldAppInfo), + Parent = rebar_app_info:parent(OldAppInfo), + Source = rebar_app_info:source(OldAppInfo), + + rebar_app_info:resource_type( + rebar_app_info:source( + rebar_app_info:parent(NewAppInfo, Parent), Source), ResourceType). maybe_upgrade(AppInfo, AppDir, Upgrade, State) -> Source = rebar_app_info:source(AppInfo), @@ -608,17 +405,6 @@ maybe_upgrade(AppInfo, AppDir, Upgrade, State) -> false end. --spec parse_goal(binary() | root, binary(), binary()) -> {binary(), binary()} | {binary(), binary(), binary()}. -parse_goal(Parent, Name, Constraint) -> - case re:run(Constraint, "([^\\d]*)(\\d.*)", [{capture, [1,2], binary}]) of - {match, [<<>>, Vsn]} -> - {Parent, Name, Vsn}; - {match, [Op, Vsn]} -> - {Parent, Name, Vsn, binary_to_atom(Op, utf8)}; - nomatch -> - throw(?PRV_ERROR({bad_constraint, Name, Constraint})) - end. - warn_skip_deps(AppInfo, State) -> Msg = "Skipping ~s (from ~p) as an app of the same name " "has already been fetched", @@ -629,25 +415,7 @@ warn_skip_deps(AppInfo, State) -> true -> ?ERROR(Msg, Args), ?FAIL end. -warn_skip_pkg({Name, Source}, State) -> - Msg = "Skipping ~s (version ~s from package index) as an app of the same " - "name has already been fetched", - Args = [Name, Source], - case rebar_state:get(State, deps_error_on_conflict, false) of - false -> ?WARN(Msg, Args); - true -> ?ERROR(Msg, Args), ?FAIL - end. - not_needs_compile(App) -> not(rebar_app_info:is_checkout(App)) andalso rebar_app_info:valid(App) andalso rebar_state:has_all_artifacts(rebar_app_info:state(App)) =:= true. - -get_package(Dep, State) -> - case rebar_state:registry(State) of - {ok, T} -> - {ok, HighestDepVsn} = rebar_packages:find_highest_matching(Dep, "0", T), - {Dep, HighestDepVsn}; - error -> - throw(?PRV_ERROR({load_registry_fail, Dep})) - end. diff --git a/src/rebar_prv_packages.erl b/src/rebar_prv_packages.erl index 82ed2f7..f5d9e38 100644 --- a/src/rebar_prv_packages.erl +++ b/src/rebar_prv_packages.erl @@ -27,19 +27,17 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - {Dict, _} = rebar_state:packages(State), - print_packages(Dict), + print_packages(), {ok, State}. -spec format_error(any()) -> iolist(). format_error(load_registry_fail) -> "Failed to load package regsitry. Try running 'rebar3 update' to fix". -print_packages(Dict) -> - Pkgs = lists:keysort(1, dict:fetch_keys(Dict)), - SortedPkgs = lists:foldl(fun({Pkg, Vsn}, Acc) -> - orddict:append_list(Pkg, [Vsn], Acc) - end, orddict:new(), Pkgs), +print_packages() -> + SortedPkgs = ets:foldl(fun({{Pkg, Vsn}, _}, Acc) -> + orddict:append_list(Pkg, [Vsn], Acc) + end, orddict:new(), ?PACKAGE_TABLE), orddict:map(fun(Name, Vsns) -> SortedVsns = lists:sort(fun(A, B) -> diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index 55c70e2..4b3a155 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -9,21 +9,14 @@ do/1, format_error/1]). +-export([hex_to_index/1]). + -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). -define(PROVIDER, update). -define(DEPS, []). -%% Ignore warning of digraph opaque type when running dialyzer --dialyzer({no_opaque, do/1}). --dialyzer({no_opaque, write_registry/3}). - -%% Ignoring the opaque type warning won't stop dialyzer from warning of -%% no return for functions that had the opaque type warnings --dialyzer({no_return, do/1}). --dialyzer({no_return, write_registry/3}). - %% =================================================================== %% Public API %% =================================================================== @@ -50,16 +43,15 @@ do(State) -> TmpDir = ec_file:insecure_mkdtemp(), TmpFile = filename:join(TmpDir, "packages.gz"), - Url = rebar_state:get(State, rebar_packages_cdn, "https://s3.amazonaws.com/s3.hex.pm/registry.ets.gz"), + Url = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_HEX_REGISTRY), {ok, _RequestId} = httpc:request(get, {Url, []}, [], [{stream, TmpFile}, {sync, true}], rebar), {ok, Data} = file:read_file(TmpFile), Unzipped = zlib:gunzip(Data), ok = file:write_file(HexFile, Unzipped), - {Dict, Graph} = hex_to_graph(HexFile), - write_registry(Dict, Graph, State), - true = digraph:delete(Graph), + + hex_to_index(State), ok catch _E:C -> @@ -73,56 +65,58 @@ do(State) -> format_error(package_index_write) -> "Failed to write package index.". --spec write_registry(rebar_dict(), {digraph, ets:tid(), ets:tid(), ets:tid(), any()}, rebar_state:t()) -> ok | {error, atom()}. -write_registry(Dict, {digraph, Edges, Vertices, Neighbors, _}, State) -> - RegistryDir = rebar_packages:package_dir(State), - filelib:ensure_dir(filename:join(RegistryDir, "dummy")), - ets:tab2file(Edges, filename:join(RegistryDir, "edges")), - ets:tab2file(Vertices, filename:join(RegistryDir, "vertices")), - ets:tab2file(Neighbors, filename:join(RegistryDir, "neighbors")), - file:write_file(filename:join(RegistryDir, "dict"), term_to_binary(Dict)). - is_supported(<<"make">>) -> true; is_supported(<<"rebar">>) -> true; +is_supported(<<"rebar3">>) -> true; is_supported(_) -> false. -hex_to_graph(Filename) -> - {ok, T} = ets:file2tab(Filename), - Graph = digraph:new(), - ets:foldl(fun({Pkg, [Versions]}, ok) when is_binary(Pkg), is_list(Versions) -> - lists:foreach(fun(Version) -> - digraph:add_vertex(Graph, {Pkg, Version}, 1) - end, Versions); - (_, ok) -> - ok - end, ok, T), - - Dict1 = ets:foldl(fun({{Pkg, PkgVsn}, [Deps, _, BuildTools | _]}, Dict) when is_list(BuildTools) -> - case lists:any(fun is_supported/1, BuildTools) of - true -> - DepsList = update_graph(Pkg, PkgVsn, Deps, T, Graph), - dict:store({Pkg, PkgVsn}, DepsList, Dict); - false -> - Dict - end; - (_, Dict) -> - Dict - end, dict:new(), T), - {Dict1, Graph}. - -update_graph(Pkg, PkgVsn, Deps, HexRegistry, Graph) -> +hex_to_index(State) -> + RegistryDir = rebar_packages:package_dir(State), + HexFile = filename:join(RegistryDir, "registry"), + try ets:file2tab(HexFile) of + {ok, Registry} -> + try + (catch ets:delete(?PACKAGE_TABLE)), + ets:new(?PACKAGE_TABLE, [named_table, public]), + ets:foldl(fun({{Pkg, PkgVsn}, [Deps, Checksum, BuildTools | _]}, _) when is_list(BuildTools) -> + case lists:any(fun is_supported/1, BuildTools) of + true -> + DepsList = update_deps_list(Deps, Registry, State), + ets:insert(?PACKAGE_TABLE, {{Pkg, PkgVsn}, DepsList, Checksum}); + false -> + true + end; + ({Pkg, [Vsns]}, _) when is_binary(Pkg) -> + ets:insert(?PACKAGE_TABLE, {Pkg, Vsns}); + (_, _) -> + true + end, true, Registry), + + ets:insert(?PACKAGE_TABLE, {package_index_version, ?PACKAGE_INDEX_VERSION}), + ets:tab2file(?PACKAGE_TABLE, filename:join(RegistryDir, "packages.idx")), + true + after + catch ets:delete(Registry) + end; + {error, Reason} -> + ?DEBUG("Error loading package registry: ~p", [Reason]), + false + catch + _:_ -> + fail + end. + +update_deps_list(Deps, HexRegistry, State) -> lists:foldl(fun([Dep, DepVsn, false, _AppName | _], DepsListAcc) -> case DepVsn of <<"~> ", Vsn/binary>> -> - case rebar_packages:find_highest_matching(Dep, Vsn, HexRegistry) of + case rebar_packages:find_highest_matching(Dep, Vsn, HexRegistry, State) of {ok, HighestDepVsn} -> - digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, HighestDepVsn}), - [{Dep, DepVsn} | DepsListAcc]; + [{Dep, HighestDepVsn} | DepsListAcc]; none -> DepsListAcc end; Vsn -> - digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, Vsn}), [{Dep, Vsn} | DepsListAcc] end; ([_Dep, _DepVsn, true, _AppName | _], DepsListAcc) -> diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl index f49eafe..7385cfe 100644 --- a/src/rebar_prv_upgrade.erl +++ b/src/rebar_prv_upgrade.erl @@ -54,7 +54,7 @@ do(State) -> Deps0 = top_level_deps(Deps, Locks), State1 = rebar_state:set(State, {deps, default}, Deps0), DepsDir = rebar_prv_install_deps:profile_dep_dir(State, default), - D = rebar_prv_install_deps:parse_deps(root, DepsDir, Deps0, State1, Locks0, 0), + D = rebar_app_utils:parse_deps(root, DepsDir, Deps0, State1, Locks0, 0), State2 = rebar_state:set(State1, {parsed_deps, default}, D), State3 = rebar_state:set(State2, {locks, default}, Locks0), State4 = rebar_state:set(State3, upgrade, true), @@ -83,7 +83,6 @@ format_error({transitive_dependency, Name}) -> format_error(Reason) -> io_lib:format("~p", [Reason]). - parse_names(Bin, Locks) -> case lists:usort(re:split(Bin, <<" *, *">>, [trim])) of %% Nothing submitted, use *all* apps diff --git a/src/rebar_state.erl b/src/rebar_state.erl index 7616151..213eb2f 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -38,9 +38,6 @@ overrides/1, overrides/2, apply_overrides/2, - packages/1, packages/2, - registry/1, registry/2, - resources/1, resources/2, add_resource/2, providers/1, providers/2, add_provider/2]). @@ -65,8 +62,6 @@ all_plugin_deps = [] :: [rebar_app_info:t()], all_deps = [] :: [rebar_app_info:t()], - packages = undefined :: {rebar_dict(), rebar_digraph()} | undefined, - registry = undefined :: {ok, ets:tid()} | error | undefined, overrides = [], resources = [], providers = []}). @@ -88,16 +83,9 @@ new(Config) when is_list(Config) -> Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins} | Config], true = rebar_config:verify_config_format(Terms), Opts = dict:from_list(Terms), - load_package_registry( - BaseState#state_t { dir = rebar_dir:get_cwd(), - default = Opts, - opts = Opts }). - -load_package_registry(Config0) -> - Registry = rebar_packages:registry(Config0), - Packages = rebar_packages:packages(Config0), - Config0#state_t{registry = Registry, - packages = Packages}. + BaseState#state_t { dir = rebar_dir:get_cwd(), + default = Opts, + opts = Opts }. -spec new(t() | atom(), list()) -> t(). new(Profile, Config) when is_atom(Profile) @@ -109,11 +97,10 @@ new(Profile, Config) when is_atom(Profile) Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins} | Config], true = rebar_config:verify_config_format(Terms), Opts = dict:from_list(Terms), - load_package_registry( - BaseState#state_t { dir = rebar_dir:get_cwd(), - current_profiles = [Profile], - default = Opts, - opts = Opts }); + BaseState#state_t { dir = rebar_dir:get_cwd(), + current_profiles = [Profile], + default = Opts, + opts = Opts }; new(ParentState=#state_t{}, Config) -> %% Load terms from rebar.config, if it exists Dir = rebar_dir:get_cwd(), @@ -446,22 +433,6 @@ namespace(#state_t{namespace=Namespace}) -> namespace(State=#state_t{}, Namespace) -> State#state_t{namespace=Namespace}. -packages(#state_t{packages=undefined}) -> - throw(packages_usage_error); -packages(#state_t{packages=Packages}) -> - Packages. - -packages(State, Packages) -> - State#state_t{packages=Packages}. - -registry(#state_t{registry=undefined}) -> - throw(registry_usage_error); -registry(#state_t{registry=Registry}) -> - Registry. - -registry(State, Registry) -> - State#state_t{registry=Registry}. - -spec resources(t()) -> [{rebar_resource:type(), module()}]. resources(#state_t{resources=Resources}) -> Resources. |