summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorTristan Sloughter <tristan.sloughter@gmail.com>2015-08-21 21:23:08 -0500
committerTristan Sloughter <tristan.sloughter@gmail.com>2015-08-21 21:23:08 -0500
commit1f6516b231260a1b2e48f9b86e37a5d317d4983c (patch)
treefdade8897340406cf52bd7807e92086856f13a60 /src
parentb52395f4aec11bacaace91ec72e100c9778d6098 (diff)
parente853e12d7c5b3f0f29670d4ae89d17d004edada1 (diff)
Merge pull request #732 from tsloughter/level_wins
this patch treats pkg and src deps as equals, so level decides winner
Diffstat (limited to 'src')
-rw-r--r--src/rebar.hrl6
-rw-r--r--src/rebar3.erl13
-rw-r--r--src/rebar_app_discover.erl18
-rw-r--r--src/rebar_app_info.erl13
-rw-r--r--src/rebar_app_utils.erl124
-rw-r--r--src/rebar_digraph.erl110
-rw-r--r--src/rebar_packages.erl158
-rw-r--r--src/rebar_plugins.erl12
-rw-r--r--src/rebar_prv_install_deps.erl388
-rw-r--r--src/rebar_prv_packages.erl12
-rw-r--r--src/rebar_prv_update.erl96
-rw-r--r--src/rebar_prv_upgrade.erl3
-rw-r--r--src/rebar_state.erl43
13 files changed, 382 insertions, 614 deletions
diff --git a/src/rebar.hrl b/src/rebar.hrl
index 8a702df..961b8ea 100644
--- a/src/rebar.hrl
+++ b/src/rebar.hrl
@@ -23,8 +23,14 @@
-define(DEFAULT_RELEASE_DIR, "rel").
-define(DEFAULT_CONFIG_FILE, "rebar.config").
-define(DEFAULT_CDN, "https://s3.amazonaws.com/s3.hex.pm/tarballs").
+-define(DEFAULT_HEX_REGISTRY, "https://s3.amazonaws.com/s3.hex.pm/registry.ets.gz").
-define(LOCK_FILE, "rebar.lock").
+-define(PACKAGE_INDEX_VERSION, 3).
+-define(PACKAGE_TABLE, package_index).
+-define(INDEX_FILE, "packages.idx").
+-define(REGISTRY_FILE, "registry").
+
-ifdef(namespaced_types).
-type rebar_dict() :: dict:dict().
-else.
diff --git a/src/rebar3.erl b/src/rebar3.erl
index 8004443..ab3e0eb 100644
--- a/src/rebar3.erl
+++ b/src/rebar3.erl
@@ -105,6 +105,7 @@ run_aux(State, RawArgs) ->
{ok, Providers} = application:get_env(rebar, providers),
%% Providers can modify profiles stored in opts, so set default after initializing providers
State4 = rebar_state:create_logic_providers(Providers, State3),
+ rebar_packages:packages(State4),
State5 = rebar_plugins:project_apps_install(State4),
State6 = rebar_state:default(State5, rebar_state:opts(State5)),
@@ -280,9 +281,15 @@ state_from_global_config(Config, GlobalConfigFile) ->
%% We don't want to worry about global plugin install state effecting later
%% usage. So we throw away the global profile state used for plugin install.
GlobalConfigThrowAway = rebar_state:current_profiles(GlobalConfig, [global]),
- GlobalState = rebar_plugins:handle_plugins(global,
- rebar_state:get(GlobalConfigThrowAway, plugins, []),
- GlobalConfigThrowAway),
+ GlobalState = case rebar_state:get(GlobalConfigThrowAway, plugins, []) of
+ [] ->
+ GlobalConfigThrowAway;
+ GlobalPluginsToInstall ->
+ rebar_packages:packages(GlobalConfigThrowAway),
+ rebar_plugins:handle_plugins(global,
+ GlobalPluginsToInstall,
+ GlobalConfigThrowAway)
+ end,
GlobalPlugins = rebar_state:providers(GlobalState),
GlobalConfig2 = rebar_state:set(GlobalConfig, plugins, []),
GlobalConfig3 = rebar_state:set(GlobalConfig2, {plugins, global}, rebar_state:get(GlobalConfigThrowAway, plugins, [])),
diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl
index e81a323..8b1e58e 100644
--- a/src/rebar_app_discover.erl
+++ b/src/rebar_app_discover.erl
@@ -96,7 +96,7 @@ merge_deps(AppInfo, State) ->
{AppInfo1, State1}.
handle_profile(Profile, Name, AppState, State) ->
- {TopSrc, TopPkg} = rebar_state:get(State, {parsed_deps, Profile}, {[], []}),
+ TopParsedDeps = rebar_state:get(State, {parsed_deps, Profile}, {[], []}),
TopLevelProfileDeps = rebar_state:get(State, {deps, Profile}, []),
AppProfileDeps = rebar_state:get(AppState, {deps, Profile}, []),
AppProfileDeps2 = rebar_utils:tup_dedup(rebar_utils:tup_sort(AppProfileDeps)),
@@ -108,18 +108,18 @@ handle_profile(Profile, Name, AppState, State) ->
%% Only deps not also specified in the top level config need
%% to be included in the parsed deps
NewDeps = ProfileDeps2 -- TopLevelProfileDeps,
- {ParsedSrc, ParsedPkg} = parse_profile_deps(Profile, Name, NewDeps, AppState, State1),
- rebar_state:set(State1, {parsed_deps, Profile}, {TopSrc++ParsedSrc, TopPkg++ParsedPkg}).
+ ParsedDeps = parse_profile_deps(Profile, Name, NewDeps, AppState, State1),
+ rebar_state:set(State1, {parsed_deps, Profile}, TopParsedDeps++ParsedDeps).
parse_profile_deps(Profile, Name, Deps, AppState, State) ->
DepsDir = rebar_prv_install_deps:profile_dep_dir(State, Profile),
Locks = rebar_state:get(State, {locks, Profile}, []),
- rebar_prv_install_deps:parse_deps(Name
- ,DepsDir
- ,Deps
- ,AppState
- ,Locks
- ,1).
+ rebar_app_utils:parse_deps(Name
+ ,DepsDir
+ ,Deps
+ ,AppState
+ ,Locks
+ ,1).
project_app_config(AppInfo, State) ->
C = rebar_config:consult(rebar_app_info:dir(AppInfo)),
diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl
index 7e31f6d..bb99584 100644
--- a/src/rebar_app_info.erl
+++ b/src/rebar_app_info.erl
@@ -34,6 +34,8 @@
dir/2,
out_dir/1,
out_dir/2,
+ resource_type/1,
+ resource_type/2,
source/1,
source/2,
state/1,
@@ -64,6 +66,7 @@
dep_level=0 :: integer(),
dir :: file:name(),
out_dir :: file:name(),
+ resource_type :: pkg | src,
source :: string() | tuple() | undefined,
state :: rebar_state:t() | undefined,
is_lock=false :: boolean(),
@@ -274,6 +277,14 @@ out_dir(AppInfo=#app_info_t{}, OutDir) ->
ebin_dir(#app_info_t{out_dir=OutDir}) ->
ec_cnv:to_list(filename:join(OutDir, "ebin")).
+-spec resource_type(t(), pkg | src) -> t().
+resource_type(AppInfo=#app_info_t{}, Type) ->
+ AppInfo#app_info_t{resource_type=Type}.
+
+-spec resource_type(t()) -> pkg | src.
+resource_type(#app_info_t{resource_type=ResourceType}) ->
+ ResourceType.
+
-spec source(t(), string() | tuple()) -> t().
source(AppInfo=#app_info_t{}, Source) ->
AppInfo#app_info_t{source=Source}.
@@ -316,7 +327,7 @@ is_checkout(#app_info_t{is_checkout=IsCheckout}) ->
-spec valid(t()) -> boolean().
valid(AppInfo=#app_info_t{valid=undefined, state=State}) ->
- case rebar_app_utils:validate_application_info(AppInfo)
+ case rebar_app_utils:validate_application_info(AppInfo) =:= true
andalso rebar_state:has_all_artifacts(State) =:= true of
true ->
true;
diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl
index e9745c3..b14c50a 100644
--- a/src/rebar_app_utils.erl
+++ b/src/rebar_app_utils.erl
@@ -32,6 +32,9 @@
app_src_to_app/2,
validate_application_info/1,
validate_application_info/2,
+ parse_deps/5,
+ parse_deps/6,
+ dep_to_app/7,
format_error/1]).
-include("rebar.hrl").
@@ -87,6 +90,109 @@ validate_application_info(AppInfo, AppDetail) ->
end
end.
+-spec parse_deps(binary(), list(), rebar_state:t(), list(), integer()) -> {[rebar_app_info:t()], [tuple()]}.
+parse_deps(DepsDir, Deps, State, Locks, Level) ->
+ parse_deps(root, DepsDir, Deps, State, Locks, Level).
+
+parse_deps(Parent, DepsDir, Deps, State, Locks, Level) ->
+ [parse_dep(Dep, Parent, DepsDir, State, Locks, Level) || Dep <- Deps].
+
+parse_dep(Dep, Parent, DepsDir, State, Locks, Level) ->
+ Name = case Dep of
+ Dep when is_tuple(Dep) ->
+ element(1, Dep);
+ Dep ->
+ Dep
+ end,
+ case lists:keyfind(ec_cnv:to_binary(Name), 1, Locks) of
+ false ->
+ parse_dep(Parent, Dep, DepsDir, false, State);
+ LockedDep ->
+ LockedLevel = element(3, LockedDep),
+ case LockedLevel > Level of
+ true ->
+ parse_dep(Parent, Dep, DepsDir, false, State);
+ false ->
+ parse_dep(Parent, LockedDep, DepsDir, true, State)
+ end
+ end.
+
+parse_dep(Parent, {Name, Vsn}, DepsDir, IsLock, State) when is_list(Vsn); is_binary(Vsn) ->
+ %% Versioned Package dependency
+ CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)),
+ case rebar_app_info:discover(CheckoutsDir) of
+ {ok, _App} ->
+ dep_to_app(root, DepsDir, Name, [], [], IsLock, State);
+ not_found ->
+ {PkgName, PkgVsn} = parse_goal(ec_cnv:to_binary(Name)
+ ,ec_cnv:to_binary(Vsn)),
+ Source = {pkg, PkgName, PkgVsn},
+ rebar_app_info:resource_type(dep_to_app(Parent, DepsDir, PkgName, PkgVsn, Source, IsLock, State), pkg)
+ end;
+parse_dep(Parent, Name, DepsDir, IsLock, State) when is_atom(Name); is_binary(Name) ->
+ %% Unversioned package dependency
+ {PkgName, PkgVsn} = get_package(ec_cnv:to_binary(Name), State),
+ CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)),
+ case rebar_app_info:discover(CheckoutsDir) of
+ {ok, _App} ->
+ dep_to_app(root, DepsDir, Name, [], [], IsLock, State);
+ not_found ->
+ Source = {pkg, PkgName, PkgVsn},
+ rebar_app_info:resource_type(dep_to_app(Parent, DepsDir, PkgName, PkgVsn, Source, IsLock, State), pkg)
+ end;
+parse_dep(Parent, {Name, Source}, DepsDir, IsLock, State) when is_tuple(Source) ->
+ dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State);
+parse_dep(Parent, {Name, _Vsn, Source}, DepsDir, IsLock, State) when is_tuple(Source) ->
+ dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State);
+parse_dep(Parent, {Name, _Vsn, Source, Opts}, DepsDir, IsLock, State) when is_tuple(Source) ->
+ ?WARN("Dependency option list ~p in ~p is not supported and will be ignored", [Opts, Name]),
+ dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State);
+parse_dep(Parent, {_Name, {pkg, Name, Vsn}, Level}, DepsDir, IsLock, State) when is_integer(Level) ->
+ CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)),
+ case rebar_app_info:discover(CheckoutsDir) of
+ {ok, _App} ->
+ dep_to_app(root, DepsDir, Name, [], [], IsLock, State);
+ not_found ->
+ Source = {pkg, Name, Vsn},
+ rebar_app_info:resource_type(dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State), pkg)
+ end;
+parse_dep(Parent, {Name, Source, Level}, DepsDir, IsLock, State) when is_tuple(Source)
+ , is_integer(Level) ->
+ dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State);
+parse_dep(_, Dep, _, _, _) ->
+ throw(?PRV_ERROR({parse_dep, Dep})).
+
+dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) ->
+ CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)),
+ BaseDir = rebar_state:get(State, base_dir, []),
+ {ok, Dep} = case rebar_app_info:discover(CheckoutsDir) of
+ {ok, App} ->
+ {ok, rebar_app_info:is_checkout(App, true)};
+ not_found ->
+ Dir = ec_cnv:to_list(filename:join(DepsDir, Name)),
+ case rebar_app_info:discover(Dir) of
+ {ok, App} ->
+ {ok, App};
+ not_found ->
+ rebar_app_info:new(Name, Vsn,
+ ec_cnv:to_list(filename:join(DepsDir, Name)))
+ end
+ end,
+ C = rebar_config:consult(rebar_app_info:dir(Dep)),
+ S = rebar_state:new(rebar_state:new(), C, rebar_app_info:dir(Dep)),
+ Overrides = rebar_state:get(State, overrides, []),
+ ParentOverrides = rebar_state:overrides(State),
+ S1 = rebar_state:set(rebar_state:overrides(S, ParentOverrides++Overrides), base_dir, BaseDir),
+ Dep1 = rebar_app_info:state(Dep, S1),
+ AppInfo = rebar_app_info:is_lock(rebar_app_info:source(Dep1, Source), IsLock),
+ ResourceType = case Source of
+ {pkg, _, _} ->
+ pkg;
+ _ ->
+ src
+ end,
+ rebar_app_info:resource_type(rebar_app_info:parent(AppInfo, Parent), ResourceType).
+
format_error(Error) ->
io_lib:format("~p", [Error]).
@@ -94,11 +200,25 @@ format_error(Error) ->
%% Internal functions
%% ===================================================================
+-spec parse_goal(binary(), binary()) -> {binary(), binary()} | {binary(), binary(), binary()}.
+parse_goal(Name, Constraint) ->
+ case re:run(Constraint, "([^\\d]*)(\\d.*)", [{capture, [1,2], binary}]) of
+ {match, [<<>>, Vsn]} ->
+ {Name, Vsn};
+ {match, [Op, Vsn]} ->
+ {Name, Vsn, binary_to_atom(Op, utf8)};
+ nomatch ->
+ throw(?PRV_ERROR({bad_constraint, Name, Constraint}))
+ end.
+
+get_package(Dep, State) ->
+ {ok, HighestDepVsn} = rebar_packages:find_highest_matching(Dep, "0", ?PACKAGE_TABLE, State),
+ {Dep, HighestDepVsn}.
+
-spec has_all_beams(file:filename_all(), [module()]) ->
true | ?PRV_ERROR({missing_module, module()}).
has_all_beams(EbinDir, [Module | ModuleList]) ->
- BeamFile = filename:join([EbinDir,
- ec_cnv:to_list(Module) ++ ".beam"]),
+ BeamFile = filename:join([EbinDir, ec_cnv:to_list(Module) ++ ".beam"]),
case filelib:is_file(BeamFile) of
true ->
has_all_beams(EbinDir, ModuleList);
diff --git a/src/rebar_digraph.erl b/src/rebar_digraph.erl
index ff0a1d2..363253a 100644
--- a/src/rebar_digraph.erl
+++ b/src/rebar_digraph.erl
@@ -2,8 +2,6 @@
-export([compile_order/1
,restore_graph/1
- ,cull_deps/2
- ,cull_deps/3
,subgraph/2
,format_error/1]).
@@ -69,17 +67,6 @@ restore_graph({Vs, Es}) ->
end, Es),
Graph.
-%% Pick packages to fullfill dependencies
-%% The first dep while traversing the graph is chosen and any conflicting
-%% dep encountered later on is ignored.
-cull_deps(Graph, Vertices) ->
- cull_deps(Graph, Vertices, sets:new()).
-
-cull_deps(Graph, Vertices, Seen) ->
- Vertices1 = lists:keysort(2, Vertices),
- {Solution, Levels, Discarded} = {dict:new(), dict:new(), sets:new()},
- cull_deps(Graph, Vertices1, Levels, Solution, Seen, Discarded).
-
format_error(no_solution) ->
io_lib:format("No solution for packages found.", []).
@@ -87,103 +74,6 @@ format_error(no_solution) ->
%% Internal Functions
%%====================================================================
-cull_deps(_Graph, [], Levels, Solution, _, Discarded) ->
- {_, Vertices} = lists:unzip(dict:to_list(Solution)),
- LvlVertices = [{Profile, {Parent, App, Vsn, dict:fetch(App, Levels)}}
- || {Profile, {Parent,App,Vsn}} <- Vertices],
- {ok, LvlVertices, sets:to_list(Discarded)};
-cull_deps(Graph, [{Profile, Level, Vs} | Vertices], Levels, Solution, Seen, Discarded) ->
- {NV, NS, LS, DS} =
- lists:foldl(fun({Parent, Name, Vsn}, {Acc, SolutionAcc, LevelsAcc, DiscardedAcc}) ->
- {SolutionAcc1, LevelsAcc1, DiscardedAcc1} =
- maybe_add_to_solution(Profile, Level, Name, {Name, Vsn}, Parent
- ,SolutionAcc
- ,LevelsAcc, Seen, DiscardedAcc),
- OutNeighbors = digraph:out_neighbours(Graph, {Name,Vsn}),
- {NewVertices, DiscardedAcc2} = handle_neighbors(Profile, Level, Name
- ,OutNeighbors, Acc, SolutionAcc1
- ,Seen, DiscardedAcc1),
- {NewVertices, SolutionAcc1, LevelsAcc1, DiscardedAcc2}
- end, {[], Solution, Levels, Discarded}, Vs),
- NewVertices = combine_profile_levels(Vertices, NV),
- cull_deps(Graph, NewVertices, LS, NS, Seen, DS).
-
-%% Combine lists of deps that have the same profile and level
-combine_profile_levels(Vertices, NewVertices) ->
- V = lists:foldl(fun({Profile, Level, Vs}, Acc) ->
- case ec_lists:find(fun({P, L, _}) ->
- P =:= Profile andalso L =:= Level
- end, Acc) of
- {ok, {_, _, OldVs}=Old} ->
- lists:delete(Old, Acc)++[{Profile, Level, lists:keysort(1, OldVs++Vs)}];
- error ->
- Acc++[{Profile, Level, Vs}]
- end
- end, Vertices, NewVertices),
- lists:keysort(2, V).
-
-%% For each outgoing edge of a dep check if it should be added to the solution
-%% and add it to the list of vertices to do the same for
-handle_neighbors(Profile, Level, Parent, OutNeighbors, Vertices
- ,Solution, Seen, Discarded) ->
- case lists:foldl(fun({Name, Vsn}=Value, {NewVertices, Discarded1}) ->
- case dict:find(Name, Solution) of
- {ok, {Profile, {Parent, Name, Vsn}}} -> % already seen
- {NewVertices,
- Discarded1};
- {ok, _} -> % conflict resolution!
- %% Warn on different version
- {NewVertices,
- sets:add_element(Value, Discarded1)};
- error ->
- %% We check Seen separately because we don't care
- %% to warn if the exact same version of a package
- %% was already part of the solution but we do
- %% if it was simply seen in source deps
- case sets:is_element(Name, Seen) of
- true ->
- {NewVertices,
- sets:add_element(Value, Discarded1)};
- false ->
- {[{Parent, Name, Vsn} | NewVertices],
- Discarded1}
- end
- end
- end, {[], Discarded}, OutNeighbors) of
- {[], DiscardedAcc2} ->
- {Vertices, DiscardedAcc2};
- {NewVertices1, DiscardedAcc2} ->
- {Vertices++[{Profile, Level+1, NewVertices1}] ,DiscardedAcc2}
- end.
-
-maybe_add_to_solution(Profile, Level, Key, {Name, Vsn}=Value, Parent
- ,Solution, Levels, Seen, Discarded) ->
- case dict:find(Key, Solution) of
- {ok, {Profile, {Parent, Name, Vsn}}} -> % already seen
- {Solution,
- Levels,
- Discarded};
- {ok, _} -> % conflict resolution!
- %% Warn on different version
- {Solution,
- Levels,
- sets:add_element(Value, Discarded)};
- error ->
- %% We check Seen separately because we don't care to warn if the exact
- %% same version of a package was already part of the solution but we do
- %% if it was simply seen in source deps
- case sets:is_element(Name, Seen) of
- true ->
- {Solution,
- Levels,
- sets:add_element(Value, Discarded)};
- false ->
- {dict:store(Key, {Profile, {Parent, Name, Vsn}}, Solution),
- dict:store(Key, Level, Levels),
- Discarded}
- end
- end.
-
subgraph(Graph, Vertices) ->
digraph_utils:subgraph(Graph, Vertices).
diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl
index ca3b676..f6993c5 100644
--- a/src/rebar_packages.erl
+++ b/src/rebar_packages.erl
@@ -1,59 +1,60 @@
-module(rebar_packages).
-export([packages/1
- ,registry/1
+ ,close_packages/0
+ ,load_and_verify_version/1
+ ,deps/3
,package_dir/1
- ,check_registry/3
,registry_checksum/2
- ,find_highest_matching/3]).
+ ,find_highest_matching/4
+ ,format_error/1]).
-export_type([package/0]).
-include("rebar.hrl").
+-include_lib("providers/include/providers.hrl").
-type pkg_name() :: binary() | atom().
-type vsn() :: binary().
-type package() :: pkg_name() | {pkg_name(), vsn()}.
--spec packages(rebar_state:t()) -> {rebar_dict(), rebar_digraph()}.
-%% DON'T USE IT! Use rebar_state:packages(State) instead.
+-spec packages(rebar_state:t()) -> ets:tid().
packages(State) ->
- RegistryDir = package_dir(State),
- DictFile = filename:join(RegistryDir, "dict"),
- Edges = filename:join(RegistryDir, "edges"),
- Vertices = filename:join(RegistryDir, "vertices"),
- Neighbors = filename:join(RegistryDir, "neighbors"),
-
- case lists:all(fun(X) -> filelib:is_file(X) end, [DictFile, Edges, Vertices, Neighbors]) of
+ catch ets:delete(?PACKAGE_TABLE),
+ case load_and_verify_version(State) of
true ->
- try
- {ok, DictBinary} = file:read_file(DictFile),
- Dict = binary_to_term(DictBinary),
- {ok, EdgesTab} = ets:file2tab(Edges),
- {ok, VerticesTab} = ets:file2tab(Vertices),
- {ok, NeighborsTab} = ets:file2tab(Neighbors),
- {Dict, {digraph, EdgesTab, VerticesTab, NeighborsTab, true}}
- catch
- _:_ ->
- ?ERROR("Bad packages index, try to fix with `rebar3 update`", []),
- {dict:new(), digraph:new()}
- end;
+ ok;
false ->
+ ?DEBUG("Error loading package index.", []),
?ERROR("Bad packages index, try to fix with `rebar3 update`", []),
- {dict:new(), digraph:new()}
+ ets:new(?PACKAGE_TABLE, [named_table, public])
end.
--spec registry(rebar_state:t()) -> {ok, ets:tid()} | {error, any()}.
-%% DON'T USE IT! Use rebar_state:registry(State) instead.
-registry(State) ->
+close_packages() ->
+ catch ets:delete(?PACKAGE_TABLE).
+
+load_and_verify_version(State) ->
RegistryDir = package_dir(State),
- HexFile = filename:join(RegistryDir, "registry"),
- case ets:file2tab(HexFile) of
- {ok, T} ->
- {ok, T};
- {error, Reason} ->
- ?DEBUG("Error loading registry: ~p", [Reason]),
- error
+ case ets:file2tab(filename:join(RegistryDir, ?INDEX_FILE)) of
+ {ok, _} ->
+ case ets:lookup_element(?PACKAGE_TABLE, package_index_version, 2) of
+ ?PACKAGE_INDEX_VERSION ->
+ true;
+ _ ->
+ (catch ets:delete(?PACKAGE_TABLE)),
+ rebar_prv_update:hex_to_index(State)
+ end;
+ _ ->
+ rebar_prv_update:hex_to_index(State)
+ end.
+
+deps(Name, Vsn, State) ->
+ try
+ verify_table(State),
+ ets:lookup_element(?PACKAGE_TABLE, {ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}, 2)
+ catch
+ _:_ ->
+ throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}))
end.
package_dir(State) ->
@@ -66,27 +67,13 @@ package_dir(State) ->
ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")),
PackageDir.
-
-check_registry(Pkg, Vsn, State) ->
- case rebar_state:registry(State) of
- {ok, T} ->
- case ets:lookup(T, Pkg) of
- [{Pkg, [Vsns]}] ->
- lists:member(Vsn, Vsns);
- _ ->
- false
- end;
- error ->
- false
- end.
-
registry_checksum({pkg, Name, Vsn}, State) ->
- {ok, Registry} = rebar_state:registry(State),
- case ets:lookup(Registry, {Name, Vsn}) of
- [{{_, _}, [_, Checksum | _]}] ->
- Checksum;
- [] ->
- none
+ try
+ verify_table(State),
+ ets:lookup_element(?PACKAGE_TABLE, {Name, Vsn}, 3)
+ catch
+ _:_ ->
+ throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}))
end.
%% Hex supports use of ~> to specify the version required for a dependency.
@@ -104,28 +91,45 @@ registry_checksum({pkg, Name, Vsn}, State) ->
%% `~> 2.1.3-dev` | `>= 2.1.3-dev and < 2.2.0`
%% `~> 2.0` | `>= 2.0.0 and < 3.0.0`
%% `~> 2.1` | `>= 2.1.0 and < 3.0.0`
-find_highest_matching(Dep, Constraint, T) ->
- case ets:lookup(T, Dep) of
- [{Dep, [[Vsn]]}] ->
- case ec_semver:pes(Vsn, Constraint) of
- true ->
- {ok, Vsn};
- false ->
- ?WARN("Only existing version of ~s is ~s which does not match constraint ~~> ~s. "
- "Using anyway, but it is not guarenteed to work.", [Dep, Vsn, Constraint]),
- {ok, Vsn}
- end;
- [{Dep, [[HeadVsn | VsnTail]]}] ->
- {ok, lists:foldl(fun(Version, Highest) ->
- case ec_semver:pes(Version, Constraint) andalso
- ec_semver:gt(Version, Highest) of
- true ->
- Version;
- false ->
- Highest
- end
- end, HeadVsn, VsnTail)};
+find_highest_matching(Dep, Constraint, Table, State) ->
+ verify_table(State),
+ case ets:lookup_element(Table, Dep, 2) of
+ [[HeadVsn | VsnTail]] ->
+ {ok, handle_vsns(Constraint, HeadVsn, VsnTail)};
+ [[Vsn]] ->
+ handle_single_vsn(Dep, Vsn, Constraint);
+ [Vsn] ->
+ handle_single_vsn(Dep, Vsn, Constraint);
+ [HeadVsn | VsnTail] ->
+ {ok, handle_vsns(Constraint, HeadVsn, VsnTail)};
[] ->
- ?WARN("Missing registry entry for package ~s", [Dep]),
+ ?WARN("Missing registry entry for package ~s. Try to fix with `rebar3 update`", [Dep]),
none
end.
+
+handle_vsns(Constraint, HeadVsn, VsnTail) ->
+ lists:foldl(fun(Version, Highest) ->
+ case ec_semver:pes(Version, Constraint) andalso
+ ec_semver:gt(Version, Highest) of
+ true ->
+ Version;
+ false ->
+ Highest
+ end
+ end, HeadVsn, VsnTail).
+
+handle_single_vsn(Dep, Vsn, Constraint) ->
+ case ec_semver:pes(Vsn, Constraint) of
+ true ->
+ {ok, Vsn};
+ false ->
+ ?WARN("Only existing version of ~s is ~s which does not match constraint ~~> ~s. "
+ "Using anyway, but it is not guarenteed to work.", [Dep, Vsn, Constraint]),
+ {ok, Vsn}
+ end.
+
+format_error({missing_package, Package, Version}) ->
+ io_lib:format("Package not found in registry: ~s-~s. Try to fix with `rebar3 update`", [Package, Version]).
+
+verify_table(State) ->
+ ets:info(?PACKAGE_TABLE, named_table) =:= true orelse ?MODULE:load_and_verify_version(State).
diff --git a/src/rebar_plugins.erl b/src/rebar_plugins.erl
index bda3fb7..6c2daef 100644
--- a/src/rebar_plugins.erl
+++ b/src/rebar_plugins.erl
@@ -24,12 +24,12 @@ project_apps_install(State) ->
StateAcc1 = handle_plugins(Profile, Plugins, StateAcc),
lists:foldl(fun(App, StateAcc2) ->
- AppDir = rebar_app_info:dir(App),
- C = rebar_config:consult(AppDir),
- S = rebar_state:new(rebar_state:new(), C, AppDir),
- Plugins2 = rebar_state:get(S, {plugins, Profile}, []),
- handle_plugins(Profile, Plugins2, StateAcc2)
- end, StateAcc1, ProjectApps)
+ AppDir = rebar_app_info:dir(App),
+ C = rebar_config:consult(AppDir),
+ S = rebar_state:new(rebar_state:new(), C, AppDir),
+ Plugins2 = rebar_state:get(S, {plugins, Profile}, []),
+ handle_plugins(Profile, Plugins2, StateAcc2)
+ end, StateAcc1, ProjectApps)
end, State, Profiles).
-spec install(rebar_state:t()) -> rebar_state:t().
diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl
index 3a5a7cd..0b0f607 100644
--- a/src/rebar_prv_install_deps.erl
+++ b/src/rebar_prv_install_deps.erl
@@ -36,8 +36,6 @@
-include_lib("providers/include/providers.hrl").
-export([handle_deps_as_profile/4,
- parse_deps/5,
- parse_deps/6,
profile_dep_dir/2,
find_cycles/1,
cull_compile/2]).
@@ -125,17 +123,9 @@ handle_deps_as_profile(Profile, State, Deps, Upgrade) ->
Locks = [],
Level = 0,
DepsDir = profile_dep_dir(State, Profile),
- {SrcDeps, PkgDeps} = parse_deps(DepsDir, Deps, State, Locks, Level),
- AllSrcProfileDeps = [{Profile, SrcDeps, Locks, Level}],
- AllPkgProfileDeps = case PkgDeps of
- [] ->
- [];
- _ ->
- [{Profile, Level, PkgDeps}]
- end,
- {AllApps, PkgDeps1, Seen, State1} = handle_profile_level(AllSrcProfileDeps, AllPkgProfileDeps, Locks, sets:new(), Upgrade, State),
-
- handle_profile_pkg_level(PkgDeps1, AllApps, Seen, Upgrade, [], State1).
+ Deps1 = rebar_app_utils:parse_deps(DepsDir, Deps, State, Locks, Level),
+ ProfileLevelDeps = [{Profile, Deps1, Level}],
+ handle_profile_level(ProfileLevelDeps, [], sets:new(), Upgrade, Locks, State).
%% ===================================================================
%% Internal functions
@@ -144,71 +134,30 @@ handle_deps_as_profile(Profile, State, Deps, Upgrade) ->
%% finds all the deps in `{deps, ...}` for each profile provided.
deps_per_profile(Profiles, Upgrade, State) ->
Level = 0,
- {AllProfileDeps, PkgDeps} = lists:foldl(fun(Profile, {SrcAcc, PkgAcc}) ->
- case parse_profile_deps(State, Profile, Level) of
- {Src, {_, _, []}} ->
- {[Src | SrcAcc], PkgAcc};
- {Src, Pkg} ->
- {[Src | SrcAcc], [Pkg | PkgAcc]}
- end
- end, {[], []}, Profiles),
- {AllApps, PkgDeps1, Seen, State1} = handle_profile_level(AllProfileDeps, PkgDeps, [], sets:new(), Upgrade, State),
Locks = rebar_state:get(State, {locks, default}, []),
- handle_profile_pkg_level(PkgDeps1, AllApps, Seen, Upgrade, Locks, State1).
+ Deps = lists:foldl(fun(Profile, DepAcc) ->
+ [parsed_profile_deps(State, Profile, Level) | DepAcc]
+ end, [], Profiles),
+ handle_profile_level(Deps, [], sets:new(), Upgrade, Locks, State).
-parse_profile_deps(State, Profile, Level) ->
- Locks = rebar_state:get(State, {locks, Profile}, []),
- {SrcDeps, PkgDeps} = rebar_state:get(State, {parsed_deps, Profile}, {[], []}),
- {{Profile, SrcDeps, Locks, Level}, {Profile, Level, PkgDeps}}.
+parsed_profile_deps(State, Profile, Level) ->
+ ParsedDeps = rebar_state:get(State, {parsed_deps, Profile}, []),
+ {Profile, ParsedDeps, Level}.
%% Level-order traversal of all dependencies, across profiles.
%% If profiles x,y,z are present, then the traversal will go:
%% x0, y0, z0, x1, y1, z1, ..., xN, yN, zN.
-handle_profile_level([], PkgDeps, SrcApps, Seen, _Upgrade, State) ->
- {SrcApps, PkgDeps, Seen, State};
-handle_profile_level([{Profile, SrcDeps, Locks, Level} | Rest], PkgDeps, SrcApps, Seen, Upgrade, State) ->
- {SrcDeps1, PkgDeps1, SrcApps1, State1, Seen1, Locks1} =
- update_src_deps(Profile, Level, SrcDeps, [], SrcApps
- ,State, Upgrade, Seen, Locks),
- SrcDeps2 = case SrcDeps1 of
+handle_profile_level([], Apps, _Seen, _Upgrade, _Locks, State) ->
+ {Apps, State};
+handle_profile_level([{Profile, Deps, Level} | Rest], Apps, Seen, Upgrade, Locks, State) ->
+ {Deps1, Apps1, State1, Seen1} =
+ update_deps(Profile, Level, Deps, Apps
+ ,State, Upgrade, Seen, Locks),
+ Deps2 = case Deps1 of
[] -> Rest;
- _ -> Rest ++ [{Profile, SrcDeps1, Locks1, Level+1}]
+ _ -> Rest ++ [{Profile, Deps1, Level+1}]
end,
- handle_profile_level(SrcDeps2, case PkgDeps1 of
- [] ->
- PkgDeps;
- _ ->
- [{Profile, Level+1, PkgDeps1} | PkgDeps]
- end, SrcApps1, sets:union(Seen, Seen1), Upgrade, State1).
-
-handle_profile_pkg_level([], AllApps, _Seen, _Upgrade, _Locks, State) ->
- {AllApps, State};
-handle_profile_pkg_level(PkgDeps, AllApps, Seen, Upgrade, Locks, State) ->
- %% Read in package index and dep graph
- {Packages, Graph} = rebar_state:packages(State),
- Registry = rebar_state:registry(State),
- State1 = rebar_state:packages(rebar_state:registry(State, Registry)
- ,{Packages, Graph}),
-
- S = case rebar_digraph:cull_deps(Graph, lists:keysort(2, PkgDeps), Seen) of
- {ok, [], []} ->
- throw({rebar_digraph, no_solution});
- {ok, [], Discarded} ->
- [warn_skip_pkg(Pkg, State) || Upgrade =:= false,
- Pkg <- Discarded,
- not(pkg_locked(Pkg, Locks))],
- [];
- {ok, Solution, []} ->
- Solution;
- {ok, Solution, Discarded} ->
- [warn_skip_pkg(Pkg, State) || Upgrade =:= false,
- Pkg <- Discarded,
- not(pkg_locked(Pkg, Locks))],
- Solution
- end,
-
- {PkgApps, State2} = update_pkg_deps(S, Packages, Upgrade, Seen, State1, Locks),
- {AllApps++PkgApps, State2}.
+ handle_profile_level(Deps2, Apps1, sets:union(Seen, Seen1), Upgrade, Locks, State1).
find_cycles(Apps) ->
case rebar_digraph:compile_order(Apps) of
@@ -220,38 +169,6 @@ find_cycles(Apps) ->
cull_compile(TopSortedDeps, ProjectApps) ->
lists:dropwhile(fun not_needs_compile/1, TopSortedDeps -- ProjectApps).
-pkg_locked({_, Name, _, _}, Locks) ->
- pkg_locked(Name, Locks);
-pkg_locked({Name, _}, Locks) ->
- pkg_locked(Name, Locks);
-pkg_locked(Name, Locks) ->
- false =/= lists:keyfind(Name, 1, Locks).
-
-update_pkg_deps(Pkgs, Packages, Upgrade, Seen, State, Locks) ->
- {Solved, _, State1}
- = lists:foldl(fun({Profile, Pkg}, {Acc, SeenAcc, StateAcc}) ->
- DepsDir = profile_dep_dir(State, Profile),
- handle_pkg_dep(Profile, Pkg, Packages, Upgrade, DepsDir, Acc, SeenAcc, Locks, StateAcc)
- end, {[], Seen, State}, Pkgs),
- {Solved, State1}.
-
-handle_pkg_dep(Profile, Pkg, Packages, Upgrade, DepsDir, Fetched, Seen, Locks, State) ->
- IsLock = pkg_locked(Pkg, Locks),
- AppInfo = package_to_app(DepsDir, Packages, Pkg, IsLock, State),
- case sets:is_element(rebar_app_info:name(AppInfo), Seen) of
- true ->
- {Fetched, Seen, State};
- false ->
- Deps = rebar_app_info:deps(AppInfo),
- Level = rebar_app_info:dep_level(AppInfo),
- {NewSeen, NewState} = maybe_lock(Profile, AppInfo, Seen, State, Level),
- {_, AppInfo1} = maybe_fetch(AppInfo, Profile, Upgrade, Seen, NewState),
- {AppInfo2, _, _, _, _} =
- handle_dep(NewState, Profile, DepsDir, AppInfo1, Locks, Level),
- AppInfo3 = rebar_app_info:deps(AppInfo2, Deps),
- {[AppInfo3 | Fetched], NewSeen, NewState}
- end.
-
maybe_lock(Profile, AppInfo, Seen, State, Level) ->
Name = rebar_app_info:name(AppInfo),
case rebar_app_info:is_checkout(AppInfo) of
@@ -278,49 +195,28 @@ maybe_lock(Profile, AppInfo, Seen, State, Level) ->
{sets:add_element(Name, Seen), State}
end.
-package_to_app(DepsDir, Packages, {Parent, Name, Vsn, Level}, IsLock, State) ->
- case dict:find({Name, Vsn}, Packages) of
- error ->
- case rebar_packages:check_registry(Name, Vsn, State) of
- true ->
- throw(?PRV_ERROR({not_rebar_package, Name, Vsn}));
- false ->
- throw(?PRV_ERROR({missing_package, Name, Vsn}))
- end;
- {ok, PkgDeps} ->
- Source = {pkg, Name, Vsn},
- AppInfo = new_dep(root, DepsDir, Name, Vsn, Source, IsLock, State),
- AppInfo1 = rebar_app_info:dep_level(rebar_app_info:deps(AppInfo, PkgDeps), Level),
- BaseDir = rebar_state:get(State, base_dir, []),
- AppState1 = rebar_state:set(rebar_app_info:state(AppInfo1), base_dir, BaseDir),
- rebar_app_info:parent(rebar_app_info:state(AppInfo1, AppState1), Parent)
- end.
-
--spec update_src_deps(atom(), non_neg_integer(), list(), list(), list(), rebar_state:t(), boolean(), sets:set(binary()), list()) -> {list(), list(), list(), rebar_state:t(), sets:set(binary()), list()}.
-update_src_deps(Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, Locks) ->
+update_deps(Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) ->
lists:foldl(
- fun(AppInfo, {SrcDepsAcc, PkgDepsAcc, SrcAppsAcc, StateAcc, SeenAcc, LocksAcc}) ->
- update_src_dep(AppInfo, Profile, Level,
- SrcDepsAcc, PkgDepsAcc, SrcAppsAcc, StateAcc,
- Upgrade, SeenAcc, Locks, LocksAcc)
+ fun(AppInfo, {DepsAcc, AppsAcc, StateAcc, SeenAcc}) ->
+ update_dep(AppInfo, Profile, Level,
+ DepsAcc, AppsAcc, StateAcc,
+ Upgrade, SeenAcc, Locks)
end,
- {[], PkgDeps, SrcApps, State, Seen, Locks},
- rebar_utils:sort_deps(SrcDeps)).
+ {[], Apps, State, Seen},
+ rebar_utils:sort_deps(Deps)).
-
-update_src_dep(AppInfo, Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, BaseLocks, Locks) ->
+update_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) ->
%% If not seen, add to list of locks to write out
Name = rebar_app_info:name(AppInfo),
case sets:is_element(Name, Seen) of
true ->
- update_seen_src_dep(AppInfo, Profile, Level,
- SrcDeps, PkgDeps, SrcApps,
- State, Upgrade, Seen, BaseLocks, Locks);
+ update_seen_dep(AppInfo, Profile, Level,
+ Deps, Apps,
+ State, Upgrade, Seen, Locks);
false ->
- update_unseen_src_dep(AppInfo, Profile, Level,
- SrcDeps, PkgDeps, SrcApps,
- State, Upgrade, Seen, Locks)
-
+ update_unseen_dep(AppInfo, Profile, Level,
+ Deps, Apps,
+ State, Upgrade, Seen, Locks)
end.
profile_dep_dir(State, Profile) ->
@@ -329,45 +225,34 @@ profile_dep_dir(State, Profile) ->
_ -> rebar_dir:deps_dir(State)
end.
-update_seen_src_dep(AppInfo, _Profile, _Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, BaseLocks, Locks) ->
+update_seen_dep(AppInfo, _Profile, _Level, Deps, Apps, State, Upgrade, Seen, Locks) ->
Name = rebar_app_info:name(AppInfo),
%% If seen from lock file or user requested an upgrade
%% don't print warning about skipping
- case lists:keymember(Name, 1, BaseLocks) of
+ case lists:keymember(Name, 1, Locks) of
false when Upgrade -> ok;
false when not Upgrade -> warn_skip_deps(AppInfo, State);
true -> ok
end,
- {SrcDeps, PkgDeps, SrcApps, State, Seen, Locks}.
+ {Deps, Apps, State, Seen}.
-update_unseen_src_dep(AppInfo, Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, Locks) ->
+update_unseen_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) ->
{NewSeen, State1} = maybe_lock(Profile, AppInfo, Seen, State, Level),
{_, AppInfo1} = maybe_fetch(AppInfo, Profile, Upgrade, Seen, State1),
- {NewSrcDeps, NewPkgDeps, NewSrcApps, State2, NewLocks} =
- handle_dep(AppInfo1, Profile, SrcDeps, PkgDeps, SrcApps,
- Level, State1, Locks),
- {NewSrcDeps, NewPkgDeps, NewSrcApps, State2, NewSeen, NewLocks}.
-
-handle_dep(AppInfo, Profile, SrcDeps, PkgDeps, SrcApps, Level, State, Locks) ->
DepsDir = profile_dep_dir(State, Profile),
- {AppInfo1, NewSrcDeps, NewPkgDeps, NewLocks, State1} =
- handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level),
- AppInfo2 = rebar_app_info:dep_level(AppInfo1, Level),
- {NewSrcDeps ++ SrcDeps
- ,NewPkgDeps++PkgDeps
- ,[AppInfo2 | SrcApps]
- ,State1
- ,NewLocks}.
-
--spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer()) ->
- {rebar_app_info:t(), [rebar_app_info:t()], [pkg_dep()], [integer()], rebar_state:t()}.
+ {AppInfo2, NewDeps, State2} =
+ handle_dep(State1, Profile, DepsDir, AppInfo1, Locks, Level),
+ AppInfo3 = rebar_app_info:dep_level(AppInfo2, Level),
+ {NewDeps ++ Deps, [AppInfo3 | Apps], State2, NewSeen}.
+
+-spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer()) -> {rebar_app_info:t(), [rebar_app_info:t()], [pkg_dep()], [integer()], rebar_state:t()}.
handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) ->
Profiles = rebar_state:current_profiles(State),
Name = rebar_app_info:name(AppInfo),
+ Vsn = rebar_app_info:original_vsn(AppInfo),
%% Deps may be under a sub project app, find it and use its state if so
- S0 = rebar_app_info:state(AppInfo),
- S = rebar_state:registry(S0, rebar_state:registry(State)),
+ S = rebar_app_info:state(AppInfo),
C = rebar_config:consult(rebar_app_info:dir(AppInfo)),
S1 = rebar_state:new(S, C, rebar_app_info:dir(AppInfo)),
S2 = rebar_state:apply_overrides(S1, Name),
@@ -375,23 +260,26 @@ handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) ->
S3 = rebar_state:apply_profiles(S2, Profiles),
Plugins = rebar_state:get(S3, plugins, []),
S4 = rebar_state:set(S3, {plugins, Profile}, Plugins),
- AppInfo1 = rebar_app_info:state(AppInfo, S4),
rebar_utils:check_min_otp_version(rebar_state:get(S4, minimum_otp_vsn, undefined)),
rebar_utils:check_blacklisted_otp_versions(rebar_state:get(S4, blacklisted_otp_vsns, [])),
%% Dep may have plugins to install. Find and install here.
S5 = rebar_plugins:install(S4),
- AppInfo2 = rebar_app_info:state(AppInfo1, S5),
+ AppInfo1 = rebar_app_info:state(AppInfo, S5),
%% Upgrade lock level to be the level the dep will have in this dep tree
- Deps = rebar_state:get(S5, {deps, default}, []),
- NewLocks = Locks++[{DepName, Source, LockLevel+Level} ||
- {DepName, Source, LockLevel} <- rebar_state:get(S5, {locks, default}, [])],
- AppInfo3 = rebar_app_info:deps(AppInfo2, rebar_state:deps_names(Deps)),
- {SrcDeps, PkgDeps} = parse_deps(rebar_app_info:name(AppInfo3), DepsDir, Deps
- ,S5, NewLocks, Level+1),
- {AppInfo3, SrcDeps, PkgDeps, NewLocks, State}.
+ case rebar_app_info:resource_type(AppInfo1) of
+ pkg ->
+ NewDeps = rebar_packages:deps(Name, Vsn, S5),
+ NewDeps1 = rebar_app_utils:parse_deps(Name, DepsDir, NewDeps, S5, Locks, Level+1),
+ {rebar_app_info:deps(AppInfo1, NewDeps), NewDeps1, State};
+ _ ->
+ Deps = rebar_state:get(S5, {deps, default}, []),
+ AppInfo2 = rebar_app_info:deps(AppInfo1, rebar_state:deps_names(Deps)),
+ Deps1 = rebar_app_utils:parse_deps(Name, DepsDir, Deps, S5, Locks, Level+1),
+ {AppInfo2, Deps1, State}
+ end.
-spec maybe_fetch(rebar_app_info:t(), atom(), boolean(),
sets:set(binary()), rebar_state:t()) -> {boolean(), rebar_app_info:t()}.
@@ -406,20 +294,20 @@ maybe_fetch(AppInfo, Profile, Upgrade, Seen, State) ->
false ->
true = fetch_app(AppInfo, AppDir, State),
maybe_symlink_default(State, Profile, AppDir, AppInfo),
- {true, update_app_info(AppDir, AppInfo)};
+ {true, rebar_app_info:valid(update_app_info(AppDir, AppInfo), false)};
{true, AppInfo1} ->
%% Preserve the state we created with overrides
+ AppInfo2 = copy_app_info(AppInfo, AppInfo1),
AppState = rebar_app_info:state(AppInfo),
- Parent = rebar_app_info:parent(AppInfo),
- Source = rebar_app_info:source(AppInfo),
- AppInfo2 = rebar_app_info:parent(rebar_app_info:state(AppInfo1, AppState), Parent),
- AppInfo3 = rebar_app_info:source(AppInfo2, Source),
- case sets:is_element(rebar_app_info:name(AppInfo), Seen) of
+ AppInfo3 = rebar_app_info:state(AppInfo2, AppState),
+ case sets:is_element(rebar_app_info:name(AppInfo3), Seen) of
true ->
{false, AppInfo3};
false ->
maybe_symlink_default(State, Profile, AppDir, AppInfo3),
- {maybe_upgrade(AppInfo, AppDir, Upgrade, State), AppInfo3}
+ MaybeUpgrade = maybe_upgrade(AppInfo, AppDir, Upgrade, State),
+ AppInfo4 = update_app_info(AppDir, AppInfo3),
+ {MaybeUpgrade, AppInfo4}
end
end
end.
@@ -467,106 +355,6 @@ make_relative_to_root(State, Path) when is_list(Path) ->
Root = rebar_dir:root_dir(State),
rebar_dir:make_relative_path(Path, Root).
--spec parse_deps(binary(), list(), rebar_state:t(), list(), integer()) -> {[rebar_app_info:t()], [tuple()]}.
-parse_deps(DepsDir, Deps, State, Locks, Level) ->
- parse_deps(root, DepsDir, Deps, State, Locks, Level).
-
-parse_deps(Parent, DepsDir, Deps, State, Locks, Level) ->
- lists:foldl(fun(Dep, Acc) ->
- Name = case Dep of
- Dep when is_tuple(Dep) ->
- element(1, Dep);
- Dep ->
- Dep
- end,
- case lists:keyfind(ec_cnv:to_binary(Name), 1, Locks) of
- false ->
- parse_dep(Parent, Dep, Acc, DepsDir, false, State);
- LockedDep ->
- LockedLevel = element(3, LockedDep),
- case LockedLevel > Level of
- true ->
- parse_dep(Parent, Dep, Acc, DepsDir, false, State);
- false ->
- parse_dep(Parent, LockedDep, Acc, DepsDir, true, State)
- end
- end
- end, {[], []}, Deps).
-
-parse_dep(Parent, {Name, Vsn}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_list(Vsn) ->
- %% Versioned Package dependency
- CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)),
- case rebar_app_info:discover(CheckoutsDir) of
- {ok, _App} ->
- Dep = new_dep(root, DepsDir, Name, [], [], IsLock, State),
- {[Dep | SrcDepsAcc], PkgDepsAcc};
- not_found ->
- {SrcDepsAcc, [parse_goal(Parent
- ,ec_cnv:to_binary(Name)
- ,ec_cnv:to_binary(Vsn)) | PkgDepsAcc]}
- end;
-parse_dep(Parent, Name, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_atom(Name); is_binary(Name) ->
- %% Unversioned package dependency
- {PkgName, PkgVsn} = get_package(ec_cnv:to_binary(Name), State),
- CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)),
- case rebar_app_info:discover(CheckoutsDir) of
- {ok, _App} ->
- Dep = new_dep(root, DepsDir, Name, [], [], IsLock, State),
- {[Dep | SrcDepsAcc], PkgDepsAcc};
- not_found ->
- {SrcDepsAcc, [{Parent, PkgName, PkgVsn} | PkgDepsAcc]}
- end;
-parse_dep(Parent, {Name, Source}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source) ->
- Dep = new_dep(Parent, DepsDir, Name, [], Source, IsLock, State),
- {[Dep | SrcDepsAcc], PkgDepsAcc};
-parse_dep(Parent, {Name, _Vsn, Source}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source) ->
- Dep = new_dep(Parent, DepsDir, Name, [], Source, IsLock, State),
- {[Dep | SrcDepsAcc], PkgDepsAcc};
-parse_dep(Parent, {Name, _Vsn, Source, Opts}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source) ->
- ?WARN("Dependency option list ~p in ~p is not supported and will be ignored", [Opts, Name]),
- Dep = new_dep(Parent, DepsDir, Name, [], Source, IsLock, State),
- {[Dep | SrcDepsAcc], PkgDepsAcc};
-parse_dep(Parent, {_Name, {pkg, Name, Vsn}, Level}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_integer(Level) ->
- CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)),
- case rebar_app_info:discover(CheckoutsDir) of
- {ok, _App} ->
- Dep = new_dep(root, DepsDir, Name, [], [], IsLock, State),
- {[Dep | SrcDepsAcc], PkgDepsAcc};
- not_found ->
- {SrcDepsAcc, [{Parent, Name, Vsn} | PkgDepsAcc]}
- end;
-parse_dep(Parent, {Name, Source, Level}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source)
- , is_integer(Level) ->
- Dep = new_dep(Parent, DepsDir, Name, [], Source, IsLock, State),
- {[Dep | SrcDepsAcc], PkgDepsAcc};
-parse_dep(_, Dep, _, _, _, _) ->
- throw(?PRV_ERROR({parse_dep, Dep})).
-
-
-new_dep(Parent, DepsDir, Name, Vsn, Source, IsLock, State) ->
- CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)),
- {ok, Dep} = case rebar_app_info:discover(CheckoutsDir) of
- {ok, App} ->
- {ok, rebar_app_info:is_checkout(App, true)};
- not_found ->
- Dir = ec_cnv:to_list(filename:join(DepsDir, Name)),
- case rebar_app_info:discover(Dir) of
- {ok, App} ->
- {ok, App};
- not_found ->
- rebar_app_info:new(Name, Vsn,
- ec_cnv:to_list(filename:join(DepsDir, Name)))
- end
- end,
- C = rebar_config:consult(rebar_app_info:dir(Dep)),
- S = rebar_state:new(rebar_state:new(), C, rebar_app_info:dir(Dep)),
- Overrides = rebar_state:get(State, overrides, []),
- ParentOverrides = rebar_state:overrides(State),
- Dep1 = rebar_app_info:state(Dep,
- rebar_state:overrides(S, ParentOverrides++Overrides)),
- AppInfo = rebar_app_info:is_lock(rebar_app_info:source(Dep1, Source), IsLock),
- rebar_app_info:parent(AppInfo, Parent).
-
fetch_app(AppInfo, AppDir, State) ->
?INFO("Fetching ~s (~p)", [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)]),
Source = rebar_app_info:source(AppInfo),
@@ -577,15 +365,24 @@ fetch_app(AppInfo, AppDir, State) ->
%% be read in an parsed.
update_app_info(AppDir, AppInfo) ->
{ok, Found} = rebar_app_info:discover(AppDir),
- Parent = rebar_app_info:parent(AppInfo),
- Source = rebar_app_info:source(AppInfo),
AppDetails = rebar_app_info:app_details(Found),
+ Vsn = rebar_app_info:original_vsn(Found),
Applications = proplists:get_value(applications, AppDetails, []),
IncludedApplications = proplists:get_value(included_applications, AppDetails, []),
- AppInfo1 = rebar_app_info:applications(
+ AppInfo1 = rebar_app_info:original_vsn(rebar_app_info:applications(
rebar_app_info:app_details(AppInfo, AppDetails),
- IncludedApplications++Applications),
- rebar_app_info:source(rebar_app_info:parent(rebar_app_info:valid(AppInfo1, false), Parent), Source).
+ IncludedApplications++Applications), Vsn),
+ AppInfo2 = copy_app_info(AppInfo, AppInfo1),
+ rebar_app_info:valid(AppInfo2, undefined).
+
+copy_app_info(OldAppInfo, NewAppInfo) ->
+ ResourceType = rebar_app_info:resource_type(OldAppInfo),
+ Parent = rebar_app_info:parent(OldAppInfo),
+ Source = rebar_app_info:source(OldAppInfo),
+
+ rebar_app_info:resource_type(
+ rebar_app_info:source(
+ rebar_app_info:parent(NewAppInfo, Parent), Source), ResourceType).
maybe_upgrade(AppInfo, AppDir, Upgrade, State) ->
Source = rebar_app_info:source(AppInfo),
@@ -608,17 +405,6 @@ maybe_upgrade(AppInfo, AppDir, Upgrade, State) ->
false
end.
--spec parse_goal(binary() | root, binary(), binary()) -> {binary(), binary()} | {binary(), binary(), binary()}.
-parse_goal(Parent, Name, Constraint) ->
- case re:run(Constraint, "([^\\d]*)(\\d.*)", [{capture, [1,2], binary}]) of
- {match, [<<>>, Vsn]} ->
- {Parent, Name, Vsn};
- {match, [Op, Vsn]} ->
- {Parent, Name, Vsn, binary_to_atom(Op, utf8)};
- nomatch ->
- throw(?PRV_ERROR({bad_constraint, Name, Constraint}))
- end.
-
warn_skip_deps(AppInfo, State) ->
Msg = "Skipping ~s (from ~p) as an app of the same name "
"has already been fetched",
@@ -629,25 +415,7 @@ warn_skip_deps(AppInfo, State) ->
true -> ?ERROR(Msg, Args), ?FAIL
end.
-warn_skip_pkg({Name, Source}, State) ->
- Msg = "Skipping ~s (version ~s from package index) as an app of the same "
- "name has already been fetched",
- Args = [Name, Source],
- case rebar_state:get(State, deps_error_on_conflict, false) of
- false -> ?WARN(Msg, Args);
- true -> ?ERROR(Msg, Args), ?FAIL
- end.
-
not_needs_compile(App) ->
not(rebar_app_info:is_checkout(App))
andalso rebar_app_info:valid(App)
andalso rebar_state:has_all_artifacts(rebar_app_info:state(App)) =:= true.
-
-get_package(Dep, State) ->
- case rebar_state:registry(State) of
- {ok, T} ->
- {ok, HighestDepVsn} = rebar_packages:find_highest_matching(Dep, "0", T),
- {Dep, HighestDepVsn};
- error ->
- throw(?PRV_ERROR({load_registry_fail, Dep}))
- end.
diff --git a/src/rebar_prv_packages.erl b/src/rebar_prv_packages.erl
index 82ed2f7..f5d9e38 100644
--- a/src/rebar_prv_packages.erl
+++ b/src/rebar_prv_packages.erl
@@ -27,19 +27,17 @@ init(State) ->
-spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
do(State) ->
- {Dict, _} = rebar_state:packages(State),
- print_packages(Dict),
+ print_packages(),
{ok, State}.
-spec format_error(any()) -> iolist().
format_error(load_registry_fail) ->
"Failed to load package regsitry. Try running 'rebar3 update' to fix".
-print_packages(Dict) ->
- Pkgs = lists:keysort(1, dict:fetch_keys(Dict)),
- SortedPkgs = lists:foldl(fun({Pkg, Vsn}, Acc) ->
- orddict:append_list(Pkg, [Vsn], Acc)
- end, orddict:new(), Pkgs),
+print_packages() ->
+ SortedPkgs = ets:foldl(fun({{Pkg, Vsn}, _}, Acc) ->
+ orddict:append_list(Pkg, [Vsn], Acc)
+ end, orddict:new(), ?PACKAGE_TABLE),
orddict:map(fun(Name, Vsns) ->
SortedVsns = lists:sort(fun(A, B) ->
diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl
index 55c70e2..4b3a155 100644
--- a/src/rebar_prv_update.erl
+++ b/src/rebar_prv_update.erl
@@ -9,21 +9,14 @@
do/1,
format_error/1]).
+-export([hex_to_index/1]).
+
-include("rebar.hrl").
-include_lib("providers/include/providers.hrl").
-define(PROVIDER, update).
-define(DEPS, []).
-%% Ignore warning of digraph opaque type when running dialyzer
--dialyzer({no_opaque, do/1}).
--dialyzer({no_opaque, write_registry/3}).
-
-%% Ignoring the opaque type warning won't stop dialyzer from warning of
-%% no return for functions that had the opaque type warnings
--dialyzer({no_return, do/1}).
--dialyzer({no_return, write_registry/3}).
-
%% ===================================================================
%% Public API
%% ===================================================================
@@ -50,16 +43,15 @@ do(State) ->
TmpDir = ec_file:insecure_mkdtemp(),
TmpFile = filename:join(TmpDir, "packages.gz"),
- Url = rebar_state:get(State, rebar_packages_cdn, "https://s3.amazonaws.com/s3.hex.pm/registry.ets.gz"),
+ Url = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_HEX_REGISTRY),
{ok, _RequestId} = httpc:request(get, {Url, []},
[], [{stream, TmpFile}, {sync, true}],
rebar),
{ok, Data} = file:read_file(TmpFile),
Unzipped = zlib:gunzip(Data),
ok = file:write_file(HexFile, Unzipped),
- {Dict, Graph} = hex_to_graph(HexFile),
- write_registry(Dict, Graph, State),
- true = digraph:delete(Graph),
+
+ hex_to_index(State),
ok
catch
_E:C ->
@@ -73,56 +65,58 @@ do(State) ->
format_error(package_index_write) ->
"Failed to write package index.".
--spec write_registry(rebar_dict(), {digraph, ets:tid(), ets:tid(), ets:tid(), any()}, rebar_state:t()) -> ok | {error, atom()}.
-write_registry(Dict, {digraph, Edges, Vertices, Neighbors, _}, State) ->
- RegistryDir = rebar_packages:package_dir(State),
- filelib:ensure_dir(filename:join(RegistryDir, "dummy")),
- ets:tab2file(Edges, filename:join(RegistryDir, "edges")),
- ets:tab2file(Vertices, filename:join(RegistryDir, "vertices")),
- ets:tab2file(Neighbors, filename:join(RegistryDir, "neighbors")),
- file:write_file(filename:join(RegistryDir, "dict"), term_to_binary(Dict)).
-
is_supported(<<"make">>) -> true;
is_supported(<<"rebar">>) -> true;
+is_supported(<<"rebar3">>) -> true;
is_supported(_) -> false.
-hex_to_graph(Filename) ->
- {ok, T} = ets:file2tab(Filename),
- Graph = digraph:new(),
- ets:foldl(fun({Pkg, [Versions]}, ok) when is_binary(Pkg), is_list(Versions) ->
- lists:foreach(fun(Version) ->
- digraph:add_vertex(Graph, {Pkg, Version}, 1)
- end, Versions);
- (_, ok) ->
- ok
- end, ok, T),
-
- Dict1 = ets:foldl(fun({{Pkg, PkgVsn}, [Deps, _, BuildTools | _]}, Dict) when is_list(BuildTools) ->
- case lists:any(fun is_supported/1, BuildTools) of
- true ->
- DepsList = update_graph(Pkg, PkgVsn, Deps, T, Graph),
- dict:store({Pkg, PkgVsn}, DepsList, Dict);
- false ->
- Dict
- end;
- (_, Dict) ->
- Dict
- end, dict:new(), T),
- {Dict1, Graph}.
-
-update_graph(Pkg, PkgVsn, Deps, HexRegistry, Graph) ->
+hex_to_index(State) ->
+ RegistryDir = rebar_packages:package_dir(State),
+ HexFile = filename:join(RegistryDir, "registry"),
+ try ets:file2tab(HexFile) of
+ {ok, Registry} ->
+ try
+ (catch ets:delete(?PACKAGE_TABLE)),
+ ets:new(?PACKAGE_TABLE, [named_table, public]),
+ ets:foldl(fun({{Pkg, PkgVsn}, [Deps, Checksum, BuildTools | _]}, _) when is_list(BuildTools) ->
+ case lists:any(fun is_supported/1, BuildTools) of
+ true ->
+ DepsList = update_deps_list(Deps, Registry, State),
+ ets:insert(?PACKAGE_TABLE, {{Pkg, PkgVsn}, DepsList, Checksum});
+ false ->
+ true
+ end;
+ ({Pkg, [Vsns]}, _) when is_binary(Pkg) ->
+ ets:insert(?PACKAGE_TABLE, {Pkg, Vsns});
+ (_, _) ->
+ true
+ end, true, Registry),
+
+ ets:insert(?PACKAGE_TABLE, {package_index_version, ?PACKAGE_INDEX_VERSION}),
+ ets:tab2file(?PACKAGE_TABLE, filename:join(RegistryDir, "packages.idx")),
+ true
+ after
+ catch ets:delete(Registry)
+ end;
+ {error, Reason} ->
+ ?DEBUG("Error loading package registry: ~p", [Reason]),
+ false
+ catch
+ _:_ ->
+ fail
+ end.
+
+update_deps_list(Deps, HexRegistry, State) ->
lists:foldl(fun([Dep, DepVsn, false, _AppName | _], DepsListAcc) ->
case DepVsn of
<<"~> ", Vsn/binary>> ->
- case rebar_packages:find_highest_matching(Dep, Vsn, HexRegistry) of
+ case rebar_packages:find_highest_matching(Dep, Vsn, HexRegistry, State) of
{ok, HighestDepVsn} ->
- digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, HighestDepVsn}),
- [{Dep, DepVsn} | DepsListAcc];
+ [{Dep, HighestDepVsn} | DepsListAcc];
none ->
DepsListAcc
end;
Vsn ->
- digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, Vsn}),
[{Dep, Vsn} | DepsListAcc]
end;
([_Dep, _DepVsn, true, _AppName | _], DepsListAcc) ->
diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl
index f49eafe..7385cfe 100644
--- a/src/rebar_prv_upgrade.erl
+++ b/src/rebar_prv_upgrade.erl
@@ -54,7 +54,7 @@ do(State) ->
Deps0 = top_level_deps(Deps, Locks),
State1 = rebar_state:set(State, {deps, default}, Deps0),
DepsDir = rebar_prv_install_deps:profile_dep_dir(State, default),
- D = rebar_prv_install_deps:parse_deps(root, DepsDir, Deps0, State1, Locks0, 0),
+ D = rebar_app_utils:parse_deps(root, DepsDir, Deps0, State1, Locks0, 0),
State2 = rebar_state:set(State1, {parsed_deps, default}, D),
State3 = rebar_state:set(State2, {locks, default}, Locks0),
State4 = rebar_state:set(State3, upgrade, true),
@@ -83,7 +83,6 @@ format_error({transitive_dependency, Name}) ->
format_error(Reason) ->
io_lib:format("~p", [Reason]).
-
parse_names(Bin, Locks) ->
case lists:usort(re:split(Bin, <<" *, *">>, [trim])) of
%% Nothing submitted, use *all* apps
diff --git a/src/rebar_state.erl b/src/rebar_state.erl
index 7616151..213eb2f 100644
--- a/src/rebar_state.erl
+++ b/src/rebar_state.erl
@@ -38,9 +38,6 @@
overrides/1, overrides/2,
apply_overrides/2,
- packages/1, packages/2,
- registry/1, registry/2,
-
resources/1, resources/2, add_resource/2,
providers/1, providers/2, add_provider/2]).
@@ -65,8 +62,6 @@
all_plugin_deps = [] :: [rebar_app_info:t()],
all_deps = [] :: [rebar_app_info:t()],
- packages = undefined :: {rebar_dict(), rebar_digraph()} | undefined,
- registry = undefined :: {ok, ets:tid()} | error | undefined,
overrides = [],
resources = [],
providers = []}).
@@ -88,16 +83,9 @@ new(Config) when is_list(Config) ->
Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins} | Config],
true = rebar_config:verify_config_format(Terms),
Opts = dict:from_list(Terms),
- load_package_registry(
- BaseState#state_t { dir = rebar_dir:get_cwd(),
- default = Opts,
- opts = Opts }).
-
-load_package_registry(Config0) ->
- Registry = rebar_packages:registry(Config0),
- Packages = rebar_packages:packages(Config0),
- Config0#state_t{registry = Registry,
- packages = Packages}.
+ BaseState#state_t { dir = rebar_dir:get_cwd(),
+ default = Opts,
+ opts = Opts }.
-spec new(t() | atom(), list()) -> t().
new(Profile, Config) when is_atom(Profile)
@@ -109,11 +97,10 @@ new(Profile, Config) when is_atom(Profile)
Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins} | Config],
true = rebar_config:verify_config_format(Terms),
Opts = dict:from_list(Terms),
- load_package_registry(
- BaseState#state_t { dir = rebar_dir:get_cwd(),
- current_profiles = [Profile],
- default = Opts,
- opts = Opts });
+ BaseState#state_t { dir = rebar_dir:get_cwd(),
+ current_profiles = [Profile],
+ default = Opts,
+ opts = Opts };
new(ParentState=#state_t{}, Config) ->
%% Load terms from rebar.config, if it exists
Dir = rebar_dir:get_cwd(),
@@ -446,22 +433,6 @@ namespace(#state_t{namespace=Namespace}) ->
namespace(State=#state_t{}, Namespace) ->
State#state_t{namespace=Namespace}.
-packages(#state_t{packages=undefined}) ->
- throw(packages_usage_error);
-packages(#state_t{packages=Packages}) ->
- Packages.
-
-packages(State, Packages) ->
- State#state_t{packages=Packages}.
-
-registry(#state_t{registry=undefined}) ->
- throw(registry_usage_error);
-registry(#state_t{registry=Registry}) ->
- Registry.
-
-registry(State, Registry) ->
- State#state_t{registry=Registry}.
-
-spec resources(t()) -> [{rebar_resource:type(), module()}].
resources(#state_t{resources=Resources}) ->
Resources.