summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/rebar.hrl6
-rw-r--r--src/rebar3.erl13
-rw-r--r--src/rebar_app_utils.erl9
-rw-r--r--src/rebar_packages.erl168
-rw-r--r--src/rebar_plugins.erl10
-rw-r--r--src/rebar_prv_install_deps.erl39
-rw-r--r--src/rebar_prv_packages.erl12
-rw-r--r--src/rebar_prv_update.erl95
-rw-r--r--src/rebar_state.erl52
-rw-r--r--test/mock_pkg_resource.erl55
-rw-r--r--test/rebar_pkg_SUITE.erl26
11 files changed, 220 insertions, 265 deletions
diff --git a/src/rebar.hrl b/src/rebar.hrl
index 8a702df..961b8ea 100644
--- a/src/rebar.hrl
+++ b/src/rebar.hrl
@@ -23,8 +23,14 @@
-define(DEFAULT_RELEASE_DIR, "rel").
-define(DEFAULT_CONFIG_FILE, "rebar.config").
-define(DEFAULT_CDN, "https://s3.amazonaws.com/s3.hex.pm/tarballs").
+-define(DEFAULT_HEX_REGISTRY, "https://s3.amazonaws.com/s3.hex.pm/registry.ets.gz").
-define(LOCK_FILE, "rebar.lock").
+-define(PACKAGE_INDEX_VERSION, 3).
+-define(PACKAGE_TABLE, package_index).
+-define(INDEX_FILE, "packages.idx").
+-define(REGISTRY_FILE, "registry").
+
-ifdef(namespaced_types).
-type rebar_dict() :: dict:dict().
-else.
diff --git a/src/rebar3.erl b/src/rebar3.erl
index 8004443..ab3e0eb 100644
--- a/src/rebar3.erl
+++ b/src/rebar3.erl
@@ -105,6 +105,7 @@ run_aux(State, RawArgs) ->
{ok, Providers} = application:get_env(rebar, providers),
%% Providers can modify profiles stored in opts, so set default after initializing providers
State4 = rebar_state:create_logic_providers(Providers, State3),
+ rebar_packages:packages(State4),
State5 = rebar_plugins:project_apps_install(State4),
State6 = rebar_state:default(State5, rebar_state:opts(State5)),
@@ -280,9 +281,15 @@ state_from_global_config(Config, GlobalConfigFile) ->
%% We don't want to worry about global plugin install state effecting later
%% usage. So we throw away the global profile state used for plugin install.
GlobalConfigThrowAway = rebar_state:current_profiles(GlobalConfig, [global]),
- GlobalState = rebar_plugins:handle_plugins(global,
- rebar_state:get(GlobalConfigThrowAway, plugins, []),
- GlobalConfigThrowAway),
+ GlobalState = case rebar_state:get(GlobalConfigThrowAway, plugins, []) of
+ [] ->
+ GlobalConfigThrowAway;
+ GlobalPluginsToInstall ->
+ rebar_packages:packages(GlobalConfigThrowAway),
+ rebar_plugins:handle_plugins(global,
+ GlobalPluginsToInstall,
+ GlobalConfigThrowAway)
+ end,
GlobalPlugins = rebar_state:providers(GlobalState),
GlobalConfig2 = rebar_state:set(GlobalConfig, plugins, []),
GlobalConfig3 = rebar_state:set(GlobalConfig2, {plugins, global}, rebar_state:get(GlobalConfigThrowAway, plugins, [])),
diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl
index 17b435c..b14c50a 100644
--- a/src/rebar_app_utils.erl
+++ b/src/rebar_app_utils.erl
@@ -212,13 +212,8 @@ parse_goal(Name, Constraint) ->
end.
get_package(Dep, State) ->
- case rebar_state:registry(State) of
- {ok, T} ->
- {ok, HighestDepVsn} = rebar_packages:find_highest_matching(Dep, "0", T),
- {Dep, HighestDepVsn};
- error ->
- throw(?PRV_ERROR({load_registry_fail, Dep}))
- end.
+ {ok, HighestDepVsn} = rebar_packages:find_highest_matching(Dep, "0", ?PACKAGE_TABLE, State),
+ {Dep, HighestDepVsn}.
-spec has_all_beams(file:filename_all(), [module()]) ->
true | ?PRV_ERROR({missing_module, module()}).
diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl
index 5f3d1c9..f6993c5 100644
--- a/src/rebar_packages.erl
+++ b/src/rebar_packages.erl
@@ -1,71 +1,60 @@
-module(rebar_packages).
-export([packages/1
- ,packages_graph/1
- ,registry/1
+ ,close_packages/0
+ ,load_and_verify_version/1
+ ,deps/3
,package_dir/1
- ,check_registry/3
,registry_checksum/2
- ,find_highest_matching/3]).
+ ,find_highest_matching/4
+ ,format_error/1]).
-export_type([package/0]).
-include("rebar.hrl").
+-include_lib("providers/include/providers.hrl").
-type pkg_name() :: binary() | atom().
-type vsn() :: binary().
-type package() :: pkg_name() | {pkg_name(), vsn()}.
--spec packages(rebar_state:t()) -> rebar_dict().
-%% DON'T USE IT! Use rebar_state:packages(State) instead.
+-spec packages(rebar_state:t()) -> ets:tid().
packages(State) ->
- RegistryDir = package_dir(State),
- DictFile = filename:join(RegistryDir, "dict"),
-
- try
- {ok, DictBinary} = file:read_file(DictFile),
- binary_to_term(DictBinary)
- catch
- _:_ ->
+ catch ets:delete(?PACKAGE_TABLE),
+ case load_and_verify_version(State) of
+ true ->
+ ok;
+ false ->
+ ?DEBUG("Error loading package index.", []),
?ERROR("Bad packages index, try to fix with `rebar3 update`", []),
- dict:new()
+ ets:new(?PACKAGE_TABLE, [named_table, public])
end.
--spec packages_graph(rebar_state:t()) -> rebar_digraph().
-packages_graph(State) ->
- RegistryDir = package_dir(State),
- Edges = filename:join(RegistryDir, "edges"),
- Vertices = filename:join(RegistryDir, "vertices"),
- Neighbors = filename:join(RegistryDir, "neighbors"),
+close_packages() ->
+ catch ets:delete(?PACKAGE_TABLE).
- case lists:all(fun(X) -> filelib:is_file(X) end, [Edges, Vertices, Neighbors]) of
- true ->
- try
- {ok, EdgesTab} = ets:file2tab(Edges),
- {ok, VerticesTab} = ets:file2tab(Vertices),
- {ok, NeighborsTab} = ets:file2tab(Neighbors),
- {digraph, EdgesTab, VerticesTab, NeighborsTab, true}
- catch
- _:_ ->
- ?ERROR("Bad packages index, try to fix with `rebar3 update`", []),
- digraph:new()
+load_and_verify_version(State) ->
+ RegistryDir = package_dir(State),
+ case ets:file2tab(filename:join(RegistryDir, ?INDEX_FILE)) of
+ {ok, _} ->
+ case ets:lookup_element(?PACKAGE_TABLE, package_index_version, 2) of
+ ?PACKAGE_INDEX_VERSION ->
+ true;
+ _ ->
+ (catch ets:delete(?PACKAGE_TABLE)),
+ rebar_prv_update:hex_to_index(State)
end;
- false ->
- ?ERROR("Bad packages index, try to fix with `rebar3 update`", []),
- digraph:new()
+ _ ->
+ rebar_prv_update:hex_to_index(State)
end.
--spec registry(rebar_state:t()) -> {ok, ets:tid()} | {error, any()}.
-%% DON'T USE IT! Use rebar_state:registry(State) instead.
-registry(State) ->
- RegistryDir = package_dir(State),
- HexFile = filename:join(RegistryDir, "registry"),
- case ets:file2tab(HexFile) of
- {ok, T} ->
- {ok, T};
- {error, Reason} ->
- ?DEBUG("Error loading registry: ~p", [Reason]),
- error
+deps(Name, Vsn, State) ->
+ try
+ verify_table(State),
+ ets:lookup_element(?PACKAGE_TABLE, {ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}, 2)
+ catch
+ _:_ ->
+ throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}))
end.
package_dir(State) ->
@@ -78,27 +67,13 @@ package_dir(State) ->
ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")),
PackageDir.
-
-check_registry(Pkg, Vsn, State) ->
- case rebar_state:registry(State) of
- {ok, T} ->
- case ets:lookup(T, Pkg) of
- [{Pkg, [Vsns]}] ->
- lists:member(Vsn, Vsns);
- _ ->
- false
- end;
- error ->
- false
- end.
-
registry_checksum({pkg, Name, Vsn}, State) ->
- {ok, Registry} = rebar_state:registry(State),
- case ets:lookup(Registry, {Name, Vsn}) of
- [{{_, _}, [_, Checksum | _]}] ->
- Checksum;
- [] ->
- none
+ try
+ verify_table(State),
+ ets:lookup_element(?PACKAGE_TABLE, {Name, Vsn}, 3)
+ catch
+ _:_ ->
+ throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}))
end.
%% Hex supports use of ~> to specify the version required for a dependency.
@@ -116,28 +91,45 @@ registry_checksum({pkg, Name, Vsn}, State) ->
%% `~> 2.1.3-dev` | `>= 2.1.3-dev and < 2.2.0`
%% `~> 2.0` | `>= 2.0.0 and < 3.0.0`
%% `~> 2.1` | `>= 2.1.0 and < 3.0.0`
-find_highest_matching(Dep, Constraint, T) ->
- case ets:lookup(T, Dep) of
- [{Dep, [[Vsn]]}] ->
- case ec_semver:pes(Vsn, Constraint) of
- true ->
- {ok, Vsn};
- false ->
- ?WARN("Only existing version of ~s is ~s which does not match constraint ~~> ~s. "
- "Using anyway, but it is not guarenteed to work.", [Dep, Vsn, Constraint]),
- {ok, Vsn}
- end;
- [{Dep, [[HeadVsn | VsnTail]]}] ->
- {ok, lists:foldl(fun(Version, Highest) ->
- case ec_semver:pes(Version, Constraint) andalso
- ec_semver:gt(Version, Highest) of
- true ->
- Version;
- false ->
- Highest
- end
- end, HeadVsn, VsnTail)};
+find_highest_matching(Dep, Constraint, Table, State) ->
+ verify_table(State),
+ case ets:lookup_element(Table, Dep, 2) of
+ [[HeadVsn | VsnTail]] ->
+ {ok, handle_vsns(Constraint, HeadVsn, VsnTail)};
+ [[Vsn]] ->
+ handle_single_vsn(Dep, Vsn, Constraint);
+ [Vsn] ->
+ handle_single_vsn(Dep, Vsn, Constraint);
+ [HeadVsn | VsnTail] ->
+ {ok, handle_vsns(Constraint, HeadVsn, VsnTail)};
[] ->
- ?WARN("Missing registry entry for package ~s", [Dep]),
+ ?WARN("Missing registry entry for package ~s. Try to fix with `rebar3 update`", [Dep]),
none
end.
+
+handle_vsns(Constraint, HeadVsn, VsnTail) ->
+ lists:foldl(fun(Version, Highest) ->
+ case ec_semver:pes(Version, Constraint) andalso
+ ec_semver:gt(Version, Highest) of
+ true ->
+ Version;
+ false ->
+ Highest
+ end
+ end, HeadVsn, VsnTail).
+
+handle_single_vsn(Dep, Vsn, Constraint) ->
+ case ec_semver:pes(Vsn, Constraint) of
+ true ->
+ {ok, Vsn};
+ false ->
+ ?WARN("Only existing version of ~s is ~s which does not match constraint ~~> ~s. "
+ "Using anyway, but it is not guarenteed to work.", [Dep, Vsn, Constraint]),
+ {ok, Vsn}
+ end.
+
+format_error({missing_package, Package, Version}) ->
+ io_lib:format("Package not found in registry: ~s-~s. Try to fix with `rebar3 update`", [Package, Version]).
+
+verify_table(State) ->
+ ets:info(?PACKAGE_TABLE, named_table) =:= true orelse ?MODULE:load_and_verify_version(State).
diff --git a/src/rebar_plugins.erl b/src/rebar_plugins.erl
index bda3fb7..742dc82 100644
--- a/src/rebar_plugins.erl
+++ b/src/rebar_plugins.erl
@@ -16,10 +16,12 @@
-spec project_apps_install(rebar_state:t()) -> rebar_state:t().
project_apps_install(State) ->
+ rebar_packages:packages(State),
+
Profiles = rebar_state:current_profiles(State),
ProjectApps = rebar_state:project_apps(State),
- lists:foldl(fun(Profile, StateAcc) ->
+ State1 = lists:foldl(fun(Profile, StateAcc) ->
Plugins = rebar_state:get(State, {plugins, Profile}, []),
StateAcc1 = handle_plugins(Profile, Plugins, StateAcc),
@@ -30,7 +32,11 @@ project_apps_install(State) ->
Plugins2 = rebar_state:get(S, {plugins, Profile}, []),
handle_plugins(Profile, Plugins2, StateAcc2)
end, StateAcc1, ProjectApps)
- end, State, Profiles).
+ end, State, Profiles),
+
+ rebar_packages:close_packages(),
+
+ State1.
-spec install(rebar_state:t()) -> rebar_state:t().
install(State) ->
diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl
index 0883fab..0b0f607 100644
--- a/src/rebar_prv_install_deps.erl
+++ b/src/rebar_prv_install_deps.erl
@@ -125,10 +125,7 @@ handle_deps_as_profile(Profile, State, Deps, Upgrade) ->
DepsDir = profile_dep_dir(State, Profile),
Deps1 = rebar_app_utils:parse_deps(DepsDir, Deps, State, Locks, Level),
ProfileLevelDeps = [{Profile, Deps1, Level}],
- Graph = rebar_state:packages_graph(State),
- Registry = rebar_packages:registry(State),
- State1 = rebar_state:packages_graph(rebar_state:registry(State, Registry), Graph),
- handle_profile_level(ProfileLevelDeps, [], sets:new(), Upgrade, Locks, State1, Graph).
+ handle_profile_level(ProfileLevelDeps, [], sets:new(), Upgrade, Locks, State).
%% ===================================================================
%% Internal functions
@@ -141,10 +138,7 @@ deps_per_profile(Profiles, Upgrade, State) ->
Deps = lists:foldl(fun(Profile, DepAcc) ->
[parsed_profile_deps(State, Profile, Level) | DepAcc]
end, [], Profiles),
- Graph = rebar_state:packages_graph(State),
- Registry = rebar_packages:registry(State),
- State1 = rebar_state:packages_graph(rebar_state:registry(State, Registry), Graph),
- handle_profile_level(Deps, [], sets:new(), Upgrade, Locks, State1, Graph).
+ handle_profile_level(Deps, [], sets:new(), Upgrade, Locks, State).
parsed_profile_deps(State, Profile, Level) ->
ParsedDeps = rebar_state:get(State, {parsed_deps, Profile}, []),
@@ -153,17 +147,17 @@ parsed_profile_deps(State, Profile, Level) ->
%% Level-order traversal of all dependencies, across profiles.
%% If profiles x,y,z are present, then the traversal will go:
%% x0, y0, z0, x1, y1, z1, ..., xN, yN, zN.
-handle_profile_level([], Apps, _Seen, _Upgrade, _Locks, State, _Graph) ->
+handle_profile_level([], Apps, _Seen, _Upgrade, _Locks, State) ->
{Apps, State};
-handle_profile_level([{Profile, Deps, Level} | Rest], Apps, Seen, Upgrade, Locks, State, Graph) ->
+handle_profile_level([{Profile, Deps, Level} | Rest], Apps, Seen, Upgrade, Locks, State) ->
{Deps1, Apps1, State1, Seen1} =
update_deps(Profile, Level, Deps, Apps
- ,State, Upgrade, Seen, Locks, Graph),
+ ,State, Upgrade, Seen, Locks),
Deps2 = case Deps1 of
[] -> Rest;
_ -> Rest ++ [{Profile, Deps1, Level+1}]
end,
- handle_profile_level(Deps2, Apps1, sets:union(Seen, Seen1), Upgrade, Locks, State1, Graph).
+ handle_profile_level(Deps2, Apps1, sets:union(Seen, Seen1), Upgrade, Locks, State1).
find_cycles(Apps) ->
case rebar_digraph:compile_order(Apps) of
@@ -201,17 +195,17 @@ maybe_lock(Profile, AppInfo, Seen, State, Level) ->
{sets:add_element(Name, Seen), State}
end.
-update_deps(Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks, Graph) ->
+update_deps(Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) ->
lists:foldl(
fun(AppInfo, {DepsAcc, AppsAcc, StateAcc, SeenAcc}) ->
update_dep(AppInfo, Profile, Level,
DepsAcc, AppsAcc, StateAcc,
- Upgrade, SeenAcc, Locks, Graph)
+ Upgrade, SeenAcc, Locks)
end,
{[], Apps, State, Seen},
rebar_utils:sort_deps(Deps)).
-update_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks, Graph) ->
+update_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) ->
%% If not seen, add to list of locks to write out
Name = rebar_app_info:name(AppInfo),
case sets:is_element(Name, Seen) of
@@ -222,7 +216,7 @@ update_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks, Gra
false ->
update_unseen_dep(AppInfo, Profile, Level,
Deps, Apps,
- State, Upgrade, Seen, Locks, Graph)
+ State, Upgrade, Seen, Locks)
end.
profile_dep_dir(State, Profile) ->
@@ -242,24 +236,23 @@ update_seen_dep(AppInfo, _Profile, _Level, Deps, Apps, State, Upgrade, Seen, Loc
end,
{Deps, Apps, State, Seen}.
-update_unseen_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks, Graph) ->
+update_unseen_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) ->
{NewSeen, State1} = maybe_lock(Profile, AppInfo, Seen, State, Level),
{_, AppInfo1} = maybe_fetch(AppInfo, Profile, Upgrade, Seen, State1),
DepsDir = profile_dep_dir(State, Profile),
{AppInfo2, NewDeps, State2} =
- handle_dep(State1, Profile, DepsDir, AppInfo1, Locks, Level, Graph),
+ handle_dep(State1, Profile, DepsDir, AppInfo1, Locks, Level),
AppInfo3 = rebar_app_info:dep_level(AppInfo2, Level),
{NewDeps ++ Deps, [AppInfo3 | Apps], State2, NewSeen}.
--spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer(), rebar_dict()) -> {rebar_app_info:t(), [rebar_app_info:t()], [pkg_dep()], [integer()], rebar_state:t()}.
-handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level, Graph) ->
+-spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer()) -> {rebar_app_info:t(), [rebar_app_info:t()], [pkg_dep()], [integer()], rebar_state:t()}.
+handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) ->
Profiles = rebar_state:current_profiles(State),
Name = rebar_app_info:name(AppInfo),
Vsn = rebar_app_info:original_vsn(AppInfo),
%% Deps may be under a sub project app, find it and use its state if so
- S0 = rebar_app_info:state(AppInfo),
- S = rebar_state:registry(S0, rebar_state:registry(State)),
+ S = rebar_app_info:state(AppInfo),
C = rebar_config:consult(rebar_app_info:dir(AppInfo)),
S1 = rebar_state:new(S, C, rebar_app_info:dir(AppInfo)),
S2 = rebar_state:apply_overrides(S1, Name),
@@ -278,7 +271,7 @@ handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level, Graph) ->
%% Upgrade lock level to be the level the dep will have in this dep tree
case rebar_app_info:resource_type(AppInfo1) of
pkg ->
- NewDeps = digraph:out_neighbours(Graph, {ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}),
+ NewDeps = rebar_packages:deps(Name, Vsn, S5),
NewDeps1 = rebar_app_utils:parse_deps(Name, DepsDir, NewDeps, S5, Locks, Level+1),
{rebar_app_info:deps(AppInfo1, NewDeps), NewDeps1, State};
_ ->
diff --git a/src/rebar_prv_packages.erl b/src/rebar_prv_packages.erl
index 44cde4e..f5d9e38 100644
--- a/src/rebar_prv_packages.erl
+++ b/src/rebar_prv_packages.erl
@@ -27,19 +27,17 @@ init(State) ->
-spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
do(State) ->
- Dict = rebar_packages:packages(State),
- print_packages(Dict),
+ print_packages(),
{ok, State}.
-spec format_error(any()) -> iolist().
format_error(load_registry_fail) ->
"Failed to load package regsitry. Try running 'rebar3 update' to fix".
-print_packages(Dict) ->
- Pkgs = lists:keysort(1, dict:fetch_keys(Dict)),
- SortedPkgs = lists:foldl(fun({Pkg, Vsn}, Acc) ->
- orddict:append_list(Pkg, [Vsn], Acc)
- end, orddict:new(), Pkgs),
+print_packages() ->
+ SortedPkgs = ets:foldl(fun({{Pkg, Vsn}, _}, Acc) ->
+ orddict:append_list(Pkg, [Vsn], Acc)
+ end, orddict:new(), ?PACKAGE_TABLE),
orddict:map(fun(Name, Vsns) ->
SortedVsns = lists:sort(fun(A, B) ->
diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl
index e73c769..4b3a155 100644
--- a/src/rebar_prv_update.erl
+++ b/src/rebar_prv_update.erl
@@ -9,21 +9,14 @@
do/1,
format_error/1]).
+-export([hex_to_index/1]).
+
-include("rebar.hrl").
-include_lib("providers/include/providers.hrl").
-define(PROVIDER, update).
-define(DEPS, []).
-%% Ignore warning of digraph opaque type when running dialyzer
--dialyzer({no_opaque, do/1}).
--dialyzer({no_opaque, write_registry/3}).
-
-%% Ignoring the opaque type warning won't stop dialyzer from warning of
-%% no return for functions that had the opaque type warnings
--dialyzer({no_return, do/1}).
--dialyzer({no_return, write_registry/3}).
-
%% ===================================================================
%% Public API
%% ===================================================================
@@ -50,16 +43,15 @@ do(State) ->
TmpDir = ec_file:insecure_mkdtemp(),
TmpFile = filename:join(TmpDir, "packages.gz"),
- Url = rebar_state:get(State, rebar_packages_cdn, "https://s3.amazonaws.com/s3.hex.pm/registry.ets.gz"),
+ Url = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_HEX_REGISTRY),
{ok, _RequestId} = httpc:request(get, {Url, []},
[], [{stream, TmpFile}, {sync, true}],
rebar),
{ok, Data} = file:read_file(TmpFile),
Unzipped = zlib:gunzip(Data),
ok = file:write_file(HexFile, Unzipped),
- {Dict, Graph} = hex_to_graph(HexFile),
- write_registry(Dict, Graph, State),
- true = digraph:delete(Graph),
+
+ hex_to_index(State),
ok
catch
_E:C ->
@@ -73,57 +65,58 @@ do(State) ->
format_error(package_index_write) ->
"Failed to write package index.".
--spec write_registry(rebar_dict(), {digraph, ets:tid(), ets:tid(), ets:tid(), any()}, rebar_state:t()) -> ok | {error, atom()}.
-write_registry(Dict, {digraph, Edges, Vertices, Neighbors, _}, State) ->
- RegistryDir = rebar_packages:package_dir(State),
- filelib:ensure_dir(filename:join(RegistryDir, "dummy")),
- ets:tab2file(Edges, filename:join(RegistryDir, "edges")),
- ets:tab2file(Vertices, filename:join(RegistryDir, "vertices")),
- ets:tab2file(Neighbors, filename:join(RegistryDir, "neighbors")),
- file:write_file(filename:join(RegistryDir, "dict"), term_to_binary(Dict)).
-
is_supported(<<"make">>) -> true;
is_supported(<<"rebar">>) -> true;
is_supported(<<"rebar3">>) -> true;
is_supported(_) -> false.
-hex_to_graph(Filename) ->
- {ok, T} = ets:file2tab(Filename),
- Graph = digraph:new(),
- ets:foldl(fun({Pkg, [Versions]}, ok) when is_binary(Pkg), is_list(Versions) ->
- lists:foreach(fun(Version) ->
- digraph:add_vertex(Graph, {Pkg, Version}, 1)
- end, Versions);
- (_, ok) ->
- ok
- end, ok, T),
-
- Dict1 = ets:foldl(fun({{Pkg, PkgVsn}, [Deps, _, BuildTools | _]}, Dict) when is_list(BuildTools) ->
- case lists:any(fun is_supported/1, BuildTools) of
- true ->
- DepsList = update_graph(Pkg, PkgVsn, Deps, T, Graph),
- dict:store({Pkg, PkgVsn}, DepsList, Dict);
- false ->
- Dict
- end;
- (_, Dict) ->
- Dict
- end, dict:new(), T),
- {Dict1, Graph}.
-
-update_graph(Pkg, PkgVsn, Deps, HexRegistry, Graph) ->
+hex_to_index(State) ->
+ RegistryDir = rebar_packages:package_dir(State),
+ HexFile = filename:join(RegistryDir, "registry"),
+ try ets:file2tab(HexFile) of
+ {ok, Registry} ->
+ try
+ (catch ets:delete(?PACKAGE_TABLE)),
+ ets:new(?PACKAGE_TABLE, [named_table, public]),
+ ets:foldl(fun({{Pkg, PkgVsn}, [Deps, Checksum, BuildTools | _]}, _) when is_list(BuildTools) ->
+ case lists:any(fun is_supported/1, BuildTools) of
+ true ->
+ DepsList = update_deps_list(Deps, Registry, State),
+ ets:insert(?PACKAGE_TABLE, {{Pkg, PkgVsn}, DepsList, Checksum});
+ false ->
+ true
+ end;
+ ({Pkg, [Vsns]}, _) when is_binary(Pkg) ->
+ ets:insert(?PACKAGE_TABLE, {Pkg, Vsns});
+ (_, _) ->
+ true
+ end, true, Registry),
+
+ ets:insert(?PACKAGE_TABLE, {package_index_version, ?PACKAGE_INDEX_VERSION}),
+ ets:tab2file(?PACKAGE_TABLE, filename:join(RegistryDir, "packages.idx")),
+ true
+ after
+ catch ets:delete(Registry)
+ end;
+ {error, Reason} ->
+ ?DEBUG("Error loading package registry: ~p", [Reason]),
+ false
+ catch
+ _:_ ->
+ fail
+ end.
+
+update_deps_list(Deps, HexRegistry, State) ->
lists:foldl(fun([Dep, DepVsn, false, _AppName | _], DepsListAcc) ->
case DepVsn of
<<"~> ", Vsn/binary>> ->
- case rebar_packages:find_highest_matching(Dep, Vsn, HexRegistry) of
+ case rebar_packages:find_highest_matching(Dep, Vsn, HexRegistry, State) of
{ok, HighestDepVsn} ->
- digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, HighestDepVsn}),
- [{Dep, DepVsn} | DepsListAcc];
+ [{Dep, HighestDepVsn} | DepsListAcc];
none ->
DepsListAcc
end;
Vsn ->
- digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, Vsn}),
[{Dep, Vsn} | DepsListAcc]
end;
([_Dep, _DepVsn, true, _AppName | _], DepsListAcc) ->
diff --git a/src/rebar_state.erl b/src/rebar_state.erl
index 54e7f13..213eb2f 100644
--- a/src/rebar_state.erl
+++ b/src/rebar_state.erl
@@ -38,10 +38,6 @@
overrides/1, overrides/2,
apply_overrides/2,
- packages_graph/1, packages_graph/2,
- packages/1,
- registry/1, registry/2,
-
resources/1, resources/2, add_resource/2,
providers/1, providers/2, add_provider/2]).
@@ -66,9 +62,6 @@
all_plugin_deps = [] :: [rebar_app_info:t()],
all_deps = [] :: [rebar_app_info:t()],
- packages = undefined :: rebar_dict(),
- packages_graph = undefined :: rebar_digraph() | undefined,
- registry = undefined :: {ok, ets:tid()} | error | undefined,
overrides = [],
resources = [],
providers = []}).
@@ -90,18 +83,9 @@ new(Config) when is_list(Config) ->
Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins} | Config],
true = rebar_config:verify_config_format(Terms),
Opts = dict:from_list(Terms),
- load_package_registry(
- BaseState#state_t { dir = rebar_dir:get_cwd(),
- default = Opts,
- opts = Opts }).
-
-load_package_registry(Config0) ->
- Registry = rebar_packages:registry(Config0),
- Packages = rebar_packages:packages(Config0),
- PackagesGraph = rebar_packages:packages_graph(Config0),
- Config0#state_t{registry = Registry,
- packages_graph = PackagesGraph,
- packages = Packages}.
+ BaseState#state_t { dir = rebar_dir:get_cwd(),
+ default = Opts,
+ opts = Opts }.
-spec new(t() | atom(), list()) -> t().
new(Profile, Config) when is_atom(Profile)
@@ -113,11 +97,10 @@ new(Profile, Config) when is_atom(Profile)
Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins} | Config],
true = rebar_config:verify_config_format(Terms),
Opts = dict:from_list(Terms),
- load_package_registry(
- BaseState#state_t { dir = rebar_dir:get_cwd(),
- current_profiles = [Profile],
- default = Opts,
- opts = Opts });
+ BaseState#state_t { dir = rebar_dir:get_cwd(),
+ current_profiles = [Profile],
+ default = Opts,
+ opts = Opts };
new(ParentState=#state_t{}, Config) ->
%% Load terms from rebar.config, if it exists
Dir = rebar_dir:get_cwd(),
@@ -450,27 +433,6 @@ namespace(#state_t{namespace=Namespace}) ->
namespace(State=#state_t{}, Namespace) ->
State#state_t{namespace=Namespace}.
-packages(#state_t{packages=undefined}) ->
- throw(packages_usage_error);
-packages(#state_t{packages=Packages}) ->
- Packages.
-
-packages_graph(#state_t{packages_graph=undefined}) ->
- throw(packages_usage_error);
-packages_graph(#state_t{packages_graph=PackagesGraph}) ->
- PackagesGraph.
-
-packages_graph(State, PackagesGraph) ->
- State#state_t{packages_graph=PackagesGraph}.
-
-registry(#state_t{registry=undefined}) ->
- throw(registry_usage_error);
-registry(#state_t{registry=Registry}) ->
- Registry.
-
-registry(State, Registry) ->
- State#state_t{registry=Registry}.
-
-spec resources(t()) -> [{rebar_resource:type(), module()}].
resources(#state_t{resources=Resources}) ->
Resources.
diff --git a/test/mock_pkg_resource.erl b/test/mock_pkg_resource.erl
index 9258f72..4ed8d8e 100644
--- a/test/mock_pkg_resource.erl
+++ b/test/mock_pkg_resource.erl
@@ -35,7 +35,7 @@ mock(Opts) ->
unmock() ->
meck:unload(?MOD),
- meck:unload(rebar_state).
+ meck:unload(rebar_packages).
%%%%%%%%%%%%%%%
%%% Private %%%
@@ -111,34 +111,19 @@ mock_pkg_index(Opts) ->
Deps = proplists:get_value(pkgdeps, Opts, []),
Skip = proplists:get_value(not_in_index, Opts, []),
%% Dict: {App, Vsn}: [{<<"link">>, <<>>}, {<<"deps">>, []}]
- %% Digraph: all apps and deps in the index
+ %% Index: all apps and deps in the index
+
Dict = find_parts(Deps, Skip),
- GraphParts = to_graph_parts(Dict),
- Digraph = rebar_digraph:restore_graph(GraphParts),
- meck:new(rebar_state, [passthrough, no_link]),
- meck:expect(rebar_state, registry,
- fun(_State) -> {ok, to_registry(Deps)} end),
- meck:expect(rebar_state, packages,
- fun(_State) -> Dict end),
- meck:expect(rebar_state, packages_graph,
- fun(_State) -> Digraph end).
+ meck:new(rebar_packages, [passthrough, no_link]),
+ meck:expect(rebar_packages, packages,
+ fun(_State) -> to_index(Deps, Dict) end),
+ meck:expect(rebar_packages, load_and_verify_version,
+ fun(_State) -> to_index(Deps, Dict), true end).
%%%%%%%%%%%%%%%
%%% Helpers %%%
%%%%%%%%%%%%%%%
-to_registry(Deps) ->
- Tid = ets:new(registry, []),
- lists:foreach(fun({{Name, Vsn}, _}) ->
- case ets:lookup(Tid, Name) of
- [{_, [Vsns]}] ->
- ets:insert(Tid, {Name, [[Vsn | Vsns]]});
- _ ->
- ets:insert(Tid, {Name, [[Vsn]]})
- end
- end, Deps),
- Tid.
-
all_files(Dir) ->
filelib:wildcard(filename:join([Dir, "**"])).
@@ -158,10 +143,20 @@ find_parts([{AppName, Deps}|Rest], Skip, Acc) ->
find_parts(Rest, Skip, AccNew)
end.
-to_graph_parts(Dict) ->
- LastUpdated = os:timestamp(),
- dict:fold(fun(K,Deps,{Ks,Vs}) ->
- {[{K,LastUpdated}|Ks],
- [{K,{list_to_binary(atom_to_list(DK)), list_to_binary(DV)}}
- || {DK,DV} <- Deps] ++ Vs}
- end, {[],[]}, Dict).
+to_index(AllDeps, Dict) ->
+ catch ets:delete(package_index),
+ ets:new(package_index, [named_table, public]),
+ dict:fold(
+ fun(K, Deps, _) ->
+ DepsList = [{ec_cnv:to_binary(DK), ec_cnv:to_binary(DV)} || {DK, DV} <- Deps],
+ ets:insert(package_index, {K, DepsList, <<"checksum">>})
+ end, ok, Dict),
+ ets:insert(package_index, {package_index_version, 3}),
+ lists:foreach(fun({{Name, Vsn}, _}) ->
+ case ets:lookup(package_index, ec_cnv:to_binary(Name)) of
+ [{_, Vsns}] ->
+ ets:insert(package_index, {ec_cnv:to_binary(Name), [ec_cnv:to_binary(Vsn) | Vsns]});
+ _ ->
+ ets:insert(package_index, {ec_cnv:to_binary(Name), [ec_cnv:to_binary(Vsn)]})
+ end
+ end, AllDeps).
diff --git a/test/rebar_pkg_SUITE.erl b/test/rebar_pkg_SUITE.erl
index e99e787..3cd3a67 100644
--- a/test/rebar_pkg_SUITE.erl
+++ b/test/rebar_pkg_SUITE.erl
@@ -159,25 +159,33 @@ mock_config(Name, Config) ->
Priv = ?config(priv_dir, Config),
CacheRoot = filename:join([Priv, "cache", atom_to_list(Name)]),
TmpDir = filename:join([Priv, "tmp", atom_to_list(Name)]),
- T = ets:new(fake_registry, [public]),
- ets:insert_new(T, [
- {{<<"badindexchk">>,<<"1.0.0">>}, [[], ?bad_checksum]},
- {{<<"goodpkg">>,<<"1.0.0">>}, [[], ?good_checksum]},
- {{<<"badpkg">>,<<"1.0.0">>}, [[], ?good_checksum]}
+ Tid = ets:new(registry_table, [public]),
+ ets:insert_new(Tid, [
+ {<<"badindexchk">>,[[<<"1.0.0">>]]},
+ {<<"goodpkg">>,[[<<"1.0.0">>]]},
+ {<<"badpkg">>,[[<<"1.0.0">>]]},
+ {{<<"badindexchk">>,<<"1.0.0">>}, [[], ?bad_checksum, [<<"rebar3">>]]},
+ {{<<"goodpkg">>,<<"1.0.0">>}, [[], ?good_checksum, [<<"rebar3">>]]},
+ {{<<"badpkg">>,<<"1.0.0">>}, [[], ?good_checksum, [<<"rebar3">>]]}
]),
CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]),
filelib:ensure_dir(filename:join([CacheDir, "registry"])),
- ok = ets:tab2file(T, filename:join([CacheDir, "registry"])),
+ ok = ets:tab2file(Tid, filename:join([CacheDir, "registry"])),
+
%% The state returns us a fake registry
meck:new(rebar_state, [passthrough]),
- meck:expect(rebar_state, registry,
- fun(_State) -> {ok, T} end),
meck:expect(rebar_state, get,
fun(_State, rebar_packages_cdn, _Default) ->
"http://test.com/"
end),
+
meck:new(rebar_dir, [passthrough]),
meck:expect(rebar_dir, global_cache_dir, fun(_) -> CacheRoot end),
+
+ meck:new(rebar_packages, [passthrough]),
+ meck:expect(rebar_packages, package_dir, fun(_) -> CacheDir end),
+ rebar_prv_update:hex_to_index(rebar_state:new()),
+
%% Cache fetches are mocked -- we assume the server and clients are
%% correctly used.
GoodCache = ?config(good_cache, Config),
@@ -194,7 +202,7 @@ mock_config(Name, Config) ->
[{cache_root, CacheRoot},
{cache_dir, CacheDir},
{tmp_dir, TmpDir},
- {mock_table, T} | Config].
+ {mock_table, Tid} | Config].
unmock_config(Config) ->
meck:unload(),
generated by cgit v1.1 at 2024-12-26 03:49:28 +0000