summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Makefile2
-rw-r--r--src/rebar_app_info.erl6
-rw-r--r--src/rebar_erlc_compiler.erl10
-rw-r--r--src/rebar_fetch.erl27
-rw-r--r--src/rebar_git_resource.erl18
-rw-r--r--src/rebar_packages.erl18
-rw-r--r--src/rebar_pkg_resource.erl8
-rw-r--r--src/rebar_prv_install_deps.erl47
-rw-r--r--src/rebar_prv_update.erl64
-rw-r--r--src/rebar_resource.erl4
-rw-r--r--test/mock_git_resource.erl2
-rw-r--r--test/mock_pkg_resource.erl19
-rw-r--r--test/rebar_deps_SUITE.erl249
-rw-r--r--test/rebar_upgrade_SUITE.erl9
14 files changed, 326 insertions, 157 deletions
diff --git a/Makefile b/Makefile
index 2e02370..f36c177 100644
--- a/Makefile
+++ b/Makefile
@@ -6,7 +6,7 @@ RETEST=$(PWD)/deps/retest/retest
DEPS_PLT=$(CURDIR)/.depsolver_plt
all:
- ./bootstrap/bootstrap
+ @rebar get-deps compile escriptize
clean:
@rm -rf rebar3 ebin/*.beam inttest/rt.work rt.work .eunit
diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl
index dca5aa8..9d779be 100644
--- a/src/rebar_app_info.erl
+++ b/src/rebar_app_info.erl
@@ -79,7 +79,7 @@ new(AppName, Vsn) ->
new(AppName, Vsn, Dir) ->
{ok, #app_info_t{name=ec_cnv:to_binary(AppName),
original_vsn=Vsn,
- dir=Dir}}.
+ dir=ec_cnv:to_list(Dir)}}.
%% @doc build a complete version of the app info with all fields set.
-spec new(atom() | binary() | string(), binary() | string(), file:name(), list()) ->
@@ -87,7 +87,7 @@ new(AppName, Vsn, Dir) ->
new(AppName, Vsn, Dir, Deps) ->
{ok, #app_info_t{name=ec_cnv:to_binary(AppName),
original_vsn=Vsn,
- dir=Dir,
+ dir=ec_cnv:to_list(Dir),
deps=Deps}}.
%% @doc discover a complete version of the app info with all fields set.
@@ -200,7 +200,7 @@ dir(#app_info_t{dir=Dir}) ->
-spec dir(t(), file:name()) -> t().
dir(AppInfo=#app_info_t{}, Dir) ->
- AppInfo#app_info_t{dir=Dir}.
+ AppInfo#app_info_t{dir=ec_cnv:to_list(Dir)}.
-spec ebin_dir(t()) -> file:name().
ebin_dir(#app_info_t{dir=Dir}) ->
diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl
index b096705..cb342a8 100644
--- a/src/rebar_erlc_compiler.erl
+++ b/src/rebar_erlc_compiler.erl
@@ -144,17 +144,13 @@ doterl_compile(Config, Dir, MoreSources, ErlOpts) ->
filename:join(Dir, X)
end, rebar_dir:src_dirs(proplists:append_values(src_dirs, ErlOpts))),
AllErlFiles = gather_src(SrcDirs, []) ++ MoreSources,
- %% NOTE: If and when erl_first_files is not inherited anymore
- %% (rebar_state:get instead of rebar_state:get_list), consider
- %% logging a warning message for any file listed in erl_first_files which
- %% wasn't found via gather_src.
-
+
%% Issue: rebar/rebar3#140 (fix matching based on same path + order of
%% erl_first_files)
ErlFirstFiles = get_erl_first_files(ErlFirstFilesConf, AllErlFiles),
RestErls = [ File || File <- AllErlFiles,
not lists:member(File, ErlFirstFiles) ],
-
+
%% Make sure that ebin/ exists and is on the path
ok = filelib:ensure_dir(filename:join([Dir, "ebin", "dummy.beam"])),
CurrPath = code:get_path(),
@@ -634,5 +630,3 @@ log_files(Prefix, Files) ->
_ ->
?DEBUG("~s:~n~p", [Prefix, Files])
end.
-
-
diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl
index 088a49a..4dfd790 100644
--- a/src/rebar_fetch.erl
+++ b/src/rebar_fetch.erl
@@ -8,12 +8,13 @@
-module(rebar_fetch).
-export([lock_source/2,
- download_source/2,
+ download_source/3,
needs_update/2]).
-export([format_error/1]).
-include("rebar.hrl").
+-include_lib("providers/include/providers.hrl").
%% map short versions of resources to module names
-define(RESOURCES, [{git, rebar_git_resource}, {pkg, rebar_pkg_resource}]).
@@ -24,13 +25,14 @@ lock_source(AppDir, Source) ->
Module = get_resource_type(Source),
Module:lock(AppDir, Source).
--spec download_source(file:filename_all(), rebar_resource:resource()) -> true | {error, any()}.
-download_source(AppDir, Source) ->
+-spec download_source(file:filename_all(), rebar_resource:resource(), rebar_state:t()) ->
+ true | {error, any()}.
+download_source(AppDir, Source, State) ->
try
Module = get_resource_type(Source),
TmpDir = ec_file:insecure_mkdtemp(),
AppDir1 = ec_cnv:to_list(AppDir),
- case Module:download(TmpDir, Source) of
+ case Module:download(TmpDir, Source, State) of
{ok, _} ->
ec_file:mkdir_p(AppDir1),
code:del_path(filename:absname(filename:join(AppDir1, "ebin"))),
@@ -38,20 +40,21 @@ download_source(AppDir, Source) ->
ok = ec_file:copy(TmpDir, filename:absname(AppDir1), [recursive]),
true;
{tarball, File} ->
+ Contents = filename:join(TmpDir, "contents"),
ec_file:mkdir_p(AppDir1),
- ok = erl_tar:extract(File, [{cwd, TmpDir}
- ,compressed]),
- BaseName = filename:basename(AppDir1),
- [FromDir] = filelib:wildcard(filename:join(TmpDir, BaseName++"-*")),
+ ec_file:mkdir_p(Contents),
+ ok = erl_tar:extract(File, [{cwd, TmpDir}]),
+ ok = erl_tar:extract(filename:join(TmpDir, "contents.tar.gz"),
+ [{cwd, Contents}, compressed]),
code:del_path(filename:absname(filename:join(AppDir1, "ebin"))),
ec_file:remove(filename:absname(AppDir1), [recursive]),
- ok = ec_file:copy(FromDir, filename:absname(AppDir1), [recursive]),
- true = code:add_patha(filename:join(AppDir1, "ebin")),
+ ok = ec_file:copy(Contents, filename:absname(AppDir1), [recursive]),
true
end
catch
- _:_ ->
- {error, {rebar_fetch, {fetch_fail, Source}}}
+ C:T ->
+ ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, erlang:get_stacktrace()]),
+ throw(?PRV_ERROR({fetch_fail, Source}))
end.
-spec needs_update(file:filename_all(), rebar_resource:resource()) -> boolean() | {error, string()}.
diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl
index bd0a978..785c362 100644
--- a/src/rebar_git_resource.erl
+++ b/src/rebar_git_resource.erl
@@ -5,7 +5,7 @@
-behaviour(rebar_resource).
-export([lock/2
- ,download/2
+ ,download/3
,needs_update/2
,make_vsn/1]).
@@ -75,28 +75,28 @@ parse_git_url("https://" ++ HostPath) ->
[Host | Path] = string:tokens(HostPath, "/"),
{Host, filename:rootname(filename:join(Path), ".git")}.
-download(Dir, {git, Url}) ->
+download(Dir, {git, Url}, State) ->
?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []),
- download(Dir, {git, Url, {branch, "master"}});
-download(Dir, {git, Url, ""}) ->
+ download(Dir, {git, Url, {branch, "master"}}, State);
+download(Dir, {git, Url, ""}, State) ->
?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []),
- download(Dir, {git, Url, {branch, "master"}});
-download(Dir, {git, Url, {branch, Branch}}) ->
+ download(Dir, {git, Url, {branch, "master"}}, State);
+download(Dir, {git, Url, {branch, Branch}}, _State) ->
ok = filelib:ensure_dir(Dir),
rebar_utils:sh(?FMT("git clone ~s ~s -b ~s --single-branch",
[Url, filename:basename(Dir), Branch]),
[{cd, filename:dirname(Dir)}]);
-download(Dir, {git, Url, {tag, Tag}}) ->
+download(Dir, {git, Url, {tag, Tag}}, _State) ->
ok = filelib:ensure_dir(Dir),
rebar_utils:sh(?FMT("git clone ~s ~s -b ~s --single-branch",
[Url, filename:basename(Dir), Tag]),
[{cd, filename:dirname(Dir)}]);
-download(Dir, {git, Url, {ref, Ref}}) ->
+download(Dir, {git, Url, {ref, Ref}}, _State) ->
ok = filelib:ensure_dir(Dir),
rebar_utils:sh(?FMT("git clone -n ~s ~s", [Url, filename:basename(Dir)]),
[{cd, filename:dirname(Dir)}]),
rebar_utils:sh(?FMT("git checkout -q ~s", [Ref]), [{cd, Dir}]);
-download(Dir, {git, Url, Rev}) ->
+download(Dir, {git, Url, Rev}, _State) ->
?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []),
ok = filelib:ensure_dir(Dir),
rebar_utils:sh(?FMT("git clone -n ~s ~s", [Url, filename:basename(Dir)]),
diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl
index 30d11c9..5400544 100644
--- a/src/rebar_packages.erl
+++ b/src/rebar_packages.erl
@@ -13,13 +13,21 @@
-spec get_packages(rebar_state:t()) -> {rebar_dict(), rebar_digraph()}.
get_packages(State) ->
RebarDir = rebar_dir:global_config_dir(State),
- PackagesFile = filename:join(RebarDir, "packages"),
- case ec_file:exists(PackagesFile) of
+ RegistryDir = filename:join(RebarDir, "packages"),
+ DictFile = filename:join(RegistryDir, "dict"),
+ Edges = filename:join(RegistryDir, "edges"),
+ Vertices = filename:join(RegistryDir, "vertices"),
+ Neighbors = filename:join(RegistryDir, "neighbors"),
+
+ case lists:all(fun(X) -> filelib:is_file(X) end, [DictFile, Edges, Vertices, Neighbors]) of
true ->
try
- {ok, Binary} = file:read_file(PackagesFile),
- {Dict, Graph} = binary_to_term(Binary),
- {Dict, rebar_digraph:restore_graph(Graph)}
+ {ok, DictBinary} = file:read_file(DictFile),
+ Dict = binary_to_term(DictBinary),
+ {ok, EdgesTab} = ets:file2tab(Edges),
+ {ok, VerticesTab} = ets:file2tab(Vertices),
+ {ok, NeighborsTab} = ets:file2tab(Neighbors),
+ {Dict, {digraph, EdgesTab, VerticesTab, NeighborsTab, true}}
catch
_:_ ->
?ERROR("Bad packages index, try to fix with `rebar3 update`", []),
diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl
index 5cd6fc8..0508e2c 100644
--- a/src/rebar_pkg_resource.erl
+++ b/src/rebar_pkg_resource.erl
@@ -5,7 +5,7 @@
-behaviour(rebar_resource).
-export([lock/2
- ,download/2
+ ,download/3
,needs_update/2
,make_vsn/1]).
@@ -23,9 +23,11 @@ needs_update(Dir, {pkg, _Name, Vsn, _Url}) ->
true
end.
-download(Dir, {pkg, _Name, _Vsn, Url}) ->
+download(Dir, {pkg, Name, Vsn}, State) ->
TmpFile = filename:join(Dir, "package.tar.gz"),
- {ok, saved_to_file} = httpc:request(get, {binary_to_list(Url), []}, [], [{stream, TmpFile}]),
+ CDN = rebar_state:get(State, rebar_packages_cdn, "https://s3.amazonaws.com/s3.hex.pm/tarballs"),
+ Url = string:join([CDN, binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>)], "/"),
+ {ok, saved_to_file} = httpc:request(get, {Url, []}, [], [{stream, TmpFile}]),
{tarball, TmpFile}.
make_vsn(_) ->
diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl
index dab4c24..61fbd70 100644
--- a/src/rebar_prv_install_deps.erl
+++ b/src/rebar_prv_install_deps.erl
@@ -75,19 +75,19 @@ do(State) ->
Profiles = rebar_state:current_profiles(State),
ProjectApps = rebar_state:project_apps(State),
- {SrcApps, State1} =
- lists:foldl(fun(Profile, {SrcAppsAcc, StateAcc}) ->
+ {Apps, State1} =
+ lists:foldl(fun(Profile, {AppsAcc, StateAcc}) ->
Locks = rebar_state:get(StateAcc, {locks, Profile}, []),
- {ok, NewSrcApps, NewState} =
+ {ok, NewApps, NewState} =
handle_deps(Profile
,StateAcc
,rebar_state:get(StateAcc, {deps, Profile}, [])
,Locks),
- {NewSrcApps++SrcAppsAcc, NewState}
+ {NewApps++AppsAcc, NewState}
end, {[], State}, lists:reverse(Profiles)),
- Source = ProjectApps ++ SrcApps,
- case find_cycles(Source ++ rebar_state:all_deps(State1)) of
+ Source = ProjectApps ++ Apps,
+ case find_cycles(Source) of
{cycles, Cycles} ->
?PRV_ERROR({cycles, Cycles});
{error, Error} ->
@@ -117,7 +117,9 @@ find_cycles(Apps) ->
-spec format_error(any()) -> iolist().
format_error({parse_dep, Dep}) ->
- io_lib:format("Failed parsing dep ~p~n", [Dep]);
+ io_lib:format("Failed parsing dep ~p", [Dep]);
+format_error({missing_package, Package, Version}) ->
+ io_lib:format("Package not found in registry: ~s-~s", [Package, Version]);
format_error({cycles, Cycles}) ->
Prints = [["applications: ",
[io_lib:format("~s ", [Dep]) || Dep <- Cycle],
@@ -178,7 +180,7 @@ handle_deps(Profile, State, Deps, Upgrade, Locks) ->
%% Sort all apps to build order
State3 = rebar_state:all_deps(State2, AllDeps),
- {ok, SrcApps, State3}.
+ {ok, AllDeps, State3}.
%% ===================================================================
%% Internal functions
@@ -193,7 +195,7 @@ update_pkg_deps(Profile, Pkgs, Packages, Upgrade, Seen, State) ->
,Packages
,Pkg),
{SeenAcc1, StateAcc1} = maybe_lock(Profile, AppInfo, SeenAcc, StateAcc, 0),
- case maybe_fetch(AppInfo, Upgrade, SeenAcc1) of
+ case maybe_fetch(AppInfo, Upgrade, SeenAcc1, State) of
true ->
{[AppInfo | Acc], SeenAcc1, StateAcc1};
false ->
@@ -227,15 +229,14 @@ maybe_lock(Profile, AppInfo, Seen, State, Level) ->
package_to_app(DepsDir, Packages, {Name, Vsn}) ->
case dict:find({Name, Vsn}, Packages) of
error ->
- {error, missing_package};
+ throw(?PRV_ERROR({missing_package, Name, Vsn}));
{ok, P} ->
PkgDeps = [{PkgName, PkgVsn}
|| {PkgName,PkgVsn} <- proplists:get_value(<<"deps">>, P, [])],
- Link = proplists:get_value(<<"link">>, P, ""),
{ok, AppInfo} = rebar_app_info:new(Name, Vsn),
AppInfo1 = rebar_app_info:deps(AppInfo, PkgDeps),
AppInfo2 = rebar_app_info:dir(AppInfo1, rebar_dir:deps_dir(DepsDir, Name)),
- rebar_app_info:source(AppInfo2, {pkg, Name, Vsn, Link})
+ rebar_app_info:source(AppInfo2, {pkg, Name, Vsn})
end.
-spec update_src_deps(atom(), non_neg_integer(), list(), list(), list(), rebar_state:t(), boolean(), sets:set(binary()), list()) -> {rebar_state:t(), list(), list(), sets:set(binary())}.
@@ -282,7 +283,7 @@ update_src_deps(Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen,
,StateAcc1
,LocksAcc);
_ ->
- maybe_fetch(AppInfo, false, SeenAcc),
+ maybe_fetch(AppInfo, false, SeenAcc, StateAcc),
handle_dep(AppInfo
,SrcDepsAcc
,PkgDepsAcc
@@ -306,7 +307,7 @@ handle_upgrade(AppInfo, SrcDeps, PkgDeps, SrcApps, Level, State, Locks) ->
Name = rebar_app_info:name(AppInfo),
case lists:keyfind(Name, 1, Locks) of
false ->
- case maybe_fetch(AppInfo, true, sets:new()) of
+ case maybe_fetch(AppInfo, true, sets:new(), State) of
true ->
handle_dep(AppInfo
,SrcDeps
@@ -357,8 +358,8 @@ handle_dep(State, DepsDir, AppInfo, Locks, Level) ->
{AppInfo2, SrcDeps, PkgDeps, Locks++NewLocks}.
-spec maybe_fetch(rebar_app_info:t(), boolean() | {true, binary(), integer()},
- sets:set(binary())) -> boolean().
-maybe_fetch(AppInfo, Upgrade, Seen) ->
+ sets:set(binary()), rebar_state:t()) -> boolean().
+maybe_fetch(AppInfo, Upgrade, Seen, State) ->
AppDir = ec_cnv:to_list(rebar_app_info:dir(AppInfo)),
Apps = rebar_app_discover:find_apps(["_checkouts"], all),
case rebar_app_utils:find(rebar_app_info:name(AppInfo), Apps) of
@@ -368,13 +369,13 @@ maybe_fetch(AppInfo, Upgrade, Seen) ->
error ->
case not app_exists(AppDir) of
true ->
- fetch_app(AppInfo, AppDir);
+ fetch_app(AppInfo, AppDir, State);
false ->
case sets:is_element(rebar_app_info:name(AppInfo), Seen) of
true ->
false;
false ->
- maybe_upgrade(AppInfo, AppDir, Upgrade)
+ maybe_upgrade(AppInfo, AppDir, Upgrade, State)
end
end
end.
@@ -457,25 +458,25 @@ app_exists(AppDir) ->
end
end.
-fetch_app(AppInfo, AppDir) ->
+fetch_app(AppInfo, AppDir, State) ->
?INFO("Fetching ~s (~p)", [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)]),
Source = rebar_app_info:source(AppInfo),
- case rebar_fetch:download_source(AppDir, Source) of
+ case rebar_fetch:download_source(AppDir, Source, State) of
{error, Reason} ->
throw(Reason);
Result ->
Result
end.
-maybe_upgrade(AppInfo, AppDir, false) ->
+maybe_upgrade(AppInfo, AppDir, false, _State) ->
Source = rebar_app_info:source(AppInfo),
rebar_fetch:needs_update(AppDir, Source);
-maybe_upgrade(AppInfo, AppDir, true) ->
+maybe_upgrade(AppInfo, AppDir, true, State) ->
Source = rebar_app_info:source(AppInfo),
case rebar_fetch:needs_update(AppDir, Source) of
true ->
?INFO("Updating ~s", [rebar_app_info:name(AppInfo)]),
- case rebar_fetch:download_source(AppDir, Source) of
+ case rebar_fetch:download_source(AppDir, Source, State) of
{error, Reason} ->
throw(Reason);
Result ->
diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl
index 524a684..655e5a3 100644
--- a/src/rebar_prv_update.erl
+++ b/src/rebar_prv_update.erl
@@ -34,18 +34,22 @@ init(State) ->
do(State) ->
?INFO("Updating package index...", []),
try
- Url = url(State),
+ Url = rebar_state:get(State, rebar_packages_cdn, "https://s3.amazonaws.com/s3.hex.pm/registry.ets.gz"),
TmpDir = ec_file:insecure_mkdtemp(),
- TmpFile = filename:join(TmpDir, "packages"),
- Home = rebar_dir:home_dir(),
- PackagesFile = filename:join([Home, ?CONFIG_DIR, "packages"]),
- filelib:ensure_dir(PackagesFile),
- {ok, _RequestId} = httpc:request(get, {Url, [{"Accept", "application/erlang"}]},
- [], [{stream, TmpFile}, {sync, true}]),
- ok = ec_file:copy(TmpFile, PackagesFile)
+ TmpFile = filename:join(TmpDir, "packages.gz"),
+ HexFile = filename:join(TmpDir, "packages"),
+ {ok, _RequestId} = httpc:request(get, {Url, []},
+ [], [{stream, TmpFile}, {sync, true}]),
+ {ok, Data} = file:read_file(TmpFile),
+ Unzipped = zlib:gunzip(Data),
+ ok = file:write_file(HexFile, Unzipped),
+ {Dict, Graph} = hex_to_graph(HexFile),
+ write_registry(Dict, Graph, State),
+ ok
catch
- _:_ ->
- {error, {?MODULE, package_index_write}}
+ E:C ->
+ io:format("E C ~p ~p~n", [E, C]),
+ throw({error, {?MODULE, package_index_write}})
end,
{ok, State}.
@@ -54,9 +58,39 @@ do(State) ->
format_error(package_index_write) ->
"Failed to write package index.".
-url(State) ->
- ErtsVsn = erlang:system_info(version),
+write_registry(Dict, {digraph, Edges, Vertices, Neighbors, _}, State) ->
+ Dir = rebar_dir:global_config_dir(State),
+ RegistryDir = filename:join(Dir, "packages"),
+ filelib:ensure_dir(filename:join(RegistryDir, "dummy")),
+ ets:tab2file(Edges, filename:join(RegistryDir, "edges")),
+ ets:tab2file(Vertices, filename:join(RegistryDir, "vertices")),
+ ets:tab2file(Neighbors, filename:join(RegistryDir, "neighbors")),
+ file:write_file(filename:join(RegistryDir, "dict"), term_to_binary(Dict)).
- Qs = [io_lib:format("~s=~s", [X, Y]) || {X, Y} <- [{"erts", ErtsVsn}]],
- Url = rebar_state:get(State, rebar_packages_url, "http://packages.rebar3.org/packages"),
- Url ++ "?" ++ string:join(Qs, "&").
+hex_to_graph(Filename) ->
+ {ok, T} = ets:file2tab(Filename),
+ Graph = digraph:new(),
+ ets:foldl(fun({Pkg, [Versions]}, ok) when is_binary(Pkg), is_list(Versions) ->
+ lists:foreach(fun(Version) ->
+ digraph:add_vertex(Graph, {Pkg, Version}, 1)
+ end, Versions);
+ (_, ok) ->
+ ok
+ end, ok, T),
+
+ Dict1 = ets:foldl(fun({{Pkg, PkgVsn}, [Deps | _]}, Dict) ->
+ DepsList = lists:foldl(fun([Dep, DepVsn, _, _], DepsListAcc) ->
+ case DepVsn of
+ <<"~> ", Vsn/binary>> ->
+ digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, Vsn}),
+ [{Dep, DepVsn} | DepsListAcc];
+ Vsn ->
+ digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, Vsn}),
+ [{Dep, Vsn} | DepsListAcc]
+ end
+ end, [], Deps),
+ dict:store({Pkg, PkgVsn}, DepsList, Dict);
+ (_, Dict) ->
+ Dict
+ end, dict:new(), T),
+ {Dict1, Graph}.
diff --git a/src/rebar_resource.erl b/src/rebar_resource.erl
index 2bc28e2..7c58135 100644
--- a/src/rebar_resource.erl
+++ b/src/rebar_resource.erl
@@ -23,7 +23,7 @@
-spec behaviour_info(atom()) -> [{atom(), arity()}] | undefined.
behaviour_info(callbacks) ->
[{lock, 2},
- {download, 2},
+ {download, 3},
{needs_update,2},
{make_vsn, 1}];
behaviour_info(_) ->
@@ -33,7 +33,7 @@ behaviour_info(_) ->
-callback lock(file:filename_all(), tuple()) ->
rebar_resource:resource().
--callback download(file:filename_all(), tuple()) ->
+-callback download(file:filename_all(), tuple(), rebar_state:t()) ->
{tarball, file:filename_all()} | {ok, any()} | {error, any()}.
-callback needs_update(file:filename_all(), tuple()) ->
boolean().
diff --git a/test/mock_git_resource.erl b/test/mock_git_resource.erl
index d2f0207..6940f54 100644
--- a/test/mock_git_resource.erl
+++ b/test/mock_git_resource.erl
@@ -103,7 +103,7 @@ mock_download(Opts) ->
Overrides = proplists:get_value(override_vsn, Opts, []),
meck:expect(
?MOD, download,
- fun (Dir, Git) ->
+ fun (Dir, Git, _) ->
filelib:ensure_dir(Dir),
{git, Url, {_, Vsn}} = normalize_git(Git, Overrides, Default),
App = app(Url),
diff --git a/test/mock_pkg_resource.erl b/test/mock_pkg_resource.erl
index ab2c0d3..c7276b6 100644
--- a/test/mock_pkg_resource.erl
+++ b/test/mock_pkg_resource.erl
@@ -21,7 +21,7 @@ mock() -> mock([]).
| {not_in_index, [{App, Vsn}]}
| {pkgdeps, [{{App,Vsn}, [Dep]}]},
App :: string(),
- Dep :: {App, string(), {pkg, App, Vsn, Url::string()}},
+ Dep :: {App, string(), {pkg, App, Vsn}},
Vsn :: string().
mock(Opts) ->
meck:new(?MOD, [no_link]),
@@ -50,7 +50,7 @@ mock_update(Opts) ->
ToUpdate = proplists:get_value(update, Opts, []),
meck:expect(
?MOD, needs_update,
- fun(_Dir, {pkg, App, _Vsn, _Url}) ->
+ fun(_Dir, {pkg, App, _Vsn}) ->
lists:member(App, ToUpdate)
end).
@@ -65,7 +65,7 @@ mock_vsn(_Opts) ->
%% @doc For each app to download, create a dummy app on disk instead.
%% The configuration for this one (passed in from `mock/1') includes:
%%
-%% - Specify a version with `{pkg, _, Vsn, _}'
+%% - Specify a version with `{pkg, _, Vsn}'
%% - Dependencies for each application must be passed of the form:
%% `{pkgdeps, [{"app1", [{app2, ".*", {pkg, ...}}]}]}' -- basically
%% the `pkgdeps' option takes a key/value list of terms to output directly
@@ -74,7 +74,7 @@ mock_download(Opts) ->
Deps = proplists:get_value(pkgdeps, Opts, []),
meck:expect(
?MOD, download,
- fun (Dir, {pkg, AppBin, Vsn, _Url}) ->
+ fun (Dir, {pkg, AppBin, Vsn}, _) ->
App = binary_to_list(AppBin),
filelib:ensure_dir(Dir),
AppDeps = proplists:get_value({App,Vsn}, Deps, []),
@@ -83,11 +83,15 @@ mock_download(Opts) ->
[element(1,D) || D <- AppDeps]
),
rebar_test_utils:create_config(Dir, [{deps, AppDeps}]),
- Tarball = filename:join([Dir, "package.tar.gz"]),
+ Tarball = filename:join([Dir, App++"-"++binary_to_list(Vsn)++".tar"]),
+ Contents = filename:join([Dir, "contents.tar.gz"]),
Files = all_files(rebar_app_info:dir(AppInfo)),
- ok = erl_tar:create(Tarball,
+ ok = erl_tar:create(Contents,
archive_names(Dir, App, Vsn, Files),
[compressed]),
+ ok = erl_tar:create(Tarball,
+ [{"contents.tar.gz", Contents}],
+ []),
[file:delete(F) || F <- Files],
{tarball, Tarball}
end).
@@ -117,8 +121,7 @@ all_files(Dir) ->
filelib:wildcard(filename:join([Dir, "**"])).
archive_names(Dir, App, Vsn, Files) ->
- ArchName = App++"-"++binary_to_list(Vsn),
- [{ArchName ++ (F -- Dir), F} || F <- Files].
+ [{(F -- Dir) -- "/", F} || F <- Files].
find_parts(Apps, Skip) -> find_parts(Apps, Skip, dict:new()).
diff --git a/test/rebar_deps_SUITE.erl b/test/rebar_deps_SUITE.erl
index 4aaa7e3..633e9a2 100644
--- a/test/rebar_deps_SUITE.erl
+++ b/test/rebar_deps_SUITE.erl
@@ -1,14 +1,16 @@
-%% TODO: add tests for:
-%% - only part of deps fetched
-%% - only part of deps locked
-%% - output only shown once
-%% - modification asterisk on locked file
-module(rebar_deps_SUITE).
-compile(export_all).
-include_lib("common_test/include/ct.hrl").
-include_lib("eunit/include/eunit.hrl").
-all() -> [default_nodep, default_lock].
+all() -> [{group, git}, {group, pkg}].
+
+groups() ->
+ [{all, [], [flat, pick_highest_left, pick_highest_right,
+ pick_smallest1, pick_smallest2,
+ circular1, circular2, circular_skip]},
+ {git, [], [{group, all}]},
+ {pkg, [], [{group, all}]}].
init_per_suite(Config) ->
application:start(meck),
@@ -17,79 +19,202 @@ init_per_suite(Config) ->
end_per_suite(_Config) ->
application:stop(meck).
+init_per_group(git, Config) ->
+ [{deps_type, git} | Config];
+init_per_group(pkg, Config) ->
+ [{deps_type, pkg} | Config];
+init_per_group(_, Config) ->
+ Config.
+
+end_per_group(_, Config) ->
+ Config.
+
init_per_testcase(Case, Config) ->
- meck:new(io, [no_link, passthrough, unstick]),
- setup_project(Case, Config).
+ {Deps, Warnings, Expect} = deps(Case),
+ Expected = case Expect of
+ {ok, List} -> {ok, format_expected_deps(List)};
+ {error, Reason} -> {error, Reason}
+ end,
+ DepsType = ?config(deps_type, Config),
+ mock_warnings(),
+ [{expect, Expected},
+ {warnings, Warnings}
+ | setup_project(Case, Config, expand_deps(DepsType, Deps))].
end_per_testcase(_, Config) ->
meck:unload(),
Config.
-config_content() ->
- [{deps, [
- {src_a, ".*", {git, "https://example.org/ferd/src_a.git", {branch, "master"}}},
- {src_b, {git, "https://example.org/ferd/src_b.git", {branch, "master"}}},
- {pkg_a, "1.0.0"}
- ]},
- {profiles,
- [{test,
- [{deps, [
- {tdep, {git, "git://example.org/ferd/tdep.git", {tag, "0.8.2"}}}
- ]}]
- }]}
- ].
-
-setup_project(Case, Config0) ->
+format_expected_deps(Deps) ->
+ [case Dep of
+ {N,V} -> {dep, N, V};
+ N -> {dep, N}
+ end || Dep <- Deps].
+
+%% format:
+%% {Spec,
+%% [Warning],
+%% {ok, Result} | {error, Reason}}
+%%
+%% Spec is a list of levelled dependencies of two possible forms:
+%% - {"Name", Spec}
+%% - {"Name", "Vsn", Spec}
+%%
+%% Warnings are going to match on mocked ?WARN(...)
+%% calls to be evaluated. An empty list means we do not care about
+%% warnings, not that no warnings will be printed. This means
+%% the list of warning isn't interpreted to be exhaustive, and more
+%% warnings may be generated than are listed.
+deps(flat) ->
+ {[{"B", []},
+ {"C", []}],
+ [],
+ {ok, ["B", "C"]}};
+deps(pick_highest_left) ->
+ {[{"B", [{"C", "2", []}]},
+ {"C", "1", []}],
+ [{"C","2"}],
+ {ok, ["B", {"C", "1"}]}};
+deps(pick_highest_right) ->
+ {[{"B", "1", []},
+ {"C", [{"B", "2", []}]}],
+ [{"B","2"}],
+ {ok, [{"B","1"}, "C"]}};
+deps(pick_smallest1) ->
+ {[{"B", [{"D", "1", []}]},
+ {"C", [{"D", "2", []}]}],
+ [{"D","2"}],
+ %% we pick D1 because B < C
+ {ok, ["B","C",{"D","1"}]}};
+deps(pick_smallest2) ->
+ {[{"C", [{"D", "2", []}]},
+ {"B", [{"D", "1", []}]}],
+ [{"D","2"}],
+ %% we pick D1 because B < C
+ {ok, ["B","C",{"D","1"}]}};
+deps(circular1) ->
+ {[{"B", [{"A", []}]}, % A is the top-level app
+ {"C", []}],
+ [],
+ {error, {rebar_prv_install_deps, {cycles, [[<<"A">>,<<"B">>]]}}}};
+deps(circular2) ->
+ {[{"B", [{"C", [{"B", []}]}]},
+ {"C", []}],
+ [],
+ {error, {rebar_prv_install_deps, {cycles, [[<<"B">>,<<"C">>]]}}}};
+deps(circular_skip) ->
+ %% Never spot the circular dep due to being to low in the deps tree
+ %% in source deps
+ {[{"B", [{"C", "2", [{"B", []}]}]},
+ {"C", "1", [{"D",[]}]}],
+ [{"C","2"}],
+ {ok, ["B", {"C","1"}, "D"]}}.
+
+expand_deps(_, []) -> [];
+expand_deps(git, [{Name, Deps} | Rest]) ->
+ Dep = {Name, ".*", {git, "https://example.org/user/"++Name++".git", "master"}},
+ [{Dep, expand_deps(git, Deps)} | expand_deps(git, Rest)];
+expand_deps(git, [{Name, Vsn, Deps} | Rest]) ->
+ Dep = {Name, Vsn, {git, "https://example.org/user/"++Name++".git", {tag, Vsn}}},
+ [{Dep, expand_deps(git, Deps)} | expand_deps(git, Rest)];
+expand_deps(pkg, [{Name, Deps} | Rest]) ->
+ Dep = {pkg, Name, "0.0.0"},
+ [{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)];
+expand_deps(pkg, [{Name, Vsn, Deps} | Rest]) ->
+ Dep = {pkg, Name, Vsn},
+ [{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)].
+
+setup_project(Case, Config0, Deps) ->
+ DepsType = ?config(deps_type, Config0),
Config = rebar_test_utils:init_rebar_state(
Config0,
- atom_to_list(Case)++"_"
+ atom_to_list(Case)++"_"++atom_to_list(DepsType)++"_"
),
AppDir = ?config(apps, Config),
rebar_test_utils:create_app(AppDir, "A", "0.0.0", [kernel, stdlib]),
- TopDeps = proplists:get_value(deps, config_content()),
- StringDeps = [erlang:setelement(1, Dep, atom_to_list(element(1,Dep)))
- || Dep <- TopDeps],
+ TopDeps = top_level_deps(Deps),
RebarConf = rebar_test_utils:create_config(AppDir, [{deps, TopDeps}]),
- mock_git_resource:mock([{deps, lists:filter(fun src_dep/1, StringDeps)}]),
- mock_pkg_resource:mock([{pkgdeps,
- [{{ec_cnv:to_binary(N),
- ec_cnv:to_binary(V)},[]}
- || {N,V} <- lists:filter(fun pkg_dep/1, StringDeps)]}]),
+ case DepsType of
+ git ->
+ mock_git_resource:mock([{deps, flat_deps(Deps)}]);
+ pkg ->
+ mock_pkg_resource:mock([{pkgdeps, flat_pkgdeps(Deps)}])
+ end,
[{rebarconfig, RebarConf} | Config].
-src_dep(Dep) ->
- case element(1, Dep) of
- "src_"++_ -> true;
- _ -> false
- end.
-pkg_dep(Dep) ->
- case element(1, Dep) of
- "pkg_"++_ -> true;
- _ -> false
- end.
+flat_deps([]) -> [];
+flat_deps([{{Name,_Vsn,Ref}, Deps} | Rest]) ->
+ [{{Name,vsn_from_ref(Ref)}, top_level_deps(Deps)}]
+ ++
+ flat_deps(Deps)
+ ++
+ flat_deps(Rest).
-default_nodep(Config) ->
- {ok, RebarConfig} = file:consult(?config(rebarconfig, Config)),
- rebar_test_utils:run_and_check(
- Config, RebarConfig, ["deps"], {ok, []}
- ),
- History = meck:history(io),
- Strings = [io_lib:format(Str, Args) || {_, {io, format, [Str, Args]}, _} <- History],
- {match, _} = re:run(Strings, "src_a\\* \\(git source\\)"),
- {match, _} = re:run(Strings, "src_b\\* \\(git source\\)"),
- {match, _} = re:run(Strings, "pkg_a\\* \\(package 1.0.0\\)").
+vsn_from_ref({git, _, {_, Vsn}}) -> Vsn;
+vsn_from_ref({git, _, Vsn}) -> Vsn.
-default_lock(Config) ->
+flat_pkgdeps([]) -> [];
+flat_pkgdeps([{{pkg, Name, Vsn}, Deps} | Rest]) ->
+ [{{iolist_to_binary(Name),iolist_to_binary(Vsn)}, top_level_deps(Deps)}]
+ ++
+ flat_pkgdeps(Deps)
+ ++
+ flat_pkgdeps(Rest).
+
+top_level_deps([]) -> [];
+top_level_deps([{{pkg, Name, Vsn}, _} | Deps]) ->
+ [{list_to_atom(Name), Vsn} | top_level_deps(Deps)];
+top_level_deps([{{Name, Vsn, Ref}, _} | Deps]) ->
+ [{list_to_atom(Name), Vsn, Ref} | top_level_deps(Deps)].
+
+app_vsn([]) -> [];
+app_vsn([{Source, Deps} | Rest]) ->
+ {Name, Vsn} = case Source of
+ {pkg, N, V} -> {N,V};
+ {N,V,_Ref} -> {N,V}
+ end,
+ [{Name, Vsn}] ++ app_vsn(Deps) ++ app_vsn(Rest).
+
+mock_warnings() ->
+ %% just let it do its thing, we check warnings through
+ %% the call log.
+ meck:new(rebar_log, [no_link, passthrough]).
+
+%%% TESTS %%%
+flat(Config) -> run(Config).
+pick_highest_left(Config) -> run(Config).
+pick_highest_right(Config) -> run(Config).
+pick_smallest1(Config) -> run(Config).
+pick_smallest2(Config) -> run(Config).
+circular1(Config) -> run(Config).
+circular2(Config) -> run(Config).
+circular_skip(Config) -> run(Config).
+
+run(Config) ->
{ok, RebarConfig} = file:consult(?config(rebarconfig, Config)),
rebar_test_utils:run_and_check(
- Config, RebarConfig, ["lock"], {ok, []}
+ Config, RebarConfig, ["install_deps"], ?config(expect, Config)
),
- rebar_test_utils:run_and_check(
- Config, RebarConfig, ["deps"], {ok, []}
- ),
- History = meck:history(io),
- Strings = [io_lib:format(Str, Args) || {_, {io, format, [Str, Args]}, _} <- History],
- {match, _} = re:run(Strings, "src_a\\ \\(locked git source\\)"),
- {match, _} = re:run(Strings, "src_b\\ \\(locked git source\\)"),
- {match, _} = re:run(Strings, "pkg_a\\ \\(locked package 1.0.0\\)").
+ check_warnings(warning_calls(), ?config(warnings, Config), ?config(deps_type, Config)).
+
+warning_calls() ->
+ History = meck:history(rebar_log),
+ [{Str, Args} || {_, {rebar_log, log, [warn, Str, Args]}, _} <- History].
+
+check_warnings(_, [], _) ->
+ ok;
+check_warnings(Warns, [{Name, Vsn} | Rest], Type) ->
+ ct:pal("Checking for warning ~p in ~p", [{Name,Vsn},Warns]),
+ ?assert(in_warnings(Type, Warns, Name, Vsn)),
+ check_warnings(Warns, Rest, Type).
+
+in_warnings(git, Warns, NameRaw, VsnRaw) ->
+ Name = iolist_to_binary(NameRaw),
+ 1 =< length([1 || {_, [AppName, {git, _, {_, Vsn}}]} <- Warns,
+ AppName =:= Name, Vsn =:= VsnRaw]);
+in_warnings(pkg, Warns, NameRaw, VsnRaw) ->
+ Name = iolist_to_binary(NameRaw),
+ Vsn = iolist_to_binary(VsnRaw),
+ 1 =< length([1 || {_, [AppName, AppVsn]} <- Warns,
+ AppName =:= Name, AppVsn =:= Vsn]).
diff --git a/test/rebar_upgrade_SUITE.erl b/test/rebar_upgrade_SUITE.erl
index 39b9687..b4daef9 100644
--- a/test/rebar_upgrade_SUITE.erl
+++ b/test/rebar_upgrade_SUITE.erl
@@ -359,7 +359,7 @@ upgrades(delete_d) ->
top_level_deps([]) -> [];
top_level_deps([{{Name, Vsn, Ref}, _} | Deps]) ->
[{list_to_atom(Name), Vsn, Ref} | top_level_deps(Deps)];
-top_level_deps([{{pkg, Name, Vsn, _URL}, _} | Deps]) ->
+top_level_deps([{{pkg, Name, Vsn}, _} | Deps]) ->
[{list_to_atom(Name), Vsn} | top_level_deps(Deps)].
mock_deps(git, Deps, Upgrades) ->
@@ -381,7 +381,7 @@ vsn_from_ref({git, _, {_, Vsn}}) -> Vsn;
vsn_from_ref({git, _, Vsn}) -> Vsn.
flat_pkgdeps([]) -> [];
-flat_pkgdeps([{{pkg, Name, Vsn, _Url}, Deps} | Rest]) ->
+flat_pkgdeps([{{pkg, Name, Vsn}, Deps} | Rest]) ->
[{{iolist_to_binary(Name),iolist_to_binary(Vsn)}, top_level_deps(Deps)}]
++
flat_pkgdeps(Deps)
@@ -396,10 +396,10 @@ expand_deps(git, [{Name, Vsn, Deps} | Rest]) ->
Dep = {Name, Vsn, {git, "https://example.org/user/"++Name++".git", {tag, Vsn}}},
[{Dep, expand_deps(git, Deps)} | expand_deps(git, Rest)];
expand_deps(pkg, [{Name, Deps} | Rest]) ->
- Dep = {pkg, Name, "0.0.0", "https://example.org/user/"++Name++".tar.gz"},
+ Dep = {pkg, Name, "0.0.0"},
[{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)];
expand_deps(pkg, [{Name, Vsn, Deps} | Rest]) ->
- Dep = {pkg, Name, Vsn, "https://example.org/user/"++Name++".tar.gz"},
+ Dep = {pkg, Name, Vsn},
[{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)].
normalize_unlocks({App, Locks}) ->
@@ -472,4 +472,3 @@ run(Config) ->
rebar_test_utils:run_and_check(
Config, NewRebarConfig, ["upgrade", App], Expectation
).
-
generated by cgit v1.1 at 2024-12-24 18:42:11 +0000