From 410e2bcdecbf10037e6430f64707ef244a738e45 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 12 Feb 2015 07:21:16 -0600 Subject: replace package management with hex.pm --- Makefile | 2 +- src/rebar_app_info.erl | 6 +- src/rebar_erlc_compiler.erl | 10 +- src/rebar_fetch.erl | 27 +++-- src/rebar_git_resource.erl | 18 +-- src/rebar_packages.erl | 18 ++- src/rebar_pkg_resource.erl | 8 +- src/rebar_prv_install_deps.erl | 47 ++++---- src/rebar_prv_update.erl | 64 ++++++++--- src/rebar_resource.erl | 4 +- test/mock_git_resource.erl | 2 +- test/mock_pkg_resource.erl | 19 ++-- test/rebar_deps_SUITE.erl | 249 +++++++++++++++++++++++++++++++---------- test/rebar_upgrade_SUITE.erl | 9 +- 14 files changed, 326 insertions(+), 157 deletions(-) diff --git a/Makefile b/Makefile index 2e02370..f36c177 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ RETEST=$(PWD)/deps/retest/retest DEPS_PLT=$(CURDIR)/.depsolver_plt all: - ./bootstrap/bootstrap + @rebar get-deps compile escriptize clean: @rm -rf rebar3 ebin/*.beam inttest/rt.work rt.work .eunit diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index dca5aa8..9d779be 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -79,7 +79,7 @@ new(AppName, Vsn) -> new(AppName, Vsn, Dir) -> {ok, #app_info_t{name=ec_cnv:to_binary(AppName), original_vsn=Vsn, - dir=Dir}}. + dir=ec_cnv:to_list(Dir)}}. %% @doc build a complete version of the app info with all fields set. -spec new(atom() | binary() | string(), binary() | string(), file:name(), list()) -> @@ -87,7 +87,7 @@ new(AppName, Vsn, Dir) -> new(AppName, Vsn, Dir, Deps) -> {ok, #app_info_t{name=ec_cnv:to_binary(AppName), original_vsn=Vsn, - dir=Dir, + dir=ec_cnv:to_list(Dir), deps=Deps}}. %% @doc discover a complete version of the app info with all fields set. @@ -200,7 +200,7 @@ dir(#app_info_t{dir=Dir}) -> -spec dir(t(), file:name()) -> t(). dir(AppInfo=#app_info_t{}, Dir) -> - AppInfo#app_info_t{dir=Dir}. + AppInfo#app_info_t{dir=ec_cnv:to_list(Dir)}. -spec ebin_dir(t()) -> file:name(). ebin_dir(#app_info_t{dir=Dir}) -> diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index b096705..cb342a8 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -144,17 +144,13 @@ doterl_compile(Config, Dir, MoreSources, ErlOpts) -> filename:join(Dir, X) end, rebar_dir:src_dirs(proplists:append_values(src_dirs, ErlOpts))), AllErlFiles = gather_src(SrcDirs, []) ++ MoreSources, - %% NOTE: If and when erl_first_files is not inherited anymore - %% (rebar_state:get instead of rebar_state:get_list), consider - %% logging a warning message for any file listed in erl_first_files which - %% wasn't found via gather_src. - + %% Issue: rebar/rebar3#140 (fix matching based on same path + order of %% erl_first_files) ErlFirstFiles = get_erl_first_files(ErlFirstFilesConf, AllErlFiles), RestErls = [ File || File <- AllErlFiles, not lists:member(File, ErlFirstFiles) ], - + %% Make sure that ebin/ exists and is on the path ok = filelib:ensure_dir(filename:join([Dir, "ebin", "dummy.beam"])), CurrPath = code:get_path(), @@ -634,5 +630,3 @@ log_files(Prefix, Files) -> _ -> ?DEBUG("~s:~n~p", [Prefix, Files]) end. - - diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index 088a49a..4dfd790 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -8,12 +8,13 @@ -module(rebar_fetch). -export([lock_source/2, - download_source/2, + download_source/3, needs_update/2]). -export([format_error/1]). -include("rebar.hrl"). +-include_lib("providers/include/providers.hrl"). %% map short versions of resources to module names -define(RESOURCES, [{git, rebar_git_resource}, {pkg, rebar_pkg_resource}]). @@ -24,13 +25,14 @@ lock_source(AppDir, Source) -> Module = get_resource_type(Source), Module:lock(AppDir, Source). --spec download_source(file:filename_all(), rebar_resource:resource()) -> true | {error, any()}. -download_source(AppDir, Source) -> +-spec download_source(file:filename_all(), rebar_resource:resource(), rebar_state:t()) -> + true | {error, any()}. +download_source(AppDir, Source, State) -> try Module = get_resource_type(Source), TmpDir = ec_file:insecure_mkdtemp(), AppDir1 = ec_cnv:to_list(AppDir), - case Module:download(TmpDir, Source) of + case Module:download(TmpDir, Source, State) of {ok, _} -> ec_file:mkdir_p(AppDir1), code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), @@ -38,20 +40,21 @@ download_source(AppDir, Source) -> ok = ec_file:copy(TmpDir, filename:absname(AppDir1), [recursive]), true; {tarball, File} -> + Contents = filename:join(TmpDir, "contents"), ec_file:mkdir_p(AppDir1), - ok = erl_tar:extract(File, [{cwd, TmpDir} - ,compressed]), - BaseName = filename:basename(AppDir1), - [FromDir] = filelib:wildcard(filename:join(TmpDir, BaseName++"-*")), + ec_file:mkdir_p(Contents), + ok = erl_tar:extract(File, [{cwd, TmpDir}]), + ok = erl_tar:extract(filename:join(TmpDir, "contents.tar.gz"), + [{cwd, Contents}, compressed]), code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), ec_file:remove(filename:absname(AppDir1), [recursive]), - ok = ec_file:copy(FromDir, filename:absname(AppDir1), [recursive]), - true = code:add_patha(filename:join(AppDir1, "ebin")), + ok = ec_file:copy(Contents, filename:absname(AppDir1), [recursive]), true end catch - _:_ -> - {error, {rebar_fetch, {fetch_fail, Source}}} + C:T -> + ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, erlang:get_stacktrace()]), + throw(?PRV_ERROR({fetch_fail, Source})) end. -spec needs_update(file:filename_all(), rebar_resource:resource()) -> boolean() | {error, string()}. diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl index bd0a978..785c362 100644 --- a/src/rebar_git_resource.erl +++ b/src/rebar_git_resource.erl @@ -5,7 +5,7 @@ -behaviour(rebar_resource). -export([lock/2 - ,download/2 + ,download/3 ,needs_update/2 ,make_vsn/1]). @@ -75,28 +75,28 @@ parse_git_url("https://" ++ HostPath) -> [Host | Path] = string:tokens(HostPath, "/"), {Host, filename:rootname(filename:join(Path), ".git")}. -download(Dir, {git, Url}) -> +download(Dir, {git, Url}, State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), - download(Dir, {git, Url, {branch, "master"}}); -download(Dir, {git, Url, ""}) -> + download(Dir, {git, Url, {branch, "master"}}, State); +download(Dir, {git, Url, ""}, State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), - download(Dir, {git, Url, {branch, "master"}}); -download(Dir, {git, Url, {branch, Branch}}) -> + download(Dir, {git, Url, {branch, "master"}}, State); +download(Dir, {git, Url, {branch, Branch}}, _State) -> ok = filelib:ensure_dir(Dir), rebar_utils:sh(?FMT("git clone ~s ~s -b ~s --single-branch", [Url, filename:basename(Dir), Branch]), [{cd, filename:dirname(Dir)}]); -download(Dir, {git, Url, {tag, Tag}}) -> +download(Dir, {git, Url, {tag, Tag}}, _State) -> ok = filelib:ensure_dir(Dir), rebar_utils:sh(?FMT("git clone ~s ~s -b ~s --single-branch", [Url, filename:basename(Dir), Tag]), [{cd, filename:dirname(Dir)}]); -download(Dir, {git, Url, {ref, Ref}}) -> +download(Dir, {git, Url, {ref, Ref}}, _State) -> ok = filelib:ensure_dir(Dir), rebar_utils:sh(?FMT("git clone -n ~s ~s", [Url, filename:basename(Dir)]), [{cd, filename:dirname(Dir)}]), rebar_utils:sh(?FMT("git checkout -q ~s", [Ref]), [{cd, Dir}]); -download(Dir, {git, Url, Rev}) -> +download(Dir, {git, Url, Rev}, _State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), ok = filelib:ensure_dir(Dir), rebar_utils:sh(?FMT("git clone -n ~s ~s", [Url, filename:basename(Dir)]), diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 30d11c9..5400544 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -13,13 +13,21 @@ -spec get_packages(rebar_state:t()) -> {rebar_dict(), rebar_digraph()}. get_packages(State) -> RebarDir = rebar_dir:global_config_dir(State), - PackagesFile = filename:join(RebarDir, "packages"), - case ec_file:exists(PackagesFile) of + RegistryDir = filename:join(RebarDir, "packages"), + DictFile = filename:join(RegistryDir, "dict"), + Edges = filename:join(RegistryDir, "edges"), + Vertices = filename:join(RegistryDir, "vertices"), + Neighbors = filename:join(RegistryDir, "neighbors"), + + case lists:all(fun(X) -> filelib:is_file(X) end, [DictFile, Edges, Vertices, Neighbors]) of true -> try - {ok, Binary} = file:read_file(PackagesFile), - {Dict, Graph} = binary_to_term(Binary), - {Dict, rebar_digraph:restore_graph(Graph)} + {ok, DictBinary} = file:read_file(DictFile), + Dict = binary_to_term(DictBinary), + {ok, EdgesTab} = ets:file2tab(Edges), + {ok, VerticesTab} = ets:file2tab(Vertices), + {ok, NeighborsTab} = ets:file2tab(Neighbors), + {Dict, {digraph, EdgesTab, VerticesTab, NeighborsTab, true}} catch _:_ -> ?ERROR("Bad packages index, try to fix with `rebar3 update`", []), diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 5cd6fc8..0508e2c 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -5,7 +5,7 @@ -behaviour(rebar_resource). -export([lock/2 - ,download/2 + ,download/3 ,needs_update/2 ,make_vsn/1]). @@ -23,9 +23,11 @@ needs_update(Dir, {pkg, _Name, Vsn, _Url}) -> true end. -download(Dir, {pkg, _Name, _Vsn, Url}) -> +download(Dir, {pkg, Name, Vsn}, State) -> TmpFile = filename:join(Dir, "package.tar.gz"), - {ok, saved_to_file} = httpc:request(get, {binary_to_list(Url), []}, [], [{stream, TmpFile}]), + CDN = rebar_state:get(State, rebar_packages_cdn, "https://s3.amazonaws.com/s3.hex.pm/tarballs"), + Url = string:join([CDN, binary_to_list(<>)], "/"), + {ok, saved_to_file} = httpc:request(get, {Url, []}, [], [{stream, TmpFile}]), {tarball, TmpFile}. make_vsn(_) -> diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index dab4c24..61fbd70 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -75,19 +75,19 @@ do(State) -> Profiles = rebar_state:current_profiles(State), ProjectApps = rebar_state:project_apps(State), - {SrcApps, State1} = - lists:foldl(fun(Profile, {SrcAppsAcc, StateAcc}) -> + {Apps, State1} = + lists:foldl(fun(Profile, {AppsAcc, StateAcc}) -> Locks = rebar_state:get(StateAcc, {locks, Profile}, []), - {ok, NewSrcApps, NewState} = + {ok, NewApps, NewState} = handle_deps(Profile ,StateAcc ,rebar_state:get(StateAcc, {deps, Profile}, []) ,Locks), - {NewSrcApps++SrcAppsAcc, NewState} + {NewApps++AppsAcc, NewState} end, {[], State}, lists:reverse(Profiles)), - Source = ProjectApps ++ SrcApps, - case find_cycles(Source ++ rebar_state:all_deps(State1)) of + Source = ProjectApps ++ Apps, + case find_cycles(Source) of {cycles, Cycles} -> ?PRV_ERROR({cycles, Cycles}); {error, Error} -> @@ -117,7 +117,9 @@ find_cycles(Apps) -> -spec format_error(any()) -> iolist(). format_error({parse_dep, Dep}) -> - io_lib:format("Failed parsing dep ~p~n", [Dep]); + io_lib:format("Failed parsing dep ~p", [Dep]); +format_error({missing_package, Package, Version}) -> + io_lib:format("Package not found in registry: ~s-~s", [Package, Version]); format_error({cycles, Cycles}) -> Prints = [["applications: ", [io_lib:format("~s ", [Dep]) || Dep <- Cycle], @@ -178,7 +180,7 @@ handle_deps(Profile, State, Deps, Upgrade, Locks) -> %% Sort all apps to build order State3 = rebar_state:all_deps(State2, AllDeps), - {ok, SrcApps, State3}. + {ok, AllDeps, State3}. %% =================================================================== %% Internal functions @@ -193,7 +195,7 @@ update_pkg_deps(Profile, Pkgs, Packages, Upgrade, Seen, State) -> ,Packages ,Pkg), {SeenAcc1, StateAcc1} = maybe_lock(Profile, AppInfo, SeenAcc, StateAcc, 0), - case maybe_fetch(AppInfo, Upgrade, SeenAcc1) of + case maybe_fetch(AppInfo, Upgrade, SeenAcc1, State) of true -> {[AppInfo | Acc], SeenAcc1, StateAcc1}; false -> @@ -227,15 +229,14 @@ maybe_lock(Profile, AppInfo, Seen, State, Level) -> package_to_app(DepsDir, Packages, {Name, Vsn}) -> case dict:find({Name, Vsn}, Packages) of error -> - {error, missing_package}; + throw(?PRV_ERROR({missing_package, Name, Vsn})); {ok, P} -> PkgDeps = [{PkgName, PkgVsn} || {PkgName,PkgVsn} <- proplists:get_value(<<"deps">>, P, [])], - Link = proplists:get_value(<<"link">>, P, ""), {ok, AppInfo} = rebar_app_info:new(Name, Vsn), AppInfo1 = rebar_app_info:deps(AppInfo, PkgDeps), AppInfo2 = rebar_app_info:dir(AppInfo1, rebar_dir:deps_dir(DepsDir, Name)), - rebar_app_info:source(AppInfo2, {pkg, Name, Vsn, Link}) + rebar_app_info:source(AppInfo2, {pkg, Name, Vsn}) end. -spec update_src_deps(atom(), non_neg_integer(), list(), list(), list(), rebar_state:t(), boolean(), sets:set(binary()), list()) -> {rebar_state:t(), list(), list(), sets:set(binary())}. @@ -282,7 +283,7 @@ update_src_deps(Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, ,StateAcc1 ,LocksAcc); _ -> - maybe_fetch(AppInfo, false, SeenAcc), + maybe_fetch(AppInfo, false, SeenAcc, StateAcc), handle_dep(AppInfo ,SrcDepsAcc ,PkgDepsAcc @@ -306,7 +307,7 @@ handle_upgrade(AppInfo, SrcDeps, PkgDeps, SrcApps, Level, State, Locks) -> Name = rebar_app_info:name(AppInfo), case lists:keyfind(Name, 1, Locks) of false -> - case maybe_fetch(AppInfo, true, sets:new()) of + case maybe_fetch(AppInfo, true, sets:new(), State) of true -> handle_dep(AppInfo ,SrcDeps @@ -357,8 +358,8 @@ handle_dep(State, DepsDir, AppInfo, Locks, Level) -> {AppInfo2, SrcDeps, PkgDeps, Locks++NewLocks}. -spec maybe_fetch(rebar_app_info:t(), boolean() | {true, binary(), integer()}, - sets:set(binary())) -> boolean(). -maybe_fetch(AppInfo, Upgrade, Seen) -> + sets:set(binary()), rebar_state:t()) -> boolean(). +maybe_fetch(AppInfo, Upgrade, Seen, State) -> AppDir = ec_cnv:to_list(rebar_app_info:dir(AppInfo)), Apps = rebar_app_discover:find_apps(["_checkouts"], all), case rebar_app_utils:find(rebar_app_info:name(AppInfo), Apps) of @@ -368,13 +369,13 @@ maybe_fetch(AppInfo, Upgrade, Seen) -> error -> case not app_exists(AppDir) of true -> - fetch_app(AppInfo, AppDir); + fetch_app(AppInfo, AppDir, State); false -> case sets:is_element(rebar_app_info:name(AppInfo), Seen) of true -> false; false -> - maybe_upgrade(AppInfo, AppDir, Upgrade) + maybe_upgrade(AppInfo, AppDir, Upgrade, State) end end end. @@ -457,25 +458,25 @@ app_exists(AppDir) -> end end. -fetch_app(AppInfo, AppDir) -> +fetch_app(AppInfo, AppDir, State) -> ?INFO("Fetching ~s (~p)", [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)]), Source = rebar_app_info:source(AppInfo), - case rebar_fetch:download_source(AppDir, Source) of + case rebar_fetch:download_source(AppDir, Source, State) of {error, Reason} -> throw(Reason); Result -> Result end. -maybe_upgrade(AppInfo, AppDir, false) -> +maybe_upgrade(AppInfo, AppDir, false, _State) -> Source = rebar_app_info:source(AppInfo), rebar_fetch:needs_update(AppDir, Source); -maybe_upgrade(AppInfo, AppDir, true) -> +maybe_upgrade(AppInfo, AppDir, true, State) -> Source = rebar_app_info:source(AppInfo), case rebar_fetch:needs_update(AppDir, Source) of true -> ?INFO("Updating ~s", [rebar_app_info:name(AppInfo)]), - case rebar_fetch:download_source(AppDir, Source) of + case rebar_fetch:download_source(AppDir, Source, State) of {error, Reason} -> throw(Reason); Result -> diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index 524a684..655e5a3 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -34,18 +34,22 @@ init(State) -> do(State) -> ?INFO("Updating package index...", []), try - Url = url(State), + Url = rebar_state:get(State, rebar_packages_cdn, "https://s3.amazonaws.com/s3.hex.pm/registry.ets.gz"), TmpDir = ec_file:insecure_mkdtemp(), - TmpFile = filename:join(TmpDir, "packages"), - Home = rebar_dir:home_dir(), - PackagesFile = filename:join([Home, ?CONFIG_DIR, "packages"]), - filelib:ensure_dir(PackagesFile), - {ok, _RequestId} = httpc:request(get, {Url, [{"Accept", "application/erlang"}]}, - [], [{stream, TmpFile}, {sync, true}]), - ok = ec_file:copy(TmpFile, PackagesFile) + TmpFile = filename:join(TmpDir, "packages.gz"), + HexFile = filename:join(TmpDir, "packages"), + {ok, _RequestId} = httpc:request(get, {Url, []}, + [], [{stream, TmpFile}, {sync, true}]), + {ok, Data} = file:read_file(TmpFile), + Unzipped = zlib:gunzip(Data), + ok = file:write_file(HexFile, Unzipped), + {Dict, Graph} = hex_to_graph(HexFile), + write_registry(Dict, Graph, State), + ok catch - _:_ -> - {error, {?MODULE, package_index_write}} + E:C -> + io:format("E C ~p ~p~n", [E, C]), + throw({error, {?MODULE, package_index_write}}) end, {ok, State}. @@ -54,9 +58,39 @@ do(State) -> format_error(package_index_write) -> "Failed to write package index.". -url(State) -> - ErtsVsn = erlang:system_info(version), +write_registry(Dict, {digraph, Edges, Vertices, Neighbors, _}, State) -> + Dir = rebar_dir:global_config_dir(State), + RegistryDir = filename:join(Dir, "packages"), + filelib:ensure_dir(filename:join(RegistryDir, "dummy")), + ets:tab2file(Edges, filename:join(RegistryDir, "edges")), + ets:tab2file(Vertices, filename:join(RegistryDir, "vertices")), + ets:tab2file(Neighbors, filename:join(RegistryDir, "neighbors")), + file:write_file(filename:join(RegistryDir, "dict"), term_to_binary(Dict)). - Qs = [io_lib:format("~s=~s", [X, Y]) || {X, Y} <- [{"erts", ErtsVsn}]], - Url = rebar_state:get(State, rebar_packages_url, "http://packages.rebar3.org/packages"), - Url ++ "?" ++ string:join(Qs, "&"). +hex_to_graph(Filename) -> + {ok, T} = ets:file2tab(Filename), + Graph = digraph:new(), + ets:foldl(fun({Pkg, [Versions]}, ok) when is_binary(Pkg), is_list(Versions) -> + lists:foreach(fun(Version) -> + digraph:add_vertex(Graph, {Pkg, Version}, 1) + end, Versions); + (_, ok) -> + ok + end, ok, T), + + Dict1 = ets:foldl(fun({{Pkg, PkgVsn}, [Deps | _]}, Dict) -> + DepsList = lists:foldl(fun([Dep, DepVsn, _, _], DepsListAcc) -> + case DepVsn of + <<"~> ", Vsn/binary>> -> + digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, Vsn}), + [{Dep, DepVsn} | DepsListAcc]; + Vsn -> + digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, Vsn}), + [{Dep, Vsn} | DepsListAcc] + end + end, [], Deps), + dict:store({Pkg, PkgVsn}, DepsList, Dict); + (_, Dict) -> + Dict + end, dict:new(), T), + {Dict1, Graph}. diff --git a/src/rebar_resource.erl b/src/rebar_resource.erl index 2bc28e2..7c58135 100644 --- a/src/rebar_resource.erl +++ b/src/rebar_resource.erl @@ -23,7 +23,7 @@ -spec behaviour_info(atom()) -> [{atom(), arity()}] | undefined. behaviour_info(callbacks) -> [{lock, 2}, - {download, 2}, + {download, 3}, {needs_update,2}, {make_vsn, 1}]; behaviour_info(_) -> @@ -33,7 +33,7 @@ behaviour_info(_) -> -callback lock(file:filename_all(), tuple()) -> rebar_resource:resource(). --callback download(file:filename_all(), tuple()) -> +-callback download(file:filename_all(), tuple(), rebar_state:t()) -> {tarball, file:filename_all()} | {ok, any()} | {error, any()}. -callback needs_update(file:filename_all(), tuple()) -> boolean(). diff --git a/test/mock_git_resource.erl b/test/mock_git_resource.erl index d2f0207..6940f54 100644 --- a/test/mock_git_resource.erl +++ b/test/mock_git_resource.erl @@ -103,7 +103,7 @@ mock_download(Opts) -> Overrides = proplists:get_value(override_vsn, Opts, []), meck:expect( ?MOD, download, - fun (Dir, Git) -> + fun (Dir, Git, _) -> filelib:ensure_dir(Dir), {git, Url, {_, Vsn}} = normalize_git(Git, Overrides, Default), App = app(Url), diff --git a/test/mock_pkg_resource.erl b/test/mock_pkg_resource.erl index ab2c0d3..c7276b6 100644 --- a/test/mock_pkg_resource.erl +++ b/test/mock_pkg_resource.erl @@ -21,7 +21,7 @@ mock() -> mock([]). | {not_in_index, [{App, Vsn}]} | {pkgdeps, [{{App,Vsn}, [Dep]}]}, App :: string(), - Dep :: {App, string(), {pkg, App, Vsn, Url::string()}}, + Dep :: {App, string(), {pkg, App, Vsn}}, Vsn :: string(). mock(Opts) -> meck:new(?MOD, [no_link]), @@ -50,7 +50,7 @@ mock_update(Opts) -> ToUpdate = proplists:get_value(update, Opts, []), meck:expect( ?MOD, needs_update, - fun(_Dir, {pkg, App, _Vsn, _Url}) -> + fun(_Dir, {pkg, App, _Vsn}) -> lists:member(App, ToUpdate) end). @@ -65,7 +65,7 @@ mock_vsn(_Opts) -> %% @doc For each app to download, create a dummy app on disk instead. %% The configuration for this one (passed in from `mock/1') includes: %% -%% - Specify a version with `{pkg, _, Vsn, _}' +%% - Specify a version with `{pkg, _, Vsn}' %% - Dependencies for each application must be passed of the form: %% `{pkgdeps, [{"app1", [{app2, ".*", {pkg, ...}}]}]}' -- basically %% the `pkgdeps' option takes a key/value list of terms to output directly @@ -74,7 +74,7 @@ mock_download(Opts) -> Deps = proplists:get_value(pkgdeps, Opts, []), meck:expect( ?MOD, download, - fun (Dir, {pkg, AppBin, Vsn, _Url}) -> + fun (Dir, {pkg, AppBin, Vsn}, _) -> App = binary_to_list(AppBin), filelib:ensure_dir(Dir), AppDeps = proplists:get_value({App,Vsn}, Deps, []), @@ -83,11 +83,15 @@ mock_download(Opts) -> [element(1,D) || D <- AppDeps] ), rebar_test_utils:create_config(Dir, [{deps, AppDeps}]), - Tarball = filename:join([Dir, "package.tar.gz"]), + Tarball = filename:join([Dir, App++"-"++binary_to_list(Vsn)++".tar"]), + Contents = filename:join([Dir, "contents.tar.gz"]), Files = all_files(rebar_app_info:dir(AppInfo)), - ok = erl_tar:create(Tarball, + ok = erl_tar:create(Contents, archive_names(Dir, App, Vsn, Files), [compressed]), + ok = erl_tar:create(Tarball, + [{"contents.tar.gz", Contents}], + []), [file:delete(F) || F <- Files], {tarball, Tarball} end). @@ -117,8 +121,7 @@ all_files(Dir) -> filelib:wildcard(filename:join([Dir, "**"])). archive_names(Dir, App, Vsn, Files) -> - ArchName = App++"-"++binary_to_list(Vsn), - [{ArchName ++ (F -- Dir), F} || F <- Files]. + [{(F -- Dir) -- "/", F} || F <- Files]. find_parts(Apps, Skip) -> find_parts(Apps, Skip, dict:new()). diff --git a/test/rebar_deps_SUITE.erl b/test/rebar_deps_SUITE.erl index 4aaa7e3..633e9a2 100644 --- a/test/rebar_deps_SUITE.erl +++ b/test/rebar_deps_SUITE.erl @@ -1,14 +1,16 @@ -%% TODO: add tests for: -%% - only part of deps fetched -%% - only part of deps locked -%% - output only shown once -%% - modification asterisk on locked file -module(rebar_deps_SUITE). -compile(export_all). -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). -all() -> [default_nodep, default_lock]. +all() -> [{group, git}, {group, pkg}]. + +groups() -> + [{all, [], [flat, pick_highest_left, pick_highest_right, + pick_smallest1, pick_smallest2, + circular1, circular2, circular_skip]}, + {git, [], [{group, all}]}, + {pkg, [], [{group, all}]}]. init_per_suite(Config) -> application:start(meck), @@ -17,79 +19,202 @@ init_per_suite(Config) -> end_per_suite(_Config) -> application:stop(meck). +init_per_group(git, Config) -> + [{deps_type, git} | Config]; +init_per_group(pkg, Config) -> + [{deps_type, pkg} | Config]; +init_per_group(_, Config) -> + Config. + +end_per_group(_, Config) -> + Config. + init_per_testcase(Case, Config) -> - meck:new(io, [no_link, passthrough, unstick]), - setup_project(Case, Config). + {Deps, Warnings, Expect} = deps(Case), + Expected = case Expect of + {ok, List} -> {ok, format_expected_deps(List)}; + {error, Reason} -> {error, Reason} + end, + DepsType = ?config(deps_type, Config), + mock_warnings(), + [{expect, Expected}, + {warnings, Warnings} + | setup_project(Case, Config, expand_deps(DepsType, Deps))]. end_per_testcase(_, Config) -> meck:unload(), Config. -config_content() -> - [{deps, [ - {src_a, ".*", {git, "https://example.org/ferd/src_a.git", {branch, "master"}}}, - {src_b, {git, "https://example.org/ferd/src_b.git", {branch, "master"}}}, - {pkg_a, "1.0.0"} - ]}, - {profiles, - [{test, - [{deps, [ - {tdep, {git, "git://example.org/ferd/tdep.git", {tag, "0.8.2"}}} - ]}] - }]} - ]. - -setup_project(Case, Config0) -> +format_expected_deps(Deps) -> + [case Dep of + {N,V} -> {dep, N, V}; + N -> {dep, N} + end || Dep <- Deps]. + +%% format: +%% {Spec, +%% [Warning], +%% {ok, Result} | {error, Reason}} +%% +%% Spec is a list of levelled dependencies of two possible forms: +%% - {"Name", Spec} +%% - {"Name", "Vsn", Spec} +%% +%% Warnings are going to match on mocked ?WARN(...) +%% calls to be evaluated. An empty list means we do not care about +%% warnings, not that no warnings will be printed. This means +%% the list of warning isn't interpreted to be exhaustive, and more +%% warnings may be generated than are listed. +deps(flat) -> + {[{"B", []}, + {"C", []}], + [], + {ok, ["B", "C"]}}; +deps(pick_highest_left) -> + {[{"B", [{"C", "2", []}]}, + {"C", "1", []}], + [{"C","2"}], + {ok, ["B", {"C", "1"}]}}; +deps(pick_highest_right) -> + {[{"B", "1", []}, + {"C", [{"B", "2", []}]}], + [{"B","2"}], + {ok, [{"B","1"}, "C"]}}; +deps(pick_smallest1) -> + {[{"B", [{"D", "1", []}]}, + {"C", [{"D", "2", []}]}], + [{"D","2"}], + %% we pick D1 because B < C + {ok, ["B","C",{"D","1"}]}}; +deps(pick_smallest2) -> + {[{"C", [{"D", "2", []}]}, + {"B", [{"D", "1", []}]}], + [{"D","2"}], + %% we pick D1 because B < C + {ok, ["B","C",{"D","1"}]}}; +deps(circular1) -> + {[{"B", [{"A", []}]}, % A is the top-level app + {"C", []}], + [], + {error, {rebar_prv_install_deps, {cycles, [[<<"A">>,<<"B">>]]}}}}; +deps(circular2) -> + {[{"B", [{"C", [{"B", []}]}]}, + {"C", []}], + [], + {error, {rebar_prv_install_deps, {cycles, [[<<"B">>,<<"C">>]]}}}}; +deps(circular_skip) -> + %% Never spot the circular dep due to being to low in the deps tree + %% in source deps + {[{"B", [{"C", "2", [{"B", []}]}]}, + {"C", "1", [{"D",[]}]}], + [{"C","2"}], + {ok, ["B", {"C","1"}, "D"]}}. + +expand_deps(_, []) -> []; +expand_deps(git, [{Name, Deps} | Rest]) -> + Dep = {Name, ".*", {git, "https://example.org/user/"++Name++".git", "master"}}, + [{Dep, expand_deps(git, Deps)} | expand_deps(git, Rest)]; +expand_deps(git, [{Name, Vsn, Deps} | Rest]) -> + Dep = {Name, Vsn, {git, "https://example.org/user/"++Name++".git", {tag, Vsn}}}, + [{Dep, expand_deps(git, Deps)} | expand_deps(git, Rest)]; +expand_deps(pkg, [{Name, Deps} | Rest]) -> + Dep = {pkg, Name, "0.0.0"}, + [{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)]; +expand_deps(pkg, [{Name, Vsn, Deps} | Rest]) -> + Dep = {pkg, Name, Vsn}, + [{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)]. + +setup_project(Case, Config0, Deps) -> + DepsType = ?config(deps_type, Config0), Config = rebar_test_utils:init_rebar_state( Config0, - atom_to_list(Case)++"_" + atom_to_list(Case)++"_"++atom_to_list(DepsType)++"_" ), AppDir = ?config(apps, Config), rebar_test_utils:create_app(AppDir, "A", "0.0.0", [kernel, stdlib]), - TopDeps = proplists:get_value(deps, config_content()), - StringDeps = [erlang:setelement(1, Dep, atom_to_list(element(1,Dep))) - || Dep <- TopDeps], + TopDeps = top_level_deps(Deps), RebarConf = rebar_test_utils:create_config(AppDir, [{deps, TopDeps}]), - mock_git_resource:mock([{deps, lists:filter(fun src_dep/1, StringDeps)}]), - mock_pkg_resource:mock([{pkgdeps, - [{{ec_cnv:to_binary(N), - ec_cnv:to_binary(V)},[]} - || {N,V} <- lists:filter(fun pkg_dep/1, StringDeps)]}]), + case DepsType of + git -> + mock_git_resource:mock([{deps, flat_deps(Deps)}]); + pkg -> + mock_pkg_resource:mock([{pkgdeps, flat_pkgdeps(Deps)}]) + end, [{rebarconfig, RebarConf} | Config]. -src_dep(Dep) -> - case element(1, Dep) of - "src_"++_ -> true; - _ -> false - end. -pkg_dep(Dep) -> - case element(1, Dep) of - "pkg_"++_ -> true; - _ -> false - end. +flat_deps([]) -> []; +flat_deps([{{Name,_Vsn,Ref}, Deps} | Rest]) -> + [{{Name,vsn_from_ref(Ref)}, top_level_deps(Deps)}] + ++ + flat_deps(Deps) + ++ + flat_deps(Rest). -default_nodep(Config) -> - {ok, RebarConfig} = file:consult(?config(rebarconfig, Config)), - rebar_test_utils:run_and_check( - Config, RebarConfig, ["deps"], {ok, []} - ), - History = meck:history(io), - Strings = [io_lib:format(Str, Args) || {_, {io, format, [Str, Args]}, _} <- History], - {match, _} = re:run(Strings, "src_a\\* \\(git source\\)"), - {match, _} = re:run(Strings, "src_b\\* \\(git source\\)"), - {match, _} = re:run(Strings, "pkg_a\\* \\(package 1.0.0\\)"). +vsn_from_ref({git, _, {_, Vsn}}) -> Vsn; +vsn_from_ref({git, _, Vsn}) -> Vsn. -default_lock(Config) -> +flat_pkgdeps([]) -> []; +flat_pkgdeps([{{pkg, Name, Vsn}, Deps} | Rest]) -> + [{{iolist_to_binary(Name),iolist_to_binary(Vsn)}, top_level_deps(Deps)}] + ++ + flat_pkgdeps(Deps) + ++ + flat_pkgdeps(Rest). + +top_level_deps([]) -> []; +top_level_deps([{{pkg, Name, Vsn}, _} | Deps]) -> + [{list_to_atom(Name), Vsn} | top_level_deps(Deps)]; +top_level_deps([{{Name, Vsn, Ref}, _} | Deps]) -> + [{list_to_atom(Name), Vsn, Ref} | top_level_deps(Deps)]. + +app_vsn([]) -> []; +app_vsn([{Source, Deps} | Rest]) -> + {Name, Vsn} = case Source of + {pkg, N, V} -> {N,V}; + {N,V,_Ref} -> {N,V} + end, + [{Name, Vsn}] ++ app_vsn(Deps) ++ app_vsn(Rest). + +mock_warnings() -> + %% just let it do its thing, we check warnings through + %% the call log. + meck:new(rebar_log, [no_link, passthrough]). + +%%% TESTS %%% +flat(Config) -> run(Config). +pick_highest_left(Config) -> run(Config). +pick_highest_right(Config) -> run(Config). +pick_smallest1(Config) -> run(Config). +pick_smallest2(Config) -> run(Config). +circular1(Config) -> run(Config). +circular2(Config) -> run(Config). +circular_skip(Config) -> run(Config). + +run(Config) -> {ok, RebarConfig} = file:consult(?config(rebarconfig, Config)), rebar_test_utils:run_and_check( - Config, RebarConfig, ["lock"], {ok, []} + Config, RebarConfig, ["install_deps"], ?config(expect, Config) ), - rebar_test_utils:run_and_check( - Config, RebarConfig, ["deps"], {ok, []} - ), - History = meck:history(io), - Strings = [io_lib:format(Str, Args) || {_, {io, format, [Str, Args]}, _} <- History], - {match, _} = re:run(Strings, "src_a\\ \\(locked git source\\)"), - {match, _} = re:run(Strings, "src_b\\ \\(locked git source\\)"), - {match, _} = re:run(Strings, "pkg_a\\ \\(locked package 1.0.0\\)"). + check_warnings(warning_calls(), ?config(warnings, Config), ?config(deps_type, Config)). + +warning_calls() -> + History = meck:history(rebar_log), + [{Str, Args} || {_, {rebar_log, log, [warn, Str, Args]}, _} <- History]. + +check_warnings(_, [], _) -> + ok; +check_warnings(Warns, [{Name, Vsn} | Rest], Type) -> + ct:pal("Checking for warning ~p in ~p", [{Name,Vsn},Warns]), + ?assert(in_warnings(Type, Warns, Name, Vsn)), + check_warnings(Warns, Rest, Type). + +in_warnings(git, Warns, NameRaw, VsnRaw) -> + Name = iolist_to_binary(NameRaw), + 1 =< length([1 || {_, [AppName, {git, _, {_, Vsn}}]} <- Warns, + AppName =:= Name, Vsn =:= VsnRaw]); +in_warnings(pkg, Warns, NameRaw, VsnRaw) -> + Name = iolist_to_binary(NameRaw), + Vsn = iolist_to_binary(VsnRaw), + 1 =< length([1 || {_, [AppName, AppVsn]} <- Warns, + AppName =:= Name, AppVsn =:= Vsn]). diff --git a/test/rebar_upgrade_SUITE.erl b/test/rebar_upgrade_SUITE.erl index 39b9687..b4daef9 100644 --- a/test/rebar_upgrade_SUITE.erl +++ b/test/rebar_upgrade_SUITE.erl @@ -359,7 +359,7 @@ upgrades(delete_d) -> top_level_deps([]) -> []; top_level_deps([{{Name, Vsn, Ref}, _} | Deps]) -> [{list_to_atom(Name), Vsn, Ref} | top_level_deps(Deps)]; -top_level_deps([{{pkg, Name, Vsn, _URL}, _} | Deps]) -> +top_level_deps([{{pkg, Name, Vsn}, _} | Deps]) -> [{list_to_atom(Name), Vsn} | top_level_deps(Deps)]. mock_deps(git, Deps, Upgrades) -> @@ -381,7 +381,7 @@ vsn_from_ref({git, _, {_, Vsn}}) -> Vsn; vsn_from_ref({git, _, Vsn}) -> Vsn. flat_pkgdeps([]) -> []; -flat_pkgdeps([{{pkg, Name, Vsn, _Url}, Deps} | Rest]) -> +flat_pkgdeps([{{pkg, Name, Vsn}, Deps} | Rest]) -> [{{iolist_to_binary(Name),iolist_to_binary(Vsn)}, top_level_deps(Deps)}] ++ flat_pkgdeps(Deps) @@ -396,10 +396,10 @@ expand_deps(git, [{Name, Vsn, Deps} | Rest]) -> Dep = {Name, Vsn, {git, "https://example.org/user/"++Name++".git", {tag, Vsn}}}, [{Dep, expand_deps(git, Deps)} | expand_deps(git, Rest)]; expand_deps(pkg, [{Name, Deps} | Rest]) -> - Dep = {pkg, Name, "0.0.0", "https://example.org/user/"++Name++".tar.gz"}, + Dep = {pkg, Name, "0.0.0"}, [{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)]; expand_deps(pkg, [{Name, Vsn, Deps} | Rest]) -> - Dep = {pkg, Name, Vsn, "https://example.org/user/"++Name++".tar.gz"}, + Dep = {pkg, Name, Vsn}, [{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)]. normalize_unlocks({App, Locks}) -> @@ -472,4 +472,3 @@ run(Config) -> rebar_test_utils:run_and_check( Config, NewRebarConfig, ["upgrade", App], Expectation ). - -- cgit v1.1