diff options
author | Tristan Sloughter <t@crashfast.com> | 2018-10-01 15:41:34 -0600 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-10-01 15:41:34 -0600 |
commit | 8bcb7da1dd294f13bc8d2040ec88f0b41b941e1b (patch) | |
tree | b43607097dd16cdc8916ac8dfb5f045e87e906d0 | |
parent | 651b54fc1b90c0603b8679a693f6f69b35bacad4 (diff) |
warn if the matched package is retired, skip prerelease (#1897)
retired packages are now used the same as any other but
a warning will be printed when it is resolved.
prerelease versions are skipped unless explicitly given
as the version in the constraint or lock file.
-rw-r--r-- | src/rebar.hrl | 6 | ||||
-rw-r--r-- | src/rebar_app_utils.erl | 41 | ||||
-rw-r--r-- | src/rebar_packages.erl | 44 | ||||
-rw-r--r-- | test/mock_pkg_resource.erl | 6 | ||||
-rw-r--r-- | test/rebar_pkg_SUITE.erl | 8 | ||||
-rw-r--r-- | test/rebar_pkg_alias_SUITE.erl | 2 | ||||
-rw-r--r-- | test/rebar_pkg_repos_SUITE.erl | 91 |
7 files changed, 134 insertions, 64 deletions
diff --git a/src/rebar.hrl b/src/rebar.hrl index 572cbe8..f11302d 100644 --- a/src/rebar.hrl +++ b/src/rebar.hrl @@ -27,9 +27,9 @@ -define(REMOTE_PACKAGE_DIR, "tarballs"). -define(LOCK_FILE, "rebar.lock"). -define(DEFAULT_COMPILER_SOURCE_FORMAT, relative). --define(PACKAGE_INDEX_VERSION, 4). --define(PACKAGE_TABLE, package_index_v4). --define(INDEX_FILE, "packages-v4.idx"). +-define(PACKAGE_INDEX_VERSION, 5). +-define(PACKAGE_TABLE, package_index). +-define(INDEX_FILE, "packages.idx"). -define(HEX_AUTH_FILE, "hex.config"). -define(PUBLIC_HEX_REPO, <<"hexpm">>). diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index 35e908c..fb08ea9 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -254,10 +254,13 @@ update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) -> {ok, Package, RepoConfig} -> #package{key={_, PkgVsn1, _}, checksum=Hash1, - dependencies=Deps} = Package, - AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName, PkgVsn1, Hash1, RepoConfig}), + dependencies=Deps, + retired=Retired} = Package, + maybe_warn_retired(PkgName, PkgVsn1, Hash, Retired), + PkgVsn2 = list_to_binary(lists:flatten(ec_semver:format(PkgVsn1))), + AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName, PkgVsn2, Hash1, RepoConfig}), AppInfo2 = rebar_app_info:update_opts_deps(AppInfo1, Deps), - rebar_app_info:original_vsn(AppInfo2, PkgVsn1); + rebar_app_info:original_vsn(AppInfo2, PkgVsn2); not_found -> throw(?PRV_ERROR({missing_package, PkgName, PkgVsn})); {error, {invalid_vsn, InvalidVsn}} -> @@ -269,10 +272,10 @@ update_source(AppInfo, Source, _State) -> %% @doc convert a given exception's payload into an io description. -spec format_error(any()) -> iolist(). format_error({missing_package, Name, undefined}) -> - io_lib:format("Package not found in any repo: ~ts.", [rebar_utils:to_binary(Name)]); + io_lib:format("Package not found in any repo: ~ts", [rebar_utils:to_binary(Name)]); format_error({missing_package, Name, Vsn}) -> - io_lib:format("Package not found in any repo: ~ts-~ts.", [rebar_utils:to_binary(Name), - rebar_utils:to_binary(Vsn)]); + io_lib:format("Package not found in any repo: ~ts ~ts", [rebar_utils:to_binary(Name), + rebar_utils:to_binary(Vsn)]); format_error({parse_dep, Dep}) -> io_lib:format("Failed parsing dep ~p", [Dep]); format_error({invalid_vsn, Dep, InvalidVsn}) -> @@ -284,6 +287,32 @@ format_error(Error) -> %% Internal functions %% =================================================================== +maybe_warn_retired(_, _, _, false) -> + ok; +maybe_warn_retired(_, _, Hash, _) when is_binary(Hash) -> + %% don't warn if this is a lock + ok; +maybe_warn_retired(Name, Vsn, _, R=#{reason := Reason}) -> + Message = maps:get(message, R, ""), + ?WARN("Warning: package ~s-~s is retired: (~s) ~s", + [Name, ec_semver:format(Vsn), retire_reason(Reason), Message]); +maybe_warn_retired(_, _, _, _) -> + ok. + +%% TODO: move to hex_core +retire_reason('RETIRED_OTHER') -> + "other"; +retire_reason('RETIRED_INVALID') -> + "invalid"; +retire_reason('RETIRED_SECURITY') -> + "security"; +retire_reason('RETIRED_DEPRECATED') -> + "deprecated"; +retire_reason('RETIRED_RENAMED') -> + "renamed"; +retire_reason(_Other) -> + "other". + %% @private checks that all the beam files have been properly %% created. -spec has_all_beams(file:filename_all(), [module()]) -> diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 8a3ffea..7533a44 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -24,10 +24,10 @@ -type package() :: pkg_name() | {pkg_name(), vsn()}. format_error({missing_package, Name, Vsn}) -> - io_lib:format("Package not found in any repo: ~ts-~ts.", [rebar_utils:to_binary(Name), - rebar_utils:to_binary(Vsn)]); + io_lib:format("Package not found in any repo: ~ts ~ts", [rebar_utils:to_binary(Name), + rebar_utils:to_binary(Vsn)]); format_error({missing_package, Pkg}) -> - io_lib:format("Package not found in any repo: ~p.", [Pkg]). + io_lib:format("Package not found in any repo: ~p", [Pkg]). -spec get(rebar_hex_repos:repo(), binary()) -> {ok, map()} | {error, term()}. get(Config, Name) -> @@ -55,29 +55,25 @@ get_all_names(State) -> _='_'}, [], ['$1']}])). --spec get_package_versions(unicode:unicode_binary(), unicode:unicode_binary(), ets:tid(), rebar_state:t()) - -> [vsn()]. +-spec get_package_versions(unicode:unicode_binary(), unicode:unicode_binary(), + ets:tid(), rebar_state:t()) -> [vsn()]. get_package_versions(Dep, Repo, Table, State) -> ?MODULE:verify_table(State), - ets:select(Table, [{#package{key={Dep,'$1', Repo}, - _='_'}, - [], ['$1']}]). - - -get_package(Dep, Vsn, Hash, Repo, Table, State) -> - get_package(Dep, Vsn, Hash, false, [Repo], Table, State). + AllowPreRelease = rebar_state:get(State, deps_allow_prerelease, false), + ets:select(Table, [{#package{key={Dep,{'$1', '$2'}, Repo}, + _='_'}, + [{'==', '$2', {{[],[]}}} || not AllowPreRelease], [{{'$1', '$2'}}]}]). -spec get_package(unicode:unicode_binary(), unicode:unicode_binary(), - binary() | undefined | '_', boolean() | '_', - unicode:unicode_binary() | '_' | list(), ets:tab(), rebar_state:t()) + binary() | undefined | '_', + [unicode:unicode_binary()] | ['_'], ets:tab(), rebar_state:t()) -> {ok, #package{}} | not_found. -get_package(Dep, Vsn, undefined, Retired, Repo, Table, State) -> - get_package(Dep, Vsn, '_', Retired, Repo, Table, State); -get_package(Dep, Vsn, Hash, Retired, Repos, Table, State) -> +get_package(Dep, Vsn, undefined, Repos, Table, State) -> + get_package(Dep, Vsn, '_', Repos, Table, State); +get_package(Dep, Vsn, Hash, Repos, Table, State) -> ?MODULE:verify_table(State), - case ets:select(Table, [{#package{key={Dep, Vsn, Repo}, + case ets:select(Table, [{#package{key={Dep, ec_semver:parse(Vsn), Repo}, checksum=Hash, - retired=Retired, _='_'}, [], ['$_']} || Repo <- Repos]) of %% have to allow multiple matches in the list for cases that Repo is `_` [Package | _] -> @@ -258,7 +254,7 @@ update_package(Name, RepoConfig=#{name := Repo}, State) -> insert_releases(Name, Releases, Repo, Table) -> [true = ets:insert(Table, - #package{key={Name, Version, Repo}, + #package{key={Name, ec_semver:parse(Version), Repo}, checksum=parse_checksum(Checksum), retired=maps:get(retired, Release, false), dependencies=parse_deps(Dependencies)}) @@ -279,7 +275,7 @@ resolve_version(Dep, DepVsn, Hash, HexRegistry, State) when is_binary(Hash) -> RepoNames = [RepoName || #{name := RepoName} <- RepoConfigs], %% allow retired packages when we have a checksum - case get_package(Dep, DepVsn, Hash, '_', RepoNames, HexRegistry, State) of + case get_package(Dep, DepVsn, Hash, RepoNames, HexRegistry, State) of {ok, Package=#package{key={_, _, RepoName}}} -> {ok, RepoConfig} = rebar_hex_repos:get_repo_config(RepoName, RepoConfigs), {ok, Package, RepoConfig}; @@ -289,7 +285,7 @@ resolve_version(Dep, DepVsn, Hash, HexRegistry, State) when is_binary(Hash) -> none -> not_found; {ok, Vsn} -> - get_package(Dep, Vsn, Hash, Repo, HexRegistry, State) + get_package(Dep, Vsn, Hash, [Repo], HexRegistry, State) end end, handle_missing_no_exception(Fun, Dep, State) @@ -300,7 +296,7 @@ resolve_version(Dep, undefined, Hash, HexRegistry, State) -> none -> not_found; {ok, Vsn} -> - get_package(Dep, Vsn, Hash, Repo, HexRegistry, State) + get_package(Dep, Vsn, Hash, [Repo], HexRegistry, State) end end, handle_missing_no_exception(Fun, Dep, State); @@ -314,7 +310,7 @@ resolve_version(Dep, DepVsn, Hash, HexRegistry, State) -> none -> not_found; {ok, Vsn} -> - get_package(Dep, Vsn, Hash, Repo, HexRegistry, State) + get_package(Dep, Vsn, Hash, [Repo], HexRegistry, State) end end, handle_missing_no_exception(Fun, Dep, State) diff --git a/test/mock_pkg_resource.erl b/test/mock_pkg_resource.erl index 9a31461..a169efd 100644 --- a/test/mock_pkg_resource.erl +++ b/test/mock_pkg_resource.erl @@ -171,7 +171,7 @@ to_index(AllDeps, Dict, Repos) -> DKB <- [ec_cnv:to_binary(DK)], DVB <- [ec_cnv:to_binary(DV)]], Repo = rebar_test_utils:random_element(Repos), - ets:insert(?PACKAGE_TABLE, #package{key={N, V, Repo}, + ets:insert(?PACKAGE_TABLE, #package{key={N, ec_semver:parse(V), Repo}, dependencies=parse_deps(DepsList), retired=false, checksum = <<"checksum">>}) @@ -179,11 +179,11 @@ to_index(AllDeps, Dict, Repos) -> lists:foreach(fun({{Name, Vsn}, _}) -> case lists:any(fun(R) -> - ets:member(?PACKAGE_TABLE, {ec_cnv:to_binary(Name), Vsn, R}) + ets:member(?PACKAGE_TABLE, {ec_cnv:to_binary(Name), ec_semver:parse(Vsn), R}) end, Repos) of false -> Repo = rebar_test_utils:random_element(Repos), - ets:insert(?PACKAGE_TABLE, #package{key={ec_cnv:to_binary(Name), Vsn, Repo}, + ets:insert(?PACKAGE_TABLE, #package{key={ec_cnv:to_binary(Name), ec_semver:parse(Vsn), Repo}, dependencies=[], retired=false, checksum = <<"checksum">>}); diff --git a/test/rebar_pkg_SUITE.erl b/test/rebar_pkg_SUITE.erl index 62bc4d1..b8495fd 100644 --- a/test/rebar_pkg_SUITE.erl +++ b/test/rebar_pkg_SUITE.erl @@ -226,13 +226,13 @@ find_highest_matching(_Config) -> State = rebar_state:new(), {ok, Vsn} = rebar_packages:find_highest_matching_( <<"goodpkg">>, <<"1.0.0">>, #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State), - ?assertEqual(<<"1.0.1">>, Vsn), + ?assertEqual({{1,0,1},{[],[]}}, Vsn), {ok, Vsn1} = rebar_packages:find_highest_matching( <<"goodpkg">>, <<"1.0">>, #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State), - ?assertEqual(<<"1.1.1">>, Vsn1), + ?assertEqual({{1,1,1},{[],[]}}, Vsn1), {ok, Vsn2} = rebar_packages:find_highest_matching( <<"goodpkg">>, <<"2.0">>, #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State), - ?assertEqual(<<"2.0.0">>, Vsn2), + ?assertEqual({{2,0,0},{[],[]}}, Vsn2), %% regression test. ~> constraints higher than the available packages would result %% in returning the first package version instead of 'none'. @@ -265,7 +265,7 @@ mock_config(Name, Config) -> lists:foreach(fun({{N, Vsn}, [Deps, Checksum, _]}) -> case ets:member(?PACKAGE_TABLE, {ec_cnv:to_binary(N), Vsn, <<"hexpm">>}) of false -> - ets:insert(?PACKAGE_TABLE, #package{key={ec_cnv:to_binary(N), Vsn, <<"hexpm">>}, + ets:insert(?PACKAGE_TABLE, #package{key={ec_cnv:to_binary(N), ec_semver:parse(Vsn), <<"hexpm">>}, dependencies=Deps, retired=false, checksum=Checksum}); diff --git a/test/rebar_pkg_alias_SUITE.erl b/test/rebar_pkg_alias_SUITE.erl index d977b64..079a3fd 100644 --- a/test/rebar_pkg_alias_SUITE.erl +++ b/test/rebar_pkg_alias_SUITE.erl @@ -229,7 +229,7 @@ mock_config(Name, Config) -> lists:foreach(fun({{N, Vsn}, [Deps, Checksum, _]}) -> case ets:member(?PACKAGE_TABLE, {ec_cnv:to_binary(N), Vsn, <<"hexpm">>}) of false -> - ets:insert(?PACKAGE_TABLE, #package{key={ec_cnv:to_binary(N), Vsn, <<"hexpm">>}, + ets:insert(?PACKAGE_TABLE, #package{key={ec_cnv:to_binary(N), ec_semver:parse(Vsn), <<"hexpm">>}, dependencies=[{DAppName, {pkg, DN, DV, undefined}} || {DN, DV, _, DAppName} <- Deps], retired=false, checksum=Checksum}); diff --git a/test/rebar_pkg_repos_SUITE.erl b/test/rebar_pkg_repos_SUITE.erl index 601566e..c808475 100644 --- a/test/rebar_pkg_repos_SUITE.erl +++ b/test/rebar_pkg_repos_SUITE.erl @@ -13,7 +13,7 @@ all() -> groups() -> [{resolve_version, [use_first_repo_match, use_exact_with_hash, fail_repo_update, - ignore_match_in_excluded_repo]}]. + ignore_match_in_excluded_repo, optional_prereleases]}]. init_per_group(resolve_version, Config) -> Repo1 = <<"test-repo-1">>, @@ -22,14 +22,17 @@ init_per_group(resolve_version, Config) -> Hexpm = <<"hexpm">>, Repos = [Repo1, Repo2, Repo3, Hexpm], - Deps = [{"A", "0.1.1", <<"good checksum">>, Repo1}, - {"A", "0.1.1", <<"good checksum">>, Repo2}, - {"B", "1.0.0", Repo1}, - {"B", "2.0.0", Repo2}, - {"B", "1.4.0", Repo3}, - {"B", "1.4.3", Hexpm}, - {"C", "1.3.1", <<"bad checksum">>, Repo1}, - {"C", "1.3.1", <<"good checksum">>, Repo2}], + Deps = [{"A", "0.1.1", <<"good checksum">>, Repo1, false}, + {"A", "0.1.1", <<"good checksum">>, Repo2, false}, + {"B", "1.0.0", Repo1, false}, + {"B", "2.0.0", Repo2, false}, + {"B", "1.4.0", Repo3, false}, + {"B", "1.4.3", Hexpm, false}, + {"B", "1.4.6", Hexpm, #{reason => 'RETIRED_INVALID'}}, + {"B", "1.5.0", Hexpm, false}, + {"B", "1.5.6-rc.0", Hexpm, true}, + {"C", "1.3.1", <<"bad checksum">>, Repo1, false}, + {"C", "1.3.1", <<"good checksum">>, Repo2, false}], [{deps, Deps}, {repos, Repos} | Config]; init_per_group(_, Config) -> Config. @@ -102,6 +105,20 @@ init_per_testcase(ignore_match_in_excluded_repo, Config) -> fun(_State) -> true end), [{state, State} | Config]; +init_per_testcase(optional_prereleases, Config) -> + Deps = ?config(deps, Config), + Repos = ?config(repos, Config), + + State = setup_deps_and_repos(Deps, Repos), + + meck:new(rebar_packages, [passthrough, no_link]), + + meck:expect(rebar_packages, update_package, + fun(_, _, _State) -> ok end), + meck:expect(rebar_packages, verify_table, + fun(_State) -> true end), + + [{state, State} | Config]; init_per_testcase(auth_merging, Config) -> meck:new(file, [passthrough, no_link, unstick]), meck:new(rebar_packages, [passthrough, no_link]), @@ -120,7 +137,8 @@ end_per_testcase(Case, _Config) when Case =:= auth_merging ; end_per_testcase(Case, _Config) when Case =:= use_first_repo_match ; Case =:= use_exact_with_hash ; Case =:= fail_repo_update ; - Case =:= ignore_match_in_excluded_repo -> + Case =:= ignore_match_in_excluded_repo ; + Case =:= optional_prereleases -> meck:unload(rebar_packages); end_per_testcase(_, _) -> ok. @@ -250,14 +268,14 @@ organization_merging(_Config) -> use_first_repo_match(Config) -> State = ?config(state, Config), - ?assertMatch({ok,{package,{<<"B">>, <<"2.0.0">>, Repo2}, + ?assertMatch({ok,{package,{<<"B">>, {{2,0,0}, {[],[]}}, Repo2}, <<"some checksum">>, false, []}, #{name := Repo2, http_adapter_config := #{profile := rebar}}}, rebar_packages:resolve_version(<<"B">>, <<"> 1.4.0">>, undefined, ?PACKAGE_TABLE, State)), - ?assertMatch({ok,{package,{<<"B">>, <<"1.4.0">>, Repo3}, + ?assertMatch({ok,{package,{<<"B">>, {{1,4,0}, {[],[]}}, Repo3}, <<"some checksum">>, false, []}, #{name := Repo3, http_adapter_config := #{profile := rebar}}}, @@ -268,7 +286,7 @@ use_first_repo_match(Config) -> use_exact_with_hash(Config) -> State = ?config(state, Config), - ?assertMatch({ok,{package,{<<"C">>, <<"1.3.1">>, Repo2}, + ?assertMatch({ok,{package,{<<"C">>, {{1,3,1}, {[],[]}}, Repo2}, <<"good checksum">>, false, []}, #{name := Repo2, http_adapter_config := #{profile := rebar}}}, @@ -278,7 +296,7 @@ use_exact_with_hash(Config) -> fail_repo_update(Config) -> State = ?config(state, Config), - ?assertMatch({ok,{package,{<<"B">>, <<"1.4.0">>, Repo3}, + ?assertMatch({ok,{package,{<<"B">>, {{1,4,0}, {[],[]}}, Repo3}, <<"some checksum">>, false, []}, #{name := Repo3, http_adapter_config := #{profile := rebar}}}, @@ -289,24 +307,51 @@ ignore_match_in_excluded_repo(Config) -> State = ?config(state, Config), Repos = ?config(repos, Config), - ?assertMatch({ok,{package,{<<"B">>, <<"1.4.3">>, Hexpm}, - <<"some checksum">>, false, []}, + ?assertMatch({ok,{package,{<<"B">>, {{1,4,6}, {[],[]}}, Hexpm}, + <<"some checksum">>, #{reason := 'RETIRED_INVALID'}, []}, #{name := Hexpm, http_adapter_config := #{profile := rebar}}}, rebar_packages:resolve_version(<<"B">>, <<"~> 1.4.0">>, undefined, ?PACKAGE_TABLE, State)), [_, Repo2 | _] = Repos, - ?assertMatch({ok,{package,{<<"A">>, <<"0.1.1">>, Repo2}, + ?assertMatch({ok,{package,{<<"A">>, {{0,1,1}, {[],[]}}, Repo2}, <<"good checksum">>, false, []}, #{name := Repo2, http_adapter_config := #{profile := rebar}}}, rebar_packages:resolve_version(<<"A">>, <<"0.1.1">>, <<"good checksum">>, ?PACKAGE_TABLE, State)). +optional_prereleases(Config) -> + State = ?config(state, Config), + + ?assertMatch({ok,{package,{<<"B">>, {{1,5,0}, {[],[]}}, Hexpm}, + <<"some checksum">>, false, []}, + #{name := Hexpm, + http_adapter_config := #{profile := rebar}}}, + rebar_packages:resolve_version(<<"B">>, <<"~> 1.5.0">>, undefined, + ?PACKAGE_TABLE, State)), + + ?assertMatch({ok,{package,{<<"B">>, {{1,5,6}, {[<<"rc">>,0],[]}}, Hexpm}, + <<"some checksum">>, true, []}, + #{name := Hexpm, + http_adapter_config := #{profile := rebar}}}, + rebar_packages:resolve_version(<<"B">>, <<"1.5.6-rc.0">>, <<"some checksum">>, + ?PACKAGE_TABLE, State)), + + %% allow prerelease through configuration + State1 = rebar_state:set(State, deps_allow_prerelease, true), + ?assertMatch({ok,{package,{<<"B">>, {{1,5,6}, {[<<"rc">>,0],[]}}, Hexpm}, + <<"some checksum">>, true, []}, + #{name := Hexpm, + http_adapter_config := #{profile := rebar}}}, + rebar_packages:resolve_version(<<"B">>, <<"~> 1.5.0">>, <<"some checksum">>, + ?PACKAGE_TABLE, State1)). + %% setup_deps_and_repos(Deps, Repos) -> + catch ets:delete(?PACKAGE_TABLE), true = rebar_packages:new_package_table(), insert_deps(Deps), State = rebar_state:new([{hex, [{repos, [#{name => R} || R <- Repos]}]}]), @@ -314,18 +359,18 @@ setup_deps_and_repos(Deps, Repos) -> insert_deps(Deps) -> - lists:foreach(fun({Name, Version, Repo}) -> + lists:foreach(fun({Name, Version, Repo, Retired}) -> ets:insert(?PACKAGE_TABLE, #package{key={rebar_utils:to_binary(Name), - rebar_utils:to_binary(Version), + ec_semver:parse(Version), rebar_utils:to_binary(Repo)}, dependencies=[], - retired=false, + retired=Retired, checksum = <<"some checksum">>}); - ({Name, Version, Checksum, Repo}) -> + ({Name, Version, Checksum, Repo, Retired}) -> ets:insert(?PACKAGE_TABLE, #package{key={rebar_utils:to_binary(Name), - rebar_utils:to_binary(Version), + ec_semver:parse(Version), rebar_utils:to_binary(Repo)}, dependencies=[], - retired=false, + retired=Retired, checksum = Checksum}) end, Deps). |