From 56b7d88975aa8da6446857cfd92de0825024bf63 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 13 Sep 2018 19:36:00 -0600 Subject: support for hex v2, multiple repository fetching, private organizations (#1884) * update to hex_core for hex-v2 repo support (#1865) * update to hex_core for hex-v2 repo support This patch adds only single repo hex-v2 support through hex_core. Packages no longer filtered out by buildtool metadata and the package index is updated per-package instead of fetched as one large ets dump. * tell travis to also build hex_core branch * support list of repos for hex packages (#1866) * support list of repos for hex packages repos are defined under the hex key in rebar configs. They can be defined at the top level of a project or globally, but not in profiles and the repos configured in dependencies are also ignored. Searching for packages involves first checking for a match in the local repo index cache, in the order repos are defined. If not found each repo is checked through the hex api for any known versions of the package and the first repo with a version that fits the constraint is used. * add {repos, replace, []} for overriding the global & default repos * add hex auth handling for repos (#1874) auth token are kept in a hex.config file that is modified by the rebar3 hex plugin. Repo names that have a : separating a parent and child are considered organizations. The parent repo's auth will be included with the child. So an organization named hexpm:rebar3_test will include any hexpm auth tokens found in the rebar3_test organization's configuration. * move packages to top level of of hexpm cache dir (#1876) * move packages to top level of of hexpm cache dir * append organization name to parent's repo_url when parsing repos * only eval config scripts and apply overrides once per app (#1879) * only eval config scripts and apply overrides once per app * move new resource behaviour to rebar_resource_v2 and keep v1 * cleanup use of rebar_resource module and unused functions * cleanup error messages and unused code * when discovering apps support mix packages as unbuilt apps (#1882) * use hex_core tarball unpacking support in pkg resource (#1883) * use hex_core tarball unpacking support in pkg resource * ignore etag if package doesn't exist and delete if checksum fails * add back tests for bad package checksums * improve bad registry checksum error message --- src/rebar_prv_install_deps.erl | 70 ++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 44 deletions(-) (limited to 'src/rebar_prv_install_deps.erl') diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index b735ed0..bad5af4 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -277,10 +277,8 @@ update_unseen_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Loc -spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer()) -> {rebar_app_info:t(), [rebar_app_info:t()], rebar_state:t()}. handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) -> Name = rebar_app_info:name(AppInfo), - C = rebar_config:consult(rebar_app_info:dir(AppInfo)), - AppInfo0 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), C), - AppInfo1 = rebar_app_info:apply_overrides(rebar_app_info:get(AppInfo, overrides, []), AppInfo0), + AppInfo1 = rebar_app_info:apply_overrides(rebar_app_info:get(AppInfo, overrides, []), AppInfo), AppInfo2 = rebar_app_info:apply_profiles(AppInfo1, [default, prod]), Plugins = rebar_app_info:get(AppInfo2, plugins, []), @@ -297,34 +295,33 @@ handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) -> AppInfo4 = rebar_app_info:deps(AppInfo3, rebar_state:deps_names(Deps)), %% Keep all overrides from the global config and this dep when parsing its deps - Overrides = rebar_app_info:get(AppInfo0, overrides, []), + Overrides = rebar_app_info:get(AppInfo, overrides, []), Deps1 = rebar_app_utils:parse_deps(Name, DepsDir, Deps, rebar_state:set(State, overrides, Overrides) ,Locks, Level+1), {AppInfo4, Deps1, State1}. -spec maybe_fetch(rebar_app_info:t(), atom(), boolean(), - sets:set(binary()), rebar_state:t()) -> {boolean(), rebar_app_info:t()}. + sets:set(binary()), rebar_state:t()) -> {ok, rebar_app_info:t()}. maybe_fetch(AppInfo, Profile, Upgrade, Seen, State) -> AppDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)), %% Don't fetch dep if it exists in the _checkouts dir case rebar_app_info:is_checkout(AppInfo) of true -> - {false, AppInfo}; + {ok, AppInfo}; false -> - case rebar_app_discover:find_app(AppInfo, AppDir, all) of + case rebar_app_info:is_available(AppInfo) of false -> - true = fetch_app(AppInfo, AppDir, State), - maybe_symlink_default(State, Profile, AppDir, AppInfo), - {true, rebar_app_info:valid(update_app_info(AppDir, AppInfo), false)}; - {true, AppInfo1} -> - case sets:is_element(rebar_app_info:name(AppInfo1), Seen) of + AppInfo1 = fetch_app(AppInfo, State), + maybe_symlink_default(State, Profile, AppDir, AppInfo1), + {ok, rebar_app_info:is_available(rebar_app_info:valid(AppInfo1, false), true)}; + true -> + case sets:is_element(rebar_app_info:name(AppInfo), Seen) of true -> - {false, AppInfo1}; + {ok, AppInfo}; false -> - maybe_symlink_default(State, Profile, AppDir, AppInfo1), - MaybeUpgrade = maybe_upgrade(AppInfo, AppDir, Upgrade, State), - AppInfo2 = update_app_info(AppDir, AppInfo1), - {MaybeUpgrade, AppInfo2} + maybe_symlink_default(State, Profile, AppDir, AppInfo), + AppInfo1 = maybe_upgrade(AppInfo, AppDir, Upgrade, State), + {ok, AppInfo1} end end end. @@ -372,52 +369,37 @@ make_relative_to_root(State, Path) when is_list(Path) -> Root = rebar_dir:root_dir(State), rebar_dir:make_relative_path(Path, Root). -fetch_app(AppInfo, AppDir, State) -> +fetch_app(AppInfo, State) -> ?INFO("Fetching ~ts (~p)", [rebar_app_info:name(AppInfo), - format_source(rebar_app_info:source(AppInfo))]), - Source = rebar_app_info:source(AppInfo), - true = rebar_fetch:download_source(AppDir, Source, State). - -format_source({pkg, Name, Vsn, _Hash}) -> {pkg, Name, Vsn}; -format_source(Source) -> Source. - -%% This is called after the dep has been downloaded and unpacked, if it hadn't been already. -%% So this is the first time for newly downloaded apps that its .app/.app.src data can -%% be read in an parsed. -update_app_info(AppDir, AppInfo) -> - case rebar_app_discover:find_app(AppInfo, AppDir, all) of - {true, AppInfo1} -> - AppInfo1; - false -> - throw(?PRV_ERROR({dep_app_not_found, AppDir, rebar_app_info:name(AppInfo)})) - end. + rebar_resource_v2:format_source(rebar_app_info:source(AppInfo))]), + rebar_fetch:download_source(AppInfo, State). -maybe_upgrade(AppInfo, AppDir, Upgrade, State) -> - Source = rebar_app_info:source(AppInfo), +maybe_upgrade(AppInfo, _AppDir, Upgrade, State) -> case Upgrade orelse rebar_app_info:is_lock(AppInfo) of true -> - case rebar_fetch:needs_update(AppDir, Source, State) of + case rebar_fetch:needs_update(AppInfo, State) of true -> - ?INFO("Upgrading ~ts (~p)", [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)]), - true = rebar_fetch:download_source(AppDir, Source, State); + ?INFO("Upgrading ~ts (~p)", [rebar_app_info:name(AppInfo), + rebar_resource_v2:format_source(rebar_app_info:source(AppInfo))]), + rebar_fetch:download_source(AppInfo, State); false -> case Upgrade of true -> ?INFO("No upgrade needed for ~ts", [rebar_app_info:name(AppInfo)]), - false; + AppInfo; false -> - false + AppInfo end end; false -> - false + AppInfo end. warn_skip_deps(AppInfo, State) -> Msg = "Skipping ~ts (from ~p) as an app of the same name " "has already been fetched", Args = [rebar_app_info:name(AppInfo), - rebar_app_info:source(AppInfo)], + rebar_resource_v2:format_source(rebar_app_info:source(AppInfo))], case rebar_state:get(State, deps_error_on_conflict, false) of false -> case rebar_state:get(State, deps_warning_on_conflict, true) of -- cgit v1.1 From 43769ae87f7c60a9b6cb3bf08085a4a4206961b7 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Mon, 17 Sep 2018 09:02:30 -0600 Subject: only print skip warning if version is different (#1886) --- src/rebar_prv_install_deps.erl | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) (limited to 'src/rebar_prv_install_deps.erl') diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index bad5af4..068c4c8 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -259,9 +259,21 @@ update_seen_dep(AppInfo, _Profile, _Level, Deps, Apps, State, Upgrade, Seen, Loc %% If seen from lock file or user requested an upgrade %% don't print warning about skipping case lists:keymember(Name, 1, Locks) of - false when Upgrade -> ok; - false when not Upgrade -> warn_skip_deps(AppInfo, State); - true -> ok + false when Upgrade -> + ok; + false when not Upgrade -> + {ok, SeenApp} = rebar_app_utils:find(Name, Apps), + Source = rebar_app_info:source(AppInfo), + case rebar_app_info:source(SeenApp) of + Source -> + %% dep is the same version and checksum as the one we already saw. + %% meaning there is no conflict, so don't warn about it. + skip; + _ -> + warn_skip_deps(Name, Source, State) + end; + true -> + ok end, {Deps, Apps, State, Seen}. @@ -395,11 +407,11 @@ maybe_upgrade(AppInfo, _AppDir, Upgrade, State) -> AppInfo end. -warn_skip_deps(AppInfo, State) -> +warn_skip_deps(Name, Source, State) -> Msg = "Skipping ~ts (from ~p) as an app of the same name " "has already been fetched", - Args = [rebar_app_info:name(AppInfo), - rebar_resource_v2:format_source(rebar_app_info:source(AppInfo))], + Args = [Name, + rebar_resource_v2:format_source(Source)], case rebar_state:get(State, deps_error_on_conflict, false) of false -> case rebar_state:get(State, deps_warning_on_conflict, true) of -- cgit v1.1