From 5411a67ee63055ff4e11abd5e8eee95d71aea997 Mon Sep 17 00:00:00 2001 From: Juan Facorro Date: Fri, 3 Mar 2017 11:14:02 +0100 Subject: Fix typo in help text for upgrade command --- src/rebar_prv_upgrade.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/rebar_prv_upgrade.erl') diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl index 18c307b..34631ff 100644 --- a/src/rebar_prv_upgrade.erl +++ b/src/rebar_prv_upgrade.erl @@ -32,7 +32,7 @@ init(State) -> {deps, ?DEPS}, {example, "rebar3 upgrade [cowboy[,ranch]]"}, {short_desc, "Upgrade dependencies."}, - {desc, "Upgrade project dependecies. Mentioning no application " + {desc, "Upgrade project dependencies. Mentioning no application " "will upgrade all dependencies. To upgrade specific dependencies, " "their names can be listed in the command."}, {opts, [ -- cgit v1.1 From c7f1554af6dbf4944d4b6a4fae2398aeb3b536d7 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 28 Apr 2017 23:35:01 -0400 Subject: Allow to upgrade profile-specific dependencies Only happens when calling `rebar3 as upgrade `, with the caveat that all profile deps get upgraded and lined up with the rebar config. --- src/rebar_prv_upgrade.erl | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) (limited to 'src/rebar_prv_upgrade.erl') diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl index 34631ff..a0ddbbc 100644 --- a/src/rebar_prv_upgrade.erl +++ b/src/rebar_prv_upgrade.erl @@ -70,7 +70,16 @@ do(State) -> is_atom(Dep) orelse is_atom(element(1, Dep))], Names = parse_names(ec_cnv:to_binary(proplists:get_value(package, Args, <<"">>)), Locks), DepsDict = deps_dict(rebar_state:all_deps(State)), - case prepare_locks(Names, Deps, Locks, [], DepsDict) of + %% Find alternative deps in non-default profiles since they may + %% need to be passed through (they are never locked) + AltProfiles = rebar_state:current_profiles(State) -- [default], + AltProfileDeps = lists:append([ + rebar_state:get(State, {deps, Profile}, []) || Profile <- AltProfiles] + ), + AltDeps = [Dep || Dep <- AltProfileDeps, + is_atom(Dep) orelse is_atom(element(1, Dep)) + andalso not lists:member(Dep, Deps)], + case prepare_locks(Names, Deps, Locks, [], DepsDict, AltDeps) of {error, Reason} -> {error, Reason}; {Locks0, _Unlocks0} -> @@ -115,20 +124,20 @@ parse_names(Bin, Locks) -> Other -> Other end. -prepare_locks([], _, Locks, Unlocks, _Dict) -> +prepare_locks([], _, Locks, Unlocks, _Dict, _AltDeps) -> {Locks, Unlocks}; -prepare_locks([Name|Names], Deps, Locks, Unlocks, Dict) -> +prepare_locks([Name|Names], Deps, Locks, Unlocks, Dict, AltDeps) -> AtomName = binary_to_atom(Name, utf8), case lists:keyfind(Name, 1, Locks) of {_, _, 0} = Lock -> case rebar_utils:tup_find(AtomName, Deps) of false -> ?WARN("Dependency ~s has been removed and will not be upgraded", [Name]), - prepare_locks(Names, Deps, Locks, Unlocks, Dict); + prepare_locks(Names, Deps, Locks, Unlocks, Dict, AltDeps); Dep -> {Source, NewLocks, NewUnlocks} = prepare_lock(Dep, Lock, Locks, Dict), prepare_locks(Names, Deps, NewLocks, - [{Name, Source, 0} | NewUnlocks ++ Unlocks], Dict) + [{Name, Source, 0} | NewUnlocks ++ Unlocks], Dict, AltDeps) end; {_, _, Level} = Lock when Level > 0 -> case rebar_utils:tup_find(AtomName, Deps) of @@ -137,10 +146,15 @@ prepare_locks([Name|Names], Deps, Locks, Unlocks, Dict) -> Dep -> % Dep has been promoted {Source, NewLocks, NewUnlocks} = prepare_lock(Dep, Lock, Locks, Dict), prepare_locks(Names, Deps, NewLocks, - [{Name, Source, 0} | NewUnlocks ++ Unlocks], Dict) + [{Name, Source, 0} | NewUnlocks ++ Unlocks], Dict, AltDeps) end; false -> - ?PRV_ERROR({unknown_dependency, Name}) + case rebar_utils:tup_find(AtomName, AltDeps) of + false -> + ?PRV_ERROR({unknown_dependency, Name}); + _ -> % non-default profile dependency found, pass through + prepare_locks(Names, Deps, Locks, Unlocks, Dict, AltDeps) + end end. prepare_lock(Dep, Lock, Locks, Dict) -> -- cgit v1.1 From 0ce1a4f5fea0a0a58b862e1e375043e7696f9bff Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 19 May 2017 22:34:26 -0400 Subject: Cull default dep names in non-default upgrade Prevents showing false upgrade messages when upgrading in a non-default profile. Also shows an info message explaining default deps don't get updated in a non-default upgrade. --- src/rebar_prv_upgrade.erl | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) (limited to 'src/rebar_prv_upgrade.erl') diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl index a0ddbbc..2de30e4 100644 --- a/src/rebar_prv_upgrade.erl +++ b/src/rebar_prv_upgrade.erl @@ -70,16 +70,9 @@ do(State) -> is_atom(Dep) orelse is_atom(element(1, Dep))], Names = parse_names(ec_cnv:to_binary(proplists:get_value(package, Args, <<"">>)), Locks), DepsDict = deps_dict(rebar_state:all_deps(State)), - %% Find alternative deps in non-default profiles since they may - %% need to be passed through (they are never locked) - AltProfiles = rebar_state:current_profiles(State) -- [default], - AltProfileDeps = lists:append([ - rebar_state:get(State, {deps, Profile}, []) || Profile <- AltProfiles] - ), - AltDeps = [Dep || Dep <- AltProfileDeps, - is_atom(Dep) orelse is_atom(element(1, Dep)) - andalso not lists:member(Dep, Deps)], - case prepare_locks(Names, Deps, Locks, [], DepsDict, AltDeps) of + AltDeps = find_non_default_deps(Deps, State), + FilteredNames = cull_default_names_if_profiles(Names, Deps, State), + case prepare_locks(FilteredNames, Deps, Locks, [], DepsDict, AltDeps) of {error, Reason} -> {error, Reason}; {Locks0, _Unlocks0} -> @@ -124,6 +117,31 @@ parse_names(Bin, Locks) -> Other -> Other end. +%% Find alternative deps in non-default profiles since they may +%% need to be passed through (they are never locked) +find_non_default_deps(Deps, State) -> + AltProfiles = rebar_state:current_profiles(State) -- [default], + AltProfileDeps = lists:append([ + rebar_state:get(State, {deps, Profile}, []) || Profile <- AltProfiles] + ), + [Dep || Dep <- AltProfileDeps, + is_atom(Dep) orelse is_atom(element(1, Dep)) + andalso not lists:member(Dep, Deps)]. + +%% If any alt profiles are used, remove the default profiles from +%% the upgrade list and warn about it. +cull_default_names_if_profiles(Names, Deps, State) -> + case rebar_state:current_profiles(State) of + [default] -> + Names; + _ -> + ?INFO("Dependencies in the default profile will not be upgraded", []), + lists:filter(fun(Name) -> + AtomName = binary_to_atom(Name, utf8), + rebar_utils:tup_find(AtomName, Deps) == false + end, Names) + end. + prepare_locks([], _, Locks, Unlocks, _Dict, _AltDeps) -> {Locks, Unlocks}; prepare_locks([Name|Names], Deps, Locks, Unlocks, Dict, AltDeps) -> -- cgit v1.1 From 963c49f5eb9ab5b34e1843fb43305743720917ac Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sun, 6 Aug 2017 07:26:21 -0400 Subject: Unicode support in all the places This is done through 3 main change groups: - replacing `~s` by `~ts` in format strings, so that strings that contain unicode are properly printed rather than crashing - adding the `unicode` argument to all function of the `re` module to ensure transformations on strings containing unicode data are valid instead of crashing (see issue #1302) - replacing `ec_cnv:to_binary/1` and `ec_cnv:to_list/1` with matching functions in `rebar_utils`. The last point has been done, rather than modifying and updating erlware commons, because binary and list conversions can be a contentious subject. For example, if what is being handled is actually bytes from a given binary stream, then forcing a byte-oriented interpretation of the data can corrupt it. As such, it does not appear safe to modify erlware commons' conversion functions since it may not be safe for all its users. Instead, rebar3 reimplements a subset of them (only converting atoms and chardata, ignoring numbers) with the explicit purpose of handling unicode string data. Tests were left as unchanged as possible. This may impact the ability to run rebar3's own suites in a unicode path, but respects a principle of least change for such a large patch. --- src/rebar_prv_upgrade.erl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'src/rebar_prv_upgrade.erl') diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl index 2de30e4..5a7dff8 100644 --- a/src/rebar_prv_upgrade.erl +++ b/src/rebar_prv_upgrade.erl @@ -68,7 +68,7 @@ do(State) -> ProfileDeps = rebar_state:get(State, {deps, default}, []), Deps = [Dep || Dep <- TopDeps ++ ProfileDeps, % TopDeps > ProfileDeps is_atom(Dep) orelse is_atom(element(1, Dep))], - Names = parse_names(ec_cnv:to_binary(proplists:get_value(package, Args, <<"">>)), Locks), + Names = parse_names(rebar_utils:to_binary(proplists:get_value(package, Args, <<"">>)), Locks), DepsDict = deps_dict(rebar_state:all_deps(State)), AltDeps = find_non_default_deps(Deps, State), FilteredNames = cull_default_names_if_profiles(Names, Deps, State), @@ -109,7 +109,7 @@ format_error(Reason) -> io_lib:format("~p", [Reason]). parse_names(Bin, Locks) -> - case lists:usort(re:split(Bin, <<" *, *">>, [trim])) of + case lists:usort(re:split(Bin, <<" *, *">>, [trim, unicode])) of %% Nothing submitted, use *all* apps [<<"">>] -> [Name || {Name, _, 0} <- Locks]; [] -> [Name || {Name, _, 0} <- Locks]; @@ -150,7 +150,7 @@ prepare_locks([Name|Names], Deps, Locks, Unlocks, Dict, AltDeps) -> {_, _, 0} = Lock -> case rebar_utils:tup_find(AtomName, Deps) of false -> - ?WARN("Dependency ~s has been removed and will not be upgraded", [Name]), + ?WARN("Dependency ~ts has been removed and will not be upgraded", [Name]), prepare_locks(Names, Deps, Locks, Unlocks, Dict, AltDeps); Dep -> {Source, NewLocks, NewUnlocks} = prepare_lock(Dep, Lock, Locks, Dict), @@ -181,7 +181,7 @@ prepare_lock(Dep, Lock, Locks, Dict) -> {Name, _, Src} -> {Name, Src}; _ when is_atom(Dep) -> %% version-free package. Must unlock whatever matches in locks - {_, Vsn, _} = lists:keyfind(ec_cnv:to_binary(Dep), 1, Locks), + {_, Vsn, _} = lists:keyfind(rebar_utils:to_binary(Dep), 1, Locks), {Dep, Vsn} end, Children = all_children(Name1, Dict), @@ -197,7 +197,7 @@ unlock_children(Children, Locks) -> unlock_children(_, [], Locks, Unlocks) -> {Locks, Unlocks}; unlock_children(Children, [App = {Name,_,_} | Apps], Locks, Unlocks) -> - case lists:member(ec_cnv:to_binary(Name), Children) of + case lists:member(rebar_utils:to_binary(Name), Children) of true -> unlock_children(Children, Apps, Locks, [App | Unlocks]); false -> @@ -215,7 +215,7 @@ all_children(Name, Dict) -> lists:flatten(all_children_(Name, Dict)). all_children_(Name, Dict) -> - case dict:find(ec_cnv:to_binary(Name), Dict) of + case dict:find(rebar_utils:to_binary(Name), Dict) of {ok, Children} -> [Children | [all_children_(Child, Dict) || Child <- Children]]; error -> -- cgit v1.1 From 71fe6c9b5f31fbd16c5f71f7b3ea51b42e11722a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jimmy=20Z=C3=B6ger?= Date: Thu, 14 Sep 2017 16:51:09 +0200 Subject: Changes word 'transient' to 'transitive' which is what it is supposed to say --- src/rebar_prv_upgrade.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/rebar_prv_upgrade.erl') diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl index 5a7dff8..cd75672 100644 --- a/src/rebar_prv_upgrade.erl +++ b/src/rebar_prv_upgrade.erl @@ -102,7 +102,7 @@ do(State) -> format_error({unknown_dependency, Name}) -> io_lib:format("Dependency ~ts not found", [Name]); format_error({transitive_dependency, Name}) -> - io_lib:format("Dependency ~ts is transient and cannot be safely upgraded. " + io_lib:format("Dependency ~ts is transitive and cannot be safely upgraded. " "Promote it to your top-level rebar.config file to upgrade it.", [Name]); format_error(Reason) -> -- cgit v1.1 From 06147af85baa1064c95f5fe5eb2c2d41a67c21a8 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Tue, 5 Jun 2018 20:43:23 -0400 Subject: Add hooks to the upgrade command As requested in #1733 --- src/rebar_prv_upgrade.erl | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'src/rebar_prv_upgrade.erl') diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl index cd75672..e4469cf 100644 --- a/src/rebar_prv_upgrade.erl +++ b/src/rebar_prv_upgrade.erl @@ -43,6 +43,19 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> + Cwd = rebar_state:dir(State), + Providers = rebar_state:providers(State), + rebar_hooks:run_project_and_app_hooks(Cwd, pre, ?PROVIDER, Providers, State), + case do_(State) of + {ok, NewState} -> + rebar_hooks:run_project_and_app_hooks(Cwd, post, ?PROVIDER, Providers, NewState), + {ok, NewState}; + Other -> + rebar_hooks:run_project_and_app_hooks(Cwd, post, ?PROVIDER, Providers, State), + Other + end. + +do_(State) -> {Args, _} = rebar_state:command_parsed_args(State), Locks = rebar_state:get(State, {locks, default}, []), %% We have 3 sources of dependencies to upgrade from: -- cgit v1.1 From 56b7d88975aa8da6446857cfd92de0825024bf63 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 13 Sep 2018 19:36:00 -0600 Subject: support for hex v2, multiple repository fetching, private organizations (#1884) * update to hex_core for hex-v2 repo support (#1865) * update to hex_core for hex-v2 repo support This patch adds only single repo hex-v2 support through hex_core. Packages no longer filtered out by buildtool metadata and the package index is updated per-package instead of fetched as one large ets dump. * tell travis to also build hex_core branch * support list of repos for hex packages (#1866) * support list of repos for hex packages repos are defined under the hex key in rebar configs. They can be defined at the top level of a project or globally, but not in profiles and the repos configured in dependencies are also ignored. Searching for packages involves first checking for a match in the local repo index cache, in the order repos are defined. If not found each repo is checked through the hex api for any known versions of the package and the first repo with a version that fits the constraint is used. * add {repos, replace, []} for overriding the global & default repos * add hex auth handling for repos (#1874) auth token are kept in a hex.config file that is modified by the rebar3 hex plugin. Repo names that have a : separating a parent and child are considered organizations. The parent repo's auth will be included with the child. So an organization named hexpm:rebar3_test will include any hexpm auth tokens found in the rebar3_test organization's configuration. * move packages to top level of of hexpm cache dir (#1876) * move packages to top level of of hexpm cache dir * append organization name to parent's repo_url when parsing repos * only eval config scripts and apply overrides once per app (#1879) * only eval config scripts and apply overrides once per app * move new resource behaviour to rebar_resource_v2 and keep v1 * cleanup use of rebar_resource module and unused functions * cleanup error messages and unused code * when discovering apps support mix packages as unbuilt apps (#1882) * use hex_core tarball unpacking support in pkg resource (#1883) * use hex_core tarball unpacking support in pkg resource * ignore etag if package doesn't exist and delete if checksum fails * add back tests for bad package checksums * improve bad registry checksum error message --- src/rebar_prv_upgrade.erl | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) (limited to 'src/rebar_prv_upgrade.erl') diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl index e4469cf..b1b1b16 100644 --- a/src/rebar_prv_upgrade.erl +++ b/src/rebar_prv_upgrade.erl @@ -82,17 +82,22 @@ do_(State) -> Deps = [Dep || Dep <- TopDeps ++ ProfileDeps, % TopDeps > ProfileDeps is_atom(Dep) orelse is_atom(element(1, Dep))], Names = parse_names(rebar_utils:to_binary(proplists:get_value(package, Args, <<"">>)), Locks), + DepsDict = deps_dict(rebar_state:all_deps(State)), AltDeps = find_non_default_deps(Deps, State), FilteredNames = cull_default_names_if_profiles(Names, Deps, State), case prepare_locks(FilteredNames, Deps, Locks, [], DepsDict, AltDeps) of {error, Reason} -> {error, Reason}; - {Locks0, _Unlocks0} -> + {Locks0, Unlocks0} -> Deps0 = top_level_deps(Deps, Locks), State1 = rebar_state:set(State, {deps, default}, Deps0), DepsDir = rebar_prv_install_deps:profile_dep_dir(State, default), D = rebar_app_utils:parse_deps(root, DepsDir, Deps0, State1, Locks0, 0), + + %% first update the package index for the packages to be upgraded + update_pkg_deps(Unlocks0, D, State1), + State2 = rebar_state:set(State1, {parsed_deps, default}, D), State3 = rebar_state:set(State2, {locks, default}, Locks0), State4 = rebar_state:set(State3, upgrade, true), @@ -121,6 +126,34 @@ format_error({transitive_dependency, Name}) -> format_error(Reason) -> io_lib:format("~p", [Reason]). +%% fetch updates for package deps that have been unlocked for upgrade +update_pkg_deps([], _, _) -> + ok; +update_pkg_deps([{Name, _, _} | Rest], AppInfos, State) -> + case rebar_app_utils:find(Name, AppInfos) of + {ok, AppInfo} -> + case element(1, rebar_app_info:source(AppInfo)) of + pkg -> + Resources = rebar_state:resources(State), + #{repos := RepoConfigs} = rebar_resource_v2:find_resource_state(pkg, Resources), + [update_package(Name, RepoConfig, State) || RepoConfig <- RepoConfigs]; + _ -> + skip + end; + _ -> + %% this should be impossible... + skip + end, + update_pkg_deps(Rest, AppInfos, State). + +update_package(Name, RepoConfig, State) -> + case rebar_packages:update_package(Name, RepoConfig, State) of + fail -> + ?WARN("Failed to fetch updates for package ~ts from repo ~ts", [Name, maps:get(name, RepoConfig)]); + _ -> + ok + end. + parse_names(Bin, Locks) -> case lists:usort(re:split(Bin, <<" *, *">>, [trim, unicode])) of %% Nothing submitted, use *all* apps -- cgit v1.1 From 4bd2b579c8b4def5353f4145f7392358f0fbaef4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andra=CC=81s=20Boroska?= Date: Sun, 2 Dec 2018 20:28:22 +0100 Subject: fix pkg name during update by using pkg_name understood by hex --- src/rebar_prv_upgrade.erl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src/rebar_prv_upgrade.erl') diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl index b1b1b16..565f342 100644 --- a/src/rebar_prv_upgrade.erl +++ b/src/rebar_prv_upgrade.erl @@ -132,11 +132,13 @@ update_pkg_deps([], _, _) -> update_pkg_deps([{Name, _, _} | Rest], AppInfos, State) -> case rebar_app_utils:find(Name, AppInfos) of {ok, AppInfo} -> - case element(1, rebar_app_info:source(AppInfo)) of + Source = rebar_app_info:source(AppInfo), + case element(1, Source) of pkg -> Resources = rebar_state:resources(State), #{repos := RepoConfigs} = rebar_resource_v2:find_resource_state(pkg, Resources), - [update_package(Name, RepoConfig, State) || RepoConfig <- RepoConfigs]; + PkgName = element(2, Source), + [update_package(PkgName, RepoConfig, State) || RepoConfig <- RepoConfigs]; _ -> skip end; -- cgit v1.1