diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/rebar.app.src | 3 | ||||
| -rw-r--r-- | src/rebar.hrl | 6 | ||||
| -rw-r--r-- | src/rebar3.erl | 13 | ||||
| -rw-r--r-- | src/rebar_app_discover.erl | 78 | ||||
| -rw-r--r-- | src/rebar_app_info.erl | 37 | ||||
| -rw-r--r-- | src/rebar_app_utils.erl | 117 | ||||
| -rw-r--r-- | src/rebar_config.erl | 9 | ||||
| -rw-r--r-- | src/rebar_digraph.erl | 111 | ||||
| -rw-r--r-- | src/rebar_erlc_compiler.erl | 64 | ||||
| -rw-r--r-- | src/rebar_fetch.erl | 2 | ||||
| -rw-r--r-- | src/rebar_file_utils.erl | 6 | ||||
| -rw-r--r-- | src/rebar_git_resource.erl | 35 | ||||
| -rw-r--r-- | src/rebar_packages.erl | 188 | ||||
| -rw-r--r-- | src/rebar_pkg_resource.erl | 4 | ||||
| -rw-r--r-- | src/rebar_plugins.erl | 12 | ||||
| -rw-r--r-- | src/rebar_prv_app_discovery.erl | 4 | ||||
| -rw-r--r-- | src/rebar_prv_common_test.erl | 8 | ||||
| -rw-r--r-- | src/rebar_prv_deps.erl | 36 | ||||
| -rw-r--r-- | src/rebar_prv_deps_tree.erl | 84 | ||||
| -rw-r--r-- | src/rebar_prv_install_deps.erl | 411 | ||||
| -rw-r--r-- | src/rebar_prv_lock.erl | 26 | ||||
| -rw-r--r-- | src/rebar_prv_packages.erl | 18 | ||||
| -rw-r--r-- | src/rebar_prv_plugins_upgrade.erl | 2 | ||||
| -rw-r--r-- | src/rebar_prv_update.erl | 103 | ||||
| -rw-r--r-- | src/rebar_prv_upgrade.erl | 21 | ||||
| -rw-r--r-- | src/rebar_relx.erl | 11 | ||||
| -rw-r--r-- | src/rebar_state.erl | 102 |
27 files changed, 780 insertions, 731 deletions
diff --git a/src/rebar.app.src b/src/rebar.app.src index 0a2e6cf..4ef493b 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -23,7 +23,7 @@ erlware_commons, providers, bbmustache, - ssl_verify_hostname, + ssl_verify_hostname, relx, inets]}, {env, [ @@ -41,6 +41,7 @@ rebar_prv_compile, rebar_prv_cover, rebar_prv_deps, + rebar_prv_deps_tree, rebar_prv_dialyzer, rebar_prv_do, rebar_prv_edoc, diff --git a/src/rebar.hrl b/src/rebar.hrl index 8a702df..961b8ea 100644 --- a/src/rebar.hrl +++ b/src/rebar.hrl @@ -23,8 +23,14 @@ -define(DEFAULT_RELEASE_DIR, "rel"). -define(DEFAULT_CONFIG_FILE, "rebar.config"). -define(DEFAULT_CDN, "https://s3.amazonaws.com/s3.hex.pm/tarballs"). +-define(DEFAULT_HEX_REGISTRY, "https://s3.amazonaws.com/s3.hex.pm/registry.ets.gz"). -define(LOCK_FILE, "rebar.lock"). +-define(PACKAGE_INDEX_VERSION, 3). +-define(PACKAGE_TABLE, package_index). +-define(INDEX_FILE, "packages.idx"). +-define(REGISTRY_FILE, "registry"). + -ifdef(namespaced_types). -type rebar_dict() :: dict:dict(). -else. diff --git a/src/rebar3.erl b/src/rebar3.erl index 8004443..ab3e0eb 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -105,6 +105,7 @@ run_aux(State, RawArgs) -> {ok, Providers} = application:get_env(rebar, providers), %% Providers can modify profiles stored in opts, so set default after initializing providers State4 = rebar_state:create_logic_providers(Providers, State3), + rebar_packages:packages(State4), State5 = rebar_plugins:project_apps_install(State4), State6 = rebar_state:default(State5, rebar_state:opts(State5)), @@ -280,9 +281,15 @@ state_from_global_config(Config, GlobalConfigFile) -> %% We don't want to worry about global plugin install state effecting later %% usage. So we throw away the global profile state used for plugin install. GlobalConfigThrowAway = rebar_state:current_profiles(GlobalConfig, [global]), - GlobalState = rebar_plugins:handle_plugins(global, - rebar_state:get(GlobalConfigThrowAway, plugins, []), - GlobalConfigThrowAway), + GlobalState = case rebar_state:get(GlobalConfigThrowAway, plugins, []) of + [] -> + GlobalConfigThrowAway; + GlobalPluginsToInstall -> + rebar_packages:packages(GlobalConfigThrowAway), + rebar_plugins:handle_plugins(global, + GlobalPluginsToInstall, + GlobalConfigThrowAway) + end, GlobalPlugins = rebar_state:providers(GlobalState), GlobalConfig2 = rebar_state:set(GlobalConfig, plugins, []), GlobalConfig3 = rebar_state:set(GlobalConfig2, {plugins, global}, rebar_state:get(GlobalConfigThrowAway, plugins, [])), diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index 3b34539..5a25a9e 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -16,7 +16,26 @@ do(State, LibDirs) -> Apps = find_apps(Dirs, all), ProjectDeps = rebar_state:deps_names(State), DepsDir = rebar_dir:deps_dir(State), + CurrentProfiles = rebar_state:current_profiles(State), + + %% There may be a top level src which is an app and there may not + %% Find it here if there is, otherwise define the deps parent as root + TopLevelApp = define_root_app(Apps, State), + + %% Handle top level deps + State1 = lists:foldl(fun(Profile, StateAcc) -> + ProfileDeps = rebar_state:get(StateAcc, {deps, Profile}, []), + ProfileDeps2 = rebar_utils:tup_dedup(rebar_utils:tup_sort(ProfileDeps)), + StateAcc1 = rebar_state:set(StateAcc, {deps, Profile}, ProfileDeps2), + ParsedDeps = parse_profile_deps(Profile + ,TopLevelApp + ,ProfileDeps2 + ,StateAcc1 + ,StateAcc1), + rebar_state:set(StateAcc1, {parsed_deps, Profile}, ParsedDeps) + end, State, lists:reverse(CurrentProfiles)), + %% Handle sub project apps deps %% Sort apps so we get the same merged deps config everytime SortedApps = rebar_utils:sort_deps(Apps), lists:foldl(fun(AppInfo, StateAcc) -> @@ -33,7 +52,19 @@ do(State, LibDirs) -> ?INFO("Ignoring ~s", [Name]), StateAcc end - end, State, SortedApps). + end, State1, SortedApps). + +define_root_app(Apps, State) -> + RootDir = rebar_dir:root_dir(State), + case ec_lists:find(fun(X) -> + ec_file:real_dir_path(rebar_app_info:dir(X)) =:= + ec_file:real_dir_path(RootDir) + end, Apps) of + {ok, App} -> + rebar_app_info:name(App); + error -> + root + end. format_error({module_list, File}) -> io_lib:format("Error reading module list from ~p~n", [File]); @@ -51,24 +82,47 @@ merge_deps(AppInfo, State) -> rebar_state:apply_profiles( rebar_state:new(reset_hooks(rebar_state:opts(State, Default)), C, rebar_app_info:dir(AppInfo)), CurrentProfiles), Name), + AppState1 = rebar_state:overrides(AppState, rebar_state:get(AppState, overrides, [])), - rebar_utils:check_min_otp_version(rebar_state:get(AppState, minimum_otp_vsn, undefined)), - rebar_utils:check_blacklisted_otp_versions(rebar_state:get(AppState, blacklisted_otp_vsns, [])), + rebar_utils:check_min_otp_version(rebar_state:get(AppState1, minimum_otp_vsn, undefined)), + rebar_utils:check_blacklisted_otp_versions(rebar_state:get(AppState1, blacklisted_otp_vsns, [])), - AppState1 = rebar_state:set(AppState, artifacts, []), - AppInfo1 = rebar_app_info:state(AppInfo, AppState1), + AppState2 = rebar_state:set(AppState1, artifacts, []), + AppInfo1 = rebar_app_info:state(AppInfo, AppState2), State1 = lists:foldl(fun(Profile, StateAcc) -> - AppProfDeps = rebar_state:get(AppState, {deps, Profile}, []), - TopLevelProfDeps = rebar_state:get(StateAcc, {deps, Profile}, []), - ProfDeps2 = rebar_utils:tup_dedup(rebar_utils:tup_umerge( - rebar_utils:tup_sort(TopLevelProfDeps) - ,rebar_utils:tup_sort(AppProfDeps))), - rebar_state:set(StateAcc, {deps, Profile}, ProfDeps2) + handle_profile(Profile, Name, AppState1, StateAcc) end, State, lists:reverse(CurrentProfiles)), {AppInfo1, State1}. +handle_profile(Profile, Name, AppState, State) -> + TopParsedDeps = rebar_state:get(State, {parsed_deps, Profile}, {[], []}), + TopLevelProfileDeps = rebar_state:get(State, {deps, Profile}, []), + AppProfileDeps = rebar_state:get(AppState, {deps, Profile}, []), + AppProfileDeps2 = rebar_utils:tup_dedup(rebar_utils:tup_sort(AppProfileDeps)), + ProfileDeps2 = rebar_utils:tup_dedup(rebar_utils:tup_umerge( + rebar_utils:tup_sort(TopLevelProfileDeps) + ,rebar_utils:tup_sort(AppProfileDeps2))), + State1 = rebar_state:set(State, {deps, Profile}, ProfileDeps2), + + %% Only deps not also specified in the top level config need + %% to be included in the parsed deps + NewDeps = ProfileDeps2 -- TopLevelProfileDeps, + ParsedDeps = parse_profile_deps(Profile, Name, NewDeps, AppState, State1), + State2 = rebar_state:set(State1, {deps, Profile}, ProfileDeps2), + rebar_state:set(State2, {parsed_deps, Profile}, TopParsedDeps++ParsedDeps). + +parse_profile_deps(Profile, Name, Deps, AppState, State) -> + DepsDir = rebar_prv_install_deps:profile_dep_dir(State, Profile), + Locks = rebar_state:get(State, {locks, Profile}, []), + rebar_app_utils:parse_deps(Name + ,DepsDir + ,Deps + ,AppState + ,Locks + ,1). + project_app_config(AppInfo, State) -> C = rebar_config:consult(rebar_app_info:dir(AppInfo)), Dir = rebar_app_info:dir(AppInfo), @@ -144,7 +198,7 @@ create_app_info(AppDir, AppFile) -> AppVsn = proplists:get_value(vsn, AppDetails), Applications = proplists:get_value(applications, AppDetails, []), IncludedApplications = proplists:get_value(included_applications, AppDetails, []), - {ok, AppInfo} = rebar_app_info:new(AppName, AppVsn, AppDir, []), + {ok, AppInfo} = rebar_app_info:new(AppName, AppVsn, AppDir), AppInfo1 = rebar_app_info:applications( rebar_app_info:app_details(AppInfo, AppDetails), IncludedApplications++Applications), diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index 6962c5a..bb5104e 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -4,6 +4,7 @@ new/2, new/3, new/4, + new/5, discover/1, name/1, name/2, @@ -17,6 +18,8 @@ app_file/2, app_details/1, app_details/2, + parent/1, + parent/2, original_vsn/1, original_vsn/2, ebin_dir/1, @@ -32,6 +35,8 @@ dir/2, out_dir/1, out_dir/2, + resource_type/1, + resource_type/2, source/1, source/2, state/1, @@ -54,6 +59,7 @@ app_file :: file:filename_all() | undefined, config :: rebar_state:t() | undefined, original_vsn :: binary() | string() | undefined, + parent=root :: binary() | root, app_details=[] :: list(), applications=[] :: list(), deps=[] :: list(), @@ -61,6 +67,7 @@ dep_level=0 :: integer(), dir :: file:name(), out_dir :: file:name(), + resource_type :: pkg | src, source :: string() | tuple() | undefined, state :: rebar_state:t() | undefined, is_lock=false :: boolean(), @@ -107,6 +114,17 @@ new(AppName, Vsn, Dir, Deps) -> out_dir=ec_cnv:to_list(Dir), deps=Deps}}. +%% @doc build a complete version of the app info with all fields set. +-spec new(atom() | binary(), atom() | binary() | string(), binary() | string(), file:name(), list()) -> + {ok, t()}. +new(Parent, AppName, Vsn, Dir, Deps) -> + {ok, #app_info_t{name=ec_cnv:to_binary(AppName), + parent=Parent, + original_vsn=Vsn, + dir=ec_cnv:to_list(Dir), + out_dir=ec_cnv:to_list(Dir), + deps=Deps}}. + %% @doc discover a complete version of the app info with all fields set. -spec discover(file:filename_all()) -> {ok, t()} | not_found. discover(Dir) -> @@ -203,6 +221,13 @@ app_details(#app_info_t{app_details=AppDetails}) -> app_details(AppInfo=#app_info_t{}, AppDetails) -> AppInfo#app_info_t{app_details=AppDetails}. +parent(#app_info_t{parent=Parent}) -> + Parent. + +-spec parent(t(), binary() | root) -> t(). +parent(AppInfo=#app_info_t{}, Parent) -> + AppInfo#app_info_t{parent=Parent}. + -spec original_vsn(t()) -> string(). original_vsn(#app_info_t{original_vsn=Vsn}) -> Vsn. @@ -264,6 +289,14 @@ out_dir(AppInfo=#app_info_t{}, OutDir) -> ebin_dir(#app_info_t{out_dir=OutDir}) -> ec_cnv:to_list(filename:join(OutDir, "ebin")). +-spec resource_type(t(), pkg | src) -> t(). +resource_type(AppInfo=#app_info_t{}, Type) -> + AppInfo#app_info_t{resource_type=Type}. + +-spec resource_type(t()) -> pkg | src. +resource_type(#app_info_t{resource_type=ResourceType}) -> + ResourceType. + -spec source(t(), string() | tuple()) -> t(). source(AppInfo=#app_info_t{}, Source) -> AppInfo#app_info_t{source=Source}. @@ -284,7 +317,7 @@ state(#app_info_t{state=State}) -> state_or_new(State, AppInfo=#app_info_t{state=undefined}) -> AppDir = dir(AppInfo), C = rebar_config:consult(AppDir), - rebar_state:new(State, C, AppDir); + rebar_state:new(State, C, AppInfo); state_or_new(_State, #app_info_t{state=State}) -> State. @@ -306,7 +339,7 @@ is_checkout(#app_info_t{is_checkout=IsCheckout}) -> -spec valid(t()) -> boolean(). valid(AppInfo=#app_info_t{valid=undefined, state=State}) -> - case rebar_app_utils:validate_application_info(AppInfo) + case rebar_app_utils:validate_application_info(AppInfo) =:= true andalso rebar_state:has_all_artifacts(State) =:= true of true -> true; diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index e9745c3..88b75ac 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -32,6 +32,9 @@ app_src_to_app/2, validate_application_info/1, validate_application_info/2, + parse_deps/5, + parse_deps/6, + dep_to_app/7, format_error/1]). -include("rebar.hrl"). @@ -87,6 +90,98 @@ validate_application_info(AppInfo, AppDetail) -> end end. +-spec parse_deps(binary(), list(), rebar_state:t(), list(), integer()) -> {[rebar_app_info:t()], [tuple()]}. +parse_deps(DepsDir, Deps, State, Locks, Level) -> + parse_deps(root, DepsDir, Deps, State, Locks, Level). + +parse_deps(Parent, DepsDir, Deps, State, Locks, Level) -> + [parse_dep(Dep, Parent, DepsDir, State, Locks, Level) || Dep <- Deps]. + +parse_dep(Dep, Parent, DepsDir, State, Locks, Level) -> + Name = case Dep of + Dep when is_tuple(Dep) -> + element(1, Dep); + Dep -> + Dep + end, + case lists:keyfind(ec_cnv:to_binary(Name), 1, Locks) of + false -> + parse_dep(Parent, Dep, DepsDir, false, State); + LockedDep -> + LockedLevel = element(3, LockedDep), + case LockedLevel > Level of + true -> + parse_dep(Parent, Dep, DepsDir, false, State); + false -> + parse_dep(Parent, LockedDep, DepsDir, true, State) + end + end. + +parse_dep(Parent, {Name, Vsn, {pkg, PkgName}}, DepsDir, IsLock, State) -> + {PkgName1, PkgVsn} = parse_goal(ec_cnv:to_binary(PkgName), ec_cnv:to_binary(Vsn)), + pkg_to_app(Parent, DepsDir, Name, PkgName1, PkgVsn, IsLock, State); +parse_dep(Parent, {Name, {pkg, PkgName}}, DepsDir, IsLock, State) -> + %% Package dependency with different package name from app name + {PkgName1, PkgVsn} = get_package(ec_cnv:to_binary(PkgName), State), + pkg_to_app(Parent, DepsDir, Name, PkgName1, PkgVsn, IsLock, State); +parse_dep(Parent, {Name, Vsn}, DepsDir, IsLock, State) when is_list(Vsn); is_binary(Vsn) -> + %% Versioned Package dependency + {PkgName, PkgVsn} = parse_goal(ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)), + pkg_to_app(Parent, DepsDir, PkgName, PkgName, PkgVsn, IsLock, State); +parse_dep(Parent, Name, DepsDir, IsLock, State) when is_atom(Name); is_binary(Name) -> + %% Unversioned package dependency + {PkgName, PkgVsn} = get_package(ec_cnv:to_binary(Name), State), + pkg_to_app(Parent, DepsDir, PkgName, PkgName, PkgVsn, IsLock, State); +parse_dep(Parent, {Name, Source}, DepsDir, IsLock, State) when is_tuple(Source) -> + dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); +parse_dep(Parent, {Name, _Vsn, Source}, DepsDir, IsLock, State) when is_tuple(Source) -> + dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); +parse_dep(Parent, {Name, _Vsn, Source, Opts}, DepsDir, IsLock, State) when is_tuple(Source) -> + ?WARN("Dependency option list ~p in ~p is not supported and will be ignored", [Opts, Name]), + dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); +parse_dep(Parent, {Name, {pkg, PkgName, Vsn}, Level}, DepsDir, IsLock, State) when is_integer(Level) -> + pkg_to_app(Parent, DepsDir, Name, PkgName, Vsn, IsLock, State); +parse_dep(Parent, {Name, Source, Level}, DepsDir, IsLock, State) when is_tuple(Source) + , is_integer(Level) -> + dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); +parse_dep(_, Dep, _, _, _) -> + throw(?PRV_ERROR({parse_dep, Dep})). + +%% Verify package exists and create the AppInfo record +pkg_to_app(Parent, DepsDir, AppName, PkgName, PkgVsn, IsLock, State) -> + %% Verify package actually exists. This will throw a missing_package exception + Deps = rebar_packages:deps(PkgName, PkgVsn, State), + Source = {pkg, PkgName, PkgVsn}, + AppInfo = dep_to_app(Parent, DepsDir, AppName, PkgVsn, Source, IsLock, State), + rebar_app_info:resource_type(rebar_app_info:deps(AppInfo, Deps), pkg). + +dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> + CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), + BaseDir = rebar_state:get(State, base_dir, []), + {ok, App1} = case rebar_app_info:discover(CheckoutsDir) of + {ok, App} -> + {ok, rebar_app_info:is_checkout(App, true)}; + not_found -> + Dir = ec_cnv:to_list(filename:join(DepsDir, Name)), + case rebar_app_info:discover(Dir) of + {ok, App} -> + {ok, App}; + not_found -> + rebar_app_info:new(Parent, Name, Vsn, Dir, []) + end + end, + C = rebar_config:consult(rebar_app_info:dir(App1)), + S = rebar_state:new(rebar_state:new(), C, App1), + Overrides = rebar_state:get(State, overrides, []), + ParentOverrides = rebar_state:overrides(State), + S1 = rebar_state:set(rebar_state:overrides(S, ParentOverrides++Overrides), base_dir, BaseDir), + App2 = rebar_app_info:state(App1, S1), + rebar_app_info:is_lock(rebar_app_info:source(App2, Source), IsLock). + +format_error({missing_package, Package}) -> + io_lib:format("Package not found in registry: ~s", [Package]); +format_error({parse_dep, Dep}) -> + io_lib:format("Failed parsing dep ~p", [Dep]); format_error(Error) -> io_lib:format("~p", [Error]). @@ -94,11 +189,29 @@ format_error(Error) -> %% Internal functions %% =================================================================== +-spec parse_goal(binary(), binary()) -> {binary(), binary()} | {binary(), binary(), binary()}. +parse_goal(Name, Constraint) -> + case re:run(Constraint, "([^\\d]*)(\\d.*)", [{capture, [1,2], binary}]) of + {match, [<<>>, Vsn]} -> + {Name, Vsn}; + {match, [Op, Vsn]} -> + {Name, Vsn, binary_to_atom(Op, utf8)}; + nomatch -> + throw(?PRV_ERROR({bad_constraint, Name, Constraint})) + end. + +get_package(Dep, State) -> + case rebar_packages:find_highest_matching(Dep, "0", ?PACKAGE_TABLE, State) of + {ok, HighestDepVsn} -> + {Dep, HighestDepVsn}; + none -> + throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Dep)})) + end. + -spec has_all_beams(file:filename_all(), [module()]) -> true | ?PRV_ERROR({missing_module, module()}). has_all_beams(EbinDir, [Module | ModuleList]) -> - BeamFile = filename:join([EbinDir, - ec_cnv:to_list(Module) ++ ".beam"]), + BeamFile = filename:join([EbinDir, ec_cnv:to_list(Module) ++ ".beam"]), case filelib:is_file(BeamFile) of true -> has_all_beams(EbinDir, ModuleList); diff --git a/src/rebar_config.erl b/src/rebar_config.erl index 554b399..b9b8b2e 100644 --- a/src/rebar_config.erl +++ b/src/rebar_config.erl @@ -128,11 +128,20 @@ find_newly_added(ConfigDeps, LockedDeps) -> check_newly_added({_, _}=Dep, LockedDeps) -> check_newly_added_(Dep, LockedDeps); +check_newly_added({_, _, {pkg, _}}=Dep, LockedDeps) -> + check_newly_added_(Dep, LockedDeps); check_newly_added({Name, _, Source}, LockedDeps) -> check_newly_added_({Name, Source}, LockedDeps); check_newly_added(Dep, LockedDeps) -> check_newly_added_(Dep, LockedDeps). +check_newly_added_({Name, Vsn, Source}, LockedDeps) -> + case check_newly_added_(Name, LockedDeps) of + {true, Name1} -> + {true, {Name1, Vsn, Source}}; + false -> + false + end; check_newly_added_({Name, Source}, LockedDeps) -> case check_newly_added_(Name, LockedDeps) of {true, Name1} -> diff --git a/src/rebar_digraph.erl b/src/rebar_digraph.erl index e989fdc..363253a 100644 --- a/src/rebar_digraph.erl +++ b/src/rebar_digraph.erl @@ -2,10 +2,7 @@ -export([compile_order/1 ,restore_graph/1 - ,cull_deps/3 - ,cull_deps/4 ,subgraph/2 - ,print_solution/2 ,format_error/1]). -include("rebar.hrl"). @@ -18,20 +15,23 @@ compile_order(Apps) -> Deps = all_apps_deps(App), add(Graph, {Name, Deps}) end, Apps), - case digraph_utils:topsort(Graph) of - false -> - case digraph_utils:is_acyclic(Graph) of - true -> - {error, no_sort}; - false -> - Cycles = lists:sort( - [lists:sort(Comp) || Comp <- digraph_utils:strong_components(Graph), - length(Comp)>1]), - {error, {cycles, Cycles}} - end; - V -> - {ok, names_to_apps(lists:reverse(V), Apps)} - end. + Order = + case digraph_utils:topsort(Graph) of + false -> + case digraph_utils:is_acyclic(Graph) of + true -> + {error, no_sort}; + false -> + Cycles = lists:sort( + [lists:sort(Comp) || Comp <- digraph_utils:strong_components(Graph), + length(Comp)>1]), + {error, {cycles, Cycles}} + end; + V -> + {ok, names_to_apps(lists:reverse(V), Apps)} + end, + true = digraph:delete(Graph), + Order. add(Graph, {PkgName, Deps}) -> case digraph:vertex(Graph, PkgName) of @@ -67,80 +67,6 @@ restore_graph({Vs, Es}) -> end, Es), Graph. -%% Pick packages to fullfill dependencies -%% The first dep while traversing the graph is chosen and any conflicting -%% dep encountered later on is ignored. - -cull_deps(Graph, Vertices, Level) -> - {ok, LvlVertices, Discarded, _} = cull_deps(Graph, Vertices, Level, none), - {ok, LvlVertices, Discarded}. - -cull_deps(Graph, Vertices, Level, SolutionGraph) -> - cull_deps(Graph, - Vertices, - Level+1, - lists:foldl(fun({Key, _}, Levels) -> - dict:store(Key, Level, Levels) - end, dict:new(), Vertices), - lists:foldl(fun({Key, _}=N, Solution) -> - dict:store(Key, N, Solution) - end, dict:new(), Vertices), - [], - SolutionGraph). - -cull_deps(_Graph, [], _Level, Levels, Solution, Discarded, SolutionGraph) -> - {_, Vertices} = lists:unzip(dict:to_list(Solution)), - LvlVertices = [{App,Vsn,dict:fetch(App,Levels)} || {App,Vsn} <- Vertices], - {ok, LvlVertices, Discarded, SolutionGraph}; -cull_deps(Graph, Vertices, Level, Levels, Solution, Discarded, SolutionGraph) -> - {NV, NS, LS, DS} = - lists:foldl(fun(V, {NewVertices, SolutionAcc, LevelsAcc, DiscardedAcc}) -> - OutNeighbors = lists:keysort(1, digraph:out_neighbours(Graph, V)), - lists:foldl(fun({Key, _}=N, {NewVertices1, SolutionAcc1, LevelsAcc1, DiscardedAcc1}) -> - case dict:find(Key, SolutionAcc1) of - {ok, N} -> % already seen - {NewVertices1, SolutionAcc1, LevelsAcc1, DiscardedAcc1}; - {ok, _} -> % conflict resolution! - {NewVertices1, SolutionAcc1, LevelsAcc1, [N|DiscardedAcc1]}; - error -> - add_to_solution_graph(N, V, SolutionGraph), - {[N | NewVertices1], - dict:store(Key, N, SolutionAcc1), - dict:store(Key, Level, LevelsAcc1), - DiscardedAcc1} - end - end, {NewVertices, SolutionAcc, LevelsAcc, DiscardedAcc}, OutNeighbors) - end, {[], Solution, Levels, Discarded}, lists:keysort(1, Vertices)), - cull_deps(Graph, NV, Level+1, LS, NS, DS, SolutionGraph). - -subgraph(Graph, Vertices) -> - digraph_utils:subgraph(Graph, Vertices). - -add_to_solution_graph(_, _, none) -> - ok; -add_to_solution_graph(N, V, SolutionGraph) -> - NewV = digraph:add_vertex(SolutionGraph, N), - digraph:add_edge(SolutionGraph, V, NewV). - -print_solution(Graph, Deps) -> - SolutionGraph = digraph:new(), - [digraph:add_vertex(SolutionGraph, V) || V <- Deps], - cull_deps(Graph, Deps, 0, SolutionGraph), - print_solution(SolutionGraph, Deps, 0). - -print_solution(_, [], _) -> - ok; -print_solution(SolutionGraph, [{N, V} | Vertices], 0) -> - ?CONSOLE("~s-~s", [N, V]), - OutNeighbors = lists:keysort(1, digraph:out_neighbours(SolutionGraph, {N,V})), - print_solution(SolutionGraph, OutNeighbors, 4), - print_solution(SolutionGraph, Vertices, 0); -print_solution(SolutionGraph, [{N, V} | Vertices], Indent) -> - ?CONSOLE("~s~s-~s", [[" " || _ <- lists:seq(0, Indent)], N, V]), - OutNeighbors = lists:keysort(1, digraph:out_neighbours(SolutionGraph, {N,V})), - print_solution(SolutionGraph, OutNeighbors, Indent+4), - print_solution(SolutionGraph, Vertices, Indent). - format_error(no_solution) -> io_lib:format("No solution for packages found.", []). @@ -148,6 +74,9 @@ format_error(no_solution) -> %% Internal Functions %%==================================================================== +subgraph(Graph, Vertices) -> + digraph_utils:subgraph(Graph, Vertices). + -spec names_to_apps([atom()], [rebar_app_info:t()]) -> [rebar_app_info:t()]. names_to_apps(Names, Apps) -> [element(2, App) || App <- [find_app_by_name(Name, Apps) || Name <- Names], App =/= error]. diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index 624fe0b..90193da 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -152,31 +152,31 @@ doterl_compile(Config, Dir, OutDir, MoreSources, ErlOpts) -> OutDir1 = proplists:get_value(outdir, ErlOpts, OutDir), - G = init_erlcinfo(proplists:get_all_values(i, ErlOpts), AllErlFiles, Dir), - - %% A source file may have been renamed or deleted. Remove it from the graph - %% and remove any beam file for that source if it exists. - Vertices = digraph:vertices(G), - [maybe_rm_beam_and_edge(G, OutDir, File) || File <- lists:sort(Vertices) -- lists:sort(AllErlFiles), - filename:extension(File) =:= ".erl"], + G = init_erlcinfo(proplists:get_all_values(i, ErlOpts), AllErlFiles, Dir, OutDir), NeededErlFiles = needed_files(G, ErlOpts, Dir, OutDir1, AllErlFiles), {ErlFirstFiles, ErlOptsFirst} = erl_first_files(Config, ErlOpts, Dir, NeededErlFiles), {DepErls, OtherErls} = lists:partition( fun(Source) -> digraph:in_degree(G, Source) > 0 end, [File || File <- NeededErlFiles, not lists:member(File, ErlFirstFiles)]), - DepErlsOrdered = digraph_utils:topsort(digraph_utils:subgraph(G, DepErls)), + SubGraph = digraph_utils:subgraph(G, DepErls), + DepErlsOrdered = digraph_utils:topsort(SubGraph), FirstErls = ErlFirstFiles ++ lists:reverse(DepErlsOrdered), ?DEBUG("Files to compile first: ~p", [FirstErls]), - rebar_base_compiler:run( - Config, FirstErls, OtherErls, - fun(S, C) -> - ErlOpts1 = case lists:member(S, ErlFirstFiles) of - true -> ErlOptsFirst; - false -> ErlOpts - end, - internal_erl_compile(C, Dir, S, OutDir1, ErlOpts1) - end), + try + rebar_base_compiler:run( + Config, FirstErls, OtherErls, + fun(S, C) -> + ErlOpts1 = case lists:member(S, ErlFirstFiles) of + true -> ErlOptsFirst; + false -> ErlOpts + end, + internal_erl_compile(C, Dir, S, OutDir1, ErlOpts1) + end) + after + true = digraph:delete(SubGraph), + true = digraph:delete(G) + end, ok. %% Get files which need to be compiled first, i.e. those specified in erl_first_files @@ -219,12 +219,13 @@ maybe_rm_beam_and_edge(G, OutDir, Source) -> case filelib:is_regular(Source) of true -> %% Actually exists, don't delete - ok; + false; false -> Target = target_base(OutDir, Source) ++ ".beam", ?DEBUG("Source ~s is gone, deleting previous beam file if it exists ~s", [Source, Target]), file:delete(Target), - digraph:del_vertex(G, Source) + digraph:del_vertex(G, Source), + true end. opts_changed(NewOpts, Target) -> @@ -250,7 +251,7 @@ erlcinfo_file(Dir) -> %% parse transforms, behaviours etc.) located in their directories or given %% InclDirs. Note that last modification times stored in vertices already respect %% dependencies induced by given graph G. -init_erlcinfo(InclDirs, Erls, Dir) -> +init_erlcinfo(InclDirs, Erls, Dir, OutDir) -> G = digraph:new([acyclic]), try restore_erlcinfo(G, InclDirs, Dir) catch @@ -259,10 +260,29 @@ init_erlcinfo(InclDirs, Erls, Dir) -> file:delete(erlcinfo_file(Dir)) end, Dirs = source_and_include_dirs(InclDirs, Erls), - Modified = lists:foldl(update_erlcinfo_fun(G, Dirs), false, Erls), - if Modified -> store_erlcinfo(G, InclDirs, Dir); not Modified -> ok end, + %% A source file may have been renamed or deleted. Remove it from the graph + %% and remove any beam file for that source if it exists. + Modified = maybe_rm_beams_and_edges(G, OutDir, Erls), + Modified1 = lists:foldl(update_erlcinfo_fun(G, Dirs), Modified, Erls), + if Modified1 -> store_erlcinfo(G, InclDirs, Dir); not Modified1 -> ok end, G. +maybe_rm_beams_and_edges(G, Dir, Files) -> + Vertices = digraph:vertices(G), + case lists:filter(fun(File) -> + case filename:extension(File) =:= ".erl" of + true -> + maybe_rm_beam_and_edge(G, Dir, File); + false -> + false + end + end, lists:sort(Vertices) -- lists:sort(Files)) of + [] -> + false; + _ -> + true + end. + source_and_include_dirs(InclDirs, Erls) -> SourceDirs = lists:map(fun filename:dirname/1, Erls), lists:usort(["include" | InclDirs ++ SourceDirs]). diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index 64c5380..b80c125 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -71,6 +71,8 @@ format_error({failed_extract, CachePath}) -> io_lib:format("Failed to extract package: ~s", [CachePath]); format_error({bad_etag, Source}) -> io_lib:format("MD5 Checksum comparison failed for: ~s", [Source]); +format_error({fetch_fail, Name, Vsn}) -> + io_lib:format("Failed to fetch and copy dep: ~s-~s", [Name, Vsn]); format_error({fetch_fail, Source}) -> io_lib:format("Failed to fetch and copy dep: ~p", [Source]); format_error({bad_checksum, File}) -> diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 07c63f1..4f8eff5 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -27,6 +27,7 @@ -module(rebar_file_utils). -export([try_consult/1, + replace_home_dir/1, format_error/1, symlink_or_copy/2, rm_rf/1, @@ -59,6 +60,10 @@ try_consult(File) -> throw(?PRV_ERROR({bad_term_file, File, Reason})) end. +replace_home_dir(Dir) -> + HomeDir = rebar_dir:home_dir(), + re:replace(Dir, [$^ | HomeDir], "~", [{return, list}]). + format_error({bad_term_file, AppFile, Reason}) -> io_lib:format("Error reading file ~s: ~s", [AppFile, file:format_error(Reason)]). @@ -318,4 +323,3 @@ cp_r_win32(Source,Dest) -> ok = cp_r_win32({filelib:is_dir(Src), Src}, Dst) end, filelib:wildcard(Source)), ok. - diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl index aec1535..2fc1ba9 100644 --- a/src/rebar_git_resource.erl +++ b/src/rebar_git_resource.erl @@ -11,6 +11,9 @@ -include("rebar.hrl"). +%% Regex used for parsing scp style remote url +-define(SCP_PATTERN, "\\A(?<username>[^@]+)@(?<host>[^:]+):(?<path>.+)\\z"). + lock(AppDir, {git, Url, _}) -> lock(AppDir, {git, Url}); lock(AppDir, {git, Url}) -> @@ -67,23 +70,27 @@ compare_url(Dir, Url) -> {ok, CurrentUrl} = rebar_utils:sh(?FMT("git config --get remote.origin.url", []), [{cd, Dir}]), CurrentUrl1 = string:strip(string:strip(CurrentUrl, both, $\n), both, $\r), - ParsedUrl = parse_git_url(Url), - ParsedCurrentUrl = parse_git_url(CurrentUrl1), + {ok, ParsedUrl} = parse_git_url(Url), + {ok, ParsedCurrentUrl} = parse_git_url(CurrentUrl1), ?DEBUG("Comparing git url ~p with ~p", [ParsedUrl, ParsedCurrentUrl]), ParsedCurrentUrl =:= ParsedUrl. -parse_git_url("git@" ++ HostPath) -> - [Host, Path] = string:tokens(HostPath, ":"), - {Host, filename:rootname(Path, ".git")}; -parse_git_url("git://" ++ HostPath) -> - [Host | Path] = string:tokens(HostPath, "/"), - {Host, filename:rootname(filename:join(Path), ".git")}; -parse_git_url("http://" ++ HostPath) -> - [Host | Path] = string:tokens(HostPath, "/"), - {Host, filename:rootname(filename:join(Path), ".git")}; -parse_git_url("https://" ++ HostPath) -> - [Host | Path] = string:tokens(HostPath, "/"), - {Host, filename:rootname(filename:join(Path), ".git")}. +parse_git_url(Url) -> + %% Checks for standard scp style git remote + case re:run(Url, ?SCP_PATTERN, [{capture, [host, path], list}]) of + {match, [Host, Path]} -> + {ok, {Host, filename:rootname(Path, ".git")}}; + nomatch -> + parse_git_url(not_scp, Url) + end. +parse_git_url(not_scp, Url) -> + UriOpts = [{scheme_defaults, [{git, 9418} | http_uri:scheme_defaults()]}], + case http_uri:parse(Url, UriOpts) of + {ok, {_Scheme, _User, Host, _Port, Path, _Query}} -> + {ok, {Host, filename:rootname(Path, ".git")}}; + {error, Reason} -> + {error, Reason} + end. download(Dir, {git, Url}, State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index e21f1fd..e3346ae 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -1,89 +1,92 @@ -module(rebar_packages). --export([get_packages/1 - ,registry/1 +-export([packages/1 + ,close_packages/0 + ,load_and_verify_version/1 + ,deps/3 + ,registry_dir/1 ,package_dir/1 - ,check_registry/3 ,registry_checksum/2 - ,find_highest_matching/3]). + ,find_highest_matching/4 + ,format_error/1]). -export_type([package/0]). -include("rebar.hrl"). +-include_lib("providers/include/providers.hrl"). -type pkg_name() :: binary() | atom(). -type vsn() :: binary(). -type package() :: pkg_name() | {pkg_name(), vsn()}. --spec get_packages(rebar_state:t()) -> {rebar_dict(), rebar_digraph()}. -get_packages(State) -> - RegistryDir = package_dir(State), - DictFile = filename:join(RegistryDir, "dict"), - Edges = filename:join(RegistryDir, "edges"), - Vertices = filename:join(RegistryDir, "vertices"), - Neighbors = filename:join(RegistryDir, "neighbors"), - - case lists:all(fun(X) -> filelib:is_file(X) end, [DictFile, Edges, Vertices, Neighbors]) of +-spec packages(rebar_state:t()) -> ets:tid(). +packages(State) -> + catch ets:delete(?PACKAGE_TABLE), + case load_and_verify_version(State) of true -> - try - {ok, DictBinary} = file:read_file(DictFile), - Dict = binary_to_term(DictBinary), - {ok, EdgesTab} = ets:file2tab(Edges), - {ok, VerticesTab} = ets:file2tab(Vertices), - {ok, NeighborsTab} = ets:file2tab(Neighbors), - {Dict, {digraph, EdgesTab, VerticesTab, NeighborsTab, true}} - catch - _:_ -> - ?ERROR("Bad packages index, try to fix with `rebar3 update`", []), - {dict:new(), digraph:new()} - end; + ok; false -> + ?DEBUG("Error loading package index.", []), ?ERROR("Bad packages index, try to fix with `rebar3 update`", []), - {dict:new(), digraph:new()} + ets:new(?PACKAGE_TABLE, [named_table, public]) end. -registry(State) -> - RegistryDir = package_dir(State), - HexFile = filename:join(RegistryDir, "registry"), - case ets:file2tab(HexFile) of - {ok, T} -> - {ok, T}; - {error, Reason} -> - ?DEBUG("Error loading registry: ~p", [Reason]), - error +close_packages() -> + catch ets:delete(?PACKAGE_TABLE). + +load_and_verify_version(State) -> + RegistryDir = registry_dir(State), + case ets:file2tab(filename:join(RegistryDir, ?INDEX_FILE)) of + {ok, _} -> + case ets:lookup_element(?PACKAGE_TABLE, package_index_version, 2) of + ?PACKAGE_INDEX_VERSION -> + true; + _ -> + (catch ets:delete(?PACKAGE_TABLE)), + rebar_prv_update:hex_to_index(State) + end; + _ -> + rebar_prv_update:hex_to_index(State) end. -package_dir(State) -> +deps(Name, Vsn, State) -> + try + verify_table(State), + ets:lookup_element(?PACKAGE_TABLE, {ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}, 2) + catch + _:_ -> + throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)})) + end. + +registry_dir(State) -> CacheDir = rebar_dir:global_cache_dir(State), - CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), - {ok, {_, _, Host, _, Path, _}} = http_uri:parse(CDN), - CDNHostPath = lists:reverse(string:tokens(Host, ".")), - CDNPath = tl(filename:split(Path)), - PackageDir = filename:join([CacheDir, "hex"] ++ CDNHostPath ++ CDNPath ++ ["packages"]), + case rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN) of + ?DEFAULT_CDN -> + RegistryDir = filename:join([CacheDir, "hex", "default"]), + ok = filelib:ensure_dir(filename:join(RegistryDir, "placeholder")), + RegistryDir; + CDN -> + {ok, {_, _, Host, _, Path, _}} = http_uri:parse(CDN), + CDNHostPath = lists:reverse(string:tokens(Host, ".")), + CDNPath = tl(filename:split(Path)), + RegistryDir = filename:join([CacheDir, "hex"] ++ CDNHostPath ++ CDNPath), + ok = filelib:ensure_dir(filename:join(RegistryDir, "placeholder")), + RegistryDir + end. + +package_dir(State) -> + RegistryDir = registry_dir(State), + PackageDir = filename:join([RegistryDir, "packages"]), ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")), PackageDir. - -check_registry(Pkg, Vsn, State) -> - case rebar_state:registry(State) of - {ok, T} -> - case ets:lookup(T, Pkg) of - [{Pkg, [Vsns]}] -> - lists:member(Vsn, Vsns); - _ -> - false - end; - error -> - false - end. - registry_checksum({pkg, Name, Vsn}, State) -> - {ok, Registry} = registry(State), - case ets:lookup(Registry, {Name, Vsn}) of - [{{_, _}, [_, Checksum | _]}] -> - Checksum; - [] -> - none + try + verify_table(State), + ets:lookup_element(?PACKAGE_TABLE, {Name, Vsn}, 3) + catch + _:_ -> + throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)})) end. %% Hex supports use of ~> to specify the version required for a dependency. @@ -101,28 +104,45 @@ registry_checksum({pkg, Name, Vsn}, State) -> %% `~> 2.1.3-dev` | `>= 2.1.3-dev and < 2.2.0` %% `~> 2.0` | `>= 2.0.0 and < 3.0.0` %% `~> 2.1` | `>= 2.1.0 and < 3.0.0` -find_highest_matching(Dep, Constraint, T) -> - case ets:lookup(T, Dep) of - [{Dep, [[Vsn]]}] -> - case ec_semver:pes(Vsn, Constraint) of - true -> - {ok, Vsn}; - false -> - ?WARN("Only existing version of ~s is ~s which does not match constraint ~~> ~s. " - "Using anyway, but it is not guarenteed to work.", [Dep, Vsn, Constraint]), - {ok, Vsn} - end; - [{Dep, [[HeadVsn | VsnTail]]}] -> - {ok, lists:foldl(fun(Version, Highest) -> - case ec_semver:pes(Version, Constraint) andalso - ec_semver:gt(Version, Highest) of - true -> - Version; - false -> - Highest - end - end, HeadVsn, VsnTail)}; - [] -> - ?WARN("Missing registry entry for package ~s", [Dep]), +find_highest_matching(Dep, Constraint, Table, State) -> + verify_table(State), + try ets:lookup_element(Table, Dep, 2) of + [[HeadVsn | VsnTail]] -> + {ok, handle_vsns(Constraint, HeadVsn, VsnTail)}; + [[Vsn]] -> + handle_single_vsn(Dep, Vsn, Constraint); + [Vsn] -> + handle_single_vsn(Dep, Vsn, Constraint); + [HeadVsn | VsnTail] -> + {ok, handle_vsns(Constraint, HeadVsn, VsnTail)} + catch + error:badarg -> none end. + +handle_vsns(Constraint, HeadVsn, VsnTail) -> + lists:foldl(fun(Version, Highest) -> + case ec_semver:pes(Version, Constraint) andalso + ec_semver:gt(Version, Highest) of + true -> + Version; + false -> + Highest + end + end, HeadVsn, VsnTail). + +handle_single_vsn(Dep, Vsn, Constraint) -> + case ec_semver:pes(Vsn, Constraint) of + true -> + {ok, Vsn}; + false -> + ?WARN("Only existing version of ~s is ~s which does not match constraint ~~> ~s. " + "Using anyway, but it is not guarenteed to work.", [Dep, Vsn, Constraint]), + {ok, Vsn} + end. + +format_error({missing_package, Package, Version}) -> + io_lib:format("Package not found in registry: ~s-~s. Try to fix with `rebar3 update`", [Package, Version]). + +verify_table(State) -> + ets:info(?PACKAGE_TABLE, named_table) =:= true orelse ?MODULE:load_and_verify_version(State). diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index fdc69e2..3430e81 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -34,7 +34,7 @@ download(TmpDir, Pkg={pkg, Name, Vsn}, State) -> Url = string:join([CDN, Package], "/"), cached_download(TmpDir, CachePath, Pkg, Url, etag(CachePath), State). -cached_download(TmpDir, CachePath, Pkg, Url, ETag, State) -> +cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn}, Url, ETag, State) -> case request(Url, ETag) of {ok, cached} -> serve_from_cache(TmpDir, CachePath, Pkg, State); @@ -44,7 +44,7 @@ cached_download(TmpDir, CachePath, Pkg, Url, ETag, State) -> ?DEBUG("Download ~s error, using ~s from cache", [Url, CachePath]), serve_from_cache(TmpDir, CachePath, Pkg, State); error -> - request_failed + {fetch_fail, Name, Vsn} end. serve_from_cache(TmpDir, CachePath, Pkg, State) -> diff --git a/src/rebar_plugins.erl b/src/rebar_plugins.erl index bda3fb7..6c2daef 100644 --- a/src/rebar_plugins.erl +++ b/src/rebar_plugins.erl @@ -24,12 +24,12 @@ project_apps_install(State) -> StateAcc1 = handle_plugins(Profile, Plugins, StateAcc), lists:foldl(fun(App, StateAcc2) -> - AppDir = rebar_app_info:dir(App), - C = rebar_config:consult(AppDir), - S = rebar_state:new(rebar_state:new(), C, AppDir), - Plugins2 = rebar_state:get(S, {plugins, Profile}, []), - handle_plugins(Profile, Plugins2, StateAcc2) - end, StateAcc1, ProjectApps) + AppDir = rebar_app_info:dir(App), + C = rebar_config:consult(AppDir), + S = rebar_state:new(rebar_state:new(), C, AppDir), + Plugins2 = rebar_state:get(S, {plugins, Profile}, []), + handle_plugins(Profile, Plugins2, StateAcc2) + end, StateAcc1, ProjectApps) end, State, Profiles). -spec install(rebar_state:t()) -> rebar_state:t(). diff --git a/src/rebar_prv_app_discovery.erl b/src/rebar_prv_app_discovery.erl index ea55e11..5449f82 100644 --- a/src/rebar_prv_app_discovery.erl +++ b/src/rebar_prv_app_discovery.erl @@ -38,6 +38,10 @@ do(State) -> State1 = rebar_app_discover:do(State, LibDirs), {ok, State1} catch + throw:{error, {rebar_packages, Error}} -> + {error, {rebar_packages, Error}}; + throw:{error, {rebar_app_utils, Error}} -> + {error, {rebar_app_utils, Error}}; throw:{error, Error} -> ?PRV_ERROR(Error) end. diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index 2b024cf..1165631 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -361,10 +361,10 @@ remove_links(Path) -> end. delete_dir_link(Path) -> - case os:type() of - {unix, _} -> file:delete(Path); - {win32, _} -> file:del_dir(Path) - end. + case os:type() of + {unix, _} -> file:delete(Path); + {win32, _} -> file:del_dir(Path) + end. dir_entries(Path) -> {ok, SubDirs} = file:list_dir(Path), diff --git a/src/rebar_prv_deps.erl b/src/rebar_prv_deps.erl index 9dc5346..9ff2bfa 100644 --- a/src/rebar_prv_deps.erl +++ b/src/rebar_prv_deps.erl @@ -24,25 +24,16 @@ init(State) -> {short_desc, "List dependencies"}, {desc, "List dependencies. Those not matching lock files " "are followed by an asterisk (*)."}, - {opts, [{tree, $t, "tree", undefined, "Display package dependencies in tree format (git and hg deps not supported)."}]}])), + {opts, []}])), {ok, State1}. -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - case display_tree(State) of - true -> - {_Packages, Graph} = rebar_state:packages(State), - List = merge_deps_per_profile(State), - {_SrcDeps, PkgDeps} = rebar_prv_install_deps:parse_deps(<<"">>, List, State, [], 0), - rebar_digraph:print_solution(Graph, PkgDeps), - {ok, State}; - false -> - Profiles = rebar_state:current_profiles(State), - List = [{Profile, rebar_state:get(State, {deps, Profile}, [])} - || Profile <- Profiles], - [display(State, Profile, Deps) || {Profile, Deps} <- List], - {ok, State} - end. + Profiles = rebar_state:current_profiles(State), + List = [{Profile, rebar_state:get(State, {deps, Profile}, [])} + || Profile <- Profiles], + [display(State, Profile, Deps) || {Profile, Deps} <- List], + {ok, State}. -spec format_error(any()) -> iolist(). format_error(Reason) -> @@ -91,7 +82,6 @@ dedup([Dep|Deps], [Name|DepNames]) -> name(T) when is_tuple(T) -> element(1, T); name(B) when is_binary(B) -> B. - display_deps(State, Deps) -> lists:foreach(fun(Dep) -> display_dep(State, Dep) end, Deps). @@ -125,17 +115,3 @@ display_dep(State, {Name, Source, Level}) when is_tuple(Source), is_integer(Leve ?CONSOLE("~s~s (locked ~s source)", [Name, NeedsUpdate, type(Source)]). type(Source) when is_tuple(Source) -> element(1, Source). - -display_tree(State) -> - {Args, _} = rebar_state:command_parsed_args(State), - proplists:get_value(tree, Args, false). - -merge_deps_per_profile(State) -> - Profiles = rebar_state:current_profiles(State), - lists:foldl(fun(Profile, Deps) -> - D = rebar_utils:deps_to_binary(rebar_state:get(State, {deps, Profile}, [])), - D1 = rebar_utils:tup_sort(D), - rebar_utils:tup_dedup( - rebar_utils:tup_umerge(D1 - ,Deps)) - end, [], Profiles). diff --git a/src/rebar_prv_deps_tree.erl b/src/rebar_prv_deps_tree.erl new file mode 100644 index 0000000..d429c52 --- /dev/null +++ b/src/rebar_prv_deps_tree.erl @@ -0,0 +1,84 @@ +-module(rebar_prv_deps_tree). + +-behaviour(provider). + +-export([init/1, + do/1, + format_error/1]). + +-include("rebar.hrl"). + +-define(PROVIDER, tree). +-define(DEPS, [lock]). + +-spec init(rebar_state:t()) -> {ok, rebar_state:t()}. +init(State) -> + State1 = rebar_state:add_provider( + State, + providers:create([{name, ?PROVIDER}, + {module, ?MODULE}, + {bare, true}, + {deps, ?DEPS}, + {example, "rebar3 tree"}, + {short_desc, "Print dependency tree."}, + {desc, ""}, + {opts, [{verbose, $v, "verbose", undefined, "Print repo and branch/tag/ref for git and hg deps"}]}])), + {ok, State1}. + +-spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. +do(State) -> + {Args, _} = rebar_state:command_parsed_args(State), + Verbose = proplists:get_value(verbose, Args, false), + print_deps_tree(rebar_state:all_deps(State), Verbose, State), + {ok, State}. + +-spec format_error(any()) -> iolist(). +format_error(Reason) -> + io_lib:format("~p", [Reason]). + +%% Internal functions + +print_deps_tree(SrcDeps, Verbose, State) -> + D = lists:foldl(fun(App, Dict) -> + Name = rebar_app_info:name(App), + Vsn = rebar_app_info:original_vsn(App), + Source = rebar_app_info:source(App), + Parent = rebar_app_info:parent(App), + dict:append_list(Parent, [{Name, Vsn, Source}], Dict) + end, dict:new(), SrcDeps), + ProjectAppNames = [{rebar_app_info:name(App) + ,rebar_app_info:original_vsn(App) + ,project} || App <- rebar_state:project_apps(State)], + case dict:find(root, D) of + {ok, Children} -> + print_children(-1, lists:keysort(1, Children++ProjectAppNames), D, Verbose); + error -> + print_children(-1, lists:keysort(1, ProjectAppNames), D, Verbose) + end. + +print_children(_, [], _, _) -> + ok; +print_children(Indent, [{Name, Vsn, Source} | Rest], Dict, Verbose) -> + + [io:format("| ") || _ <- lists:seq(0, Indent, 2)], + io:format("|- "), + io:format("~s-~s (~s)~n", [Name, Vsn, type(Source, Verbose)]), + case dict:find(Name, Dict) of + {ok, Children} -> + print_children(Indent+2, lists:keysort(1, Children), Dict, Verbose), + print_children(Indent, Rest, Dict, Verbose); + error -> + print_children(Indent, Rest, Dict, Verbose) + end. + +type(project, _) -> + "project app"; +type(Source, Verbose) when is_tuple(Source) -> + case {element(1, Source), Verbose} of + {pkg, _} -> + "hex package"; + {Other, false} -> + io_lib:format("~s repo", [Other]); + {_, true} -> + io_lib:format("~s", [element(2, Source)]) + end. diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index 72350d6..ca9344b 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -36,7 +36,7 @@ -include_lib("providers/include/providers.hrl"). -export([handle_deps_as_profile/4, - parse_deps/5, + profile_dep_dir/2, find_cycles/1, cull_compile/2]). @@ -98,6 +98,8 @@ do(State) -> end. -spec format_error(any()) -> iolist(). +format_error({dep_app_not_found, AppDir, AppName}) -> + io_lib:format("Dependency failure: Application ~s not found at the top level of directory ~s", [AppName, AppDir]); format_error({load_registry_fail, Dep}) -> io_lib:format("Error loading registry to resolve version of ~s. Try fixing by running 'rebar3 update'", [Dep]); format_error({bad_constraint, Name, Constraint}) -> @@ -108,6 +110,8 @@ format_error({not_rebar_package, Package, Version}) -> io_lib:format("Package not buildable with rebar3: ~s-~s", [Package, Version]); format_error({missing_package, Package, Version}) -> io_lib:format("Package not found in registry: ~s-~s", [Package, Version]); +format_error({missing_package, Package}) -> + io_lib:format("Package not found in registry: ~s", [Package]); format_error({cycles, Cycles}) -> Prints = [["applications: ", [io_lib:format("~s ", [Dep]) || Dep <- Cycle], @@ -123,13 +127,9 @@ handle_deps_as_profile(Profile, State, Deps, Upgrade) -> Locks = [], Level = 0, DepsDir = profile_dep_dir(State, Profile), - {SrcDeps, PkgDeps} = parse_deps(DepsDir, Deps, State, Locks, Level), - AllSrcProfileDeps = [{Profile, SrcDeps, Locks, Level}], - AllPkgProfileDeps = [{Profile, Locks, PkgDeps, Level}], - {AllApps, PkgDeps1, Seen, State1} = handle_profile_level(AllSrcProfileDeps, AllPkgProfileDeps, Locks, sets:new(), Upgrade, State), - - handle_profile_pkg_level(PkgDeps1, AllApps, Seen, Upgrade, State1). - + Deps1 = rebar_app_utils:parse_deps(DepsDir, Deps, State, Locks, Level), + ProfileLevelDeps = [{Profile, Deps1, Level}], + handle_profile_level(ProfileLevelDeps, [], sets:new(), Upgrade, Locks, State). %% =================================================================== %% Internal functions @@ -138,56 +138,30 @@ handle_deps_as_profile(Profile, State, Deps, Upgrade) -> %% finds all the deps in `{deps, ...}` for each profile provided. deps_per_profile(Profiles, Upgrade, State) -> Level = 0, - {AllProfileDeps, PkgDeps} = lists:foldl(fun(Profile, {SrcAcc, PkgAcc}) -> - {Src, Pkg} = parse_profile_deps(State, Profile, Level), - {[Src | SrcAcc], [Pkg | PkgAcc]} - end, {[], []}, Profiles), - {AllApps, PkgDeps1, Seen, State1} = handle_profile_level(AllProfileDeps, PkgDeps, [], sets:new(), Upgrade, State), - - handle_profile_pkg_level(PkgDeps1, AllApps, Seen, Upgrade, State1). + Locks = rebar_state:get(State, {locks, default}, []), + Deps = lists:foldl(fun(Profile, DepAcc) -> + [parsed_profile_deps(State, Profile, Level) | DepAcc] + end, [], Profiles), + handle_profile_level(Deps, [], sets:new(), Upgrade, Locks, State). -parse_profile_deps(State, Profile, Level) -> - DepsDir = profile_dep_dir(State, Profile), - Locks = rebar_state:get(State, {locks, Profile}, []), - Deps = rebar_state:get(State, {deps, Profile}, []), - {SrcDeps, PkgDeps} = parse_deps(DepsDir, Deps, State, Locks, Level), - {{Profile, SrcDeps, Locks, Level}, {Profile, Locks, PkgDeps, Level}}. +parsed_profile_deps(State, Profile, Level) -> + ParsedDeps = rebar_state:get(State, {parsed_deps, Profile}, []), + {Profile, ParsedDeps, Level}. %% Level-order traversal of all dependencies, across profiles. %% If profiles x,y,z are present, then the traversal will go: %% x0, y0, z0, x1, y1, z1, ..., xN, yN, zN. -handle_profile_level([], PkgDeps, SrcApps, Seen, _Upgrade, State) -> - {SrcApps, PkgDeps, Seen, State}; -handle_profile_level([{Profile, SrcDeps, Locks, Level} | Rest], PkgDeps, SrcApps, Seen, Upgrade, State) -> - {SrcDeps1, PkgDeps1, SrcApps1, State1, Seen1, Locks1} = - update_src_deps(Profile, Level, SrcDeps, [], SrcApps - ,State, Upgrade, Seen, Locks), - SrcDeps2 = case SrcDeps1 of +handle_profile_level([], Apps, _Seen, _Upgrade, _Locks, State) -> + {Apps, State}; +handle_profile_level([{Profile, Deps, Level} | Rest], Apps, Seen, Upgrade, Locks, State) -> + {Deps1, Apps1, State1, Seen1} = + update_deps(Profile, Level, Deps, Apps + ,State, Upgrade, Seen, Locks), + Deps2 = case Deps1 of [] -> Rest; - _ -> Rest ++ [{Profile, SrcDeps1, Locks1, Level+1}] + _ -> Rest ++ [{Profile, Deps1, Level+1}] end, - handle_profile_level(SrcDeps2, [{Profile, Locks1, PkgDeps1, Level+1} | PkgDeps], SrcApps1++SrcApps, sets:union(Seen, Seen1), Upgrade, State1). - -handle_profile_pkg_level(PkgDeps, AllApps, Seen, Upgrade, State) -> - %% Read in package index and dep graph - {Packages, Graph} = rebar_state:packages(State), - Registry = rebar_packages:registry(State), - State1 = rebar_state:packages(rebar_state:registry(State, Registry) - ,{Packages, Graph}), - - lists:foldl(fun({_Profile, _, [], _}, {AllAcc, StateAcc}) -> - {AllAcc, StateAcc}; - ({Profile1, Locks, PkgDeps2, Level}, {AllAcc, StateAcc}) -> - {Solved, StateAcc2} = update_pkg_deps(Profile1, Packages, PkgDeps2 - ,Graph, Upgrade, Seen, StateAcc, Locks - ,Level), - - AllDeps = lists:ukeymerge(2 - ,lists:ukeysort(2, AllAcc) - ,lists:ukeysort(2, Solved)), - - {AllDeps, StateAcc2} - end, {AllApps, State1}, PkgDeps). + handle_profile_level(Deps2, Apps1, sets:union(Seen, Seen1), Upgrade, Locks, State1). find_cycles(Apps) -> case rebar_digraph:compile_order(Apps) of @@ -199,52 +173,6 @@ find_cycles(Apps) -> cull_compile(TopSortedDeps, ProjectApps) -> lists:dropwhile(fun not_needs_compile/1, TopSortedDeps -- ProjectApps). -update_pkg_deps(Profile, Packages, PkgDeps, Graph, Upgrade, Seen, State, Locks, Level) -> - case PkgDeps of - [] -> %% No pkg deps - {[], State}; - PkgDeps -> - %% Find pkg deps needed - S = case rebar_digraph:cull_deps(Graph, PkgDeps, Level) of - {ok, [], _} -> - throw({rebar_digraph, no_solution}); - {ok, Solution, []} -> - Solution; - {ok, Solution, Discarded} -> - [warn_skip_pkg(Pkg, State) || Pkg <- Discarded, not(pkg_locked(Pkg, Locks))], - Solution - end, - update_pkg_deps(Profile, S, Packages, Upgrade, Seen, State, Locks) - end. - -pkg_locked({Name, _, _}, Locks) -> - pkg_locked(Name, Locks); -pkg_locked({Name, _}, Locks) -> - pkg_locked(Name, Locks); -pkg_locked(Name, Locks) -> - false =/= lists:keyfind(Name, 1, Locks). - -update_pkg_deps(Profile, Pkgs, Packages, Upgrade, Seen, State, Locks) -> - %% Create app_info record for each pkg dep - DepsDir = profile_dep_dir(State, Profile), - {Solved, _, State1} - = lists:foldl(fun(Pkg, {Acc, SeenAcc, StateAcc}) -> - handle_pkg_dep(Profile, Pkg, Packages, Upgrade, DepsDir, Acc, SeenAcc, Locks, StateAcc) - end, {[], Seen, State}, Pkgs), - {Solved, State1}. - -handle_pkg_dep(Profile, Pkg, Packages, Upgrade, DepsDir, Fetched, Seen, Locks, State) -> - IsLock = pkg_locked(Pkg, Locks), - AppInfo = package_to_app(DepsDir, Packages, Pkg, IsLock, State), - Deps = rebar_app_info:deps(AppInfo), - Level = rebar_app_info:dep_level(AppInfo), - {NewSeen, NewState} = maybe_lock(Profile, AppInfo, Seen, State, Level), - {_, AppInfo1} = maybe_fetch(AppInfo, Profile, Upgrade, Seen, NewState), - {AppInfo2, _, _, _, _} = - handle_dep(NewState, Profile, DepsDir, AppInfo1, Locks, Level), - AppInfo3 = rebar_app_info:deps(AppInfo2, Deps), - {[AppInfo3 | Fetched], NewSeen, NewState}. - maybe_lock(Profile, AppInfo, Seen, State, Level) -> Name = rebar_app_info:name(AppInfo), case rebar_app_info:is_checkout(AppInfo) of @@ -271,49 +199,28 @@ maybe_lock(Profile, AppInfo, Seen, State, Level) -> {sets:add_element(Name, Seen), State} end. -package_to_app(DepsDir, Packages, {Name, Vsn, Level}, IsLock, State) -> - case dict:find({Name, Vsn}, Packages) of - error -> - case rebar_packages:check_registry(Name, Vsn, State) of - true -> - throw(?PRV_ERROR({not_rebar_package, Name, Vsn})); - false -> - throw(?PRV_ERROR({missing_package, Name, Vsn})) - end; - {ok, PkgDeps} -> - Source = {pkg, Name, Vsn}, - AppInfo = new_dep(DepsDir, Name, Vsn, Source, IsLock, State), - AppInfo1 = rebar_app_info:dep_level(rebar_app_info:deps(AppInfo, PkgDeps), Level), - BaseDir = rebar_state:get(State, base_dir, []), - AppState1 = rebar_state:set(rebar_app_info:state(AppInfo1), base_dir, BaseDir), - rebar_app_info:state(AppInfo1, AppState1) - end. - --spec update_src_deps(atom(), non_neg_integer(), list(), list(), list(), rebar_state:t(), boolean(), sets:set(binary()), list()) -> {list(), list(), list(), rebar_state:t(), sets:set(binary()), list()}. -update_src_deps(Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, Locks) -> +update_deps(Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) -> lists:foldl( - fun(AppInfo, {SrcDepsAcc, PkgDepsAcc, SrcAppsAcc, StateAcc, SeenAcc, LocksAcc}) -> - update_src_dep(AppInfo, Profile, Level, - SrcDepsAcc, PkgDepsAcc, SrcAppsAcc, StateAcc, - Upgrade, SeenAcc, Locks, LocksAcc) + fun(AppInfo, {DepsAcc, AppsAcc, StateAcc, SeenAcc}) -> + update_dep(AppInfo, Profile, Level, + DepsAcc, AppsAcc, StateAcc, + Upgrade, SeenAcc, Locks) end, - {[], PkgDeps, SrcApps, State, Seen, Locks}, - rebar_utils:sort_deps(SrcDeps)). + {[], Apps, State, Seen}, + rebar_utils:sort_deps(Deps)). - -update_src_dep(AppInfo, Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, BaseLocks, Locks) -> +update_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) -> %% If not seen, add to list of locks to write out Name = rebar_app_info:name(AppInfo), case sets:is_element(Name, Seen) of true -> - update_seen_src_dep(AppInfo, Profile, Level, - SrcDeps, PkgDeps, SrcApps, - State, Upgrade, Seen, BaseLocks, Locks); + update_seen_dep(AppInfo, Profile, Level, + Deps, Apps, + State, Upgrade, Seen, Locks); false -> - update_unseen_src_dep(AppInfo, Profile, Level, - SrcDeps, PkgDeps, SrcApps, - State, Upgrade, Seen, Locks) - + update_unseen_dep(AppInfo, Profile, Level, + Deps, Apps, + State, Upgrade, Seen, Locks) end. profile_dep_dir(State, Profile) -> @@ -322,85 +229,53 @@ profile_dep_dir(State, Profile) -> _ -> rebar_dir:deps_dir(State) end. -update_seen_src_dep(AppInfo, _Profile, _Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, BaseLocks, Locks) -> +update_seen_dep(AppInfo, _Profile, _Level, Deps, Apps, State, Upgrade, Seen, Locks) -> Name = rebar_app_info:name(AppInfo), %% If seen from lock file or user requested an upgrade %% don't print warning about skipping - case lists:keymember(Name, 1, BaseLocks) of + case lists:keymember(Name, 1, Locks) of false when Upgrade -> ok; false when not Upgrade -> warn_skip_deps(AppInfo, State); true -> ok end, - {SrcDeps, PkgDeps, SrcApps, State, Seen, Locks}. + {Deps, Apps, State, Seen}. -update_unseen_src_dep(AppInfo, Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, Locks) -> +update_unseen_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) -> {NewSeen, State1} = maybe_lock(Profile, AppInfo, Seen, State, Level), - {NewSrcDeps, NewPkgDeps, NewSrcApps, State2, NewLocks} - = case Upgrade of - true -> - handle_upgrade(AppInfo, Profile, SrcDeps, PkgDeps, SrcApps, - Level, State1, Seen, Locks); - _ -> - {_, AppInfo1} = maybe_fetch(AppInfo, Profile, false, Seen, State1), - handle_dep(AppInfo1, Profile, SrcDeps, PkgDeps, SrcApps, - Level, State1, Locks) - end, - {NewSrcDeps, NewPkgDeps, NewSrcApps, State2, NewSeen, NewLocks}. - -handle_upgrade(AppInfo, Profile, SrcDeps, PkgDeps, SrcApps, Level, State, Seen, Locks) -> - Name = rebar_app_info:name(AppInfo), - case lists:keyfind(Name, 1, Locks) of - false -> - {_, AppInfo1} = maybe_fetch(AppInfo, Profile, true, Seen, State), - handle_dep(AppInfo1, Profile, SrcDeps, PkgDeps, SrcApps, - Level, State, Locks); - _StillLocked -> - handle_dep(AppInfo, Profile, SrcDeps, PkgDeps, SrcApps, - Level, State, Locks) - end. - -handle_dep(AppInfo, Profile, SrcDeps, PkgDeps, SrcApps, Level, State, Locks) -> + {_, AppInfo1} = maybe_fetch(AppInfo, Profile, Upgrade, Seen, State1), DepsDir = profile_dep_dir(State, Profile), - {AppInfo1, NewSrcDeps, NewPkgDeps, NewLocks, State1} = - handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level), - AppInfo2 = rebar_app_info:dep_level(AppInfo1, Level), - {NewSrcDeps ++ SrcDeps - ,NewPkgDeps++PkgDeps - ,[AppInfo2 | SrcApps] - ,State1 - ,NewLocks}. - --spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer()) -> - {rebar_app_info:t(), [rebar_app_info:t()], [pkg_dep()], [integer()], rebar_state:t()}. + {AppInfo2, NewDeps, State2} = + handle_dep(State1, Profile, DepsDir, AppInfo1, Locks, Level), + AppInfo3 = rebar_app_info:dep_level(AppInfo2, Level), + {NewDeps ++ Deps, [AppInfo3 | Apps], State2, NewSeen}. + +-spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer()) -> {rebar_app_info:t(), [rebar_app_info:t()], [pkg_dep()], [integer()], rebar_state:t()}. handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) -> Profiles = rebar_state:current_profiles(State), Name = rebar_app_info:name(AppInfo), - C = rebar_config:consult(rebar_app_info:dir(AppInfo)), - + %% Deps may be under a sub project app, find it and use its state if so S = rebar_app_info:state(AppInfo), - S1 = rebar_state:new(S, C, rebar_app_info:dir(AppInfo)), + C = rebar_config:consult(rebar_app_info:dir(AppInfo)), + S1 = rebar_state:new(S, C, AppInfo), S2 = rebar_state:apply_overrides(S1, Name), S3 = rebar_state:apply_profiles(S2, Profiles), Plugins = rebar_state:get(S3, plugins, []), S4 = rebar_state:set(S3, {plugins, Profile}, Plugins), - AppInfo1 = rebar_app_info:state(AppInfo, S4), rebar_utils:check_min_otp_version(rebar_state:get(S4, minimum_otp_vsn, undefined)), rebar_utils:check_blacklisted_otp_versions(rebar_state:get(S4, blacklisted_otp_vsns, [])), %% Dep may have plugins to install. Find and install here. S5 = rebar_plugins:install(S4), - AppInfo2 = rebar_app_info:state(AppInfo1, S5), + AppInfo1 = rebar_app_info:state(AppInfo, S5), %% Upgrade lock level to be the level the dep will have in this dep tree Deps = rebar_state:get(S5, {deps, default}, []), - NewLocks = [{DepName, Source, LockLevel+Level} || - {DepName, Source, LockLevel} <- rebar_state:get(S5, {locks, default}, [])], - AppInfo3 = rebar_app_info:deps(AppInfo2, rebar_state:deps_names(Deps)), - {SrcDeps, PkgDeps} = parse_deps(DepsDir, Deps, S5, Locks, Level+1), - {AppInfo3, SrcDeps, PkgDeps, Locks++NewLocks, State}. + AppInfo2 = rebar_app_info:deps(AppInfo1, rebar_state:deps_names(Deps)), + Deps1 = rebar_app_utils:parse_deps(Name, DepsDir, Deps, S5, Locks, Level+1), + {AppInfo2, Deps1, State}. -spec maybe_fetch(rebar_app_info:t(), atom(), boolean(), sets:set(binary()), rebar_state:t()) -> {boolean(), rebar_app_info:t()}. @@ -415,17 +290,20 @@ maybe_fetch(AppInfo, Profile, Upgrade, Seen, State) -> false -> true = fetch_app(AppInfo, AppDir, State), maybe_symlink_default(State, Profile, AppDir, AppInfo), - {true, update_app_info(AppDir, AppInfo)}; + {true, rebar_app_info:valid(update_app_info(AppDir, AppInfo), false)}; {true, AppInfo1} -> %% Preserve the state we created with overrides + AppInfo2 = copy_app_info(AppInfo, AppInfo1), AppState = rebar_app_info:state(AppInfo), - AppInfo2 = rebar_app_info:state(AppInfo1, AppState), - case sets:is_element(rebar_app_info:name(AppInfo), Seen) of + AppInfo3 = rebar_app_info:state(AppInfo2, AppState), + case sets:is_element(rebar_app_info:name(AppInfo3), Seen) of true -> - {false, AppInfo2}; + {false, AppInfo3}; false -> - maybe_symlink_default(State, Profile, AppDir, AppInfo2), - {maybe_upgrade(AppInfo, AppDir, Upgrade, State), AppInfo2} + maybe_symlink_default(State, Profile, AppDir, AppInfo3), + MaybeUpgrade = maybe_upgrade(AppInfo, AppDir, Upgrade, State), + AppInfo4 = update_app_info(AppDir, AppInfo3), + {MaybeUpgrade, AppInfo4} end end end. @@ -473,101 +351,6 @@ make_relative_to_root(State, Path) when is_list(Path) -> Root = rebar_dir:root_dir(State), rebar_dir:make_relative_path(Path, Root). --spec parse_deps(binary(), list(), rebar_state:t(), list(), integer()) -> {[rebar_app_info:t()], [tuple()]}. -parse_deps(DepsDir, Deps, State, Locks, Level) -> - lists:foldl(fun(Dep, Acc) -> - Name = case Dep of - Dep when is_tuple(Dep) -> - element(1, Dep); - Dep -> - Dep - end, - case lists:keyfind(ec_cnv:to_binary(Name), 1, Locks) of - false -> - parse_dep(Dep, Acc, DepsDir, false, State); - LockedDep -> - LockedLevel = element(3, LockedDep), - case LockedLevel > Level of - true -> - parse_dep(Dep, Acc, DepsDir, false, State); - false -> - parse_dep(LockedDep, Acc, DepsDir, true, State) - end - end - end, {[], []}, Deps). - -parse_dep({Name, Vsn}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_list(Vsn) -> - %% Versioned Package dependency - CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), - case rebar_app_info:discover(CheckoutsDir) of - {ok, _App} -> - Dep = new_dep(DepsDir, Name, [], [], IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; - not_found -> - {SrcDepsAcc, [parse_goal(ec_cnv:to_binary(Name) - ,ec_cnv:to_binary(Vsn)) | PkgDepsAcc]} - end; -parse_dep(Name, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_atom(Name); is_binary(Name) -> - %% Unversioned package dependency - {PkgName, PkgVsn} = get_package(ec_cnv:to_binary(Name), State), - CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), - case rebar_app_info:discover(CheckoutsDir) of - {ok, _App} -> - Dep = new_dep(DepsDir, Name, [], [], IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; - not_found -> - {SrcDepsAcc, [{PkgName, PkgVsn} | PkgDepsAcc]} - end; -parse_dep({Name, Source}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source) -> - Dep = new_dep(DepsDir, Name, [], Source, IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; -parse_dep({Name, _Vsn, Source}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source) -> - Dep = new_dep(DepsDir, Name, [], Source, IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; -parse_dep({Name, _Vsn, Source, Opts}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source) -> - ?WARN("Dependency option list ~p in ~p is not supported and will be ignored", [Opts, Name]), - Dep = new_dep(DepsDir, Name, [], Source, IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; -parse_dep({_Name, {pkg, Name, Vsn}, Level}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_integer(Level) -> - CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), - case rebar_app_info:discover(CheckoutsDir) of - {ok, _App} -> - Dep = new_dep(DepsDir, Name, [], [], IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; - not_found -> - {SrcDepsAcc, [{Name, Vsn} | PkgDepsAcc]} - end; -parse_dep({Name, Source, Level}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source) - , is_integer(Level) -> - Dep = new_dep(DepsDir, Name, [], Source, IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; -parse_dep(Dep, _, _, _, _) -> - throw(?PRV_ERROR({parse_dep, Dep})). - - -new_dep(DepsDir, Name, Vsn, Source, IsLock, State) -> - CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), - {ok, Dep} = case rebar_app_info:discover(CheckoutsDir) of - {ok, App} -> - {ok, rebar_app_info:is_checkout(App, true)}; - not_found -> - Dir = ec_cnv:to_list(filename:join(DepsDir, Name)), - case rebar_app_info:discover(Dir) of - {ok, App} -> - {ok, App}; - not_found -> - rebar_app_info:new(Name, Vsn, - ec_cnv:to_list(filename:join(DepsDir, Name))) - end - end, - C = rebar_config:consult(rebar_app_info:dir(Dep)), - S = rebar_state:new(rebar_state:new(), C, rebar_app_info:dir(Dep)), - Overrides = rebar_state:get(State, overrides, []), - ParentOverrides = rebar_state:overrides(State), - Dep1 = rebar_app_info:state(Dep, - rebar_state:overrides(S, ParentOverrides++Overrides)), - rebar_app_info:is_lock(rebar_app_info:source(Dep1, Source), IsLock). - fetch_app(AppInfo, AppDir, State) -> ?INFO("Fetching ~s (~p)", [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)]), Source = rebar_app_info:source(AppInfo), @@ -577,14 +360,31 @@ fetch_app(AppInfo, AppDir, State) -> %% So this is the first time for newly downloaded apps that its .app/.app.src data can %% be read in an parsed. update_app_info(AppDir, AppInfo) -> - {ok, Found} = rebar_app_info:discover(AppDir), - AppDetails = rebar_app_info:app_details(Found), - Applications = proplists:get_value(applications, AppDetails, []), - IncludedApplications = proplists:get_value(included_applications, AppDetails, []), - AppInfo1 = rebar_app_info:applications( - rebar_app_info:app_details(AppInfo, AppDetails), - IncludedApplications++Applications), - rebar_app_info:valid(AppInfo1, false). + case rebar_app_info:discover(AppDir) of + {ok, Found} -> + AppDetails = rebar_app_info:app_details(Found), + Vsn = rebar_app_info:original_vsn(Found), + Applications = proplists:get_value(applications, AppDetails, []), + IncludedApplications = proplists:get_value(included_applications, AppDetails, []), + AppInfo1 = rebar_app_info:original_vsn(rebar_app_info:applications( + rebar_app_info:app_details(AppInfo, AppDetails), + IncludedApplications++Applications), Vsn), + AppInfo2 = copy_app_info(AppInfo, AppInfo1), + rebar_app_info:valid(AppInfo2, undefined); + not_found -> + throw(?PRV_ERROR({dep_app_not_found, AppDir, rebar_app_info:name(AppInfo)})) + end. + +copy_app_info(OldAppInfo, NewAppInfo) -> + Deps = rebar_app_info:deps(OldAppInfo), + ResourceType = rebar_app_info:resource_type(OldAppInfo), + Parent = rebar_app_info:parent(OldAppInfo), + Source = rebar_app_info:source(OldAppInfo), + + rebar_app_info:deps( + rebar_app_info:resource_type( + rebar_app_info:source( + rebar_app_info:parent(NewAppInfo, Parent), Source), ResourceType), Deps). maybe_upgrade(AppInfo, AppDir, Upgrade, State) -> Source = rebar_app_info:source(AppInfo), @@ -607,17 +407,6 @@ maybe_upgrade(AppInfo, AppDir, Upgrade, State) -> false end. --spec parse_goal(binary(), binary()) -> {binary(), binary()} | {binary(), binary(), binary()}. -parse_goal(Name, Constraint) -> - case re:run(Constraint, "([^\\d]*)(\\d.*)", [{capture, [1,2], binary}]) of - {match, [<<>>, Vsn]} -> - {Name, Vsn}; - {match, [Op, Vsn]} -> - {Name, Vsn, binary_to_atom(Op, utf8)}; - nomatch -> - throw(?PRV_ERROR({bad_constraint, Name, Constraint})) - end. - warn_skip_deps(AppInfo, State) -> Msg = "Skipping ~s (from ~p) as an app of the same name " "has already been fetched", @@ -628,25 +417,7 @@ warn_skip_deps(AppInfo, State) -> true -> ?ERROR(Msg, Args), ?FAIL end. -warn_skip_pkg({Name, Source}, State) -> - Msg = "Skipping ~s (version ~s from package index) as an app of the same " - "name has already been fetched", - Args = [Name, Source], - case rebar_state:get(State, deps_error_on_conflict, false) of - false -> ?WARN(Msg, Args); - true -> ?ERROR(Msg, Args), ?FAIL - end. - not_needs_compile(App) -> not(rebar_app_info:is_checkout(App)) andalso rebar_app_info:valid(App) andalso rebar_state:has_all_artifacts(rebar_app_info:state(App)) =:= true. - -get_package(Dep, State) -> - case rebar_state:registry(State) of - {ok, T} -> - {ok, HighestDepVsn} = rebar_packages:find_highest_matching(Dep, "0", T), - {Dep, HighestDepVsn}; - error -> - throw(?PRV_ERROR({load_registry_fail, Dep})) - end. diff --git a/src/rebar_prv_lock.erl b/src/rebar_prv_lock.erl index 1844934..8578979 100644 --- a/src/rebar_prv_lock.erl +++ b/src/rebar_prv_lock.erl @@ -29,18 +29,24 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - OldLocks = rebar_state:get(State, {locks, default}, []), - Locks = build_locks(State), - Dir = rebar_state:dir(State), - file:write_file(filename:join(Dir, ?LOCK_FILE), - io_lib:format("~p.~n", [Locks])), - State1 = rebar_state:set(State, {locks, default}, Locks), + %% Only lock default profile run + case rebar_state:current_profiles(State) of + [default] -> + OldLocks = rebar_state:get(State, {locks, default}, []), + Locks = lists:keysort(1, build_locks(State)), + Dir = rebar_state:dir(State), + file:write_file(filename:join(Dir, ?LOCK_FILE), + io_lib:format("~p.~n", [Locks])), + State1 = rebar_state:set(State, {locks, default}, Locks), - OldLockNames = [element(1,L) || L <- OldLocks], - NewLockNames = [element(1,L) || L <- Locks], - rebar_utils:info_useless(OldLockNames, NewLockNames), + OldLockNames = [element(1,L) || L <- OldLocks], + NewLockNames = [element(1,L) || L <- Locks], + rebar_utils:info_useless(OldLockNames, NewLockNames), - {ok, State1}. + {ok, State1}; + _ -> + {ok, State} + end. -spec format_error(any()) -> iolist(). format_error(Reason) -> diff --git a/src/rebar_prv_packages.erl b/src/rebar_prv_packages.erl index 880d4a6..5b8ea66 100644 --- a/src/rebar_prv_packages.erl +++ b/src/rebar_prv_packages.erl @@ -27,19 +27,21 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - {Dict, _} = rebar_packages:get_packages(State), - print_packages(Dict), + print_packages(), {ok, State}. -spec format_error(any()) -> iolist(). format_error(load_registry_fail) -> "Failed to load package regsitry. Try running 'rebar3 update' to fix". -print_packages(Dict) -> - Pkgs = lists:keysort(1, dict:fetch_keys(Dict)), - SortedPkgs = lists:foldl(fun({Pkg, Vsn}, Acc) -> - orddict:append_list(Pkg, [Vsn], Acc) - end, orddict:new(), Pkgs), +print_packages() -> + SortedPkgs = ets:foldl(fun({package_index_version, _}, Acc) -> + Acc; + ({Pkg, Vsns}, Acc) -> + orddict:store(Pkg, Vsns, Acc); + (_, Acc) -> + Acc + end, orddict:new(), ?PACKAGE_TABLE), orddict:map(fun(Name, Vsns) -> SortedVsns = lists:sort(fun(A, B) -> @@ -47,7 +49,7 @@ print_packages(Dict) -> ,ec_semver:parse(B)) end, Vsns), VsnStr = join(SortedVsns, <<", ">>), - io:format("~s:~n Versions: ~s~n~n", [Name, VsnStr]) + ?CONSOLE("~s:~n Versions: ~s~n", [Name, VsnStr]) end, SortedPkgs). -spec join([binary()], binary()) -> binary(). diff --git a/src/rebar_prv_plugins_upgrade.erl b/src/rebar_prv_plugins_upgrade.erl index dfc9990..fabfa5b 100644 --- a/src/rebar_prv_plugins_upgrade.erl +++ b/src/rebar_prv_plugins_upgrade.erl @@ -91,5 +91,5 @@ build_plugin(AppInfo, Apps, State) -> Providers = rebar_state:providers(State), AppDir = rebar_app_info:dir(AppInfo), C = rebar_config:consult(AppDir), - S = rebar_state:new(rebar_state:all_deps(rebar_state:new(), Apps), C, AppDir), + S = rebar_state:new(rebar_state:all_deps(rebar_state:new(), Apps), C, AppInfo), rebar_prv_compile:compile(S, Providers, AppInfo). diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index 049e78c..10faf4d 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -9,21 +9,14 @@ do/1, format_error/1]). +-export([hex_to_index/1]). + -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). -define(PROVIDER, update). -define(DEPS, []). -%% Ignore warning of digraph opaque type when running dialyzer --dialyzer({no_opaque, do/1}). --dialyzer({no_opaque, write_registry/3}). - -%% Ignoring the opaque type warning won't stop dialyzer from warning of -%% no return for functions that had the opaque type warnings --dialyzer({no_return, do/1}). --dialyzer({no_return, write_registry/3}). - %% =================================================================== %% Public API %% =================================================================== @@ -42,23 +35,23 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - ?INFO("Updating package index...", []), try - RegistryDir = rebar_packages:package_dir(State), + RegistryDir = rebar_packages:registry_dir(State), filelib:ensure_dir(filename:join(RegistryDir, "dummy")), HexFile = filename:join(RegistryDir, "registry"), + ?INFO("Updating package registry...", []), TmpDir = ec_file:insecure_mkdtemp(), TmpFile = filename:join(TmpDir, "packages.gz"), - Url = rebar_state:get(State, rebar_packages_cdn, "https://s3.amazonaws.com/s3.hex.pm/registry.ets.gz"), + Url = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_HEX_REGISTRY), {ok, _RequestId} = httpc:request(get, {Url, []}, [], [{stream, TmpFile}, {sync, true}], rebar), {ok, Data} = file:read_file(TmpFile), Unzipped = zlib:gunzip(Data), ok = file:write_file(HexFile, Unzipped), - {Dict, Graph} = hex_to_graph(HexFile), - write_registry(Dict, Graph, State), + ?INFO("Writing registry to ~s", [rebar_file_utils:replace_home_dir(HexFile)]), + hex_to_index(State), ok catch _E:C -> @@ -72,56 +65,62 @@ do(State) -> format_error(package_index_write) -> "Failed to write package index.". --spec write_registry(rebar_dict(), {digraph, ets:tid(), ets:tid(), ets:tid(), any()}, rebar_state:t()) -> ok | {error, atom()}. -write_registry(Dict, {digraph, Edges, Vertices, Neighbors, _}, State) -> - RegistryDir = rebar_packages:package_dir(State), - filelib:ensure_dir(filename:join(RegistryDir, "dummy")), - ets:tab2file(Edges, filename:join(RegistryDir, "edges")), - ets:tab2file(Vertices, filename:join(RegistryDir, "vertices")), - ets:tab2file(Neighbors, filename:join(RegistryDir, "neighbors")), - file:write_file(filename:join(RegistryDir, "dict"), term_to_binary(Dict)). - is_supported(<<"make">>) -> true; is_supported(<<"rebar">>) -> true; +is_supported(<<"rebar3">>) -> true; is_supported(_) -> false. -hex_to_graph(Filename) -> - {ok, T} = ets:file2tab(Filename), - Graph = digraph:new(), - ets:foldl(fun({Pkg, [Versions]}, ok) when is_binary(Pkg), is_list(Versions) -> - lists:foreach(fun(Version) -> - digraph:add_vertex(Graph, {Pkg, Version}, 1) - end, Versions); - (_, ok) -> - ok - end, ok, T), - - Dict1 = ets:foldl(fun({{Pkg, PkgVsn}, [Deps, _, BuildTools | _]}, Dict) when is_list(BuildTools) -> - case lists:any(fun is_supported/1, BuildTools) of - true -> - DepsList = update_graph(Pkg, PkgVsn, Deps, T, Graph), - dict:store({Pkg, PkgVsn}, DepsList, Dict); - false -> - Dict - end; - (_, Dict) -> - Dict - end, dict:new(), T), - {Dict1, Graph}. - -update_graph(Pkg, PkgVsn, Deps, HexRegistry, Graph) -> +hex_to_index(State) -> + RegistryDir = rebar_packages:registry_dir(State), + HexFile = filename:join(RegistryDir, "registry"), + try ets:file2tab(HexFile) of + {ok, Registry} -> + try + PackageIndex = filename:join(RegistryDir, "packages.idx"), + ?INFO("Generating package index...", []), + (catch ets:delete(?PACKAGE_TABLE)), + ets:new(?PACKAGE_TABLE, [named_table, public]), + ets:foldl(fun({{Pkg, PkgVsn}, [Deps, Checksum, BuildTools | _]}, _) when is_list(BuildTools) -> + case lists:any(fun is_supported/1, BuildTools) of + true -> + DepsList = update_deps_list(Deps, Registry, State), + ets:insert(?PACKAGE_TABLE, {{Pkg, PkgVsn}, DepsList, Checksum}); + false -> + true + end; + ({Pkg, [Vsns]}, _) when is_binary(Pkg) -> + ets:insert(?PACKAGE_TABLE, {Pkg, Vsns}); + (_, _) -> + true + end, true, Registry), + + ets:insert(?PACKAGE_TABLE, {package_index_version, ?PACKAGE_INDEX_VERSION}), + ?INFO("Writing index to ~s", [rebar_file_utils:replace_home_dir(PackageIndex)]), + ets:tab2file(?PACKAGE_TABLE, PackageIndex), + true + after + catch ets:delete(Registry) + end; + {error, Reason} -> + ?DEBUG("Error loading package registry: ~p", [Reason]), + false + catch + _:_ -> + fail + end. + +update_deps_list(Deps, HexRegistry, State) -> lists:foldl(fun([Dep, DepVsn, false, _AppName | _], DepsListAcc) -> case DepVsn of <<"~> ", Vsn/binary>> -> - case rebar_packages:find_highest_matching(Dep, Vsn, HexRegistry) of + case rebar_packages:find_highest_matching(Dep, Vsn, HexRegistry, State) of {ok, HighestDepVsn} -> - digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, HighestDepVsn}), - [{Dep, DepVsn} | DepsListAcc]; + [{Dep, HighestDepVsn} | DepsListAcc]; none -> + ?WARN("Missing registry entry for package ~s. Try to fix with `rebar3 update`", [Dep]), DepsListAcc end; Vsn -> - digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, Vsn}), [{Dep, Vsn} | DepsListAcc] end; ([_Dep, _DepVsn, true, _AppName | _], DepsListAcc) -> diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl index 3a371ca..97d1953 100644 --- a/src/rebar_prv_upgrade.erl +++ b/src/rebar_prv_upgrade.erl @@ -53,18 +53,21 @@ do(State) -> {Locks0, _Unlocks0} -> Deps0 = top_level_deps(Deps, Locks), State1 = rebar_state:set(State, {deps, default}, Deps0), - State2 = rebar_state:set(State1, {locks, default}, Locks0), - State3 = rebar_state:set(State2, upgrade, true), - UpdatedLocks = [L || L <- rebar_state:lock(State3), + DepsDir = rebar_prv_install_deps:profile_dep_dir(State, default), + D = rebar_app_utils:parse_deps(root, DepsDir, Deps0, State1, Locks0, 0), + State2 = rebar_state:set(State1, {parsed_deps, default}, D), + State3 = rebar_state:set(State2, {locks, default}, Locks0), + State4 = rebar_state:set(State3, upgrade, true), + UpdatedLocks = [L || L <- rebar_state:lock(State4), lists:keymember(rebar_app_info:name(L), 1, Locks0)], - Res = rebar_prv_install_deps:do(rebar_state:lock(State3, UpdatedLocks)), + Res = rebar_prv_install_deps:do(rebar_state:lock(State4, UpdatedLocks)), case Res of - {ok, State4} -> + {ok, State5} -> rebar_utils:info_useless( [element(1,Lock) || Lock <- Locks], - [rebar_app_info:name(App) || App <- rebar_state:lock(State4)] + [rebar_app_info:name(App) || App <- rebar_state:lock(State5)] ), - rebar_prv_lock:do(State4); + rebar_prv_lock:do(State5); _ -> Res end @@ -80,7 +83,6 @@ format_error({transitive_dependency, Name}) -> format_error(Reason) -> io_lib:format("~p", [Reason]). - parse_names(Bin, Locks) -> case lists:usort(re:split(Bin, <<" *, *">>, [trim])) of %% Nothing submitted, use *all* apps @@ -98,7 +100,8 @@ prepare_locks([Name|Names], Deps, Locks, Unlocks) -> {_, _, 0} = Lock -> case rebar_utils:tup_find(AtomName, Deps) of false -> - ?PRV_ERROR({unknown_dependency, Name}); + ?WARN("Dependency ~s has been removed and will not be upgraded", [Name]), + prepare_locks(Names, Deps, Locks, Unlocks); Dep -> {Source, NewLocks, NewUnlocks} = prepare_lock(Dep, Lock, Locks), prepare_locks(Names, Deps, NewLocks, diff --git a/src/rebar_relx.erl b/src/rebar_relx.erl index a3adedd..699c780 100644 --- a/src/rebar_relx.erl +++ b/src/rebar_relx.erl @@ -28,14 +28,12 @@ do(Module, Command, Provider, State) -> case rebar_state:get(State, relx, []) of [] -> relx:main([{lib_dirs, LibDirs} - ,{output_dir, OutputDir} - ,{caller, api}], AllOptions); + ,{caller, api} | output_dir(OutputDir, Options)], AllOptions); Config -> Config1 = update_config(Config), relx:main([{lib_dirs, LibDirs} ,{config, Config1} - ,{output_dir, OutputDir} - ,{caller, api}], AllOptions) + ,{caller, api} | output_dir(OutputDir, Options)], AllOptions) end, rebar_hooks:run_all_hooks(Cwd, post, Provider, Providers, State), {ok, State} @@ -67,3 +65,8 @@ update_config(Config) -> end end, {[], []}, Config), lists:reverse(Special) ++ Other. + +%% Don't override output_dir if the user passed one on the command line +output_dir(OutputDir, Options) -> + [{output_dir, OutputDir} || not(lists:member("-o", Options)) + andalso not(lists:member("--output-dir", Options))]. diff --git a/src/rebar_state.erl b/src/rebar_state.erl index 59a9588..e31b01b 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -38,9 +38,6 @@ overrides/1, overrides/2, apply_overrides/2, - packages/1, packages/2, - registry/1, registry/2, - resources/1, resources/2, add_resource/2, providers/1, providers/2, add_provider/2]). @@ -65,8 +62,6 @@ all_plugin_deps = [] :: [rebar_app_info:t()], all_deps = [] :: [rebar_app_info:t()], - packages = undefined :: {rebar_dict(), rebar_digraph()} | undefined, - registry = undefined :: {ok, ets:tid()} | error | undefined, overrides = [], resources = [], providers = []}). @@ -83,11 +78,7 @@ new() -> -spec new(list()) -> t(). new(Config) when is_list(Config) -> BaseState = base_state(), - Deps = proplists:get_value(deps, Config, []), - Plugins = proplists:get_value(plugins, Config, []), - Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins} | Config], - true = rebar_config:verify_config_format(Terms), - Opts = dict:from_list(Terms), + Opts = base_opts(Config), BaseState#state_t { dir = rebar_dir:get_cwd(), default = Opts, opts = Opts }. @@ -96,12 +87,7 @@ new(Config) when is_list(Config) -> new(Profile, Config) when is_atom(Profile) , is_list(Config) -> BaseState = base_state(), - Deps = proplists:get_value(deps, Config, []), - - Plugins = proplists:get_value(plugins, Config, []), - Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins} | Config], - true = rebar_config:verify_config_format(Terms), - Opts = dict:from_list(Terms), + Opts = base_opts(Config), BaseState#state_t { dir = rebar_dir:get_cwd(), current_profiles = [Profile], default = Opts, @@ -111,25 +97,26 @@ new(ParentState=#state_t{}, Config) -> Dir = rebar_dir:get_cwd(), new(ParentState, Config, Dir). --spec new(t(), list(), file:name()) -> t(). -new(ParentState, Config, Dir) -> +-spec new(t(), list(), rebar_app_info:t() | file:filename_all()) -> t(). +new(ParentState, Config, Dir) when is_list(Dir) -> + new(ParentState, Config, deps_from_config(Dir, Config), Dir); +new(ParentState, Config, AppInfo) -> + Dir = rebar_app_info:dir(AppInfo), + DepLocks = case rebar_app_info:resource_type(AppInfo) of + pkg -> + Deps = rebar_app_info:deps(AppInfo), + [{{locks, default}, Deps}, {{deps, default}, Deps}]; + _ -> + deps_from_config(Dir, Config) + end, + new(ParentState, Config, DepLocks, Dir). + +new(ParentState, Config, Deps, Dir) -> Opts = ParentState#state_t.opts, - LocalOpts = case rebar_config:consult_lock_file(filename:join(Dir, ?LOCK_FILE)) of - [D] -> - %% We want the top level deps only from the lock file. - %% This ensures deterministic overrides for configs. - Deps = [X || X <- D, element(3, X) =:= 0], - Plugins = proplists:get_value(plugins, Config, []), - Terms = [{{locks, default}, D}, {{deps, default}, Deps}, {{plugins, default}, Plugins} | Config], - true = rebar_config:verify_config_format(Terms), - dict:from_list(Terms); - _ -> - D = proplists:get_value(deps, Config, []), - Plugins = proplists:get_value(plugins, Config, []), - Terms = [{{deps, default}, D}, {{plugins, default}, Plugins} | Config], - true = rebar_config:verify_config_format(Terms), - dict:from_list(Terms) - end, + Plugins = proplists:get_value(plugins, Config, []), + Terms = Deps++[{{plugins, default}, Plugins} | Config], + true = rebar_config:verify_config_format(Terms), + LocalOpts = dict:from_list(Terms), NewOpts = merge_opts(LocalOpts, Opts), @@ -137,6 +124,17 @@ new(ParentState, Config, Dir) -> ,opts=NewOpts ,default=NewOpts}. +deps_from_config(Dir, Config) -> + case rebar_config:consult_lock_file(filename:join(Dir, ?LOCK_FILE)) of + [D] -> + %% We want the top level deps only from the lock file. + %% This ensures deterministic overrides for configs. + Deps = [X || X <- D, element(3, X) =:= 0], + [{{locks, default}, D}, {{deps, default}, Deps}]; + _ -> + [{{deps, default}, proplists:get_value(deps, Config, [])}] + end. + base_state() -> case application:get_env(rebar, resources) of undefined -> @@ -146,6 +144,13 @@ base_state() -> end, #state_t{resources=Resources}. +base_opts(Config) -> + Deps = proplists:get_value(deps, Config, []), + Plugins = proplists:get_value(plugins, Config, []), + Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins} | Config], + true = rebar_config:verify_config_format(Terms), + dict:from_list(Terms). + get(State, Key) -> {ok, Value} = dict:find(Key, State#state_t.opts), Value. @@ -253,7 +258,9 @@ apply_overrides(State=#state_t{overrides=Overrides}, AppName) -> %% Inefficient. We want the order we get here though. State1 = lists:foldl(fun({override, O}, StateAcc) -> - lists:foldl(fun({Key, Value}, StateAcc1) -> + lists:foldl(fun({deps, Value}, StateAcc1) -> + rebar_state:set(StateAcc1, {deps,default}, Value); + ({Key, Value}, StateAcc1) -> rebar_state:set(StateAcc1, Key, Value) end, StateAcc, O); (_, StateAcc) -> @@ -261,7 +268,9 @@ apply_overrides(State=#state_t{overrides=Overrides}, AppName) -> end, State, Overrides), State2 = lists:foldl(fun({override, N, O}, StateAcc) when N =:= Name -> - lists:foldl(fun({Key, Value}, StateAcc1) -> + lists:foldl(fun({deps, Value}, StateAcc1) -> + rebar_state:set(StateAcc1, {deps,default}, Value); + ({Key, Value}, StateAcc1) -> rebar_state:set(StateAcc1, Key, Value) end, StateAcc, O); (_, StateAcc) -> @@ -269,7 +278,10 @@ apply_overrides(State=#state_t{overrides=Overrides}, AppName) -> end, State1, Overrides), State3 = lists:foldl(fun({add, N, O}, StateAcc) when N =:= Name -> - lists:foldl(fun({Key, Value}, StateAcc1) -> + lists:foldl(fun({deps, Value}, StateAcc1) -> + OldValue = rebar_state:get(StateAcc1, {deps,default}, []), + rebar_state:set(StateAcc1, {deps,default}, Value++OldValue); + ({Key, Value}, StateAcc1) -> OldValue = rebar_state:get(StateAcc1, Key, []), rebar_state:set(StateAcc1, Key, Value++OldValue) end, StateAcc, O); @@ -438,22 +450,6 @@ namespace(#state_t{namespace=Namespace}) -> namespace(State=#state_t{}, Namespace) -> State#state_t{namespace=Namespace}. -packages(State=#state_t{packages=undefined}) -> - rebar_packages:get_packages(State); -packages(#state_t{packages=Packages}) -> - Packages. - -packages(State, Packages) -> - State#state_t{packages=Packages}. - -registry(State=#state_t{registry=undefined}) -> - rebar_packages:registry(State); -registry(#state_t{registry=Registry}) -> - Registry. - -registry(State, Registry) -> - State#state_t{registry=Registry}. - -spec resources(t()) -> [{rebar_resource:type(), module()}]. resources(#state_t{resources=Resources}) -> Resources. |