diff options
45 files changed, 1349 insertions, 882 deletions
@@ -42,6 +42,7 @@ locations ([hex.pm](http://hex.pm), git, hg, and so on). | report | Report on environment and versions for bug reports | | shell | Run shell with project apps in path | | tar | Package release into tarball | +| tree | Print dependency tree | | unlock | Unlock dependencies | | update | Update package index | | upgrade | Fetch latest version of dep | diff --git a/priv/shell-completion/bash/rebar3 b/priv/shell-completion/bash/rebar3 index 4e28d3d..40009b7 100644 --- a/priv/shell-completion/bash/rebar3 +++ b/priv/shell-completion/bash/rebar3 @@ -15,6 +15,7 @@ _rebar3() compile \ cover \ ct \ + deps \ dialyzer \ do \ edoc \ @@ -29,6 +30,7 @@ _rebar3() report \ shell \ tar \ + tree \ unlock \ update \ upgrade \ @@ -77,6 +79,8 @@ _rebar3() --basic_html \ --ct_hooks \ --verbose" + elif [[ ${prev} == deps ]] ; then + : elif [[ ${prev} == dialyzer ]] ; then sopts="-u -s" lopts="--update-plt --succ-typings" @@ -168,6 +172,9 @@ _rebar3() --system_libs \ --version \ --root" + elif [[ ${prev} == tree ]] ; then + sopts="-v" + lopts="--verbose" elif [[ ${prev} == update ]] ; then : elif [[ ${prev} == upgrade ]] ; then diff --git a/priv/shell-completion/fish/rebar3.fish b/priv/shell-completion/fish/rebar3.fish index ad3dad9..df1697e 100644 --- a/priv/shell-completion/fish/rebar3.fish +++ b/priv/shell-completion/fish/rebar3.fish @@ -50,6 +50,7 @@ end ## report Provide a crash report to be sent to the rebar3 issues page. ## shell Run shell with project apps and deps in path. ## tar Tar archive of release built of project. +## tree Print dependency tree. ## unlock Unlock dependencies. ## update Update package index. ## upgrade Upgrade dependencies. @@ -149,6 +150,10 @@ complete -f -c 'rebar3' -n '__fish_rebar3_using_command tar' -l system_libs complete -f -c 'rebar3' -n '__fish_rebar3_using_command tar' -l version -d "Print relx version" complete -f -c 'rebar3' -n '__fish_rebar3_using_command tar' -s r -l root -d "The project root directory" +complete -f -c 'rebar3' -n '__fish_rebar3_needs_command' -a tree -d "Print depdency tree." + +complete -f -c 'rebar3' -n '__fish_rebar3_needs_command tree' -s v -l verbose -d "Print repo and branch/tag/ref for git and hg deps." + complete -f -c 'rebar3' -n '__fish_rebar3_needs_command' -a unlock -d "Unlock dependencies." complete -f -c 'rebar3' -n '__fish_rebar3_needs_command' -a update -d "Update package index." @@ -158,4 +163,3 @@ complete -f -c 'rebar3' -n '__fish_rebar3_needs_command' -a upgrade -d "Upgrade complete -f -c 'rebar3' -n '__fish_rebar3_needs_command' -a version -d "Print version for rebar and current Erlang." complete -f -c 'rebar3' -n '__fish_rebar3_needs_command' -a xref -d "Run cross reference analysis." - diff --git a/priv/shell-completion/zsh/_rebar3 b/priv/shell-completion/zsh/_rebar3 index b03b7c9..04575bc 100644 --- a/priv/shell-completion/zsh/_rebar3 +++ b/priv/shell-completion/zsh/_rebar3 @@ -191,6 +191,11 @@ _rebar3 () { '(-r --root)'{-r,--root}'[The project root directory]:system libs:_files -/' \ && ret=0 ;; + (tree) + _arguments \ + '(-v --verbose)'{-v,--verbose}'[Print repo and branch/tag/ref for git and hg deps]' \ + && ret=0 + ;; (unlock) _arguments \ '*: :_rebar3_list_deps' \ @@ -236,6 +241,7 @@ _rebar3_tasks() { 'report:Provide a crash report to be sent to the rebar3 issues page.' 'shell:Run shell with project apps and deps in path.' 'tar:Tar archive of release built of project.' + 'tree:Print dependency tree.' 'unlock:Unlock dependencies.' 'update:Update package index.' 'upgrade:Upgrade dependencies.' diff --git a/rebar.config b/rebar.config index 2ce40e2..1b3a8c8 100644 --- a/rebar.config +++ b/rebar.config @@ -5,8 +5,8 @@ {ssl_verify_hostname, "1.0.5"}, {providers, "1.4.1"}, {getopt, "0.8.2"}, - {bbmustache, "1.0.3"}, - {relx, "3.4.0"}]}. + {bbmustache, "1.0.4"}, + {relx, "3.5.0"}]}. {escript_name, rebar3}. {escript_emu_args, "%%! +sbtu +A0\n"}. @@ -1,6 +1,6 @@ -[{<<"bbmustache">>,{pkg,<<"bbmustache">>,<<"1.0.3">>},0}, - {<<"providers">>,{pkg,<<"providers">>,<<"1.4.1">>},0}, +[{<<"bbmustache">>,{pkg,<<"bbmustache">>,<<"1.0.4">>},0}, {<<"erlware_commons">>,{pkg,<<"erlware_commons">>,<<"0.15.0">>},0}, - {<<"relx">>,{pkg,<<"relx">>,<<"3.4.0">>},0}, - {<<"ssl_verify_hostname">>,{pkg,<<"ssl_verify_hostname">>,<<"1.0.5">>},0}, - {<<"getopt">>,{pkg,<<"getopt">>,<<"0.8.2">>},0}]. + {<<"getopt">>,{pkg,<<"getopt">>,<<"0.8.2">>},0}, + {<<"providers">>,{pkg,<<"providers">>,<<"1.4.1">>},0}, + {<<"relx">>,{pkg,<<"relx">>,<<"3.5.0">>},0}, + {<<"ssl_verify_hostname">>,{pkg,<<"ssl_verify_hostname">>,<<"1.0.5">>},0}]. diff --git a/src/rebar.app.src b/src/rebar.app.src index 0a2e6cf..4ef493b 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -23,7 +23,7 @@ erlware_commons, providers, bbmustache, - ssl_verify_hostname, + ssl_verify_hostname, relx, inets]}, {env, [ @@ -41,6 +41,7 @@ rebar_prv_compile, rebar_prv_cover, rebar_prv_deps, + rebar_prv_deps_tree, rebar_prv_dialyzer, rebar_prv_do, rebar_prv_edoc, diff --git a/src/rebar.hrl b/src/rebar.hrl index 8a702df..961b8ea 100644 --- a/src/rebar.hrl +++ b/src/rebar.hrl @@ -23,8 +23,14 @@ -define(DEFAULT_RELEASE_DIR, "rel"). -define(DEFAULT_CONFIG_FILE, "rebar.config"). -define(DEFAULT_CDN, "https://s3.amazonaws.com/s3.hex.pm/tarballs"). +-define(DEFAULT_HEX_REGISTRY, "https://s3.amazonaws.com/s3.hex.pm/registry.ets.gz"). -define(LOCK_FILE, "rebar.lock"). +-define(PACKAGE_INDEX_VERSION, 3). +-define(PACKAGE_TABLE, package_index). +-define(INDEX_FILE, "packages.idx"). +-define(REGISTRY_FILE, "registry"). + -ifdef(namespaced_types). -type rebar_dict() :: dict:dict(). -else. diff --git a/src/rebar3.erl b/src/rebar3.erl index 8004443..ab3e0eb 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -105,6 +105,7 @@ run_aux(State, RawArgs) -> {ok, Providers} = application:get_env(rebar, providers), %% Providers can modify profiles stored in opts, so set default after initializing providers State4 = rebar_state:create_logic_providers(Providers, State3), + rebar_packages:packages(State4), State5 = rebar_plugins:project_apps_install(State4), State6 = rebar_state:default(State5, rebar_state:opts(State5)), @@ -280,9 +281,15 @@ state_from_global_config(Config, GlobalConfigFile) -> %% We don't want to worry about global plugin install state effecting later %% usage. So we throw away the global profile state used for plugin install. GlobalConfigThrowAway = rebar_state:current_profiles(GlobalConfig, [global]), - GlobalState = rebar_plugins:handle_plugins(global, - rebar_state:get(GlobalConfigThrowAway, plugins, []), - GlobalConfigThrowAway), + GlobalState = case rebar_state:get(GlobalConfigThrowAway, plugins, []) of + [] -> + GlobalConfigThrowAway; + GlobalPluginsToInstall -> + rebar_packages:packages(GlobalConfigThrowAway), + rebar_plugins:handle_plugins(global, + GlobalPluginsToInstall, + GlobalConfigThrowAway) + end, GlobalPlugins = rebar_state:providers(GlobalState), GlobalConfig2 = rebar_state:set(GlobalConfig, plugins, []), GlobalConfig3 = rebar_state:set(GlobalConfig2, {plugins, global}, rebar_state:get(GlobalConfigThrowAway, plugins, [])), diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index 3b34539..5a25a9e 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -16,7 +16,26 @@ do(State, LibDirs) -> Apps = find_apps(Dirs, all), ProjectDeps = rebar_state:deps_names(State), DepsDir = rebar_dir:deps_dir(State), + CurrentProfiles = rebar_state:current_profiles(State), + + %% There may be a top level src which is an app and there may not + %% Find it here if there is, otherwise define the deps parent as root + TopLevelApp = define_root_app(Apps, State), + + %% Handle top level deps + State1 = lists:foldl(fun(Profile, StateAcc) -> + ProfileDeps = rebar_state:get(StateAcc, {deps, Profile}, []), + ProfileDeps2 = rebar_utils:tup_dedup(rebar_utils:tup_sort(ProfileDeps)), + StateAcc1 = rebar_state:set(StateAcc, {deps, Profile}, ProfileDeps2), + ParsedDeps = parse_profile_deps(Profile + ,TopLevelApp + ,ProfileDeps2 + ,StateAcc1 + ,StateAcc1), + rebar_state:set(StateAcc1, {parsed_deps, Profile}, ParsedDeps) + end, State, lists:reverse(CurrentProfiles)), + %% Handle sub project apps deps %% Sort apps so we get the same merged deps config everytime SortedApps = rebar_utils:sort_deps(Apps), lists:foldl(fun(AppInfo, StateAcc) -> @@ -33,7 +52,19 @@ do(State, LibDirs) -> ?INFO("Ignoring ~s", [Name]), StateAcc end - end, State, SortedApps). + end, State1, SortedApps). + +define_root_app(Apps, State) -> + RootDir = rebar_dir:root_dir(State), + case ec_lists:find(fun(X) -> + ec_file:real_dir_path(rebar_app_info:dir(X)) =:= + ec_file:real_dir_path(RootDir) + end, Apps) of + {ok, App} -> + rebar_app_info:name(App); + error -> + root + end. format_error({module_list, File}) -> io_lib:format("Error reading module list from ~p~n", [File]); @@ -51,24 +82,47 @@ merge_deps(AppInfo, State) -> rebar_state:apply_profiles( rebar_state:new(reset_hooks(rebar_state:opts(State, Default)), C, rebar_app_info:dir(AppInfo)), CurrentProfiles), Name), + AppState1 = rebar_state:overrides(AppState, rebar_state:get(AppState, overrides, [])), - rebar_utils:check_min_otp_version(rebar_state:get(AppState, minimum_otp_vsn, undefined)), - rebar_utils:check_blacklisted_otp_versions(rebar_state:get(AppState, blacklisted_otp_vsns, [])), + rebar_utils:check_min_otp_version(rebar_state:get(AppState1, minimum_otp_vsn, undefined)), + rebar_utils:check_blacklisted_otp_versions(rebar_state:get(AppState1, blacklisted_otp_vsns, [])), - AppState1 = rebar_state:set(AppState, artifacts, []), - AppInfo1 = rebar_app_info:state(AppInfo, AppState1), + AppState2 = rebar_state:set(AppState1, artifacts, []), + AppInfo1 = rebar_app_info:state(AppInfo, AppState2), State1 = lists:foldl(fun(Profile, StateAcc) -> - AppProfDeps = rebar_state:get(AppState, {deps, Profile}, []), - TopLevelProfDeps = rebar_state:get(StateAcc, {deps, Profile}, []), - ProfDeps2 = rebar_utils:tup_dedup(rebar_utils:tup_umerge( - rebar_utils:tup_sort(TopLevelProfDeps) - ,rebar_utils:tup_sort(AppProfDeps))), - rebar_state:set(StateAcc, {deps, Profile}, ProfDeps2) + handle_profile(Profile, Name, AppState1, StateAcc) end, State, lists:reverse(CurrentProfiles)), {AppInfo1, State1}. +handle_profile(Profile, Name, AppState, State) -> + TopParsedDeps = rebar_state:get(State, {parsed_deps, Profile}, {[], []}), + TopLevelProfileDeps = rebar_state:get(State, {deps, Profile}, []), + AppProfileDeps = rebar_state:get(AppState, {deps, Profile}, []), + AppProfileDeps2 = rebar_utils:tup_dedup(rebar_utils:tup_sort(AppProfileDeps)), + ProfileDeps2 = rebar_utils:tup_dedup(rebar_utils:tup_umerge( + rebar_utils:tup_sort(TopLevelProfileDeps) + ,rebar_utils:tup_sort(AppProfileDeps2))), + State1 = rebar_state:set(State, {deps, Profile}, ProfileDeps2), + + %% Only deps not also specified in the top level config need + %% to be included in the parsed deps + NewDeps = ProfileDeps2 -- TopLevelProfileDeps, + ParsedDeps = parse_profile_deps(Profile, Name, NewDeps, AppState, State1), + State2 = rebar_state:set(State1, {deps, Profile}, ProfileDeps2), + rebar_state:set(State2, {parsed_deps, Profile}, TopParsedDeps++ParsedDeps). + +parse_profile_deps(Profile, Name, Deps, AppState, State) -> + DepsDir = rebar_prv_install_deps:profile_dep_dir(State, Profile), + Locks = rebar_state:get(State, {locks, Profile}, []), + rebar_app_utils:parse_deps(Name + ,DepsDir + ,Deps + ,AppState + ,Locks + ,1). + project_app_config(AppInfo, State) -> C = rebar_config:consult(rebar_app_info:dir(AppInfo)), Dir = rebar_app_info:dir(AppInfo), @@ -144,7 +198,7 @@ create_app_info(AppDir, AppFile) -> AppVsn = proplists:get_value(vsn, AppDetails), Applications = proplists:get_value(applications, AppDetails, []), IncludedApplications = proplists:get_value(included_applications, AppDetails, []), - {ok, AppInfo} = rebar_app_info:new(AppName, AppVsn, AppDir, []), + {ok, AppInfo} = rebar_app_info:new(AppName, AppVsn, AppDir), AppInfo1 = rebar_app_info:applications( rebar_app_info:app_details(AppInfo, AppDetails), IncludedApplications++Applications), diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index 6962c5a..bb5104e 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -4,6 +4,7 @@ new/2, new/3, new/4, + new/5, discover/1, name/1, name/2, @@ -17,6 +18,8 @@ app_file/2, app_details/1, app_details/2, + parent/1, + parent/2, original_vsn/1, original_vsn/2, ebin_dir/1, @@ -32,6 +35,8 @@ dir/2, out_dir/1, out_dir/2, + resource_type/1, + resource_type/2, source/1, source/2, state/1, @@ -54,6 +59,7 @@ app_file :: file:filename_all() | undefined, config :: rebar_state:t() | undefined, original_vsn :: binary() | string() | undefined, + parent=root :: binary() | root, app_details=[] :: list(), applications=[] :: list(), deps=[] :: list(), @@ -61,6 +67,7 @@ dep_level=0 :: integer(), dir :: file:name(), out_dir :: file:name(), + resource_type :: pkg | src, source :: string() | tuple() | undefined, state :: rebar_state:t() | undefined, is_lock=false :: boolean(), @@ -107,6 +114,17 @@ new(AppName, Vsn, Dir, Deps) -> out_dir=ec_cnv:to_list(Dir), deps=Deps}}. +%% @doc build a complete version of the app info with all fields set. +-spec new(atom() | binary(), atom() | binary() | string(), binary() | string(), file:name(), list()) -> + {ok, t()}. +new(Parent, AppName, Vsn, Dir, Deps) -> + {ok, #app_info_t{name=ec_cnv:to_binary(AppName), + parent=Parent, + original_vsn=Vsn, + dir=ec_cnv:to_list(Dir), + out_dir=ec_cnv:to_list(Dir), + deps=Deps}}. + %% @doc discover a complete version of the app info with all fields set. -spec discover(file:filename_all()) -> {ok, t()} | not_found. discover(Dir) -> @@ -203,6 +221,13 @@ app_details(#app_info_t{app_details=AppDetails}) -> app_details(AppInfo=#app_info_t{}, AppDetails) -> AppInfo#app_info_t{app_details=AppDetails}. +parent(#app_info_t{parent=Parent}) -> + Parent. + +-spec parent(t(), binary() | root) -> t(). +parent(AppInfo=#app_info_t{}, Parent) -> + AppInfo#app_info_t{parent=Parent}. + -spec original_vsn(t()) -> string(). original_vsn(#app_info_t{original_vsn=Vsn}) -> Vsn. @@ -264,6 +289,14 @@ out_dir(AppInfo=#app_info_t{}, OutDir) -> ebin_dir(#app_info_t{out_dir=OutDir}) -> ec_cnv:to_list(filename:join(OutDir, "ebin")). +-spec resource_type(t(), pkg | src) -> t(). +resource_type(AppInfo=#app_info_t{}, Type) -> + AppInfo#app_info_t{resource_type=Type}. + +-spec resource_type(t()) -> pkg | src. +resource_type(#app_info_t{resource_type=ResourceType}) -> + ResourceType. + -spec source(t(), string() | tuple()) -> t(). source(AppInfo=#app_info_t{}, Source) -> AppInfo#app_info_t{source=Source}. @@ -284,7 +317,7 @@ state(#app_info_t{state=State}) -> state_or_new(State, AppInfo=#app_info_t{state=undefined}) -> AppDir = dir(AppInfo), C = rebar_config:consult(AppDir), - rebar_state:new(State, C, AppDir); + rebar_state:new(State, C, AppInfo); state_or_new(_State, #app_info_t{state=State}) -> State. @@ -306,7 +339,7 @@ is_checkout(#app_info_t{is_checkout=IsCheckout}) -> -spec valid(t()) -> boolean(). valid(AppInfo=#app_info_t{valid=undefined, state=State}) -> - case rebar_app_utils:validate_application_info(AppInfo) + case rebar_app_utils:validate_application_info(AppInfo) =:= true andalso rebar_state:has_all_artifacts(State) =:= true of true -> true; diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index e9745c3..88b75ac 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -32,6 +32,9 @@ app_src_to_app/2, validate_application_info/1, validate_application_info/2, + parse_deps/5, + parse_deps/6, + dep_to_app/7, format_error/1]). -include("rebar.hrl"). @@ -87,6 +90,98 @@ validate_application_info(AppInfo, AppDetail) -> end end. +-spec parse_deps(binary(), list(), rebar_state:t(), list(), integer()) -> {[rebar_app_info:t()], [tuple()]}. +parse_deps(DepsDir, Deps, State, Locks, Level) -> + parse_deps(root, DepsDir, Deps, State, Locks, Level). + +parse_deps(Parent, DepsDir, Deps, State, Locks, Level) -> + [parse_dep(Dep, Parent, DepsDir, State, Locks, Level) || Dep <- Deps]. + +parse_dep(Dep, Parent, DepsDir, State, Locks, Level) -> + Name = case Dep of + Dep when is_tuple(Dep) -> + element(1, Dep); + Dep -> + Dep + end, + case lists:keyfind(ec_cnv:to_binary(Name), 1, Locks) of + false -> + parse_dep(Parent, Dep, DepsDir, false, State); + LockedDep -> + LockedLevel = element(3, LockedDep), + case LockedLevel > Level of + true -> + parse_dep(Parent, Dep, DepsDir, false, State); + false -> + parse_dep(Parent, LockedDep, DepsDir, true, State) + end + end. + +parse_dep(Parent, {Name, Vsn, {pkg, PkgName}}, DepsDir, IsLock, State) -> + {PkgName1, PkgVsn} = parse_goal(ec_cnv:to_binary(PkgName), ec_cnv:to_binary(Vsn)), + pkg_to_app(Parent, DepsDir, Name, PkgName1, PkgVsn, IsLock, State); +parse_dep(Parent, {Name, {pkg, PkgName}}, DepsDir, IsLock, State) -> + %% Package dependency with different package name from app name + {PkgName1, PkgVsn} = get_package(ec_cnv:to_binary(PkgName), State), + pkg_to_app(Parent, DepsDir, Name, PkgName1, PkgVsn, IsLock, State); +parse_dep(Parent, {Name, Vsn}, DepsDir, IsLock, State) when is_list(Vsn); is_binary(Vsn) -> + %% Versioned Package dependency + {PkgName, PkgVsn} = parse_goal(ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)), + pkg_to_app(Parent, DepsDir, PkgName, PkgName, PkgVsn, IsLock, State); +parse_dep(Parent, Name, DepsDir, IsLock, State) when is_atom(Name); is_binary(Name) -> + %% Unversioned package dependency + {PkgName, PkgVsn} = get_package(ec_cnv:to_binary(Name), State), + pkg_to_app(Parent, DepsDir, PkgName, PkgName, PkgVsn, IsLock, State); +parse_dep(Parent, {Name, Source}, DepsDir, IsLock, State) when is_tuple(Source) -> + dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); +parse_dep(Parent, {Name, _Vsn, Source}, DepsDir, IsLock, State) when is_tuple(Source) -> + dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); +parse_dep(Parent, {Name, _Vsn, Source, Opts}, DepsDir, IsLock, State) when is_tuple(Source) -> + ?WARN("Dependency option list ~p in ~p is not supported and will be ignored", [Opts, Name]), + dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); +parse_dep(Parent, {Name, {pkg, PkgName, Vsn}, Level}, DepsDir, IsLock, State) when is_integer(Level) -> + pkg_to_app(Parent, DepsDir, Name, PkgName, Vsn, IsLock, State); +parse_dep(Parent, {Name, Source, Level}, DepsDir, IsLock, State) when is_tuple(Source) + , is_integer(Level) -> + dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); +parse_dep(_, Dep, _, _, _) -> + throw(?PRV_ERROR({parse_dep, Dep})). + +%% Verify package exists and create the AppInfo record +pkg_to_app(Parent, DepsDir, AppName, PkgName, PkgVsn, IsLock, State) -> + %% Verify package actually exists. This will throw a missing_package exception + Deps = rebar_packages:deps(PkgName, PkgVsn, State), + Source = {pkg, PkgName, PkgVsn}, + AppInfo = dep_to_app(Parent, DepsDir, AppName, PkgVsn, Source, IsLock, State), + rebar_app_info:resource_type(rebar_app_info:deps(AppInfo, Deps), pkg). + +dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> + CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), + BaseDir = rebar_state:get(State, base_dir, []), + {ok, App1} = case rebar_app_info:discover(CheckoutsDir) of + {ok, App} -> + {ok, rebar_app_info:is_checkout(App, true)}; + not_found -> + Dir = ec_cnv:to_list(filename:join(DepsDir, Name)), + case rebar_app_info:discover(Dir) of + {ok, App} -> + {ok, App}; + not_found -> + rebar_app_info:new(Parent, Name, Vsn, Dir, []) + end + end, + C = rebar_config:consult(rebar_app_info:dir(App1)), + S = rebar_state:new(rebar_state:new(), C, App1), + Overrides = rebar_state:get(State, overrides, []), + ParentOverrides = rebar_state:overrides(State), + S1 = rebar_state:set(rebar_state:overrides(S, ParentOverrides++Overrides), base_dir, BaseDir), + App2 = rebar_app_info:state(App1, S1), + rebar_app_info:is_lock(rebar_app_info:source(App2, Source), IsLock). + +format_error({missing_package, Package}) -> + io_lib:format("Package not found in registry: ~s", [Package]); +format_error({parse_dep, Dep}) -> + io_lib:format("Failed parsing dep ~p", [Dep]); format_error(Error) -> io_lib:format("~p", [Error]). @@ -94,11 +189,29 @@ format_error(Error) -> %% Internal functions %% =================================================================== +-spec parse_goal(binary(), binary()) -> {binary(), binary()} | {binary(), binary(), binary()}. +parse_goal(Name, Constraint) -> + case re:run(Constraint, "([^\\d]*)(\\d.*)", [{capture, [1,2], binary}]) of + {match, [<<>>, Vsn]} -> + {Name, Vsn}; + {match, [Op, Vsn]} -> + {Name, Vsn, binary_to_atom(Op, utf8)}; + nomatch -> + throw(?PRV_ERROR({bad_constraint, Name, Constraint})) + end. + +get_package(Dep, State) -> + case rebar_packages:find_highest_matching(Dep, "0", ?PACKAGE_TABLE, State) of + {ok, HighestDepVsn} -> + {Dep, HighestDepVsn}; + none -> + throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Dep)})) + end. + -spec has_all_beams(file:filename_all(), [module()]) -> true | ?PRV_ERROR({missing_module, module()}). has_all_beams(EbinDir, [Module | ModuleList]) -> - BeamFile = filename:join([EbinDir, - ec_cnv:to_list(Module) ++ ".beam"]), + BeamFile = filename:join([EbinDir, ec_cnv:to_list(Module) ++ ".beam"]), case filelib:is_file(BeamFile) of true -> has_all_beams(EbinDir, ModuleList); diff --git a/src/rebar_config.erl b/src/rebar_config.erl index 554b399..b9b8b2e 100644 --- a/src/rebar_config.erl +++ b/src/rebar_config.erl @@ -128,11 +128,20 @@ find_newly_added(ConfigDeps, LockedDeps) -> check_newly_added({_, _}=Dep, LockedDeps) -> check_newly_added_(Dep, LockedDeps); +check_newly_added({_, _, {pkg, _}}=Dep, LockedDeps) -> + check_newly_added_(Dep, LockedDeps); check_newly_added({Name, _, Source}, LockedDeps) -> check_newly_added_({Name, Source}, LockedDeps); check_newly_added(Dep, LockedDeps) -> check_newly_added_(Dep, LockedDeps). +check_newly_added_({Name, Vsn, Source}, LockedDeps) -> + case check_newly_added_(Name, LockedDeps) of + {true, Name1} -> + {true, {Name1, Vsn, Source}}; + false -> + false + end; check_newly_added_({Name, Source}, LockedDeps) -> case check_newly_added_(Name, LockedDeps) of {true, Name1} -> diff --git a/src/rebar_digraph.erl b/src/rebar_digraph.erl index e989fdc..363253a 100644 --- a/src/rebar_digraph.erl +++ b/src/rebar_digraph.erl @@ -2,10 +2,7 @@ -export([compile_order/1 ,restore_graph/1 - ,cull_deps/3 - ,cull_deps/4 ,subgraph/2 - ,print_solution/2 ,format_error/1]). -include("rebar.hrl"). @@ -18,20 +15,23 @@ compile_order(Apps) -> Deps = all_apps_deps(App), add(Graph, {Name, Deps}) end, Apps), - case digraph_utils:topsort(Graph) of - false -> - case digraph_utils:is_acyclic(Graph) of - true -> - {error, no_sort}; - false -> - Cycles = lists:sort( - [lists:sort(Comp) || Comp <- digraph_utils:strong_components(Graph), - length(Comp)>1]), - {error, {cycles, Cycles}} - end; - V -> - {ok, names_to_apps(lists:reverse(V), Apps)} - end. + Order = + case digraph_utils:topsort(Graph) of + false -> + case digraph_utils:is_acyclic(Graph) of + true -> + {error, no_sort}; + false -> + Cycles = lists:sort( + [lists:sort(Comp) || Comp <- digraph_utils:strong_components(Graph), + length(Comp)>1]), + {error, {cycles, Cycles}} + end; + V -> + {ok, names_to_apps(lists:reverse(V), Apps)} + end, + true = digraph:delete(Graph), + Order. add(Graph, {PkgName, Deps}) -> case digraph:vertex(Graph, PkgName) of @@ -67,80 +67,6 @@ restore_graph({Vs, Es}) -> end, Es), Graph. -%% Pick packages to fullfill dependencies -%% The first dep while traversing the graph is chosen and any conflicting -%% dep encountered later on is ignored. - -cull_deps(Graph, Vertices, Level) -> - {ok, LvlVertices, Discarded, _} = cull_deps(Graph, Vertices, Level, none), - {ok, LvlVertices, Discarded}. - -cull_deps(Graph, Vertices, Level, SolutionGraph) -> - cull_deps(Graph, - Vertices, - Level+1, - lists:foldl(fun({Key, _}, Levels) -> - dict:store(Key, Level, Levels) - end, dict:new(), Vertices), - lists:foldl(fun({Key, _}=N, Solution) -> - dict:store(Key, N, Solution) - end, dict:new(), Vertices), - [], - SolutionGraph). - -cull_deps(_Graph, [], _Level, Levels, Solution, Discarded, SolutionGraph) -> - {_, Vertices} = lists:unzip(dict:to_list(Solution)), - LvlVertices = [{App,Vsn,dict:fetch(App,Levels)} || {App,Vsn} <- Vertices], - {ok, LvlVertices, Discarded, SolutionGraph}; -cull_deps(Graph, Vertices, Level, Levels, Solution, Discarded, SolutionGraph) -> - {NV, NS, LS, DS} = - lists:foldl(fun(V, {NewVertices, SolutionAcc, LevelsAcc, DiscardedAcc}) -> - OutNeighbors = lists:keysort(1, digraph:out_neighbours(Graph, V)), - lists:foldl(fun({Key, _}=N, {NewVertices1, SolutionAcc1, LevelsAcc1, DiscardedAcc1}) -> - case dict:find(Key, SolutionAcc1) of - {ok, N} -> % already seen - {NewVertices1, SolutionAcc1, LevelsAcc1, DiscardedAcc1}; - {ok, _} -> % conflict resolution! - {NewVertices1, SolutionAcc1, LevelsAcc1, [N|DiscardedAcc1]}; - error -> - add_to_solution_graph(N, V, SolutionGraph), - {[N | NewVertices1], - dict:store(Key, N, SolutionAcc1), - dict:store(Key, Level, LevelsAcc1), - DiscardedAcc1} - end - end, {NewVertices, SolutionAcc, LevelsAcc, DiscardedAcc}, OutNeighbors) - end, {[], Solution, Levels, Discarded}, lists:keysort(1, Vertices)), - cull_deps(Graph, NV, Level+1, LS, NS, DS, SolutionGraph). - -subgraph(Graph, Vertices) -> - digraph_utils:subgraph(Graph, Vertices). - -add_to_solution_graph(_, _, none) -> - ok; -add_to_solution_graph(N, V, SolutionGraph) -> - NewV = digraph:add_vertex(SolutionGraph, N), - digraph:add_edge(SolutionGraph, V, NewV). - -print_solution(Graph, Deps) -> - SolutionGraph = digraph:new(), - [digraph:add_vertex(SolutionGraph, V) || V <- Deps], - cull_deps(Graph, Deps, 0, SolutionGraph), - print_solution(SolutionGraph, Deps, 0). - -print_solution(_, [], _) -> - ok; -print_solution(SolutionGraph, [{N, V} | Vertices], 0) -> - ?CONSOLE("~s-~s", [N, V]), - OutNeighbors = lists:keysort(1, digraph:out_neighbours(SolutionGraph, {N,V})), - print_solution(SolutionGraph, OutNeighbors, 4), - print_solution(SolutionGraph, Vertices, 0); -print_solution(SolutionGraph, [{N, V} | Vertices], Indent) -> - ?CONSOLE("~s~s-~s", [[" " || _ <- lists:seq(0, Indent)], N, V]), - OutNeighbors = lists:keysort(1, digraph:out_neighbours(SolutionGraph, {N,V})), - print_solution(SolutionGraph, OutNeighbors, Indent+4), - print_solution(SolutionGraph, Vertices, Indent). - format_error(no_solution) -> io_lib:format("No solution for packages found.", []). @@ -148,6 +74,9 @@ format_error(no_solution) -> %% Internal Functions %%==================================================================== +subgraph(Graph, Vertices) -> + digraph_utils:subgraph(Graph, Vertices). + -spec names_to_apps([atom()], [rebar_app_info:t()]) -> [rebar_app_info:t()]. names_to_apps(Names, Apps) -> [element(2, App) || App <- [find_app_by_name(Name, Apps) || Name <- Names], App =/= error]. diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index 624fe0b..90193da 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -152,31 +152,31 @@ doterl_compile(Config, Dir, OutDir, MoreSources, ErlOpts) -> OutDir1 = proplists:get_value(outdir, ErlOpts, OutDir), - G = init_erlcinfo(proplists:get_all_values(i, ErlOpts), AllErlFiles, Dir), - - %% A source file may have been renamed or deleted. Remove it from the graph - %% and remove any beam file for that source if it exists. - Vertices = digraph:vertices(G), - [maybe_rm_beam_and_edge(G, OutDir, File) || File <- lists:sort(Vertices) -- lists:sort(AllErlFiles), - filename:extension(File) =:= ".erl"], + G = init_erlcinfo(proplists:get_all_values(i, ErlOpts), AllErlFiles, Dir, OutDir), NeededErlFiles = needed_files(G, ErlOpts, Dir, OutDir1, AllErlFiles), {ErlFirstFiles, ErlOptsFirst} = erl_first_files(Config, ErlOpts, Dir, NeededErlFiles), {DepErls, OtherErls} = lists:partition( fun(Source) -> digraph:in_degree(G, Source) > 0 end, [File || File <- NeededErlFiles, not lists:member(File, ErlFirstFiles)]), - DepErlsOrdered = digraph_utils:topsort(digraph_utils:subgraph(G, DepErls)), + SubGraph = digraph_utils:subgraph(G, DepErls), + DepErlsOrdered = digraph_utils:topsort(SubGraph), FirstErls = ErlFirstFiles ++ lists:reverse(DepErlsOrdered), ?DEBUG("Files to compile first: ~p", [FirstErls]), - rebar_base_compiler:run( - Config, FirstErls, OtherErls, - fun(S, C) -> - ErlOpts1 = case lists:member(S, ErlFirstFiles) of - true -> ErlOptsFirst; - false -> ErlOpts - end, - internal_erl_compile(C, Dir, S, OutDir1, ErlOpts1) - end), + try + rebar_base_compiler:run( + Config, FirstErls, OtherErls, + fun(S, C) -> + ErlOpts1 = case lists:member(S, ErlFirstFiles) of + true -> ErlOptsFirst; + false -> ErlOpts + end, + internal_erl_compile(C, Dir, S, OutDir1, ErlOpts1) + end) + after + true = digraph:delete(SubGraph), + true = digraph:delete(G) + end, ok. %% Get files which need to be compiled first, i.e. those specified in erl_first_files @@ -219,12 +219,13 @@ maybe_rm_beam_and_edge(G, OutDir, Source) -> case filelib:is_regular(Source) of true -> %% Actually exists, don't delete - ok; + false; false -> Target = target_base(OutDir, Source) ++ ".beam", ?DEBUG("Source ~s is gone, deleting previous beam file if it exists ~s", [Source, Target]), file:delete(Target), - digraph:del_vertex(G, Source) + digraph:del_vertex(G, Source), + true end. opts_changed(NewOpts, Target) -> @@ -250,7 +251,7 @@ erlcinfo_file(Dir) -> %% parse transforms, behaviours etc.) located in their directories or given %% InclDirs. Note that last modification times stored in vertices already respect %% dependencies induced by given graph G. -init_erlcinfo(InclDirs, Erls, Dir) -> +init_erlcinfo(InclDirs, Erls, Dir, OutDir) -> G = digraph:new([acyclic]), try restore_erlcinfo(G, InclDirs, Dir) catch @@ -259,10 +260,29 @@ init_erlcinfo(InclDirs, Erls, Dir) -> file:delete(erlcinfo_file(Dir)) end, Dirs = source_and_include_dirs(InclDirs, Erls), - Modified = lists:foldl(update_erlcinfo_fun(G, Dirs), false, Erls), - if Modified -> store_erlcinfo(G, InclDirs, Dir); not Modified -> ok end, + %% A source file may have been renamed or deleted. Remove it from the graph + %% and remove any beam file for that source if it exists. + Modified = maybe_rm_beams_and_edges(G, OutDir, Erls), + Modified1 = lists:foldl(update_erlcinfo_fun(G, Dirs), Modified, Erls), + if Modified1 -> store_erlcinfo(G, InclDirs, Dir); not Modified1 -> ok end, G. +maybe_rm_beams_and_edges(G, Dir, Files) -> + Vertices = digraph:vertices(G), + case lists:filter(fun(File) -> + case filename:extension(File) =:= ".erl" of + true -> + maybe_rm_beam_and_edge(G, Dir, File); + false -> + false + end + end, lists:sort(Vertices) -- lists:sort(Files)) of + [] -> + false; + _ -> + true + end. + source_and_include_dirs(InclDirs, Erls) -> SourceDirs = lists:map(fun filename:dirname/1, Erls), lists:usort(["include" | InclDirs ++ SourceDirs]). diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index 64c5380..b80c125 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -71,6 +71,8 @@ format_error({failed_extract, CachePath}) -> io_lib:format("Failed to extract package: ~s", [CachePath]); format_error({bad_etag, Source}) -> io_lib:format("MD5 Checksum comparison failed for: ~s", [Source]); +format_error({fetch_fail, Name, Vsn}) -> + io_lib:format("Failed to fetch and copy dep: ~s-~s", [Name, Vsn]); format_error({fetch_fail, Source}) -> io_lib:format("Failed to fetch and copy dep: ~p", [Source]); format_error({bad_checksum, File}) -> diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 07c63f1..4f8eff5 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -27,6 +27,7 @@ -module(rebar_file_utils). -export([try_consult/1, + replace_home_dir/1, format_error/1, symlink_or_copy/2, rm_rf/1, @@ -59,6 +60,10 @@ try_consult(File) -> throw(?PRV_ERROR({bad_term_file, File, Reason})) end. +replace_home_dir(Dir) -> + HomeDir = rebar_dir:home_dir(), + re:replace(Dir, [$^ | HomeDir], "~", [{return, list}]). + format_error({bad_term_file, AppFile, Reason}) -> io_lib:format("Error reading file ~s: ~s", [AppFile, file:format_error(Reason)]). @@ -318,4 +323,3 @@ cp_r_win32(Source,Dest) -> ok = cp_r_win32({filelib:is_dir(Src), Src}, Dst) end, filelib:wildcard(Source)), ok. - diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl index aec1535..2fc1ba9 100644 --- a/src/rebar_git_resource.erl +++ b/src/rebar_git_resource.erl @@ -11,6 +11,9 @@ -include("rebar.hrl"). +%% Regex used for parsing scp style remote url +-define(SCP_PATTERN, "\\A(?<username>[^@]+)@(?<host>[^:]+):(?<path>.+)\\z"). + lock(AppDir, {git, Url, _}) -> lock(AppDir, {git, Url}); lock(AppDir, {git, Url}) -> @@ -67,23 +70,27 @@ compare_url(Dir, Url) -> {ok, CurrentUrl} = rebar_utils:sh(?FMT("git config --get remote.origin.url", []), [{cd, Dir}]), CurrentUrl1 = string:strip(string:strip(CurrentUrl, both, $\n), both, $\r), - ParsedUrl = parse_git_url(Url), - ParsedCurrentUrl = parse_git_url(CurrentUrl1), + {ok, ParsedUrl} = parse_git_url(Url), + {ok, ParsedCurrentUrl} = parse_git_url(CurrentUrl1), ?DEBUG("Comparing git url ~p with ~p", [ParsedUrl, ParsedCurrentUrl]), ParsedCurrentUrl =:= ParsedUrl. -parse_git_url("git@" ++ HostPath) -> - [Host, Path] = string:tokens(HostPath, ":"), - {Host, filename:rootname(Path, ".git")}; -parse_git_url("git://" ++ HostPath) -> - [Host | Path] = string:tokens(HostPath, "/"), - {Host, filename:rootname(filename:join(Path), ".git")}; -parse_git_url("http://" ++ HostPath) -> - [Host | Path] = string:tokens(HostPath, "/"), - {Host, filename:rootname(filename:join(Path), ".git")}; -parse_git_url("https://" ++ HostPath) -> - [Host | Path] = string:tokens(HostPath, "/"), - {Host, filename:rootname(filename:join(Path), ".git")}. +parse_git_url(Url) -> + %% Checks for standard scp style git remote + case re:run(Url, ?SCP_PATTERN, [{capture, [host, path], list}]) of + {match, [Host, Path]} -> + {ok, {Host, filename:rootname(Path, ".git")}}; + nomatch -> + parse_git_url(not_scp, Url) + end. +parse_git_url(not_scp, Url) -> + UriOpts = [{scheme_defaults, [{git, 9418} | http_uri:scheme_defaults()]}], + case http_uri:parse(Url, UriOpts) of + {ok, {_Scheme, _User, Host, _Port, Path, _Query}} -> + {ok, {Host, filename:rootname(Path, ".git")}}; + {error, Reason} -> + {error, Reason} + end. download(Dir, {git, Url}, State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index e21f1fd..e3346ae 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -1,89 +1,92 @@ -module(rebar_packages). --export([get_packages/1 - ,registry/1 +-export([packages/1 + ,close_packages/0 + ,load_and_verify_version/1 + ,deps/3 + ,registry_dir/1 ,package_dir/1 - ,check_registry/3 ,registry_checksum/2 - ,find_highest_matching/3]). + ,find_highest_matching/4 + ,format_error/1]). -export_type([package/0]). -include("rebar.hrl"). +-include_lib("providers/include/providers.hrl"). -type pkg_name() :: binary() | atom(). -type vsn() :: binary(). -type package() :: pkg_name() | {pkg_name(), vsn()}. --spec get_packages(rebar_state:t()) -> {rebar_dict(), rebar_digraph()}. -get_packages(State) -> - RegistryDir = package_dir(State), - DictFile = filename:join(RegistryDir, "dict"), - Edges = filename:join(RegistryDir, "edges"), - Vertices = filename:join(RegistryDir, "vertices"), - Neighbors = filename:join(RegistryDir, "neighbors"), - - case lists:all(fun(X) -> filelib:is_file(X) end, [DictFile, Edges, Vertices, Neighbors]) of +-spec packages(rebar_state:t()) -> ets:tid(). +packages(State) -> + catch ets:delete(?PACKAGE_TABLE), + case load_and_verify_version(State) of true -> - try - {ok, DictBinary} = file:read_file(DictFile), - Dict = binary_to_term(DictBinary), - {ok, EdgesTab} = ets:file2tab(Edges), - {ok, VerticesTab} = ets:file2tab(Vertices), - {ok, NeighborsTab} = ets:file2tab(Neighbors), - {Dict, {digraph, EdgesTab, VerticesTab, NeighborsTab, true}} - catch - _:_ -> - ?ERROR("Bad packages index, try to fix with `rebar3 update`", []), - {dict:new(), digraph:new()} - end; + ok; false -> + ?DEBUG("Error loading package index.", []), ?ERROR("Bad packages index, try to fix with `rebar3 update`", []), - {dict:new(), digraph:new()} + ets:new(?PACKAGE_TABLE, [named_table, public]) end. -registry(State) -> - RegistryDir = package_dir(State), - HexFile = filename:join(RegistryDir, "registry"), - case ets:file2tab(HexFile) of - {ok, T} -> - {ok, T}; - {error, Reason} -> - ?DEBUG("Error loading registry: ~p", [Reason]), - error +close_packages() -> + catch ets:delete(?PACKAGE_TABLE). + +load_and_verify_version(State) -> + RegistryDir = registry_dir(State), + case ets:file2tab(filename:join(RegistryDir, ?INDEX_FILE)) of + {ok, _} -> + case ets:lookup_element(?PACKAGE_TABLE, package_index_version, 2) of + ?PACKAGE_INDEX_VERSION -> + true; + _ -> + (catch ets:delete(?PACKAGE_TABLE)), + rebar_prv_update:hex_to_index(State) + end; + _ -> + rebar_prv_update:hex_to_index(State) end. -package_dir(State) -> +deps(Name, Vsn, State) -> + try + verify_table(State), + ets:lookup_element(?PACKAGE_TABLE, {ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}, 2) + catch + _:_ -> + throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)})) + end. + +registry_dir(State) -> CacheDir = rebar_dir:global_cache_dir(State), - CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), - {ok, {_, _, Host, _, Path, _}} = http_uri:parse(CDN), - CDNHostPath = lists:reverse(string:tokens(Host, ".")), - CDNPath = tl(filename:split(Path)), - PackageDir = filename:join([CacheDir, "hex"] ++ CDNHostPath ++ CDNPath ++ ["packages"]), + case rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN) of + ?DEFAULT_CDN -> + RegistryDir = filename:join([CacheDir, "hex", "default"]), + ok = filelib:ensure_dir(filename:join(RegistryDir, "placeholder")), + RegistryDir; + CDN -> + {ok, {_, _, Host, _, Path, _}} = http_uri:parse(CDN), + CDNHostPath = lists:reverse(string:tokens(Host, ".")), + CDNPath = tl(filename:split(Path)), + RegistryDir = filename:join([CacheDir, "hex"] ++ CDNHostPath ++ CDNPath), + ok = filelib:ensure_dir(filename:join(RegistryDir, "placeholder")), + RegistryDir + end. + +package_dir(State) -> + RegistryDir = registry_dir(State), + PackageDir = filename:join([RegistryDir, "packages"]), ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")), PackageDir. - -check_registry(Pkg, Vsn, State) -> - case rebar_state:registry(State) of - {ok, T} -> - case ets:lookup(T, Pkg) of - [{Pkg, [Vsns]}] -> - lists:member(Vsn, Vsns); - _ -> - false - end; - error -> - false - end. - registry_checksum({pkg, Name, Vsn}, State) -> - {ok, Registry} = registry(State), - case ets:lookup(Registry, {Name, Vsn}) of - [{{_, _}, [_, Checksum | _]}] -> - Checksum; - [] -> - none + try + verify_table(State), + ets:lookup_element(?PACKAGE_TABLE, {Name, Vsn}, 3) + catch + _:_ -> + throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)})) end. %% Hex supports use of ~> to specify the version required for a dependency. @@ -101,28 +104,45 @@ registry_checksum({pkg, Name, Vsn}, State) -> %% `~> 2.1.3-dev` | `>= 2.1.3-dev and < 2.2.0` %% `~> 2.0` | `>= 2.0.0 and < 3.0.0` %% `~> 2.1` | `>= 2.1.0 and < 3.0.0` -find_highest_matching(Dep, Constraint, T) -> - case ets:lookup(T, Dep) of - [{Dep, [[Vsn]]}] -> - case ec_semver:pes(Vsn, Constraint) of - true -> - {ok, Vsn}; - false -> - ?WARN("Only existing version of ~s is ~s which does not match constraint ~~> ~s. " - "Using anyway, but it is not guarenteed to work.", [Dep, Vsn, Constraint]), - {ok, Vsn} - end; - [{Dep, [[HeadVsn | VsnTail]]}] -> - {ok, lists:foldl(fun(Version, Highest) -> - case ec_semver:pes(Version, Constraint) andalso - ec_semver:gt(Version, Highest) of - true -> - Version; - false -> - Highest - end - end, HeadVsn, VsnTail)}; - [] -> - ?WARN("Missing registry entry for package ~s", [Dep]), +find_highest_matching(Dep, Constraint, Table, State) -> + verify_table(State), + try ets:lookup_element(Table, Dep, 2) of + [[HeadVsn | VsnTail]] -> + {ok, handle_vsns(Constraint, HeadVsn, VsnTail)}; + [[Vsn]] -> + handle_single_vsn(Dep, Vsn, Constraint); + [Vsn] -> + handle_single_vsn(Dep, Vsn, Constraint); + [HeadVsn | VsnTail] -> + {ok, handle_vsns(Constraint, HeadVsn, VsnTail)} + catch + error:badarg -> none end. + +handle_vsns(Constraint, HeadVsn, VsnTail) -> + lists:foldl(fun(Version, Highest) -> + case ec_semver:pes(Version, Constraint) andalso + ec_semver:gt(Version, Highest) of + true -> + Version; + false -> + Highest + end + end, HeadVsn, VsnTail). + +handle_single_vsn(Dep, Vsn, Constraint) -> + case ec_semver:pes(Vsn, Constraint) of + true -> + {ok, Vsn}; + false -> + ?WARN("Only existing version of ~s is ~s which does not match constraint ~~> ~s. " + "Using anyway, but it is not guarenteed to work.", [Dep, Vsn, Constraint]), + {ok, Vsn} + end. + +format_error({missing_package, Package, Version}) -> + io_lib:format("Package not found in registry: ~s-~s. Try to fix with `rebar3 update`", [Package, Version]). + +verify_table(State) -> + ets:info(?PACKAGE_TABLE, named_table) =:= true orelse ?MODULE:load_and_verify_version(State). diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index fdc69e2..3430e81 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -34,7 +34,7 @@ download(TmpDir, Pkg={pkg, Name, Vsn}, State) -> Url = string:join([CDN, Package], "/"), cached_download(TmpDir, CachePath, Pkg, Url, etag(CachePath), State). -cached_download(TmpDir, CachePath, Pkg, Url, ETag, State) -> +cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn}, Url, ETag, State) -> case request(Url, ETag) of {ok, cached} -> serve_from_cache(TmpDir, CachePath, Pkg, State); @@ -44,7 +44,7 @@ cached_download(TmpDir, CachePath, Pkg, Url, ETag, State) -> ?DEBUG("Download ~s error, using ~s from cache", [Url, CachePath]), serve_from_cache(TmpDir, CachePath, Pkg, State); error -> - request_failed + {fetch_fail, Name, Vsn} end. serve_from_cache(TmpDir, CachePath, Pkg, State) -> diff --git a/src/rebar_plugins.erl b/src/rebar_plugins.erl index bda3fb7..6c2daef 100644 --- a/src/rebar_plugins.erl +++ b/src/rebar_plugins.erl @@ -24,12 +24,12 @@ project_apps_install(State) -> StateAcc1 = handle_plugins(Profile, Plugins, StateAcc), lists:foldl(fun(App, StateAcc2) -> - AppDir = rebar_app_info:dir(App), - C = rebar_config:consult(AppDir), - S = rebar_state:new(rebar_state:new(), C, AppDir), - Plugins2 = rebar_state:get(S, {plugins, Profile}, []), - handle_plugins(Profile, Plugins2, StateAcc2) - end, StateAcc1, ProjectApps) + AppDir = rebar_app_info:dir(App), + C = rebar_config:consult(AppDir), + S = rebar_state:new(rebar_state:new(), C, AppDir), + Plugins2 = rebar_state:get(S, {plugins, Profile}, []), + handle_plugins(Profile, Plugins2, StateAcc2) + end, StateAcc1, ProjectApps) end, State, Profiles). -spec install(rebar_state:t()) -> rebar_state:t(). diff --git a/src/rebar_prv_app_discovery.erl b/src/rebar_prv_app_discovery.erl index ea55e11..5449f82 100644 --- a/src/rebar_prv_app_discovery.erl +++ b/src/rebar_prv_app_discovery.erl @@ -38,6 +38,10 @@ do(State) -> State1 = rebar_app_discover:do(State, LibDirs), {ok, State1} catch + throw:{error, {rebar_packages, Error}} -> + {error, {rebar_packages, Error}}; + throw:{error, {rebar_app_utils, Error}} -> + {error, {rebar_app_utils, Error}}; throw:{error, Error} -> ?PRV_ERROR(Error) end. diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index 2b024cf..1165631 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -361,10 +361,10 @@ remove_links(Path) -> end. delete_dir_link(Path) -> - case os:type() of - {unix, _} -> file:delete(Path); - {win32, _} -> file:del_dir(Path) - end. + case os:type() of + {unix, _} -> file:delete(Path); + {win32, _} -> file:del_dir(Path) + end. dir_entries(Path) -> {ok, SubDirs} = file:list_dir(Path), diff --git a/src/rebar_prv_deps.erl b/src/rebar_prv_deps.erl index 9dc5346..9ff2bfa 100644 --- a/src/rebar_prv_deps.erl +++ b/src/rebar_prv_deps.erl @@ -24,25 +24,16 @@ init(State) -> {short_desc, "List dependencies"}, {desc, "List dependencies. Those not matching lock files " "are followed by an asterisk (*)."}, - {opts, [{tree, $t, "tree", undefined, "Display package dependencies in tree format (git and hg deps not supported)."}]}])), + {opts, []}])), {ok, State1}. -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - case display_tree(State) of - true -> - {_Packages, Graph} = rebar_state:packages(State), - List = merge_deps_per_profile(State), - {_SrcDeps, PkgDeps} = rebar_prv_install_deps:parse_deps(<<"">>, List, State, [], 0), - rebar_digraph:print_solution(Graph, PkgDeps), - {ok, State}; - false -> - Profiles = rebar_state:current_profiles(State), - List = [{Profile, rebar_state:get(State, {deps, Profile}, [])} - || Profile <- Profiles], - [display(State, Profile, Deps) || {Profile, Deps} <- List], - {ok, State} - end. + Profiles = rebar_state:current_profiles(State), + List = [{Profile, rebar_state:get(State, {deps, Profile}, [])} + || Profile <- Profiles], + [display(State, Profile, Deps) || {Profile, Deps} <- List], + {ok, State}. -spec format_error(any()) -> iolist(). format_error(Reason) -> @@ -91,7 +82,6 @@ dedup([Dep|Deps], [Name|DepNames]) -> name(T) when is_tuple(T) -> element(1, T); name(B) when is_binary(B) -> B. - display_deps(State, Deps) -> lists:foreach(fun(Dep) -> display_dep(State, Dep) end, Deps). @@ -125,17 +115,3 @@ display_dep(State, {Name, Source, Level}) when is_tuple(Source), is_integer(Leve ?CONSOLE("~s~s (locked ~s source)", [Name, NeedsUpdate, type(Source)]). type(Source) when is_tuple(Source) -> element(1, Source). - -display_tree(State) -> - {Args, _} = rebar_state:command_parsed_args(State), - proplists:get_value(tree, Args, false). - -merge_deps_per_profile(State) -> - Profiles = rebar_state:current_profiles(State), - lists:foldl(fun(Profile, Deps) -> - D = rebar_utils:deps_to_binary(rebar_state:get(State, {deps, Profile}, [])), - D1 = rebar_utils:tup_sort(D), - rebar_utils:tup_dedup( - rebar_utils:tup_umerge(D1 - ,Deps)) - end, [], Profiles). diff --git a/src/rebar_prv_deps_tree.erl b/src/rebar_prv_deps_tree.erl new file mode 100644 index 0000000..d429c52 --- /dev/null +++ b/src/rebar_prv_deps_tree.erl @@ -0,0 +1,84 @@ +-module(rebar_prv_deps_tree). + +-behaviour(provider). + +-export([init/1, + do/1, + format_error/1]). + +-include("rebar.hrl"). + +-define(PROVIDER, tree). +-define(DEPS, [lock]). + +-spec init(rebar_state:t()) -> {ok, rebar_state:t()}. +init(State) -> + State1 = rebar_state:add_provider( + State, + providers:create([{name, ?PROVIDER}, + {module, ?MODULE}, + {bare, true}, + {deps, ?DEPS}, + {example, "rebar3 tree"}, + {short_desc, "Print dependency tree."}, + {desc, ""}, + {opts, [{verbose, $v, "verbose", undefined, "Print repo and branch/tag/ref for git and hg deps"}]}])), + {ok, State1}. + +-spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. +do(State) -> + {Args, _} = rebar_state:command_parsed_args(State), + Verbose = proplists:get_value(verbose, Args, false), + print_deps_tree(rebar_state:all_deps(State), Verbose, State), + {ok, State}. + +-spec format_error(any()) -> iolist(). +format_error(Reason) -> + io_lib:format("~p", [Reason]). + +%% Internal functions + +print_deps_tree(SrcDeps, Verbose, State) -> + D = lists:foldl(fun(App, Dict) -> + Name = rebar_app_info:name(App), + Vsn = rebar_app_info:original_vsn(App), + Source = rebar_app_info:source(App), + Parent = rebar_app_info:parent(App), + dict:append_list(Parent, [{Name, Vsn, Source}], Dict) + end, dict:new(), SrcDeps), + ProjectAppNames = [{rebar_app_info:name(App) + ,rebar_app_info:original_vsn(App) + ,project} || App <- rebar_state:project_apps(State)], + case dict:find(root, D) of + {ok, Children} -> + print_children(-1, lists:keysort(1, Children++ProjectAppNames), D, Verbose); + error -> + print_children(-1, lists:keysort(1, ProjectAppNames), D, Verbose) + end. + +print_children(_, [], _, _) -> + ok; +print_children(Indent, [{Name, Vsn, Source} | Rest], Dict, Verbose) -> + + [io:format("| ") || _ <- lists:seq(0, Indent, 2)], + io:format("|- "), + io:format("~s-~s (~s)~n", [Name, Vsn, type(Source, Verbose)]), + case dict:find(Name, Dict) of + {ok, Children} -> + print_children(Indent+2, lists:keysort(1, Children), Dict, Verbose), + print_children(Indent, Rest, Dict, Verbose); + error -> + print_children(Indent, Rest, Dict, Verbose) + end. + +type(project, _) -> + "project app"; +type(Source, Verbose) when is_tuple(Source) -> + case {element(1, Source), Verbose} of + {pkg, _} -> + "hex package"; + {Other, false} -> + io_lib:format("~s repo", [Other]); + {_, true} -> + io_lib:format("~s", [element(2, Source)]) + end. diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index 72350d6..ca9344b 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -36,7 +36,7 @@ -include_lib("providers/include/providers.hrl"). -export([handle_deps_as_profile/4, - parse_deps/5, + profile_dep_dir/2, find_cycles/1, cull_compile/2]). @@ -98,6 +98,8 @@ do(State) -> end. -spec format_error(any()) -> iolist(). +format_error({dep_app_not_found, AppDir, AppName}) -> + io_lib:format("Dependency failure: Application ~s not found at the top level of directory ~s", [AppName, AppDir]); format_error({load_registry_fail, Dep}) -> io_lib:format("Error loading registry to resolve version of ~s. Try fixing by running 'rebar3 update'", [Dep]); format_error({bad_constraint, Name, Constraint}) -> @@ -108,6 +110,8 @@ format_error({not_rebar_package, Package, Version}) -> io_lib:format("Package not buildable with rebar3: ~s-~s", [Package, Version]); format_error({missing_package, Package, Version}) -> io_lib:format("Package not found in registry: ~s-~s", [Package, Version]); +format_error({missing_package, Package}) -> + io_lib:format("Package not found in registry: ~s", [Package]); format_error({cycles, Cycles}) -> Prints = [["applications: ", [io_lib:format("~s ", [Dep]) || Dep <- Cycle], @@ -123,13 +127,9 @@ handle_deps_as_profile(Profile, State, Deps, Upgrade) -> Locks = [], Level = 0, DepsDir = profile_dep_dir(State, Profile), - {SrcDeps, PkgDeps} = parse_deps(DepsDir, Deps, State, Locks, Level), - AllSrcProfileDeps = [{Profile, SrcDeps, Locks, Level}], - AllPkgProfileDeps = [{Profile, Locks, PkgDeps, Level}], - {AllApps, PkgDeps1, Seen, State1} = handle_profile_level(AllSrcProfileDeps, AllPkgProfileDeps, Locks, sets:new(), Upgrade, State), - - handle_profile_pkg_level(PkgDeps1, AllApps, Seen, Upgrade, State1). - + Deps1 = rebar_app_utils:parse_deps(DepsDir, Deps, State, Locks, Level), + ProfileLevelDeps = [{Profile, Deps1, Level}], + handle_profile_level(ProfileLevelDeps, [], sets:new(), Upgrade, Locks, State). %% =================================================================== %% Internal functions @@ -138,56 +138,30 @@ handle_deps_as_profile(Profile, State, Deps, Upgrade) -> %% finds all the deps in `{deps, ...}` for each profile provided. deps_per_profile(Profiles, Upgrade, State) -> Level = 0, - {AllProfileDeps, PkgDeps} = lists:foldl(fun(Profile, {SrcAcc, PkgAcc}) -> - {Src, Pkg} = parse_profile_deps(State, Profile, Level), - {[Src | SrcAcc], [Pkg | PkgAcc]} - end, {[], []}, Profiles), - {AllApps, PkgDeps1, Seen, State1} = handle_profile_level(AllProfileDeps, PkgDeps, [], sets:new(), Upgrade, State), - - handle_profile_pkg_level(PkgDeps1, AllApps, Seen, Upgrade, State1). + Locks = rebar_state:get(State, {locks, default}, []), + Deps = lists:foldl(fun(Profile, DepAcc) -> + [parsed_profile_deps(State, Profile, Level) | DepAcc] + end, [], Profiles), + handle_profile_level(Deps, [], sets:new(), Upgrade, Locks, State). -parse_profile_deps(State, Profile, Level) -> - DepsDir = profile_dep_dir(State, Profile), - Locks = rebar_state:get(State, {locks, Profile}, []), - Deps = rebar_state:get(State, {deps, Profile}, []), - {SrcDeps, PkgDeps} = parse_deps(DepsDir, Deps, State, Locks, Level), - {{Profile, SrcDeps, Locks, Level}, {Profile, Locks, PkgDeps, Level}}. +parsed_profile_deps(State, Profile, Level) -> + ParsedDeps = rebar_state:get(State, {parsed_deps, Profile}, []), + {Profile, ParsedDeps, Level}. %% Level-order traversal of all dependencies, across profiles. %% If profiles x,y,z are present, then the traversal will go: %% x0, y0, z0, x1, y1, z1, ..., xN, yN, zN. -handle_profile_level([], PkgDeps, SrcApps, Seen, _Upgrade, State) -> - {SrcApps, PkgDeps, Seen, State}; -handle_profile_level([{Profile, SrcDeps, Locks, Level} | Rest], PkgDeps, SrcApps, Seen, Upgrade, State) -> - {SrcDeps1, PkgDeps1, SrcApps1, State1, Seen1, Locks1} = - update_src_deps(Profile, Level, SrcDeps, [], SrcApps - ,State, Upgrade, Seen, Locks), - SrcDeps2 = case SrcDeps1 of +handle_profile_level([], Apps, _Seen, _Upgrade, _Locks, State) -> + {Apps, State}; +handle_profile_level([{Profile, Deps, Level} | Rest], Apps, Seen, Upgrade, Locks, State) -> + {Deps1, Apps1, State1, Seen1} = + update_deps(Profile, Level, Deps, Apps + ,State, Upgrade, Seen, Locks), + Deps2 = case Deps1 of [] -> Rest; - _ -> Rest ++ [{Profile, SrcDeps1, Locks1, Level+1}] + _ -> Rest ++ [{Profile, Deps1, Level+1}] end, - handle_profile_level(SrcDeps2, [{Profile, Locks1, PkgDeps1, Level+1} | PkgDeps], SrcApps1++SrcApps, sets:union(Seen, Seen1), Upgrade, State1). - -handle_profile_pkg_level(PkgDeps, AllApps, Seen, Upgrade, State) -> - %% Read in package index and dep graph - {Packages, Graph} = rebar_state:packages(State), - Registry = rebar_packages:registry(State), - State1 = rebar_state:packages(rebar_state:registry(State, Registry) - ,{Packages, Graph}), - - lists:foldl(fun({_Profile, _, [], _}, {AllAcc, StateAcc}) -> - {AllAcc, StateAcc}; - ({Profile1, Locks, PkgDeps2, Level}, {AllAcc, StateAcc}) -> - {Solved, StateAcc2} = update_pkg_deps(Profile1, Packages, PkgDeps2 - ,Graph, Upgrade, Seen, StateAcc, Locks - ,Level), - - AllDeps = lists:ukeymerge(2 - ,lists:ukeysort(2, AllAcc) - ,lists:ukeysort(2, Solved)), - - {AllDeps, StateAcc2} - end, {AllApps, State1}, PkgDeps). + handle_profile_level(Deps2, Apps1, sets:union(Seen, Seen1), Upgrade, Locks, State1). find_cycles(Apps) -> case rebar_digraph:compile_order(Apps) of @@ -199,52 +173,6 @@ find_cycles(Apps) -> cull_compile(TopSortedDeps, ProjectApps) -> lists:dropwhile(fun not_needs_compile/1, TopSortedDeps -- ProjectApps). -update_pkg_deps(Profile, Packages, PkgDeps, Graph, Upgrade, Seen, State, Locks, Level) -> - case PkgDeps of - [] -> %% No pkg deps - {[], State}; - PkgDeps -> - %% Find pkg deps needed - S = case rebar_digraph:cull_deps(Graph, PkgDeps, Level) of - {ok, [], _} -> - throw({rebar_digraph, no_solution}); - {ok, Solution, []} -> - Solution; - {ok, Solution, Discarded} -> - [warn_skip_pkg(Pkg, State) || Pkg <- Discarded, not(pkg_locked(Pkg, Locks))], - Solution - end, - update_pkg_deps(Profile, S, Packages, Upgrade, Seen, State, Locks) - end. - -pkg_locked({Name, _, _}, Locks) -> - pkg_locked(Name, Locks); -pkg_locked({Name, _}, Locks) -> - pkg_locked(Name, Locks); -pkg_locked(Name, Locks) -> - false =/= lists:keyfind(Name, 1, Locks). - -update_pkg_deps(Profile, Pkgs, Packages, Upgrade, Seen, State, Locks) -> - %% Create app_info record for each pkg dep - DepsDir = profile_dep_dir(State, Profile), - {Solved, _, State1} - = lists:foldl(fun(Pkg, {Acc, SeenAcc, StateAcc}) -> - handle_pkg_dep(Profile, Pkg, Packages, Upgrade, DepsDir, Acc, SeenAcc, Locks, StateAcc) - end, {[], Seen, State}, Pkgs), - {Solved, State1}. - -handle_pkg_dep(Profile, Pkg, Packages, Upgrade, DepsDir, Fetched, Seen, Locks, State) -> - IsLock = pkg_locked(Pkg, Locks), - AppInfo = package_to_app(DepsDir, Packages, Pkg, IsLock, State), - Deps = rebar_app_info:deps(AppInfo), - Level = rebar_app_info:dep_level(AppInfo), - {NewSeen, NewState} = maybe_lock(Profile, AppInfo, Seen, State, Level), - {_, AppInfo1} = maybe_fetch(AppInfo, Profile, Upgrade, Seen, NewState), - {AppInfo2, _, _, _, _} = - handle_dep(NewState, Profile, DepsDir, AppInfo1, Locks, Level), - AppInfo3 = rebar_app_info:deps(AppInfo2, Deps), - {[AppInfo3 | Fetched], NewSeen, NewState}. - maybe_lock(Profile, AppInfo, Seen, State, Level) -> Name = rebar_app_info:name(AppInfo), case rebar_app_info:is_checkout(AppInfo) of @@ -271,49 +199,28 @@ maybe_lock(Profile, AppInfo, Seen, State, Level) -> {sets:add_element(Name, Seen), State} end. -package_to_app(DepsDir, Packages, {Name, Vsn, Level}, IsLock, State) -> - case dict:find({Name, Vsn}, Packages) of - error -> - case rebar_packages:check_registry(Name, Vsn, State) of - true -> - throw(?PRV_ERROR({not_rebar_package, Name, Vsn})); - false -> - throw(?PRV_ERROR({missing_package, Name, Vsn})) - end; - {ok, PkgDeps} -> - Source = {pkg, Name, Vsn}, - AppInfo = new_dep(DepsDir, Name, Vsn, Source, IsLock, State), - AppInfo1 = rebar_app_info:dep_level(rebar_app_info:deps(AppInfo, PkgDeps), Level), - BaseDir = rebar_state:get(State, base_dir, []), - AppState1 = rebar_state:set(rebar_app_info:state(AppInfo1), base_dir, BaseDir), - rebar_app_info:state(AppInfo1, AppState1) - end. - --spec update_src_deps(atom(), non_neg_integer(), list(), list(), list(), rebar_state:t(), boolean(), sets:set(binary()), list()) -> {list(), list(), list(), rebar_state:t(), sets:set(binary()), list()}. -update_src_deps(Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, Locks) -> +update_deps(Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) -> lists:foldl( - fun(AppInfo, {SrcDepsAcc, PkgDepsAcc, SrcAppsAcc, StateAcc, SeenAcc, LocksAcc}) -> - update_src_dep(AppInfo, Profile, Level, - SrcDepsAcc, PkgDepsAcc, SrcAppsAcc, StateAcc, - Upgrade, SeenAcc, Locks, LocksAcc) + fun(AppInfo, {DepsAcc, AppsAcc, StateAcc, SeenAcc}) -> + update_dep(AppInfo, Profile, Level, + DepsAcc, AppsAcc, StateAcc, + Upgrade, SeenAcc, Locks) end, - {[], PkgDeps, SrcApps, State, Seen, Locks}, - rebar_utils:sort_deps(SrcDeps)). + {[], Apps, State, Seen}, + rebar_utils:sort_deps(Deps)). - -update_src_dep(AppInfo, Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, BaseLocks, Locks) -> +update_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) -> %% If not seen, add to list of locks to write out Name = rebar_app_info:name(AppInfo), case sets:is_element(Name, Seen) of true -> - update_seen_src_dep(AppInfo, Profile, Level, - SrcDeps, PkgDeps, SrcApps, - State, Upgrade, Seen, BaseLocks, Locks); + update_seen_dep(AppInfo, Profile, Level, + Deps, Apps, + State, Upgrade, Seen, Locks); false -> - update_unseen_src_dep(AppInfo, Profile, Level, - SrcDeps, PkgDeps, SrcApps, - State, Upgrade, Seen, Locks) - + update_unseen_dep(AppInfo, Profile, Level, + Deps, Apps, + State, Upgrade, Seen, Locks) end. profile_dep_dir(State, Profile) -> @@ -322,85 +229,53 @@ profile_dep_dir(State, Profile) -> _ -> rebar_dir:deps_dir(State) end. -update_seen_src_dep(AppInfo, _Profile, _Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, BaseLocks, Locks) -> +update_seen_dep(AppInfo, _Profile, _Level, Deps, Apps, State, Upgrade, Seen, Locks) -> Name = rebar_app_info:name(AppInfo), %% If seen from lock file or user requested an upgrade %% don't print warning about skipping - case lists:keymember(Name, 1, BaseLocks) of + case lists:keymember(Name, 1, Locks) of false when Upgrade -> ok; false when not Upgrade -> warn_skip_deps(AppInfo, State); true -> ok end, - {SrcDeps, PkgDeps, SrcApps, State, Seen, Locks}. + {Deps, Apps, State, Seen}. -update_unseen_src_dep(AppInfo, Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, Locks) -> +update_unseen_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Locks) -> {NewSeen, State1} = maybe_lock(Profile, AppInfo, Seen, State, Level), - {NewSrcDeps, NewPkgDeps, NewSrcApps, State2, NewLocks} - = case Upgrade of - true -> - handle_upgrade(AppInfo, Profile, SrcDeps, PkgDeps, SrcApps, - Level, State1, Seen, Locks); - _ -> - {_, AppInfo1} = maybe_fetch(AppInfo, Profile, false, Seen, State1), - handle_dep(AppInfo1, Profile, SrcDeps, PkgDeps, SrcApps, - Level, State1, Locks) - end, - {NewSrcDeps, NewPkgDeps, NewSrcApps, State2, NewSeen, NewLocks}. - -handle_upgrade(AppInfo, Profile, SrcDeps, PkgDeps, SrcApps, Level, State, Seen, Locks) -> - Name = rebar_app_info:name(AppInfo), - case lists:keyfind(Name, 1, Locks) of - false -> - {_, AppInfo1} = maybe_fetch(AppInfo, Profile, true, Seen, State), - handle_dep(AppInfo1, Profile, SrcDeps, PkgDeps, SrcApps, - Level, State, Locks); - _StillLocked -> - handle_dep(AppInfo, Profile, SrcDeps, PkgDeps, SrcApps, - Level, State, Locks) - end. - -handle_dep(AppInfo, Profile, SrcDeps, PkgDeps, SrcApps, Level, State, Locks) -> + {_, AppInfo1} = maybe_fetch(AppInfo, Profile, Upgrade, Seen, State1), DepsDir = profile_dep_dir(State, Profile), - {AppInfo1, NewSrcDeps, NewPkgDeps, NewLocks, State1} = - handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level), - AppInfo2 = rebar_app_info:dep_level(AppInfo1, Level), - {NewSrcDeps ++ SrcDeps - ,NewPkgDeps++PkgDeps - ,[AppInfo2 | SrcApps] - ,State1 - ,NewLocks}. - --spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer()) -> - {rebar_app_info:t(), [rebar_app_info:t()], [pkg_dep()], [integer()], rebar_state:t()}. + {AppInfo2, NewDeps, State2} = + handle_dep(State1, Profile, DepsDir, AppInfo1, Locks, Level), + AppInfo3 = rebar_app_info:dep_level(AppInfo2, Level), + {NewDeps ++ Deps, [AppInfo3 | Apps], State2, NewSeen}. + +-spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer()) -> {rebar_app_info:t(), [rebar_app_info:t()], [pkg_dep()], [integer()], rebar_state:t()}. handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) -> Profiles = rebar_state:current_profiles(State), Name = rebar_app_info:name(AppInfo), - C = rebar_config:consult(rebar_app_info:dir(AppInfo)), - + %% Deps may be under a sub project app, find it and use its state if so S = rebar_app_info:state(AppInfo), - S1 = rebar_state:new(S, C, rebar_app_info:dir(AppInfo)), + C = rebar_config:consult(rebar_app_info:dir(AppInfo)), + S1 = rebar_state:new(S, C, AppInfo), S2 = rebar_state:apply_overrides(S1, Name), S3 = rebar_state:apply_profiles(S2, Profiles), Plugins = rebar_state:get(S3, plugins, []), S4 = rebar_state:set(S3, {plugins, Profile}, Plugins), - AppInfo1 = rebar_app_info:state(AppInfo, S4), rebar_utils:check_min_otp_version(rebar_state:get(S4, minimum_otp_vsn, undefined)), rebar_utils:check_blacklisted_otp_versions(rebar_state:get(S4, blacklisted_otp_vsns, [])), %% Dep may have plugins to install. Find and install here. S5 = rebar_plugins:install(S4), - AppInfo2 = rebar_app_info:state(AppInfo1, S5), + AppInfo1 = rebar_app_info:state(AppInfo, S5), %% Upgrade lock level to be the level the dep will have in this dep tree Deps = rebar_state:get(S5, {deps, default}, []), - NewLocks = [{DepName, Source, LockLevel+Level} || - {DepName, Source, LockLevel} <- rebar_state:get(S5, {locks, default}, [])], - AppInfo3 = rebar_app_info:deps(AppInfo2, rebar_state:deps_names(Deps)), - {SrcDeps, PkgDeps} = parse_deps(DepsDir, Deps, S5, Locks, Level+1), - {AppInfo3, SrcDeps, PkgDeps, Locks++NewLocks, State}. + AppInfo2 = rebar_app_info:deps(AppInfo1, rebar_state:deps_names(Deps)), + Deps1 = rebar_app_utils:parse_deps(Name, DepsDir, Deps, S5, Locks, Level+1), + {AppInfo2, Deps1, State}. -spec maybe_fetch(rebar_app_info:t(), atom(), boolean(), sets:set(binary()), rebar_state:t()) -> {boolean(), rebar_app_info:t()}. @@ -415,17 +290,20 @@ maybe_fetch(AppInfo, Profile, Upgrade, Seen, State) -> false -> true = fetch_app(AppInfo, AppDir, State), maybe_symlink_default(State, Profile, AppDir, AppInfo), - {true, update_app_info(AppDir, AppInfo)}; + {true, rebar_app_info:valid(update_app_info(AppDir, AppInfo), false)}; {true, AppInfo1} -> %% Preserve the state we created with overrides + AppInfo2 = copy_app_info(AppInfo, AppInfo1), AppState = rebar_app_info:state(AppInfo), - AppInfo2 = rebar_app_info:state(AppInfo1, AppState), - case sets:is_element(rebar_app_info:name(AppInfo), Seen) of + AppInfo3 = rebar_app_info:state(AppInfo2, AppState), + case sets:is_element(rebar_app_info:name(AppInfo3), Seen) of true -> - {false, AppInfo2}; + {false, AppInfo3}; false -> - maybe_symlink_default(State, Profile, AppDir, AppInfo2), - {maybe_upgrade(AppInfo, AppDir, Upgrade, State), AppInfo2} + maybe_symlink_default(State, Profile, AppDir, AppInfo3), + MaybeUpgrade = maybe_upgrade(AppInfo, AppDir, Upgrade, State), + AppInfo4 = update_app_info(AppDir, AppInfo3), + {MaybeUpgrade, AppInfo4} end end end. @@ -473,101 +351,6 @@ make_relative_to_root(State, Path) when is_list(Path) -> Root = rebar_dir:root_dir(State), rebar_dir:make_relative_path(Path, Root). --spec parse_deps(binary(), list(), rebar_state:t(), list(), integer()) -> {[rebar_app_info:t()], [tuple()]}. -parse_deps(DepsDir, Deps, State, Locks, Level) -> - lists:foldl(fun(Dep, Acc) -> - Name = case Dep of - Dep when is_tuple(Dep) -> - element(1, Dep); - Dep -> - Dep - end, - case lists:keyfind(ec_cnv:to_binary(Name), 1, Locks) of - false -> - parse_dep(Dep, Acc, DepsDir, false, State); - LockedDep -> - LockedLevel = element(3, LockedDep), - case LockedLevel > Level of - true -> - parse_dep(Dep, Acc, DepsDir, false, State); - false -> - parse_dep(LockedDep, Acc, DepsDir, true, State) - end - end - end, {[], []}, Deps). - -parse_dep({Name, Vsn}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_list(Vsn) -> - %% Versioned Package dependency - CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), - case rebar_app_info:discover(CheckoutsDir) of - {ok, _App} -> - Dep = new_dep(DepsDir, Name, [], [], IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; - not_found -> - {SrcDepsAcc, [parse_goal(ec_cnv:to_binary(Name) - ,ec_cnv:to_binary(Vsn)) | PkgDepsAcc]} - end; -parse_dep(Name, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_atom(Name); is_binary(Name) -> - %% Unversioned package dependency - {PkgName, PkgVsn} = get_package(ec_cnv:to_binary(Name), State), - CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), - case rebar_app_info:discover(CheckoutsDir) of - {ok, _App} -> - Dep = new_dep(DepsDir, Name, [], [], IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; - not_found -> - {SrcDepsAcc, [{PkgName, PkgVsn} | PkgDepsAcc]} - end; -parse_dep({Name, Source}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source) -> - Dep = new_dep(DepsDir, Name, [], Source, IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; -parse_dep({Name, _Vsn, Source}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source) -> - Dep = new_dep(DepsDir, Name, [], Source, IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; -parse_dep({Name, _Vsn, Source, Opts}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source) -> - ?WARN("Dependency option list ~p in ~p is not supported and will be ignored", [Opts, Name]), - Dep = new_dep(DepsDir, Name, [], Source, IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; -parse_dep({_Name, {pkg, Name, Vsn}, Level}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_integer(Level) -> - CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), - case rebar_app_info:discover(CheckoutsDir) of - {ok, _App} -> - Dep = new_dep(DepsDir, Name, [], [], IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; - not_found -> - {SrcDepsAcc, [{Name, Vsn} | PkgDepsAcc]} - end; -parse_dep({Name, Source, Level}, {SrcDepsAcc, PkgDepsAcc}, DepsDir, IsLock, State) when is_tuple(Source) - , is_integer(Level) -> - Dep = new_dep(DepsDir, Name, [], Source, IsLock, State), - {[Dep | SrcDepsAcc], PkgDepsAcc}; -parse_dep(Dep, _, _, _, _) -> - throw(?PRV_ERROR({parse_dep, Dep})). - - -new_dep(DepsDir, Name, Vsn, Source, IsLock, State) -> - CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), - {ok, Dep} = case rebar_app_info:discover(CheckoutsDir) of - {ok, App} -> - {ok, rebar_app_info:is_checkout(App, true)}; - not_found -> - Dir = ec_cnv:to_list(filename:join(DepsDir, Name)), - case rebar_app_info:discover(Dir) of - {ok, App} -> - {ok, App}; - not_found -> - rebar_app_info:new(Name, Vsn, - ec_cnv:to_list(filename:join(DepsDir, Name))) - end - end, - C = rebar_config:consult(rebar_app_info:dir(Dep)), - S = rebar_state:new(rebar_state:new(), C, rebar_app_info:dir(Dep)), - Overrides = rebar_state:get(State, overrides, []), - ParentOverrides = rebar_state:overrides(State), - Dep1 = rebar_app_info:state(Dep, - rebar_state:overrides(S, ParentOverrides++Overrides)), - rebar_app_info:is_lock(rebar_app_info:source(Dep1, Source), IsLock). - fetch_app(AppInfo, AppDir, State) -> ?INFO("Fetching ~s (~p)", [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)]), Source = rebar_app_info:source(AppInfo), @@ -577,14 +360,31 @@ fetch_app(AppInfo, AppDir, State) -> %% So this is the first time for newly downloaded apps that its .app/.app.src data can %% be read in an parsed. update_app_info(AppDir, AppInfo) -> - {ok, Found} = rebar_app_info:discover(AppDir), - AppDetails = rebar_app_info:app_details(Found), - Applications = proplists:get_value(applications, AppDetails, []), - IncludedApplications = proplists:get_value(included_applications, AppDetails, []), - AppInfo1 = rebar_app_info:applications( - rebar_app_info:app_details(AppInfo, AppDetails), - IncludedApplications++Applications), - rebar_app_info:valid(AppInfo1, false). + case rebar_app_info:discover(AppDir) of + {ok, Found} -> + AppDetails = rebar_app_info:app_details(Found), + Vsn = rebar_app_info:original_vsn(Found), + Applications = proplists:get_value(applications, AppDetails, []), + IncludedApplications = proplists:get_value(included_applications, AppDetails, []), + AppInfo1 = rebar_app_info:original_vsn(rebar_app_info:applications( + rebar_app_info:app_details(AppInfo, AppDetails), + IncludedApplications++Applications), Vsn), + AppInfo2 = copy_app_info(AppInfo, AppInfo1), + rebar_app_info:valid(AppInfo2, undefined); + not_found -> + throw(?PRV_ERROR({dep_app_not_found, AppDir, rebar_app_info:name(AppInfo)})) + end. + +copy_app_info(OldAppInfo, NewAppInfo) -> + Deps = rebar_app_info:deps(OldAppInfo), + ResourceType = rebar_app_info:resource_type(OldAppInfo), + Parent = rebar_app_info:parent(OldAppInfo), + Source = rebar_app_info:source(OldAppInfo), + + rebar_app_info:deps( + rebar_app_info:resource_type( + rebar_app_info:source( + rebar_app_info:parent(NewAppInfo, Parent), Source), ResourceType), Deps). maybe_upgrade(AppInfo, AppDir, Upgrade, State) -> Source = rebar_app_info:source(AppInfo), @@ -607,17 +407,6 @@ maybe_upgrade(AppInfo, AppDir, Upgrade, State) -> false end. --spec parse_goal(binary(), binary()) -> {binary(), binary()} | {binary(), binary(), binary()}. -parse_goal(Name, Constraint) -> - case re:run(Constraint, "([^\\d]*)(\\d.*)", [{capture, [1,2], binary}]) of - {match, [<<>>, Vsn]} -> - {Name, Vsn}; - {match, [Op, Vsn]} -> - {Name, Vsn, binary_to_atom(Op, utf8)}; - nomatch -> - throw(?PRV_ERROR({bad_constraint, Name, Constraint})) - end. - warn_skip_deps(AppInfo, State) -> Msg = "Skipping ~s (from ~p) as an app of the same name " "has already been fetched", @@ -628,25 +417,7 @@ warn_skip_deps(AppInfo, State) -> true -> ?ERROR(Msg, Args), ?FAIL end. -warn_skip_pkg({Name, Source}, State) -> - Msg = "Skipping ~s (version ~s from package index) as an app of the same " - "name has already been fetched", - Args = [Name, Source], - case rebar_state:get(State, deps_error_on_conflict, false) of - false -> ?WARN(Msg, Args); - true -> ?ERROR(Msg, Args), ?FAIL - end. - not_needs_compile(App) -> not(rebar_app_info:is_checkout(App)) andalso rebar_app_info:valid(App) andalso rebar_state:has_all_artifacts(rebar_app_info:state(App)) =:= true. - -get_package(Dep, State) -> - case rebar_state:registry(State) of - {ok, T} -> - {ok, HighestDepVsn} = rebar_packages:find_highest_matching(Dep, "0", T), - {Dep, HighestDepVsn}; - error -> - throw(?PRV_ERROR({load_registry_fail, Dep})) - end. diff --git a/src/rebar_prv_lock.erl b/src/rebar_prv_lock.erl index 1844934..8578979 100644 --- a/src/rebar_prv_lock.erl +++ b/src/rebar_prv_lock.erl @@ -29,18 +29,24 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - OldLocks = rebar_state:get(State, {locks, default}, []), - Locks = build_locks(State), - Dir = rebar_state:dir(State), - file:write_file(filename:join(Dir, ?LOCK_FILE), - io_lib:format("~p.~n", [Locks])), - State1 = rebar_state:set(State, {locks, default}, Locks), + %% Only lock default profile run + case rebar_state:current_profiles(State) of + [default] -> + OldLocks = rebar_state:get(State, {locks, default}, []), + Locks = lists:keysort(1, build_locks(State)), + Dir = rebar_state:dir(State), + file:write_file(filename:join(Dir, ?LOCK_FILE), + io_lib:format("~p.~n", [Locks])), + State1 = rebar_state:set(State, {locks, default}, Locks), - OldLockNames = [element(1,L) || L <- OldLocks], - NewLockNames = [element(1,L) || L <- Locks], - rebar_utils:info_useless(OldLockNames, NewLockNames), + OldLockNames = [element(1,L) || L <- OldLocks], + NewLockNames = [element(1,L) || L <- Locks], + rebar_utils:info_useless(OldLockNames, NewLockNames), - {ok, State1}. + {ok, State1}; + _ -> + {ok, State} + end. -spec format_error(any()) -> iolist(). format_error(Reason) -> diff --git a/src/rebar_prv_packages.erl b/src/rebar_prv_packages.erl index 880d4a6..5b8ea66 100644 --- a/src/rebar_prv_packages.erl +++ b/src/rebar_prv_packages.erl @@ -27,19 +27,21 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - {Dict, _} = rebar_packages:get_packages(State), - print_packages(Dict), + print_packages(), {ok, State}. -spec format_error(any()) -> iolist(). format_error(load_registry_fail) -> "Failed to load package regsitry. Try running 'rebar3 update' to fix". -print_packages(Dict) -> - Pkgs = lists:keysort(1, dict:fetch_keys(Dict)), - SortedPkgs = lists:foldl(fun({Pkg, Vsn}, Acc) -> - orddict:append_list(Pkg, [Vsn], Acc) - end, orddict:new(), Pkgs), +print_packages() -> + SortedPkgs = ets:foldl(fun({package_index_version, _}, Acc) -> + Acc; + ({Pkg, Vsns}, Acc) -> + orddict:store(Pkg, Vsns, Acc); + (_, Acc) -> + Acc + end, orddict:new(), ?PACKAGE_TABLE), orddict:map(fun(Name, Vsns) -> SortedVsns = lists:sort(fun(A, B) -> @@ -47,7 +49,7 @@ print_packages(Dict) -> ,ec_semver:parse(B)) end, Vsns), VsnStr = join(SortedVsns, <<", ">>), - io:format("~s:~n Versions: ~s~n~n", [Name, VsnStr]) + ?CONSOLE("~s:~n Versions: ~s~n", [Name, VsnStr]) end, SortedPkgs). -spec join([binary()], binary()) -> binary(). diff --git a/src/rebar_prv_plugins_upgrade.erl b/src/rebar_prv_plugins_upgrade.erl index dfc9990..fabfa5b 100644 --- a/src/rebar_prv_plugins_upgrade.erl +++ b/src/rebar_prv_plugins_upgrade.erl @@ -91,5 +91,5 @@ build_plugin(AppInfo, Apps, State) -> Providers = rebar_state:providers(State), AppDir = rebar_app_info:dir(AppInfo), C = rebar_config:consult(AppDir), - S = rebar_state:new(rebar_state:all_deps(rebar_state:new(), Apps), C, AppDir), + S = rebar_state:new(rebar_state:all_deps(rebar_state:new(), Apps), C, AppInfo), rebar_prv_compile:compile(S, Providers, AppInfo). diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index 049e78c..10faf4d 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -9,21 +9,14 @@ do/1, format_error/1]). +-export([hex_to_index/1]). + -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). -define(PROVIDER, update). -define(DEPS, []). -%% Ignore warning of digraph opaque type when running dialyzer --dialyzer({no_opaque, do/1}). --dialyzer({no_opaque, write_registry/3}). - -%% Ignoring the opaque type warning won't stop dialyzer from warning of -%% no return for functions that had the opaque type warnings --dialyzer({no_return, do/1}). --dialyzer({no_return, write_registry/3}). - %% =================================================================== %% Public API %% =================================================================== @@ -42,23 +35,23 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - ?INFO("Updating package index...", []), try - RegistryDir = rebar_packages:package_dir(State), + RegistryDir = rebar_packages:registry_dir(State), filelib:ensure_dir(filename:join(RegistryDir, "dummy")), HexFile = filename:join(RegistryDir, "registry"), + ?INFO("Updating package registry...", []), TmpDir = ec_file:insecure_mkdtemp(), TmpFile = filename:join(TmpDir, "packages.gz"), - Url = rebar_state:get(State, rebar_packages_cdn, "https://s3.amazonaws.com/s3.hex.pm/registry.ets.gz"), + Url = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_HEX_REGISTRY), {ok, _RequestId} = httpc:request(get, {Url, []}, [], [{stream, TmpFile}, {sync, true}], rebar), {ok, Data} = file:read_file(TmpFile), Unzipped = zlib:gunzip(Data), ok = file:write_file(HexFile, Unzipped), - {Dict, Graph} = hex_to_graph(HexFile), - write_registry(Dict, Graph, State), + ?INFO("Writing registry to ~s", [rebar_file_utils:replace_home_dir(HexFile)]), + hex_to_index(State), ok catch _E:C -> @@ -72,56 +65,62 @@ do(State) -> format_error(package_index_write) -> "Failed to write package index.". --spec write_registry(rebar_dict(), {digraph, ets:tid(), ets:tid(), ets:tid(), any()}, rebar_state:t()) -> ok | {error, atom()}. -write_registry(Dict, {digraph, Edges, Vertices, Neighbors, _}, State) -> - RegistryDir = rebar_packages:package_dir(State), - filelib:ensure_dir(filename:join(RegistryDir, "dummy")), - ets:tab2file(Edges, filename:join(RegistryDir, "edges")), - ets:tab2file(Vertices, filename:join(RegistryDir, "vertices")), - ets:tab2file(Neighbors, filename:join(RegistryDir, "neighbors")), - file:write_file(filename:join(RegistryDir, "dict"), term_to_binary(Dict)). - is_supported(<<"make">>) -> true; is_supported(<<"rebar">>) -> true; +is_supported(<<"rebar3">>) -> true; is_supported(_) -> false. -hex_to_graph(Filename) -> - {ok, T} = ets:file2tab(Filename), - Graph = digraph:new(), - ets:foldl(fun({Pkg, [Versions]}, ok) when is_binary(Pkg), is_list(Versions) -> - lists:foreach(fun(Version) -> - digraph:add_vertex(Graph, {Pkg, Version}, 1) - end, Versions); - (_, ok) -> - ok - end, ok, T), - - Dict1 = ets:foldl(fun({{Pkg, PkgVsn}, [Deps, _, BuildTools | _]}, Dict) when is_list(BuildTools) -> - case lists:any(fun is_supported/1, BuildTools) of - true -> - DepsList = update_graph(Pkg, PkgVsn, Deps, T, Graph), - dict:store({Pkg, PkgVsn}, DepsList, Dict); - false -> - Dict - end; - (_, Dict) -> - Dict - end, dict:new(), T), - {Dict1, Graph}. - -update_graph(Pkg, PkgVsn, Deps, HexRegistry, Graph) -> +hex_to_index(State) -> + RegistryDir = rebar_packages:registry_dir(State), + HexFile = filename:join(RegistryDir, "registry"), + try ets:file2tab(HexFile) of + {ok, Registry} -> + try + PackageIndex = filename:join(RegistryDir, "packages.idx"), + ?INFO("Generating package index...", []), + (catch ets:delete(?PACKAGE_TABLE)), + ets:new(?PACKAGE_TABLE, [named_table, public]), + ets:foldl(fun({{Pkg, PkgVsn}, [Deps, Checksum, BuildTools | _]}, _) when is_list(BuildTools) -> + case lists:any(fun is_supported/1, BuildTools) of + true -> + DepsList = update_deps_list(Deps, Registry, State), + ets:insert(?PACKAGE_TABLE, {{Pkg, PkgVsn}, DepsList, Checksum}); + false -> + true + end; + ({Pkg, [Vsns]}, _) when is_binary(Pkg) -> + ets:insert(?PACKAGE_TABLE, {Pkg, Vsns}); + (_, _) -> + true + end, true, Registry), + + ets:insert(?PACKAGE_TABLE, {package_index_version, ?PACKAGE_INDEX_VERSION}), + ?INFO("Writing index to ~s", [rebar_file_utils:replace_home_dir(PackageIndex)]), + ets:tab2file(?PACKAGE_TABLE, PackageIndex), + true + after + catch ets:delete(Registry) + end; + {error, Reason} -> + ?DEBUG("Error loading package registry: ~p", [Reason]), + false + catch + _:_ -> + fail + end. + +update_deps_list(Deps, HexRegistry, State) -> lists:foldl(fun([Dep, DepVsn, false, _AppName | _], DepsListAcc) -> case DepVsn of <<"~> ", Vsn/binary>> -> - case rebar_packages:find_highest_matching(Dep, Vsn, HexRegistry) of + case rebar_packages:find_highest_matching(Dep, Vsn, HexRegistry, State) of {ok, HighestDepVsn} -> - digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, HighestDepVsn}), - [{Dep, DepVsn} | DepsListAcc]; + [{Dep, HighestDepVsn} | DepsListAcc]; none -> + ?WARN("Missing registry entry for package ~s. Try to fix with `rebar3 update`", [Dep]), DepsListAcc end; Vsn -> - digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, Vsn}), [{Dep, Vsn} | DepsListAcc] end; ([_Dep, _DepVsn, true, _AppName | _], DepsListAcc) -> diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl index 3a371ca..97d1953 100644 --- a/src/rebar_prv_upgrade.erl +++ b/src/rebar_prv_upgrade.erl @@ -53,18 +53,21 @@ do(State) -> {Locks0, _Unlocks0} -> Deps0 = top_level_deps(Deps, Locks), State1 = rebar_state:set(State, {deps, default}, Deps0), - State2 = rebar_state:set(State1, {locks, default}, Locks0), - State3 = rebar_state:set(State2, upgrade, true), - UpdatedLocks = [L || L <- rebar_state:lock(State3), + DepsDir = rebar_prv_install_deps:profile_dep_dir(State, default), + D = rebar_app_utils:parse_deps(root, DepsDir, Deps0, State1, Locks0, 0), + State2 = rebar_state:set(State1, {parsed_deps, default}, D), + State3 = rebar_state:set(State2, {locks, default}, Locks0), + State4 = rebar_state:set(State3, upgrade, true), + UpdatedLocks = [L || L <- rebar_state:lock(State4), lists:keymember(rebar_app_info:name(L), 1, Locks0)], - Res = rebar_prv_install_deps:do(rebar_state:lock(State3, UpdatedLocks)), + Res = rebar_prv_install_deps:do(rebar_state:lock(State4, UpdatedLocks)), case Res of - {ok, State4} -> + {ok, State5} -> rebar_utils:info_useless( [element(1,Lock) || Lock <- Locks], - [rebar_app_info:name(App) || App <- rebar_state:lock(State4)] + [rebar_app_info:name(App) || App <- rebar_state:lock(State5)] ), - rebar_prv_lock:do(State4); + rebar_prv_lock:do(State5); _ -> Res end @@ -80,7 +83,6 @@ format_error({transitive_dependency, Name}) -> format_error(Reason) -> io_lib:format("~p", [Reason]). - parse_names(Bin, Locks) -> case lists:usort(re:split(Bin, <<" *, *">>, [trim])) of %% Nothing submitted, use *all* apps @@ -98,7 +100,8 @@ prepare_locks([Name|Names], Deps, Locks, Unlocks) -> {_, _, 0} = Lock -> case rebar_utils:tup_find(AtomName, Deps) of false -> - ?PRV_ERROR({unknown_dependency, Name}); + ?WARN("Dependency ~s has been removed and will not be upgraded", [Name]), + prepare_locks(Names, Deps, Locks, Unlocks); Dep -> {Source, NewLocks, NewUnlocks} = prepare_lock(Dep, Lock, Locks), prepare_locks(Names, Deps, NewLocks, diff --git a/src/rebar_relx.erl b/src/rebar_relx.erl index a3adedd..699c780 100644 --- a/src/rebar_relx.erl +++ b/src/rebar_relx.erl @@ -28,14 +28,12 @@ do(Module, Command, Provider, State) -> case rebar_state:get(State, relx, []) of [] -> relx:main([{lib_dirs, LibDirs} - ,{output_dir, OutputDir} - ,{caller, api}], AllOptions); + ,{caller, api} | output_dir(OutputDir, Options)], AllOptions); Config -> Config1 = update_config(Config), relx:main([{lib_dirs, LibDirs} ,{config, Config1} - ,{output_dir, OutputDir} - ,{caller, api}], AllOptions) + ,{caller, api} | output_dir(OutputDir, Options)], AllOptions) end, rebar_hooks:run_all_hooks(Cwd, post, Provider, Providers, State), {ok, State} @@ -67,3 +65,8 @@ update_config(Config) -> end end, {[], []}, Config), lists:reverse(Special) ++ Other. + +%% Don't override output_dir if the user passed one on the command line +output_dir(OutputDir, Options) -> + [{output_dir, OutputDir} || not(lists:member("-o", Options)) + andalso not(lists:member("--output-dir", Options))]. diff --git a/src/rebar_state.erl b/src/rebar_state.erl index 59a9588..e31b01b 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -38,9 +38,6 @@ overrides/1, overrides/2, apply_overrides/2, - packages/1, packages/2, - registry/1, registry/2, - resources/1, resources/2, add_resource/2, providers/1, providers/2, add_provider/2]). @@ -65,8 +62,6 @@ all_plugin_deps = [] :: [rebar_app_info:t()], all_deps = [] :: [rebar_app_info:t()], - packages = undefined :: {rebar_dict(), rebar_digraph()} | undefined, - registry = undefined :: {ok, ets:tid()} | error | undefined, overrides = [], resources = [], providers = []}). @@ -83,11 +78,7 @@ new() -> -spec new(list()) -> t(). new(Config) when is_list(Config) -> BaseState = base_state(), - Deps = proplists:get_value(deps, Config, []), - Plugins = proplists:get_value(plugins, Config, []), - Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins} | Config], - true = rebar_config:verify_config_format(Terms), - Opts = dict:from_list(Terms), + Opts = base_opts(Config), BaseState#state_t { dir = rebar_dir:get_cwd(), default = Opts, opts = Opts }. @@ -96,12 +87,7 @@ new(Config) when is_list(Config) -> new(Profile, Config) when is_atom(Profile) , is_list(Config) -> BaseState = base_state(), - Deps = proplists:get_value(deps, Config, []), - - Plugins = proplists:get_value(plugins, Config, []), - Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins} | Config], - true = rebar_config:verify_config_format(Terms), - Opts = dict:from_list(Terms), + Opts = base_opts(Config), BaseState#state_t { dir = rebar_dir:get_cwd(), current_profiles = [Profile], default = Opts, @@ -111,25 +97,26 @@ new(ParentState=#state_t{}, Config) -> Dir = rebar_dir:get_cwd(), new(ParentState, Config, Dir). --spec new(t(), list(), file:name()) -> t(). -new(ParentState, Config, Dir) -> +-spec new(t(), list(), rebar_app_info:t() | file:filename_all()) -> t(). +new(ParentState, Config, Dir) when is_list(Dir) -> + new(ParentState, Config, deps_from_config(Dir, Config), Dir); +new(ParentState, Config, AppInfo) -> + Dir = rebar_app_info:dir(AppInfo), + DepLocks = case rebar_app_info:resource_type(AppInfo) of + pkg -> + Deps = rebar_app_info:deps(AppInfo), + [{{locks, default}, Deps}, {{deps, default}, Deps}]; + _ -> + deps_from_config(Dir, Config) + end, + new(ParentState, Config, DepLocks, Dir). + +new(ParentState, Config, Deps, Dir) -> Opts = ParentState#state_t.opts, - LocalOpts = case rebar_config:consult_lock_file(filename:join(Dir, ?LOCK_FILE)) of - [D] -> - %% We want the top level deps only from the lock file. - %% This ensures deterministic overrides for configs. - Deps = [X || X <- D, element(3, X) =:= 0], - Plugins = proplists:get_value(plugins, Config, []), - Terms = [{{locks, default}, D}, {{deps, default}, Deps}, {{plugins, default}, Plugins} | Config], - true = rebar_config:verify_config_format(Terms), - dict:from_list(Terms); - _ -> - D = proplists:get_value(deps, Config, []), - Plugins = proplists:get_value(plugins, Config, []), - Terms = [{{deps, default}, D}, {{plugins, default}, Plugins} | Config], - true = rebar_config:verify_config_format(Terms), - dict:from_list(Terms) - end, + Plugins = proplists:get_value(plugins, Config, []), + Terms = Deps++[{{plugins, default}, Plugins} | Config], + true = rebar_config:verify_config_format(Terms), + LocalOpts = dict:from_list(Terms), NewOpts = merge_opts(LocalOpts, Opts), @@ -137,6 +124,17 @@ new(ParentState, Config, Dir) -> ,opts=NewOpts ,default=NewOpts}. +deps_from_config(Dir, Config) -> + case rebar_config:consult_lock_file(filename:join(Dir, ?LOCK_FILE)) of + [D] -> + %% We want the top level deps only from the lock file. + %% This ensures deterministic overrides for configs. + Deps = [X || X <- D, element(3, X) =:= 0], + [{{locks, default}, D}, {{deps, default}, Deps}]; + _ -> + [{{deps, default}, proplists:get_value(deps, Config, [])}] + end. + base_state() -> case application:get_env(rebar, resources) of undefined -> @@ -146,6 +144,13 @@ base_state() -> end, #state_t{resources=Resources}. +base_opts(Config) -> + Deps = proplists:get_value(deps, Config, []), + Plugins = proplists:get_value(plugins, Config, []), + Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins} | Config], + true = rebar_config:verify_config_format(Terms), + dict:from_list(Terms). + get(State, Key) -> {ok, Value} = dict:find(Key, State#state_t.opts), Value. @@ -253,7 +258,9 @@ apply_overrides(State=#state_t{overrides=Overrides}, AppName) -> %% Inefficient. We want the order we get here though. State1 = lists:foldl(fun({override, O}, StateAcc) -> - lists:foldl(fun({Key, Value}, StateAcc1) -> + lists:foldl(fun({deps, Value}, StateAcc1) -> + rebar_state:set(StateAcc1, {deps,default}, Value); + ({Key, Value}, StateAcc1) -> rebar_state:set(StateAcc1, Key, Value) end, StateAcc, O); (_, StateAcc) -> @@ -261,7 +268,9 @@ apply_overrides(State=#state_t{overrides=Overrides}, AppName) -> end, State, Overrides), State2 = lists:foldl(fun({override, N, O}, StateAcc) when N =:= Name -> - lists:foldl(fun({Key, Value}, StateAcc1) -> + lists:foldl(fun({deps, Value}, StateAcc1) -> + rebar_state:set(StateAcc1, {deps,default}, Value); + ({Key, Value}, StateAcc1) -> rebar_state:set(StateAcc1, Key, Value) end, StateAcc, O); (_, StateAcc) -> @@ -269,7 +278,10 @@ apply_overrides(State=#state_t{overrides=Overrides}, AppName) -> end, State1, Overrides), State3 = lists:foldl(fun({add, N, O}, StateAcc) when N =:= Name -> - lists:foldl(fun({Key, Value}, StateAcc1) -> + lists:foldl(fun({deps, Value}, StateAcc1) -> + OldValue = rebar_state:get(StateAcc1, {deps,default}, []), + rebar_state:set(StateAcc1, {deps,default}, Value++OldValue); + ({Key, Value}, StateAcc1) -> OldValue = rebar_state:get(StateAcc1, Key, []), rebar_state:set(StateAcc1, Key, Value++OldValue) end, StateAcc, O); @@ -438,22 +450,6 @@ namespace(#state_t{namespace=Namespace}) -> namespace(State=#state_t{}, Namespace) -> State#state_t{namespace=Namespace}. -packages(State=#state_t{packages=undefined}) -> - rebar_packages:get_packages(State); -packages(#state_t{packages=Packages}) -> - Packages. - -packages(State, Packages) -> - State#state_t{packages=Packages}. - -registry(State=#state_t{registry=undefined}) -> - rebar_packages:registry(State); -registry(#state_t{registry=Registry}) -> - Registry. - -registry(State, Registry) -> - State#state_t{registry=Registry}. - -spec resources(t()) -> [{rebar_resource:type(), module()}]. resources(#state_t{resources=Resources}) -> Resources. diff --git a/test/mock_git_resource.erl b/test/mock_git_resource.erl index d8f747b..0f4aff6 100644 --- a/test/mock_git_resource.erl +++ b/test/mock_git_resource.erl @@ -20,7 +20,8 @@ mock() -> mock([]). | {override_vsn, [{App, Vsn}]} | {deps, [{App, [Dep]}]}, App :: string(), - Dep :: {App, string(), {git, string()} | {git, string(), term()}}, + Dep :: {App, string(), {git, string()} | {git, string(), term()}} + | {pkg, App, term()}, Vsn :: string(). mock(Opts) -> meck:new(?MOD, [no_link]), @@ -46,8 +47,8 @@ mock_lock(_) -> case Git of {git, Url, {tag, Ref}} -> {git, Url, {ref, Ref}}; {git, Url, {ref, Ref}} -> {git, Url, {ref, Ref}}; - {git, Url} -> {git, Url, {ref, "fake-ref"}}; - {git, Url, _} -> {git, Url, {ref, "fake-ref"}} + {git, Url} -> {git, Url, {ref, "0.0.0"}}; + {git, Url, _} -> {git, Url, {ref, "0.0.0"}} end end). diff --git a/test/mock_pkg_resource.erl b/test/mock_pkg_resource.erl index eda863b..4ed8d8e 100644 --- a/test/mock_pkg_resource.erl +++ b/test/mock_pkg_resource.erl @@ -111,32 +111,19 @@ mock_pkg_index(Opts) -> Deps = proplists:get_value(pkgdeps, Opts, []), Skip = proplists:get_value(not_in_index, Opts, []), %% Dict: {App, Vsn}: [{<<"link">>, <<>>}, {<<"deps">>, []}] - %% Digraph: all apps and deps in the index + %% Index: all apps and deps in the index + Dict = find_parts(Deps, Skip), - GraphParts = to_graph_parts(Dict), - Digraph = rebar_digraph:restore_graph(GraphParts), meck:new(rebar_packages, [passthrough, no_link]), - meck:expect(rebar_packages, registry, - fun(_State) -> {ok, to_registry(Deps)} end), - meck:expect(rebar_packages, get_packages, - fun(_State) -> {Dict, Digraph} end). + meck:expect(rebar_packages, packages, + fun(_State) -> to_index(Deps, Dict) end), + meck:expect(rebar_packages, load_and_verify_version, + fun(_State) -> to_index(Deps, Dict), true end). %%%%%%%%%%%%%%% %%% Helpers %%% %%%%%%%%%%%%%%% -to_registry(Deps) -> - Tid = ets:new(registry, []), - lists:foreach(fun({{Name, Vsn}, _}) -> - case ets:lookup(Tid, Name) of - [{_, [Vsns]}] -> - ets:insert(Tid, {Name, [[Vsn | Vsns]]}); - _ -> - ets:insert(Tid, {Name, [[Vsn]]}) - end - end, Deps), - Tid. - all_files(Dir) -> filelib:wildcard(filename:join([Dir, "**"])). @@ -156,10 +143,20 @@ find_parts([{AppName, Deps}|Rest], Skip, Acc) -> find_parts(Rest, Skip, AccNew) end. -to_graph_parts(Dict) -> - LastUpdated = os:timestamp(), - dict:fold(fun(K,Deps,{Ks,Vs}) -> - {[{K,LastUpdated}|Ks], - [{K,{list_to_binary(atom_to_list(DK)), list_to_binary(DV)}} - || {DK,DV} <- Deps] ++ Vs} - end, {[],[]}, Dict). +to_index(AllDeps, Dict) -> + catch ets:delete(package_index), + ets:new(package_index, [named_table, public]), + dict:fold( + fun(K, Deps, _) -> + DepsList = [{ec_cnv:to_binary(DK), ec_cnv:to_binary(DV)} || {DK, DV} <- Deps], + ets:insert(package_index, {K, DepsList, <<"checksum">>}) + end, ok, Dict), + ets:insert(package_index, {package_index_version, 3}), + lists:foreach(fun({{Name, Vsn}, _}) -> + case ets:lookup(package_index, ec_cnv:to_binary(Name)) of + [{_, Vsns}] -> + ets:insert(package_index, {ec_cnv:to_binary(Name), [ec_cnv:to_binary(Vsn) | Vsns]}); + _ -> + ets:insert(package_index, {ec_cnv:to_binary(Name), [ec_cnv:to_binary(Vsn)]}) + end + end, AllDeps). diff --git a/test/rebar_compile_SUITE.erl b/test/rebar_compile_SUITE.erl index 0aaa899..f726943 100644 --- a/test/rebar_compile_SUITE.erl +++ b/test/rebar_compile_SUITE.erl @@ -23,7 +23,9 @@ erl_first_files_test/1, mib_test/1, only_default_transitive_deps/1, - clean_all/1]). + clean_all/1, + override_deps/1, + profile_override_deps/1]). -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -52,7 +54,7 @@ all() -> dont_recompile_yrl_or_xrl, delete_beam_if_source_deleted, deps_in_path, checkout_priority, highest_version_of_pkg_dep, parse_transform_test, erl_first_files_test, mib_test, only_default_transitive_deps, - clean_all]. + clean_all, override_deps, profile_override_deps]. build_basic_app(Config) -> AppDir = ?config(apps, Config), @@ -544,7 +546,8 @@ only_default_transitive_deps(Config) -> GitDeps = rebar_test_utils:expand_deps(git, [{"a", "1.0.0", []}]), PkgName = rebar_test_utils:create_random_name("pkg1_"), - mock_git_resource:mock([{deps, rebar_test_utils:flat_deps(GitDeps)}, + {SrcDeps, _} = rebar_test_utils:flat_deps(GitDeps), + mock_git_resource:mock([{deps, SrcDeps}, {config, [{profiles, [{test, [{deps, [list_to_atom(PkgName)]}]}]}]}]), mock_pkg_resource:mock([{pkgdeps, [{{iolist_to_binary(PkgName), <<"0.1.0">>}, []}]}]), @@ -590,3 +593,41 @@ clean_all(Config) -> Config, RConf, ["clean", "--all"], {ok, [{app, Name, invalid}, {app, DepName, invalid}, {app, PkgName, invalid}]} ). + +override_deps(Config) -> + mock_git_resource:mock([{deps, [{some_dep, "0.0.1"},{other_dep, "0.0.1"}]}]), + Deps = rebar_test_utils:expand_deps(git, [{"some_dep", "0.0.1", [{"other_dep", "0.0.1", []}]}]), + TopDeps = rebar_test_utils:top_level_deps(Deps), + + RebarConfig = [ + {deps, TopDeps}, + {overrides, [ + {override, some_dep, [ + {deps, []} + ]} + ]} + ], + rebar_test_utils:run_and_check( + Config, RebarConfig, ["compile"], + {ok, [{dep, "some_dep"},{dep_not_exist, "other_dep"}]} + ). + +profile_override_deps(Config) -> + mock_git_resource:mock([{deps, [{some_dep, "0.0.1"},{other_dep, "0.0.1"}]}]), + Deps = rebar_test_utils:expand_deps(git, [{"some_dep", "0.0.1", [{"other_dep", "0.0.1", []}]}]), + TopDeps = rebar_test_utils:top_level_deps(Deps), + + RebarConfig = [ + {deps, TopDeps}, + {profiles, [{a, + [{overrides, [ + {override, some_dep, [ + {deps, []} + ]} + ]} + ]} + ]}], + rebar_test_utils:run_and_check( + Config, RebarConfig, ["as", "a", "compile"], + {ok, [{dep, "some_dep"},{dep_not_exist, "other_dep"}]} + ). diff --git a/test/rebar_deps_SUITE.erl b/test/rebar_deps_SUITE.erl index 004d50b..fd86226 100644 --- a/test/rebar_deps_SUITE.erl +++ b/test/rebar_deps_SUITE.erl @@ -171,30 +171,11 @@ setup_project(Case, Config0, Deps) -> rebar_test_utils:create_app(AppDir, "A", "0.0.0", [kernel, stdlib]), TopDeps = rebar_test_utils:top_level_deps(Deps), RebarConf = rebar_test_utils:create_config(AppDir, [{deps, TopDeps}]), - case DepsType of - git -> - mock_git_resource:mock([{deps, rebar_test_utils:flat_deps(Deps)}]); - pkg -> - mock_pkg_resource:mock([{pkgdeps, flat_pkgdeps(Deps)}]) - end, + {SrcDeps, PkgDeps} = rebar_test_utils:flat_deps(Deps), + mock_git_resource:mock([{deps, SrcDeps}]), + mock_pkg_resource:mock([{pkgdeps, PkgDeps}]), [{rebarconfig, RebarConf} | Config]. -flat_pkgdeps([]) -> []; -flat_pkgdeps([{{pkg, Name, Vsn}, Deps} | Rest]) -> - [{{iolist_to_binary(Name),iolist_to_binary(Vsn)}, rebar_test_utils:top_level_deps(Deps)}] - ++ - flat_pkgdeps(Deps) - ++ - flat_pkgdeps(Rest). - -app_vsn([]) -> []; -app_vsn([{Source, Deps} | Rest]) -> - {Name, Vsn} = case Source of - {pkg, N, V} -> {N,V}; - {N,V,_Ref} -> {N,V} - end, - [{Name, Vsn}] ++ app_vsn(Deps) ++ app_vsn(Rest). - mock_warnings() -> %% just let it do its thing, we check warnings through %% the call log. @@ -217,7 +198,8 @@ sub_app_deps(Config) -> Deps = rebar_test_utils:expand_deps(git, [{"a", "1.0.0", []} ,{"b", "1.0.0", []} ,{"b", "2.0.0", []}]), - mock_git_resource:mock([{deps, rebar_test_utils:flat_deps(Deps)}]), + {SrcDeps, _} = rebar_test_utils:flat_deps(Deps), + mock_git_resource:mock([{deps, SrcDeps}]), Name = rebar_test_utils:create_random_name("sub_app1_"), Vsn = rebar_test_utils:create_random_vsn(), @@ -239,9 +221,10 @@ sub_app_deps(Config) -> newly_added_dep(Config) -> AppDir = ?config(apps, Config), Deps = rebar_test_utils:expand_deps(git, [{"a", "1.0.0", []} - ,{"b", "1.0.0", [{"c", "1.0.0", []}]} - ,{"c", "2.0.0", []}]), - mock_git_resource:mock([{deps, rebar_test_utils:flat_deps(Deps)}]), + ,{"b", "1.0.0", [{"c", "1.0.0", []}]} + ,{"c", "2.0.0", []}]), + {SrcDeps, _} = rebar_test_utils:flat_deps(Deps), + mock_git_resource:mock([{deps, SrcDeps}]), Name = rebar_test_utils:create_random_name("app_"), Vsn = rebar_test_utils:create_random_vsn(), @@ -314,5 +297,5 @@ in_warnings(git, Warns, NameRaw, VsnRaw) -> in_warnings(pkg, Warns, NameRaw, VsnRaw) -> Name = iolist_to_binary(NameRaw), Vsn = iolist_to_binary(VsnRaw), - 1 =< length([1 || {_, [AppName, AppVsn]} <- Warns, + 1 =< length([1 || {_, [AppName, {pkg, _, AppVsn}]} <- Warns, AppName =:= Name, AppVsn =:= Vsn]). diff --git a/test/rebar_install_deps_SUITE.erl b/test/rebar_install_deps_SUITE.erl index be42e68..b8b70b3 100644 --- a/test/rebar_install_deps_SUITE.erl +++ b/test/rebar_install_deps_SUITE.erl @@ -4,16 +4,23 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("kernel/include/file.hrl"). -all() -> [{group, git}, {group, pkg}]. +all() -> [{group, git}, {group, pkg}, {group, mixed}]. groups() -> - [{all, [], [flat, pick_highest_left, pick_highest_right, - pick_smallest1, pick_smallest2, - circular1, circular2, circular_skip, - fail_conflict, default_profile, nondefault_profile, - nondefault_pick_highest]}, - {git, [], [{group, all}]}, - {pkg, [], [{group, all}]}]. + [{unique, [], [flat, pick_highest_left, pick_highest_right, + pick_smallest1, pick_smallest2, + circular1, circular2, circular_skip, + fail_conflict, default_profile, nondefault_profile, + nondefault_pick_highest]}, + {git, [], [{group, unique}]}, + {pkg, [], [{group, unique}]}, + {mixed, [], [ + m_flat1, m_flat2, m_circular1, m_circular2, + m_pick_source1, m_pick_source2, m_pick_source3, + m_pick_source4, m_pick_source5, m_source_to_pkg, + m_pkg_level1, m_pkg_level2, m_pkg_level3, m_pkg_level3_alpha_order + ]} + ]. init_per_suite(Config) -> application:start(meck), @@ -26,19 +33,33 @@ init_per_group(git, Config) -> [{deps_type, git} | Config]; init_per_group(pkg, Config) -> [{deps_type, pkg} | Config]; +init_per_group(mixed, Config) -> + [{deps_type, mixed} | Config]; init_per_group(_, Config) -> Config. end_per_group(_, Config) -> Config. -init_per_testcase(Case, Config) -> +init_per_testcase(Case, Config) when is_atom(Case) -> + DepsType = ?config(deps_type, Config), + init_per_testcase({DepsType, Case}, Config); +init_per_testcase({mixed, Case}, Config) -> + {Deps, Warnings, Expect} = mdeps(Case), + Expected = case Expect of + {ok, List} -> {ok, format_expected_mdeps(List)}; + Other -> Other + end, + mock_warnings(), + [{expect, Expected}, + {warnings, format_expected_mixed_warnings(Warnings)} + | setup_project(Case, Config, rebar_test_utils:expand_deps(mixed, Deps))]; +init_per_testcase({DepsType, Case}, Config) -> {Deps, Warnings, Expect} = deps(Case), Expected = case Expect of {ok, List} -> {ok, format_expected_deps(List)}; Other -> Other end, - DepsType = ?config(deps_type, Config), mock_warnings(), [{expect, Expected}, {warnings, Warnings} @@ -54,6 +75,32 @@ format_expected_deps(Deps) -> N -> [{dep, N}, {lock, N}] end || Dep <- Deps]). +format_expected_mdeps(Deps) -> + %% for mixed deps, lowercase is a package, uppercase is source. + %% We can't check which was used from the dep, but the lock contains + %% the type and we can use that information. + lists:append([ + case Dep of + {N,V} when hd(N) >= $a, hd(N) =< $z -> + UN = string:to_upper(N), + [{dep, UN, V}, {lock, pkg, UN, V}]; + {N,V} when hd(N) >= $A, hd(N) =< $Z -> + [{dep, N, V}, {lock, src, N, V}]; + N when hd(N) >= $a, hd(N) =< $z -> + UN = string:to_upper(N), + [{dep, UN}, {lock, pkg, UN, "0.0.0"}]; + N when hd(N) >= $A, hd(N) =< $Z -> + [{dep, N}, {lock, src, N, "0.0.0"}] + end || Dep <- Deps]). + +format_expected_mixed_warnings(Warnings) -> + [case W of + {N, Vsn} when hd(N) >= $a, hd(N) =< $z -> {pkg, string:to_upper(N), Vsn}; + {N, Vsn} when hd(N) >= $A, hd(N) =< $Z -> {git, N, Vsn}; + N when hd(N) >= $a, hd(N) =< $z -> {pkg, string:to_upper(N), "0.0.0"}; + N when hd(N) >= $A, hd(N) =< $Z -> {git, N, "0.0.0"} + end || W <- Warnings]. + %% format: %% {Spec, %% [Warning], @@ -131,6 +178,78 @@ deps(nondefault_pick_highest) -> %% This is all handled in setup_project {[],[],{ok,[]}}. +%% format: +%% Same as `deps/1' except "A" is a source dep +%% and "a" is a package dep. +mdeps(m_flat1) -> + {[{"c", []}, + {"B", []}], + [], + {ok, ["B","c"]}}; +mdeps(m_flat2) -> + {[{"B", []}, + {"c", []}], + [], + {ok, ["B","c"]}}; +mdeps(m_circular1) -> + {[{"b", [{"a",[]}]}], % "A" is the top app + [], + {error, {rebar_prv_install_deps, {cycles, [[<<"A">>,<<"B">>]]}}}}; +mdeps(m_circular2) -> + {[{"B", [{"c", [{"b", []}]}]}], + [], + {error, {rebar_prv_install_deps, {cycles, [[<<"B">>,<<"C">>]]}}}}; +mdeps(m_pick_source1) -> + {[{"B", [{"D", []}]}, + {"c", [{"d", []}]}], + ["d"], + {ok, ["B", "c", "D"]}}; +mdeps(m_pick_source2) -> + %% The order of declaration is important. + {[{"b", []}, + {"B", []}], + [], + {ok, ["b"]}}; +mdeps(m_pick_source3) -> + {[{"B", []}, + {"b", []}], + [], + {ok, ["B"]}}; +mdeps(m_pick_source4) -> + {[{"b", [{"d", "1", []}]}, + {"C", [{"D", "1", []}]}], + [{"D", "1"}], + {ok, ["b", "C", {"d", "1"}]}}; +mdeps(m_pick_source5) -> + {[{"B", [{"d", "1", []}]}, + {"C", [{"D", "1", []}]}], + [{"D", "1"}], + {ok, ["B", "C", {"d", "1"}]}}; +mdeps(m_source_to_pkg) -> + {[{"B", [{"c",[{"d", []}]}]}], + [], + {ok, ["B", "c", "d"]}}; +mdeps(m_pkg_level1) -> + {[{"B", [{"D", [{"e", "2", []}]}]}, + {"C", [{"e", "1", []}]}], + [{"e","2"}], + {ok, ["B","C","D",{"e","1"}]}}; +mdeps(m_pkg_level2) -> + {[{"B", [{"e", "1", []}]}, + {"C", [{"D", [{"e", "2", []}]}]}], + [{"e","2"}], + {ok, ["B","C","D",{"e","1"}]}}; +mdeps(m_pkg_level3_alpha_order) -> + {[{"B", [{"d", [{"f", "1", []}]}]}, + {"C", [{"E", [{"f", "2", []}]}]}], + [{"f","2"}], + {ok, ["B","C","d","E",{"f","1"}]}}; +mdeps(m_pkg_level3) -> + {[{"B", [{"d", [{"f", "1", []}]}]}, + {"C", [{"E", [{"G", [{"f", "2", []}]}]}]}], + [{"f","2"}], + {ok, ["B","C","d","E","G",{"f","1"}]}}. + setup_project(fail_conflict, Config0, Deps) -> DepsType = ?config(deps_type, Config0), Config = rebar_test_utils:init_rebar_state( @@ -142,12 +261,9 @@ setup_project(fail_conflict, Config0, Deps) -> TopDeps = rebar_test_utils:top_level_deps(Deps), RebarConf = rebar_test_utils:create_config(AppDir, [{deps, TopDeps}, {deps_error_on_conflict, true}]), - case DepsType of - git -> - mock_git_resource:mock([{deps, rebar_test_utils:flat_deps(Deps)}]); - pkg -> - mock_pkg_resource:mock([{pkgdeps, rebar_test_utils:flat_pkgdeps(Deps)}]) - end, + {SrcDeps, PkgDeps} = rebar_test_utils:flat_deps(Deps), + mock_git_resource:mock([{deps, SrcDeps}]), + mock_pkg_resource:mock([{pkgdeps, PkgDeps}]), [{rebarconfig, RebarConf} | Config]; setup_project(nondefault_profile, Config0, Deps) -> DepsType = ?config(deps_type, Config0), @@ -161,12 +277,9 @@ setup_project(nondefault_profile, Config0, Deps) -> RebarConf = rebar_test_utils:create_config(AppDir, [{profiles, [ {nondef, [{deps, TopDeps}]} ]}]), - case DepsType of - git -> - mock_git_resource:mock([{deps, rebar_test_utils:flat_deps(Deps)}]); - pkg -> - mock_pkg_resource:mock([{pkgdeps, rebar_test_utils:flat_pkgdeps(Deps)}]) - end, + {SrcDeps, PkgDeps} = rebar_test_utils:flat_deps(Deps), + mock_git_resource:mock([{deps, SrcDeps}]), + mock_pkg_resource:mock([{pkgdeps, PkgDeps}]), [{rebarconfig, RebarConf} | Config]; setup_project(nondefault_pick_highest, Config0, _) -> DepsType = ?config(deps_type, Config0), @@ -187,13 +300,11 @@ setup_project(nondefault_pick_highest, Config0, _) -> ), case DepsType of git -> - mock_git_resource:mock( - [{deps, rebar_test_utils:flat_deps(DefaultDeps ++ ProfileDeps)}] - ); + {SrcDeps, _} = rebar_test_utils:flat_deps(DefaultDeps++ProfileDeps), + mock_git_resource:mock([{deps, SrcDeps}]); pkg -> - mock_pkg_resource:mock( - [{pkgdeps, rebar_test_utils:flat_pkgdeps(DefaultDeps ++ ProfileDeps)}] - ) + {_, PkgDeps} = rebar_test_utils:flat_deps(DefaultDeps++ProfileDeps), + mock_pkg_resource:mock([{pkgdeps, PkgDeps}]) end, [{rebarconfig, RebarConf} | Config]; setup_project(Case, Config0, Deps) -> @@ -206,12 +317,9 @@ setup_project(Case, Config0, Deps) -> rebar_test_utils:create_app(AppDir, "A", "0.0.0", [kernel, stdlib]), TopDeps = rebar_test_utils:top_level_deps(Deps), RebarConf = rebar_test_utils:create_config(AppDir, [{deps, TopDeps}]), - case DepsType of - git -> - mock_git_resource:mock([{deps, rebar_test_utils:flat_deps(Deps)}]); - pkg -> - mock_pkg_resource:mock([{pkgdeps, rebar_test_utils:flat_pkgdeps(Deps)}]) - end, + {SrcDeps, PkgDeps} = rebar_test_utils:flat_deps(Deps), + mock_git_resource:mock([{deps, SrcDeps}]), + mock_pkg_resource:mock([{pkgdeps, PkgDeps}]), [{rebarconfig, RebarConf} | Config]. mock_warnings() -> @@ -241,6 +349,9 @@ default_profile(Config) -> AppDir = ?config(apps, Config), {ok, Apps} = Expect = ?config(expect, Config), rebar_test_utils:run_and_check( + Config, RebarConfig, ["lock"], Expect + ), + rebar_test_utils:run_and_check( Config, RebarConfig, ["as", "profile", "lock"], Expect ), check_warnings(error_calls(), ?config(warnings, Config), ?config(deps_type, Config)), @@ -253,6 +364,9 @@ default_profile(Config) -> || {dep, App} <- Apps], %% A second run to another profile also links default to the right spot rebar_test_utils:run_and_check( + Config, RebarConfig, ["lock"], Expect + ), + rebar_test_utils:run_and_check( Config, RebarConfig, ["as", "other", "lock"], Expect ), [?assertMatch({ok, #file_info{type=directory}}, % somehow symlinks return dirs @@ -296,10 +410,13 @@ nondefault_profile(Config) -> nondefault_pick_highest(Config) -> {ok, RebarConfig} = file:consult(?config(rebarconfig, Config)), - %AppDir = ?config(apps, Config), + rebar_test_utils:run_and_check( + Config, RebarConfig, ["lock"], + {ok, [{dep, "B"}, {lock, "B"}, {lock, "C", "1"}, {dep, "C", "1"}], "default"} + ), rebar_test_utils:run_and_check( Config, RebarConfig, ["as", "nondef", "lock"], - {ok, [{dep, "B"}, {lock, "B"}, {dep, "C", "2"}], "nondef"} + {ok, [{dep, "B"}, {lock, "B"}, {lock, "C", "1"}, {dep, "C", "2"}], "nondef"} ), rebar_test_utils:run_and_check( Config, RebarConfig, ["lock"], @@ -307,9 +424,24 @@ nondefault_pick_highest(Config) -> ), rebar_test_utils:run_and_check( Config, RebarConfig, ["as", "nondef", "lock"], - {ok, [{dep, "B"}, {lock, "B"}, {dep, "C", "2"}], "nondef"} + {ok, [{dep, "B"}, {lock, "B"}, {lock, "C", "1"}, {dep, "C", "2"}], "nondef"} ). +m_flat1(Config) -> run(Config). +m_flat2(Config) -> run(Config). +m_circular1(Config) -> run(Config). +m_circular2(Config) -> run(Config). +m_pick_source1(Config) -> run(Config). +m_pick_source2(Config) -> run(Config). +m_pick_source3(Config) -> run(Config). +m_pick_source4(Config) -> run(Config). +m_pick_source5(Config) -> run(Config). +m_source_to_pkg(Config) -> run(Config). +m_pkg_level1(Config) -> run(Config). +m_pkg_level2(Config) -> run(Config). +m_pkg_level3(Config) -> run(Config). +m_pkg_level3_alpha_order(Config) -> run(Config). + run(Config) -> {ok, RebarConfig} = file:consult(?config(rebarconfig, Config)), rebar_test_utils:run_and_check( @@ -327,6 +459,10 @@ error_calls() -> check_warnings(_, [], _) -> ok; +check_warnings(Warns, [{Type, Name, Vsn} | Rest], mixed) -> + ct:pal("Checking for warning ~p in ~p", [{Name,Vsn},Warns]), + ?assert(in_warnings(Type, Warns, Name, Vsn)), + check_warnings(Warns, Rest, mixed); check_warnings(Warns, [{Name, Vsn} | Rest], Type) -> ct:pal("Checking for warning ~p in ~p", [{Name,Vsn},Warns]), ?assert(in_warnings(Type, Warns, Name, Vsn)), @@ -339,5 +475,5 @@ in_warnings(git, Warns, NameRaw, VsnRaw) -> in_warnings(pkg, Warns, NameRaw, VsnRaw) -> Name = iolist_to_binary(NameRaw), Vsn = iolist_to_binary(VsnRaw), - 1 =< length([1 || {_, [AppName, AppVsn]} <- Warns, + 1 =< length([1 || {_, [AppName, {pkg, _, AppVsn}]} <- Warns, AppName =:= Name, AppVsn =:= Vsn]). diff --git a/test/rebar_pkg_SUITE.erl b/test/rebar_pkg_SUITE.erl index 85bd6f0..b3201ad 100644 --- a/test/rebar_pkg_SUITE.erl +++ b/test/rebar_pkg_SUITE.erl @@ -11,7 +11,7 @@ -define(good_checksum, <<"1C6CE379D191FBAB41B7905075E0BF87CBBE23C77CECE775C5A0B786B2244C35">>). all() -> [good_uncached, good_cached, badindexchk, badpkg, - bad_to_good, good_disconnect, bad_disconnect]. + bad_to_good, good_disconnect, bad_disconnect, pkgs_provider]. init_per_suite(Config) -> application:start(meck), @@ -20,6 +20,8 @@ init_per_suite(Config) -> end_per_suite(_Config) -> application:stop(meck). +init_per_testcase(pkgs_provider, Config) -> + Config; init_per_testcase(good_uncached=Name, Config0) -> Config = [{good_cache, false}, {pkg, {<<"goodpkg">>, <<"1.0.0">>}} @@ -74,6 +76,8 @@ init_per_testcase(bad_disconnect=Name, Config0) -> meck:expect(httpc, request, fun(_, _, _, _, _) -> {error, econnrefused} end), Config. +end_per_testcase(pkgs_provider, Config) -> + Config; end_per_testcase(_, Config) -> unmock_config(Config), Config. @@ -148,9 +152,15 @@ bad_disconnect(Config) -> Tmp = ?config(tmp_dir, Config), {Pkg,Vsn} = ?config(pkg, Config), State = ?config(state, Config), - ?assertEqual(request_failed, + ?assertEqual({fetch_fail, Pkg, Vsn}, rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn}, State)). +pkgs_provider(Config) -> + Config1 = rebar_test_utils:init_rebar_state(Config), + rebar_test_utils:run_and_check( + Config1, [], ["pkgs"], + {ok, []} + ). %%%%%%%%%%%%%%% %%% Helpers %%% @@ -159,25 +169,34 @@ mock_config(Name, Config) -> Priv = ?config(priv_dir, Config), CacheRoot = filename:join([Priv, "cache", atom_to_list(Name)]), TmpDir = filename:join([Priv, "tmp", atom_to_list(Name)]), - T = ets:new(fake_registry, [public]), - ets:insert_new(T, [ - {{<<"badindexchk">>,<<"1.0.0">>}, [[], ?bad_checksum]}, - {{<<"goodpkg">>,<<"1.0.0">>}, [[], ?good_checksum]}, - {{<<"badpkg">>,<<"1.0.0">>}, [[], ?good_checksum]} + Tid = ets:new(registry_table, [public]), + ets:insert_new(Tid, [ + {<<"badindexchk">>,[[<<"1.0.0">>]]}, + {<<"goodpkg">>,[[<<"1.0.0">>]]}, + {<<"badpkg">>,[[<<"1.0.0">>]]}, + {{<<"badindexchk">>,<<"1.0.0">>}, [[], ?bad_checksum, [<<"rebar3">>]]}, + {{<<"goodpkg">>,<<"1.0.0">>}, [[], ?good_checksum, [<<"rebar3">>]]}, + {{<<"badpkg">>,<<"1.0.0">>}, [[], ?good_checksum, [<<"rebar3">>]]} ]), CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]), filelib:ensure_dir(filename:join([CacheDir, "registry"])), - ok = ets:tab2file(T, filename:join([CacheDir, "registry"])), + ok = ets:tab2file(Tid, filename:join([CacheDir, "registry"])), + %% The state returns us a fake registry meck:new(rebar_state, [passthrough]), - meck:expect(rebar_state, registry, - fun(_State) -> {ok, fake_registry} end), meck:expect(rebar_state, get, fun(_State, rebar_packages_cdn, _Default) -> "http://test.com/" end), + meck:new(rebar_dir, [passthrough]), meck:expect(rebar_dir, global_cache_dir, fun(_) -> CacheRoot end), + + meck:new(rebar_packages, [passthrough]), + meck:expect(rebar_packages, registry_dir, fun(_) -> CacheDir end), + meck:expect(rebar_packages, package_dir, fun(_) -> CacheDir end), + rebar_prv_update:hex_to_index(rebar_state:new()), + %% Cache fetches are mocked -- we assume the server and clients are %% correctly used. GoodCache = ?config(good_cache, Config), @@ -194,7 +213,7 @@ mock_config(Name, Config) -> [{cache_root, CacheRoot}, {cache_dir, CacheDir}, {tmp_dir, TmpDir}, - {mock_table, T} | Config]. + {mock_table, Tid} | Config]. unmock_config(Config) -> meck:unload(), diff --git a/test/rebar_pkg_alias_SUITE.erl b/test/rebar_pkg_alias_SUITE.erl new file mode 100644 index 0000000..f7fa5d4 --- /dev/null +++ b/test/rebar_pkg_alias_SUITE.erl @@ -0,0 +1,121 @@ +-module(rebar_pkg_alias_SUITE). +-compile(export_all). +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("kernel/include/file.hrl"). + +all() -> [same_alias, diff_alias, diff_alias_vsn]. + +%% {uuid, {pkg, uuid}} = uuid +%% {uuid, {pkg, alias}} = uuid on disk +%% another run should yield the same lock file without error +init_per_suite(Config) -> + mock_config(?MODULE, Config). + +end_per_suite(Config) -> + unmock_config(Config). + +init_per_testcase(same_alias, Config0) -> + Config = rebar_test_utils:init_rebar_state(Config0,"same_alias_"), + AppDir = ?config(apps, Config), + rebar_test_utils:create_app(AppDir, "A", "0.0.0", [kernel, stdlib]), + RebarConf = rebar_test_utils:create_config(AppDir, [{deps, [{fakelib, {pkg, fakelib}}]}]), + [{rebarconfig, RebarConf} | Config]; +init_per_testcase(diff_alias, Config0) -> + Config = rebar_test_utils:init_rebar_state(Config0,"diff_alias_"), + AppDir = ?config(apps, Config), + rebar_test_utils:create_app(AppDir, "A", "0.0.0", [kernel, stdlib]), + RebarConf = rebar_test_utils:create_config(AppDir, [{deps, [{fakelib, {pkg, goodpkg}}]}]), + [{rebarconfig, RebarConf} | Config]; +init_per_testcase(diff_alias_vsn, Config0) -> + Config = rebar_test_utils:init_rebar_state(Config0,"diff_alias_vsn_"), + AppDir = ?config(apps, Config), + rebar_test_utils:create_app(AppDir, "A", "0.0.0", [kernel, stdlib]), + RebarConf = rebar_test_utils:create_config(AppDir, [{deps, [{fakelib, "1.0.0", {pkg, goodpkg}}]}]), + [{rebarconfig, RebarConf} | Config]. + +end_per_testcase(_, Config) -> + Config. + +same_alias(Config) -> + {ok, RebarConfig} = file:consult(?config(rebarconfig, Config)), + rebar_test_utils:run_and_check( + Config, RebarConfig, ["lock"], + {ok, [{lock, "fakelib"}, {dep, "fakelib"}]} + ). + +diff_alias(Config) -> + %% even though the dep is 'fakelib' aliased as 'goodpkg' all + %% internal records use 'fakelib' as a value. Just make sure + %% the lock actually maintains the proper source as 'goodpkg' + AppDir = ?config(apps, Config), + Lockfile = filename:join([AppDir, "rebar.lock"]), + {ok, RebarConfig} = file:consult(?config(rebarconfig, Config)), + rebar_test_utils:run_and_check( + Config, RebarConfig, ["lock"], + {ok, [{lock, "fakelib"},{dep, "fakelib"}]} + ), + {ok, [LockData]} = file:consult(Lockfile), + ?assert(lists:any(fun({<<"fakelib">>,{pkg,<<"goodpkg">>,_},_}) -> true + ; (_) -> false end, LockData)), + %% An second run yields the same + rebar_test_utils:run_and_check( + Config, RebarConfig, ["lock"], + {ok, [{lock, "fakelib"},{dep, "fakelib"}]} + ), + {ok, [LockData]} = file:consult(Lockfile), + %% So does an upgrade + rebar_test_utils:run_and_check( + Config, RebarConfig, ["upgrade"], + {ok, [{lock, "fakelib"},{dep, "fakelib"}]} + ), + {ok, [LockData]} = file:consult(Lockfile). + +diff_alias_vsn(Config) -> diff_alias(Config). + +mock_config(Name, Config) -> + Priv = ?config(priv_dir, Config), + AppDir = filename:join([Priv, "fakelib"]), + CacheRoot = filename:join([Priv, "cache", atom_to_list(Name)]), + TmpDir = filename:join([Priv, "tmp", atom_to_list(Name)]), + CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]), + filelib:ensure_dir(filename:join([CacheDir, "registry"])), + rebar_test_utils:create_app(AppDir, "fakelib", "1.0.0", [kernel, stdlib]), + {Chk,Etag} = rebar_test_utils:package_app(AppDir, CacheDir, "fakelib-1.0.0"), + {Chk,Etag} = rebar_test_utils:package_app(AppDir, CacheDir, "goodpkg-1.0.0"), + + Tid = ets:new(registry_table, [public]), + ets:insert_new(Tid, [ + {<<"fakelib">>,[[<<"1.0.0">>]]}, + {<<"goodpkg">>,[[<<"1.0.0">>]]}, + {{<<"fakelib">>,<<"1.0.0">>}, [[], Chk, [<<"rebar3">>]]}, + {{<<"goodpkg">>,<<"1.0.0">>}, [[], Chk, [<<"rebar3">>]]} + ]), + ok = ets:tab2file(Tid, filename:join([CacheDir, "registry"])), + ets:delete(Tid), + %% The state returns us a fake registry + meck:new(rebar_dir, [passthrough, no_link]), + meck:expect(rebar_dir, global_cache_dir, fun(_) -> CacheRoot end), + + meck:new(rebar_packages, [passthrough, no_link]), + meck:expect(rebar_packages, registry_dir, fun(_) -> CacheDir end), + meck:expect(rebar_packages, package_dir, fun(_) -> CacheDir end), + rebar_prv_update:hex_to_index(rebar_state:new()), + + %% Cache fetches are mocked -- we assume the server and clients are + %% correctly used. + meck:new(httpc, [passthrough, unsticky, no_link]), + meck:expect(httpc, request, + fun(get, {_Url, _Opts}, _, _, _) -> + {ok, {{<<"1.0.0">>, 304, <<"Not Modified">>}, [{"etag", Etag}], <<>>}} + end), + %% Move all packages to cache + NewConf = [{cache_root, CacheRoot}, + {cache_dir, CacheDir}, + {tmp_dir, TmpDir}, + {mock_table, Tid} | Config], + NewConf. + +unmock_config(Config) -> + meck:unload(), + Config. diff --git a/test/rebar_plugins_SUITE.erl b/test/rebar_plugins_SUITE.erl index adfeafe..5e2c782 100644 --- a/test/rebar_plugins_SUITE.erl +++ b/test/rebar_plugins_SUITE.erl @@ -45,7 +45,8 @@ compile_plugins(Config) -> PluginName = rebar_test_utils:create_random_name("plugin1_"), Plugins = rebar_test_utils:expand_deps(git, [{PluginName, Vsn, []}]), - mock_git_resource:mock([{deps, rebar_test_utils:flat_deps(Plugins)}]), + {SrcDeps, _} = rebar_test_utils:flat_deps(Plugins), + mock_git_resource:mock([{deps, SrcDeps}]), mock_pkg_resource:mock([{pkgdeps, [{{list_to_binary(DepName), list_to_binary(Vsn)}, []}]}, {config, [{plugins, [ @@ -137,7 +138,8 @@ complex_plugins(Config) -> Deps = rebar_test_utils:expand_deps(git, [{PluginName, Vsn2, [{DepName2, Vsn, [{DepName3, Vsn, []}]}]} ,{DepName, Vsn, []}]), - mock_git_resource:mock([{deps, rebar_test_utils:flat_deps(Deps)}]), + {SrcDeps, _} = rebar_test_utils:flat_deps(Deps), + mock_git_resource:mock([{deps, SrcDeps}]), RConfFile = rebar_test_utils:create_config(AppDir, diff --git a/test/rebar_profiles_SUITE.erl b/test/rebar_profiles_SUITE.erl index b42df39..41bb535 100644 --- a/test/rebar_profiles_SUITE.erl +++ b/test/rebar_profiles_SUITE.erl @@ -57,7 +57,8 @@ profile_new_key(Config) -> AllDeps = rebar_test_utils:expand_deps(git, [{"a", "1.0.0", []} ,{"b", "1.0.0", []}]), - mock_git_resource:mock([{deps, rebar_test_utils:flat_deps(AllDeps)}]), + {SrcDeps, []} = rebar_test_utils:flat_deps(AllDeps), + mock_git_resource:mock([{deps, SrcDeps}]), Name = rebar_test_utils:create_random_name("profile_new_key_"), Vsn = rebar_test_utils:create_random_vsn(), @@ -82,7 +83,8 @@ profile_merge_keys(Config) -> AllDeps = rebar_test_utils:expand_deps(git, [{"a", "1.0.0", []} ,{"b", "1.0.0", []} ,{"b", "2.0.0", []}]), - mock_git_resource:mock([{deps, rebar_test_utils:flat_deps(AllDeps)}]), + {SrcDeps, []} = rebar_test_utils:flat_deps(AllDeps), + mock_git_resource:mock([{deps, SrcDeps}]), Name = rebar_test_utils:create_random_name("profile_new_key_"), Vsn = rebar_test_utils:create_random_vsn(), @@ -111,7 +113,8 @@ explicit_profile_deduplicate_deps(Config) -> ,{"a", "2.0.0", []} ,{"b", "1.0.0", []} ,{"b", "2.0.0", []}]), - mock_git_resource:mock([{deps, rebar_test_utils:flat_deps(AllDeps)}]), + {SrcDeps, []} = rebar_test_utils:flat_deps(AllDeps), + mock_git_resource:mock([{deps, SrcDeps}]), Name = rebar_test_utils:create_random_name("explicit_profile_deduplicate_deps_"), Vsn = rebar_test_utils:create_random_vsn(), @@ -141,7 +144,8 @@ implicit_profile_deduplicate_deps(Config) -> ,{"a", "2.0.0", []} ,{"b", "1.0.0", []} ,{"b", "2.0.0", []}]), - mock_git_resource:mock([{deps, rebar_test_utils:flat_deps(AllDeps)}]), + {SrcDeps, []} = rebar_test_utils:flat_deps(AllDeps), + mock_git_resource:mock([{deps, SrcDeps}]), Name = rebar_test_utils:create_random_name("implicit_profile_deduplicate_deps_"), Vsn = rebar_test_utils:create_random_vsn(), @@ -169,7 +173,8 @@ all_deps_code_paths(Config) -> AllDeps = rebar_test_utils:expand_deps(git, [{"a", "1.0.0", []} ,{"b", "2.0.0", []}]), - mock_git_resource:mock([{deps, rebar_test_utils:flat_deps(AllDeps)}]), + {SrcDeps, []} = rebar_test_utils:flat_deps(AllDeps), + mock_git_resource:mock([{deps, SrcDeps}]), Name = rebar_test_utils:create_random_name("all_deps_code_paths"), Vsn = rebar_test_utils:create_random_vsn(), diff --git a/test/rebar_release_SUITE.erl b/test/rebar_release_SUITE.erl index 1ef0771..f6fe8ff 100644 --- a/test/rebar_release_SUITE.erl +++ b/test/rebar_release_SUITE.erl @@ -4,10 +4,11 @@ -include_lib("eunit/include/eunit.hrl"). all() -> [release, - dev_mode_release, - profile_dev_mode_override_release, - tar, - extend_release]. + dev_mode_release, + profile_dev_mode_override_release, + tar, + extend_release, + user_output_dir]. init_per_testcase(Case, Config0) -> Config = rebar_test_utils:init_rebar_state(Config0), @@ -109,3 +110,27 @@ extend_release(Config) -> ["release", "-n", "extended"], {ok, [{release, extended, Vsn, false}]} ). + +user_output_dir(Config) -> + AppDir = ?config(apps, Config), + Name = ?config(name, Config), + ReleaseDir = filename:join(AppDir, "./_rel"), + Vsn = "1.0.0", + + {ok, RebarConfig} = + file:consult(rebar_test_utils:create_config(AppDir, + [{relx, [{release, {list_to_atom(Name), Vsn}, + [list_to_atom(Name)]}, + {lib_dirs, [AppDir]}, + {dev_mode, true}]}])), + rebar_test_utils:run_and_check( + Config, RebarConfig, + ["release", "-o", ReleaseDir], + {ok, []} + ), + + RelxState = rlx_state:new("", [], []), + RelxState1 = rlx_state:base_output_dir(RelxState, ReleaseDir), + {ok, RelxState2} = rlx_prv_app_discover:do(RelxState1), + {ok, RelxState3} = rlx_prv_rel_discover:do(RelxState2), + rlx_state:get_realized_release(RelxState3, list_to_atom(Name), Vsn). diff --git a/test/rebar_test_utils.erl b/test/rebar_test_utils.erl index be52e81..8d1d408 100644 --- a/test/rebar_test_utils.erl +++ b/test/rebar_test_utils.erl @@ -2,8 +2,9 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). -export([init_rebar_state/1, init_rebar_state/2, run_and_check/4]). --export([expand_deps/2, flat_deps/1, flat_pkgdeps/1, top_level_deps/1]). --export([create_app/4, create_eunit_app/4, create_empty_app/4, create_config/2]). +-export([expand_deps/2, flat_deps/1, top_level_deps/1]). +-export([create_app/4, create_eunit_app/4, create_empty_app/4, create_config/2, + package_app/3]). -export([create_random_name/1, create_random_vsn/0, write_src_file/2]). %%%%%%%%%%%%%% @@ -25,7 +26,9 @@ init_rebar_state(Config, Name) -> ok = ec_file:mkdir_p(CheckoutsDir), Verbosity = rebar3:log_level(), rebar_log:init(command_line, Verbosity), + GlobalDir = filename:join([DataDir, "cache"]), State = rebar_state:new([{base_dir, filename:join([AppsDir, "_build"])} + ,{global_rebar_dir, GlobalDir} ,{root_dir, AppsDir}]), [{apps, AppsDir}, {checkouts, CheckoutsDir}, {state, State} | Config]. @@ -137,24 +140,43 @@ expand_deps(pkg, [{Name, Deps} | Rest]) -> [{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)]; expand_deps(pkg, [{Name, Vsn, Deps} | Rest]) -> Dep = {pkg, Name, Vsn}, - [{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)]. - -flat_deps([]) -> []; -flat_deps([{{Name,_Vsn,Ref}, Deps} | Rest]) -> - [{{Name,vsn_from_ref(Ref)}, top_level_deps(Deps)}] - ++ - flat_deps(Deps) - ++ - flat_deps(Rest). - -flat_pkgdeps([]) -> []; -flat_pkgdeps([{{pkg, Name, Vsn}, Deps} | Rest]) -> - [{{iolist_to_binary(Name),iolist_to_binary(Vsn)}, top_level_deps(Deps)}] - ++ - flat_pkgdeps(Deps) - ++ - flat_pkgdeps(Rest). - + [{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)]; +expand_deps(mixed, [{Name, Deps} | Rest]) -> + Dep = if hd(Name) >= $a, hd(Name) =< $z -> + {pkg, string:to_upper(Name), "0.0.0"} + ; hd(Name) >= $A, hd(Name) =< $Z -> + {Name, ".*", {git, "https://example.org/user/"++Name++".git", "master"}} + end, + [{Dep, expand_deps(mixed, Deps)} | expand_deps(mixed, Rest)]; +expand_deps(mixed, [{Name, Vsn, Deps} | Rest]) -> + Dep = if hd(Name) >= $a, hd(Name) =< $z -> + {pkg, string:to_upper(Name), Vsn} + ; hd(Name) >= $A, hd(Name) =< $Z -> + {Name, Vsn, {git, "https://example.org/user/"++Name++".git", {tag, Vsn}}} + end, + [{Dep, expand_deps(mixed, Deps)} | expand_deps(mixed, Rest)]. + +%% Source deps can depend on both source and package dependencies; +%% package deps can only depend on package deps. +%% For things to work we have to go down the dep tree and find all +%% lineages of pkg deps and return them, whereas the source deps +%% can be left as is. +flat_deps(Deps) -> flat_deps(Deps, [], []). + +flat_deps([], Src, Pkg) -> {Src, Pkg}; +flat_deps([{{pkg, Name, Vsn}, PkgDeps} | Rest], Src, Pkg) -> + Current = {{iolist_to_binary(Name), iolist_to_binary(Vsn)}, + top_level_deps(PkgDeps)}, + {[], FlatPkgDeps} = flat_deps(PkgDeps), + flat_deps(Rest, + Src, + Pkg ++ [Current | FlatPkgDeps]); +flat_deps([{{Name,_Vsn,Ref}, Deps} | Rest], Src, Pkg) -> + Current = {{Name,vsn_from_ref(Ref)}, top_level_deps(Deps)}, + {FlatDeps, FlatPkgDeps} = flat_deps(Deps), + flat_deps(Rest, + Src ++ [Current | FlatDeps], + Pkg ++ FlatPkgDeps). vsn_from_ref({git, _, {_, Vsn}}) -> Vsn; vsn_from_ref({git, _, Vsn}) -> Vsn. @@ -278,6 +300,28 @@ check_results(AppDir, Expected, ProfileRun) -> ?assertEqual(iolist_to_binary(Vsn), iolist_to_binary(LockVsn)) end + ; ({lock, pkg, Name, Vsn}) -> + ct:pal("Pkg Lock Name: ~p, Vsn: ~p", [Name, Vsn]), + case lists:keyfind(iolist_to_binary(Name), 1, Locks) of + false -> + error({lock_not_found, Name}); + {_LockName, {pkg, _, LockVsn}, _} -> + ?assertEqual(iolist_to_binary(Vsn), + iolist_to_binary(LockVsn)); + {_LockName, {_, _, {ref, LockVsn}}, _} -> + error({source_lock, {Name, LockVsn}}) + end + ; ({lock, src, Name, Vsn}) -> + ct:pal("Src Lock Name: ~p, Vsn: ~p", [Name, Vsn]), + case lists:keyfind(iolist_to_binary(Name), 1, Locks) of + false -> + error({lock_not_found, Name}); + {_LockName, {pkg, _, LockVsn}, _} -> + error({pkg_lock, {Name, LockVsn}}); + {_LockName, {_, _, {ref, LockVsn}}, _} -> + ?assertEqual(iolist_to_binary(Vsn), + iolist_to_binary(LockVsn)) + end ; ({release, Name, Vsn, ExpectedDevMode}) -> ct:pal("Release: ~p-~s", [Name, Vsn]), {ok, Cwd} = file:get_cwd(), @@ -375,3 +419,25 @@ get_app_metadata(Name, Vsn, Deps) -> {included_applications, []}, {registered, []}, {applications, Deps}]}. + +package_app(AppDir, DestDir, PkgName) -> + Name = PkgName++".tar", + {ok, Fs} = file:list_dir(AppDir), + ok = erl_tar:create(filename:join(DestDir, "contents.tar.gz"), + lists:zip(Fs, [filename:join(AppDir,F) || F <- Fs]), + [compressed]), + ok = file:write_file(filename:join(DestDir, "metadata.config"), "who cares"), + ok = file:write_file(filename:join(DestDir, "VERSION"), "3"), + {ok, Contents} = file:read_file(filename:join(DestDir, "contents.tar.gz")), + Blob = <<"3who cares", Contents/binary>>, + <<X:256/big-unsigned>> = crypto:hash(sha256, Blob), + BinChecksum = list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))), + ok = file:write_file(filename:join(DestDir, "CHECKSUM"), BinChecksum), + PkgFiles = ["contents.tar.gz", "VERSION", "metadata.config", "CHECKSUM"], + Archive = filename:join(DestDir, Name), + ok = erl_tar:create(Archive, + lists:zip(PkgFiles, [filename:join(DestDir,F) || F <- PkgFiles])), + {ok, BinFull} = file:read_file(Archive), + <<E:128/big-unsigned-integer>> = crypto:hash(md5, BinFull), + Etag = string:to_lower(lists:flatten(io_lib:format("~32.16.0b", [E]))), + {BinChecksum, Etag}. diff --git a/test/rebar_upgrade_SUITE.erl b/test/rebar_upgrade_SUITE.erl index 4ab99c7..54f16da 100644 --- a/test/rebar_upgrade_SUITE.erl +++ b/test/rebar_upgrade_SUITE.erl @@ -425,17 +425,21 @@ upgrades(compile_upgrade_parity) -> mock_deps(git, Deps, Upgrades) -> catch mock_git_resource:unmock(), - mock_git_resource:mock([{deps, rebar_test_utils:flat_deps(Deps)}, {upgrade, Upgrades}]); + {SrcDeps, _} = rebar_test_utils:flat_deps(Deps), + mock_git_resource:mock([{deps, SrcDeps}, {upgrade, Upgrades}]); mock_deps(pkg, Deps, Upgrades) -> catch mock_pkg_resource:unmock(), - mock_pkg_resource:mock([{pkgdeps, rebar_test_utils:flat_pkgdeps(Deps)}, {upgrade, Upgrades}]). + {_, PkgDeps} = rebar_test_utils:flat_deps(Deps), + mock_pkg_resource:mock([{pkgdeps, PkgDeps}, {upgrade, Upgrades}]). mock_deps(git, OldDeps, Deps, Upgrades) -> catch mock_git_resource:unmock(), - mock_git_resource:mock([{deps, rebar_test_utils:flat_deps(Deps++OldDeps)}, {upgrade, Upgrades}]); + {SrcDeps, _} = rebar_test_utils:flat_deps(Deps++OldDeps), + mock_git_resource:mock([{deps, SrcDeps}, {upgrade, Upgrades}]); mock_deps(pkg, OldDeps, Deps, Upgrades) -> catch mock_pkg_resource:unmock(), - mock_pkg_resource:mock([{pkgdeps, rebar_test_utils:flat_pkgdeps(Deps++OldDeps)}, {upgrade, Upgrades}]). + {_, PkgDeps} = rebar_test_utils:flat_deps(Deps++OldDeps), + mock_pkg_resource:mock([{pkgdeps, PkgDeps}, {upgrade, Upgrades}]). normalize_unlocks({App, Locks}) -> {iolist_to_binary(App), |