diff options
-rw-r--r-- | .travis.yml | 27 | ||||
-rw-r--r-- | README.md | 2 | ||||
-rwxr-xr-x | bootstrap | 10 | ||||
-rw-r--r-- | rebar.config | 3 | ||||
-rw-r--r-- | rebar.lock | 2 | ||||
-rw-r--r-- | src/rebar.hrl | 1 | ||||
-rw-r--r-- | src/rebar3.erl | 10 | ||||
-rw-r--r-- | src/rebar_app_utils.erl | 7 | ||||
-rw-r--r-- | src/rebar_config.erl | 18 | ||||
-rw-r--r-- | src/rebar_file_utils.erl | 15 | ||||
-rw-r--r-- | src/rebar_git_resource.erl | 13 | ||||
-rw-r--r-- | src/rebar_hooks.erl | 28 | ||||
-rw-r--r-- | src/rebar_log.erl | 12 | ||||
-rw-r--r-- | src/rebar_prv_common_test.erl | 25 | ||||
-rw-r--r-- | src/rebar_prv_cover.erl | 24 | ||||
-rw-r--r-- | src/rebar_prv_path.erl | 8 | ||||
-rw-r--r-- | src/rebar_prv_shell.erl | 34 | ||||
-rw-r--r-- | src/rebar_prv_update.erl | 12 | ||||
-rw-r--r-- | src/rebar_prv_upgrade.erl | 24 | ||||
-rw-r--r-- | src/rebar_state.erl | 3 | ||||
-rw-r--r-- | src/rebar_utils.erl | 20 | ||||
-rw-r--r-- | test/rebar_cover_SUITE.erl | 8 | ||||
-rw-r--r-- | test/rebar_ct_SUITE.erl | 50 | ||||
-rw-r--r-- | test/rebar_deps_SUITE.erl | 73 | ||||
-rw-r--r-- | test/rebar_upgrade_SUITE.erl | 52 |
25 files changed, 367 insertions, 114 deletions
diff --git a/.travis.yml b/.travis.yml index ab2d28d..a45cb05 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,27 +1,24 @@ sudo: false language: erlang -install: "true" +install: 'true' otp_release: - - 18.0 - - 17.5 - - R16B03-1 - - R15B03 +- 18.0 +- 17.5 +- R16B03-1 +- R15B03 before_script: "./bootstrap" script: "./rebar3 ct" branches: only: - - master + - master cache: directories: - - $HOME/.cache/rebar3/hex/default -before_deploy: "rm -rf !(rebar3)" + - "$HOME/.cache/rebar3/hex/default" deploy: + provider: releases + api_key: + secure: MjloYuaQF3cd3Oab57zqwPDLPqt5MDgBIrRLpXOQwNovr2tnkKd4aJK3QJ3pTxvZievjgl+qIYI1IZyjuRV37nkjAfMw14iig959wi0k8XTJoMdylVxE5X7hk4SiWhX/ycnJx3C28PPw1OitGTF76HAJDMgEelNdoNt+hvjvDEo= + file: rebar3 on: + repo: erlang/rebar3 tags: true - condition: $TRAVIS_OTP_RELEASE = R16B03-1 - provider: s3 - access_key_id: AKIAJAPYAQEFYCYSNL7Q - secret_access_key: - secure: "BUv2KQABv0Q4e8DAVNBRTc/lXHWt27yCN46Fdgo1IrcSSIiP+hq2yXzQcXLbPwkEu6pxUZQtL3mvKbt6l7uw3wFrcRfFAi1PGTITAW8MTmxtwcZIBcHSk3XOzDbkK+fYYcaddszmt7hDzzEFPtmYXiNgnaMIVeynhQLgcCcIRRQ=" - bucket: "rebar3" - skip_cleanup: true @@ -53,7 +53,7 @@ best experience you can get. ## Getting Started -A [getting started guide is maintained on the offcial documentation website](http://www.rebar3.org/docs/getting-started), +A [getting started guide is maintained on the official documentation website](http://www.rebar3.org/docs/getting-started), but installing rebar3 can be done by any of the ways described below Nightly compiled version: @@ -24,7 +24,7 @@ main(_) -> bootstrap_rebar3(), %% Build rebar.app from rebar.app.src - {ok, App} = rebar_app_info:new(rebar, "3.1.0", filename:absname("_build/default/lib/rebar/")), + {ok, App} = rebar_app_info:new(rebar, "3.1.1", filename:absname("_build/default/lib/rebar/")), rebar_otp_app:compile(rebar_state:new(), App), %% Because we are compiling files that are loaded already we want to silence @@ -133,7 +133,11 @@ get_rebar_config() -> end. get_http_vars(Scheme) -> - proplists:get_value(Scheme, get_rebar_config(), []). + OS = case os:getenv(atom_to_list(Scheme)) of + Str when is_list(Str) -> Str; + _ -> [] + end, + proplists:get_value(Scheme, get_rebar_config(), OS). set_httpc_options() -> set_httpc_options(https_proxy, get_http_vars(https_proxy)), @@ -350,7 +354,7 @@ format_error(AbsSource, Extra, {Mod, Desc}) -> io_lib:format("~s: ~s~s~n", [AbsSource, Extra, ErrorDesc]). additional_defines() -> - [{d, D} || {Re, D} <- [{"^[0-9]+", namespaced_types}, {"^R1[4|5]", deprecated_crypto}], is_otp_release(Re)]. + [{d, D} || {Re, D} <- [{"^[0-9]+", namespaced_types}, {"^R1[4|5]", deprecated_crypto}, {"^((1[8|9])|2)", rand_module}], is_otp_release(Re)]. is_otp_release(ArchRegex) -> case re:run(otp_release(), ArchRegex, [{capture, none}]) of diff --git a/rebar.config b/rebar.config index 5d3b310..ea5af27 100644 --- a/rebar.config +++ b/rebar.config @@ -1,7 +1,7 @@ %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- %% ex: ts=4 sw=4 ft=erlang et -{deps, [{erlware_commons, "0.20.0"}, +{deps, [{erlware_commons, "0.21.0"}, {ssl_verify_hostname, "1.0.5"}, {certifi, "0.4.0"}, {providers, "1.6.0"}, @@ -51,6 +51,7 @@ %% Overrides {overrides, [{override, erlware_commons, [{erl_opts, [{platform_define, "^[0-9]+", namespaced_types}, {platform_define, "^R1[4|5]", deprecated_crypto}, + {platform_define, "^((1[8|9])|2)", rand_module}, no_debug_info, warnings_as_errors]}, {deps, []}, {plugins, []}]}, @@ -2,7 +2,7 @@ {<<"certifi">>,{pkg,<<"certifi">>,<<"0.4.0">>},0}, {<<"cf">>,{pkg,<<"cf">>,<<"0.2.1">>},0}, {<<"cth_readable">>,{pkg,<<"cth_readable">>,<<"1.2.2">>},0}, - {<<"erlware_commons">>,{pkg,<<"erlware_commons">>,<<"0.20.0">>},0}, + {<<"erlware_commons">>,{pkg,<<"erlware_commons">>,<<"0.21.0">>},0}, {<<"eunit_formatters">>,{pkg,<<"eunit_formatters">>,<<"0.3.1">>},0}, {<<"getopt">>,{pkg,<<"getopt">>,<<"0.8.2">>},0}, {<<"providers">>,{pkg,<<"providers">>,<<"1.6.0">>},0}, diff --git a/src/rebar.hrl b/src/rebar.hrl index 0b7f0b1..4d69c7b 100644 --- a/src/rebar.hrl +++ b/src/rebar.hrl @@ -10,6 +10,7 @@ -define(INFO(Str, Args), rebar_log:log(info, Str, Args)). -define(WARN(Str, Args), rebar_log:log(warn, Str, Args)). -define(ERROR(Str, Args), rebar_log:log(error, Str, Args)). +-define(CRASHDUMP(Str, Args), rebar_log:crashdump(Str, Args)). -define(FMT(Str, Args), lists:flatten(io_lib:format(Str, Args))). diff --git a/src/rebar3.erl b/src/rebar3.erl index ff0ab6a..82b4472 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -272,20 +272,22 @@ handle_error({error, rebar_abort}) -> handle_error({error, {Module, Reason}}) -> case code:which(Module) of non_existing -> - ?ERROR("Uncaught error in rebar_core. Run with DEBUG=1 to see stacktrace", []), + ?CRASHDUMP("~p: ~p~n~p~n~n", [Module, Reason, erlang:get_stacktrace()]), + ?ERROR("Uncaught error in rebar_core. Run with DEBUG=1 to stacktrace or consult rebar3.crashdump", []), ?DEBUG("Uncaught error: ~p ~p", [Module, Reason]), ?INFO("When submitting a bug report, please include the output of `rebar3 report \"your command\"`", []); _ -> - ?ERROR(Module:format_error(Reason), []) + ?ERROR("~s", [Module:format_error(Reason)]) end, erlang:halt(1); handle_error({error, Error}) when is_list(Error) -> - ?ERROR(Error, []), + ?ERROR("~s", [Error]), erlang:halt(1); handle_error(Error) -> %% Nothing should percolate up from rebar_core; %% Dump this error to console - ?ERROR("Uncaught error in rebar_core. Run with DEBUG=1 to see stacktrace", []), + ?CRASHDUMP("Error: ~p~n~p~n~n", [Error, erlang:get_stacktrace()]), + ?ERROR("Uncaught error in rebar_core. Run with DEBUG=1 to see stacktrace or consult rebar3.crashdump", []), ?DEBUG("Uncaught error: ~p", [Error]), case erlang:get_stacktrace() of [] -> ok; diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index d3ef841..957526e 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -134,7 +134,12 @@ parse_dep(Parent, {Name, Source}, DepsDir, IsLock, State) when is_tuple(Source) dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); parse_dep(Parent, {Name, _Vsn, Source}, DepsDir, IsLock, State) when is_tuple(Source) -> dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); -parse_dep(Parent, {Name, _Vsn, Source, Opts}, DepsDir, IsLock, State) when is_tuple(Source) -> +parse_dep(Parent, {Name, _Vsn, Source, Opts}, DepsDir, IsLock, State) when is_tuple(Source), + is_list(Opts) -> + ?WARN("Dependency option list ~p in ~p is not supported and will be ignored", [Opts, Name]), + dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); +parse_dep(Parent, {Name, Source, Opts}, DepsDir, IsLock, State) when is_tuple(Source), + is_list(Opts) -> ?WARN("Dependency option list ~p in ~p is not supported and will be ignored", [Opts, Name]), dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); parse_dep(Parent, {Name, {pkg, PkgName, Vsn}, Level}, DepsDir, IsLock, State) when is_integer(Level) -> diff --git a/src/rebar_config.erl b/src/rebar_config.erl index 8d7bcf4..836980e 100644 --- a/src/rebar_config.erl +++ b/src/rebar_config.erl @@ -180,6 +180,24 @@ check_newly_added({Name, _, Source}, LockedDeps) -> check_newly_added(Dep, LockedDeps) -> check_newly_added_(Dep, LockedDeps). +%% get [raw] deps out of the way +check_newly_added_({Name, Source, Opts}, LockedDeps) when is_tuple(Source), + is_list(Opts) -> + case check_newly_added_(Name, LockedDeps) of + {true, Name1} -> + {true, {Name1, Source}}; + false -> + false + end; +check_newly_added_({Name,_Vsn,Source,Opts}, LockedDeps) when is_tuple(Source), + is_list(Opts) -> + case check_newly_added_(Name, LockedDeps) of + {true, Name1} -> + {true, {Name1, Source}}; + false -> + false + end; +%% and on to regular deps check_newly_added_({Name, Vsn, Source}, LockedDeps) -> case check_newly_added_(Name, LockedDeps) of {true, Name1} -> diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 19cc94d..667be62 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -27,6 +27,7 @@ -module(rebar_file_utils). -export([try_consult/1, + consult_config/2, format_error/1, symlink_or_copy/2, rm_rf/1, @@ -63,6 +64,20 @@ try_consult(File) -> throw(?PRV_ERROR({bad_term_file, File, Reason})) end. +-spec consult_config(rebar_state:t(), string()) -> [[tuple()]]. +consult_config(State, Filename) -> + Fullpath = filename:join(rebar_dir:root_dir(State), Filename), + ?DEBUG("Loading configuration from ~p", [Fullpath]), + Config = case try_consult(Fullpath) of + [T] -> T; + [] -> [] + end, + SubConfigs = [consult_config(State, Entry ++ ".config") || + Entry <- Config, is_list(Entry) + ], + + [Config | lists:merge(SubConfigs)]. + format_error({bad_term_file, AppFile, Reason}) -> io_lib:format("Error reading file ~s: ~s", [AppFile, file:format_error(Reason)]). diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl index 5a6a5ef..f0f5f03 100644 --- a/src/rebar_git_resource.erl +++ b/src/rebar_git_resource.erl @@ -17,10 +17,17 @@ lock(AppDir, {git, Url, _}) -> lock(AppDir, {git, Url}); lock(AppDir, {git, Url}) -> - AbortMsg = io_lib:format("Locking of git dependency failed in ~s", [AppDir]), + AbortMsg = lists:flatten(io_lib:format("Locking of git dependency failed in ~s", [AppDir])), + Dir = rebar_utils:escape_double_quotes(AppDir), {ok, VsnString} = - rebar_utils:sh("git --git-dir=\"" ++ rebar_utils:escape_double_quotes(AppDir) ++ "/.git\" rev-parse --verify HEAD", - [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]), + case os:type() of + {win32, _} -> + rebar_utils:sh("git rev-parse --git-dir=\"" ++ Dir ++ "/.git\" --work-tree=\"" ++ Dir ++ "\" --verify HEAD", + [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]); + _ -> + rebar_utils:sh("git --git-dir=\"" ++ Dir ++ "/.git\" rev-parse --verify HEAD", + [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]) + end, Ref = string:strip(VsnString, both, $\n), {git, Url, {ref, Ref}}. diff --git a/src/rebar_hooks.erl b/src/rebar_hooks.erl index 3af17ca..d6a0e2b 100644 --- a/src/rebar_hooks.erl +++ b/src/rebar_hooks.erl @@ -38,18 +38,22 @@ run_provider_hooks(Dir, Type, Command, Providers, Opts, State) -> run_provider_hooks_(_Dir, _Type, _Command, _Providers, [], State) -> State; run_provider_hooks_(Dir, Type, Command, Providers, TypeHooks, State) -> - PluginDepsPaths = rebar_state:code_paths(State, all_plugin_deps), - code:add_pathsa(PluginDepsPaths), - Providers1 = rebar_state:providers(State), - State1 = rebar_state:providers(rebar_state:dir(State, Dir), Providers++Providers1), - HookProviders = proplists:get_all_values(Command, TypeHooks), - case rebar_core:do(HookProviders, State1) of - {error, ProviderName} -> - ?DEBUG(format_error({bad_provider, Type, Command, ProviderName}), []), - throw(?PRV_ERROR({bad_provider, Type, Command, ProviderName})); - {ok, State2} -> - rebar_utils:remove_from_code_path(PluginDepsPaths), - State2 + case proplists:get_all_values(Command, TypeHooks) of + [] -> + State; + HookProviders -> + PluginDepsPaths = lists:usort(rebar_state:code_paths(State, all_plugin_deps)), + code:add_pathsa(PluginDepsPaths), + Providers1 = rebar_state:providers(State), + State1 = rebar_state:providers(rebar_state:dir(State, Dir), Providers++Providers1), + case rebar_core:do(HookProviders, State1) of + {error, ProviderName} -> + ?DEBUG(format_error({bad_provider, Type, Command, ProviderName}), []), + throw(?PRV_ERROR({bad_provider, Type, Command, ProviderName})); + {ok, State2} -> + rebar_utils:remove_from_code_path(PluginDepsPaths), + State2 + end end. format_error({bad_provider, Type, Command, {Name, Namespace}}) -> diff --git a/src/rebar_log.erl b/src/rebar_log.erl index 23ae81e..b1a70c2 100644 --- a/src/rebar_log.erl +++ b/src/rebar_log.erl @@ -27,6 +27,7 @@ -module(rebar_log). -export([init/2, + crashdump/2, set_level/1, get_level/0, error_level/0, @@ -73,6 +74,7 @@ init(Caller, Verbosity) -> ?DEBUG_LEVEL -> debug end, Intensity = intensity(), + application:set_env(rebar, log_caller, Caller), Log = ec_cmd_log:new(Level, Caller, Intensity), set_level(valid_level(Verbosity)), application:set_env(rebar, log, Log). @@ -95,6 +97,16 @@ log(Level, Str, Args) -> {ok, LogState} = application:get_env(rebar, log), ec_cmd_log:Level(LogState, Str++"~n", Args). +crashdump(Str, Args) -> + crashdump("rebar3.crashdump", Str, Args). +crashdump(File, Str, Args) -> + case application:get_env(rebar, log_caller) of + {ok, api} -> + ok; + _ -> + file:write_file(File, io_lib:fwrite(Str, Args)) + end. + error_level() -> ?ERROR_LEVEL. default_level() -> ?INFO_LEVEL. diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index 5712fbf..fbd0e89 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -116,6 +116,7 @@ prepare_tests(State) -> %% rebar.config test options CfgOpts = cfgopts(State), ProjectApps = rebar_state:project_apps(State), + %% prioritize tests to run first trying any command line specified %% tests falling back to tests specified in the config file finally %% running a default set if no other tests are present @@ -215,6 +216,14 @@ add_hooks(Opts, State) -> select_tests(_, _, {error, _} = Error, _) -> Error; select_tests(_, _, _, {error, _} = Error) -> Error; select_tests(State, ProjectApps, CmdOpts, CfgOpts) -> + %% set application env if sys_config argument is provided + SysConfigs = sys_config_list(CmdOpts, CfgOpts), + Configs = lists:flatmap(fun(Filename) -> + rebar_file_utils:consult_config(State, Filename) + end, SysConfigs), + [application:load(Application) || Config <- SysConfigs, {Application, _} <- Config], + rebar_utils:reread_config(Configs), + Merged = lists:ukeymerge(1, lists:ukeysort(1, CmdOpts), lists:ukeysort(1, CfgOpts)), @@ -229,6 +238,17 @@ select_tests(State, ProjectApps, CmdOpts, CfgOpts) -> end, discover_tests(State, ProjectApps, Opts). +sys_config_list(CmdOpts, CfgOpts) -> + CmdSysConfigs = split_string(proplists:get_value(sys_config, CmdOpts, "")), + case proplists:get_value(sys_config, CfgOpts, []) of + [H | _]=Configs when is_list(H) -> + Configs ++ CmdSysConfigs; + [] -> + CmdSysConfigs; + Configs -> + [Configs | CmdSysConfigs] + end. + discover_tests(State, ProjectApps, Opts) -> case {proplists:get_value(suite, Opts), proplists:get_value(dir, Opts)} of %% no dirs or suites defined, try using `$APP/test` and `$ROOT/test` @@ -647,7 +667,8 @@ ct_opts(_State) -> {verbose, $v, "verbose", boolean, help(verbose)}, {name, undefined, "name", atom, help(name)}, {sname, undefined, "sname", atom, help(sname)}, - {setcookie, undefined, "setcookie", atom, help(setcookie)} + {setcookie, undefined, "setcookie", atom, help(setcookie)}, + {sys_config, undefined, "sys_config", string, help(sys_config)} %% comma-seperated list ]. help(dir) -> @@ -662,6 +683,8 @@ help(label) -> "Test label"; help(config) -> "List of config files"; +help(sys_config) -> + "List of application config files"; help(allow_user_terms) -> "Allow user defined config values in config files"; help(logdir) -> diff --git a/src/rebar_prv_cover.erl b/src/rebar_prv_cover.erl index c915141..464967b 100644 --- a/src/rebar_prv_cover.erl +++ b/src/rebar_prv_cover.erl @@ -296,9 +296,9 @@ strip_coverdir(File) -> cover_compile(State, apps) -> Apps = filter_checkouts(rebar_state:project_apps(State)), AppDirs = app_dirs(Apps), - ExtraDirs = extra_src_dirs(State, Apps), - cover_compile(State, lists:filter(fun(D) -> ec_file:is_dir(D) end, AppDirs ++ ExtraDirs)); + cover_compile(State, lists:filter(fun(D) -> ec_file:is_dir(D) end, AppDirs)); cover_compile(State, Dirs) -> + rebar_utils:update_code(rebar_state:code_paths(State, all_deps), [soft_purge]), %% start the cover server if necessary {ok, CoverPid} = start_cover(), %% redirect cover output @@ -316,27 +316,15 @@ cover_compile(State, Dirs) -> %% print any warnings about modules that failed to cover compile lists:foreach(fun print_cover_warnings/1, lists:flatten(Results)) end - end, Dirs). + end, Dirs), + rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), + ok. app_dirs(Apps) -> lists:foldl(fun app_ebin_dirs/2, [], Apps). app_ebin_dirs(App, Acc) -> - AppDir = rebar_app_info:ebin_dir(App), - ExtraDirs = rebar_dir:extra_src_dirs(rebar_app_info:opts(App), []), - OutDir = rebar_app_info:out_dir(App), - [AppDir] ++ [filename:join([OutDir, D]) || D <- ExtraDirs] ++ Acc. - -extra_src_dirs(State, Apps) -> - BaseDir = rebar_state:dir(State), - F = fun(App) -> rebar_app_info:dir(App) == BaseDir end, - %% check that this app hasn't already been dealt with - Extras = case lists:any(F, Apps) of - false -> rebar_dir:extra_src_dirs(rebar_state:opts(State), []); - true -> [] - end, - OutDir = rebar_dir:base_dir(State), - [filename:join([OutDir, "extras", D]) || D <- Extras]. + [rebar_app_info:ebin_dir(App)|Acc]. filter_checkouts(Apps) -> filter_checkouts(Apps, []). diff --git a/src/rebar_prv_path.erl b/src/rebar_prv_path.erl index 4e88496..4259eec 100644 --- a/src/rebar_prv_path.erl +++ b/src/rebar_prv_path.erl @@ -77,15 +77,15 @@ paths([{rel, true}|Rest], Apps, State, Acc) -> base_dir(State) -> io_lib:format("~s", [rebar_dir:base_dir(State)]). bin_dir(State) -> io_lib:format("~s/bin", [rebar_dir:base_dir(State)]). -lib_dir(State) -> io_lib:format("~s/lib", [rebar_dir:base_dir(State)]). +lib_dir(State) -> io_lib:format("~s", [rebar_dir:deps_dir(State)]). rel_dir(State) -> io_lib:format("~s/rel", [rebar_dir:base_dir(State)]). ebin_dirs(Apps, State) -> - lists:map(fun(App) -> io_lib:format("~s/lib/~s/ebin", [rebar_dir:base_dir(State), App]) end, Apps). + lists:map(fun(App) -> io_lib:format("~s/~s/ebin", [rebar_dir:deps_dir(State), App]) end, Apps). priv_dirs(Apps, State) -> - lists:map(fun(App) -> io_lib:format("~s/lib/~s/priv", [rebar_dir:base_dir(State), App]) end, Apps). + lists:map(fun(App) -> io_lib:format("~s/~s/priv", [rebar_dir:deps_dir(State), App]) end, Apps). src_dirs(Apps, State) -> - lists:map(fun(App) -> io_lib:format("~s/lib/~s/src", [rebar_dir:base_dir(State), App]) end, Apps). + lists:map(fun(App) -> io_lib:format("~s/~s/src", [rebar_dir:deps_dir(State), App]) end, Apps). print_paths_if_exist(Paths, State) -> {RawOpts, _} = rebar_state:command_parsed_args(State), diff --git a/src/rebar_prv_shell.erl b/src/rebar_prv_shell.erl index 0ede495..2e6c296 100644 --- a/src/rebar_prv_shell.erl +++ b/src/rebar_prv_shell.erl @@ -108,6 +108,10 @@ shell(State) -> simulate_proc_lib(), true = register(rebar_agent, self()), {ok, GenState} = rebar_agent:init(State), + %% Hack to fool the init process into thinking we have stopped and the normal + %% node start process can go on. Without it, init:get_status() always return + %% '{starting, started}' instead of '{started, started}' + init ! {'EXIT', self(), normal}, gen_server:enter_loop(rebar_agent, [], GenState, {local, rebar_agent}, hibernate). info() -> @@ -328,16 +332,7 @@ reread_config(State) -> no_config -> ok; ConfigList -> - try - [application:set_env(Application, Key, Val) - || Config <- ConfigList, - {Application, Items} <- Config, - {Key, Val} <- Items] - catch _:_ -> - ?ERROR("The configuration file submitted could not be read " - "and will be ignored.", []) - end, - ok + rebar_utils:reread_config(ConfigList) end. boot_apps(Apps) -> @@ -402,7 +397,7 @@ find_config(State) -> no_value -> no_config; Filename when is_list(Filename) -> - consult_config(State, Filename) + rebar_file_utils:consult_config(State, Filename) end. -spec first_value([Fun], State) -> no_value | Value when @@ -410,7 +405,7 @@ find_config(State) -> State :: rebar_state:t(), Fun :: fun ((State) -> no_value | Value). first_value([], _) -> no_value; -first_value([Fun | Rest], State) -> +first_value([Fun | Rest], State) -> case Fun(State) of no_value -> first_value(Rest, State); @@ -441,18 +436,3 @@ find_config_rebar(State) -> find_config_relx(State) -> debug_get_value(sys_config, rebar_state:get(State, relx, []), no_value, "Found config from relx."). - --spec consult_config(rebar_state:t(), string()) -> [[tuple()]]. -consult_config(State, Filename) -> - Fullpath = filename:join(rebar_dir:root_dir(State), Filename), - ?DEBUG("Loading configuration from ~p", [Fullpath]), - Config = case rebar_file_utils:try_consult(Fullpath) of - [T] -> T; - [] -> [] - end, - SubConfigs = [consult_config(State, Entry ++ ".config") || - Entry <- Config, is_list(Entry) - ], - - [Config | lists:merge(SubConfigs)]. - diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index 5e1e253..54b135e 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -114,14 +114,14 @@ hex_to_index(State) -> ets:foldl(fun({Pkg, [[]]}, _) when is_binary(Pkg) -> true; - ({Pkg, [Vsns=[Vsn | _Rest]]}, _) when is_binary(Pkg) -> + ({Pkg, [Vsns=[_Vsn | _Rest]]}, _) when is_binary(Pkg) -> %% Verify the package is of the right build tool by checking if the first %% version exists in the table from the foldl above - case ets:member(?PACKAGE_TABLE, {Pkg, Vsn}) of - true -> - ets:insert(?PACKAGE_TABLE, {Pkg, Vsns}); - false -> - true + case [V || V <- Vsns, ets:member(?PACKAGE_TABLE, {Pkg, V})] of + [] -> + true; + Vsns1 -> + ets:insert(?PACKAGE_TABLE, {Pkg, Vsns1}) end; (_, _) -> true diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl index c5c43e4..18c307b 100644 --- a/src/rebar_prv_upgrade.erl +++ b/src/rebar_prv_upgrade.erl @@ -45,7 +45,29 @@ init(State) -> do(State) -> {Args, _} = rebar_state:command_parsed_args(State), Locks = rebar_state:get(State, {locks, default}, []), - Deps = rebar_state:get(State, deps, []), + %% We have 3 sources of dependencies to upgrade from: + %% 1. the top-level rebar.config (in `deps', dep name is an atom) + %% 2. the app-level rebar.config in umbrella apps (in `{deps, default}', + %% where the dep name is an atom) + %% 3. the formatted sources for all after app-parsing (in `{deps, default}', + %% where the reprocessed app name is a binary) + %% + %% The gotcha with these is that the selection of dependencies with a + %% binary name (those that are stable and usable internally) is done with + %% in the profile deps only, but while accounting for locks. + %% Because our job here is to unlock those that have changed, we must + %% instead use the atom-based names, both in `deps' and `{deps, default}', + %% as those are the dependencies that may have changed but have been + %% disregarded by locks. + %% + %% As such, the working set of dependencies is the addition of + %% `deps' and `{deps, default}' entries with an atom name, as those + %% disregard locks and parsed values post-selection altogether. + %% Packages without versions can of course be a single atom. + TopDeps = rebar_state:get(State, deps, []), + ProfileDeps = rebar_state:get(State, {deps, default}, []), + Deps = [Dep || Dep <- TopDeps ++ ProfileDeps, % TopDeps > ProfileDeps + is_atom(Dep) orelse is_atom(element(1, Dep))], Names = parse_names(ec_cnv:to_binary(proplists:get_value(package, Args, <<"">>)), Locks), DepsDict = deps_dict(rebar_state:all_deps(State)), case prepare_locks(Names, Deps, Locks, [], DepsDict) of diff --git a/src/rebar_state.erl b/src/rebar_state.erl index a613a00..bdd4aeb 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -417,7 +417,8 @@ create_logic_providers(ProviderModules, State0) -> catch C:T -> ?DEBUG("~p: ~p ~p", [C, T, erlang:get_stacktrace()]), - throw({error, "Failed creating providers. Run with DEBUG=1 for stacktrace."}) + ?CRASHDUMP("~p: ~p~n~p~n~n~p", [C, T, erlang:get_stacktrace(), State0]), + throw({error, "Failed creating providers. Run with DEBUG=1 for stacktrace or consult rebar3.crashdump."}) end. to_list(#state_t{} = State) -> diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 56a3940..aa9e268 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -69,7 +69,8 @@ check_blacklisted_otp_versions/1, info_useless/2, list_dir/1, - user_agent/0]). + user_agent/0, + reread_config/1]). %% for internal use only -export([otp_release/0]). @@ -412,6 +413,17 @@ user_agent() -> {ok, Vsn} = application:get_key(rebar, vsn), ?FMT("Rebar/~s (OTP/~s)", [Vsn, otp_release()]). +reread_config(ConfigList) -> + try + [application:set_env(Application, Key, Val) + || Config <- ConfigList, + {Application, Items} <- Config, + {Key, Val} <- Items] + catch _:_ -> + ?ERROR("The configuration file submitted could not be read " + "and will be ignored.", []) + end. + %% ==================================================================== %% Internal functions %% ==================================================================== @@ -789,9 +801,13 @@ maybe_ends_in_comma(H) -> end. get_http_vars(Scheme) -> + OS = case os:getenv(atom_to_list(Scheme)) of + Str when is_list(Str) -> Str; + _ -> [] + end, GlobalConfigFile = rebar_dir:global_config(), Config = rebar_config:consult_file(GlobalConfigFile), - proplists:get_value(Scheme, Config, []). + proplists:get_value(Scheme, Config, OS). set_httpc_options() -> set_httpc_options(https_proxy, get_http_vars(https_proxy)), diff --git a/test/rebar_cover_SUITE.erl b/test/rebar_cover_SUITE.erl index a838d7d..841e29f 100644 --- a/test/rebar_cover_SUITE.erl +++ b/test/rebar_cover_SUITE.erl @@ -90,7 +90,7 @@ basic_extra_src_dirs(Config) -> {file, _} = cover:is_compiled(Mod), ExtraMod = list_to_atom(lists:flatten(io_lib:format("~ts_extra", [Name]))), - {file, _} = cover:is_compiled(ExtraMod). + false = cover:is_compiled(ExtraMod). release_extra_src_dirs(Config) -> AppDir = ?config(apps, Config), @@ -129,9 +129,9 @@ release_extra_src_dirs(Config) -> {file, _} = cover:is_compiled(Mod2), ExtraMod1 = list_to_atom(lists:flatten(io_lib:format("~ts_extra", [Name1]))), - {file, _} = cover:is_compiled(ExtraMod1), + false = cover:is_compiled(ExtraMod1), ExtraMod2 = list_to_atom(lists:flatten(io_lib:format("~ts_extra", [Name2]))), - {file, _} = cover:is_compiled(ExtraMod2). + false = cover:is_compiled(ExtraMod2). root_extra_src_dirs(Config) -> AppDir = ?config(apps, Config), @@ -160,7 +160,7 @@ root_extra_src_dirs(Config) -> Mod2 = list_to_atom(Name2), {file, _} = cover:is_compiled(Mod2), - {file, _} = cover:is_compiled(extra). + false = cover:is_compiled(extra). index_written(Config) -> AppDir = ?config(apps, Config), diff --git a/test/rebar_ct_SUITE.erl b/test/rebar_ct_SUITE.erl index 1da7571..c10875b 100644 --- a/test/rebar_ct_SUITE.erl +++ b/test/rebar_ct_SUITE.erl @@ -41,6 +41,7 @@ cmd_scale_timetraps/1, cmd_create_priv_dir/1, cmd_include_dir/1, + cmd_sys_config/1, cfg_opts/1, cfg_arbitrary_opts/1, cfg_test_spec/1, @@ -51,6 +52,7 @@ misspecified_ct_compile_opts/1, misspecified_ct_first_files/1]). +-include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). all() -> [{group, basic_app}, @@ -102,7 +104,8 @@ groups() -> [{basic_app, [], [basic_app_default_dirs, cmd_multiply_timetraps, cmd_scale_timetraps, cmd_create_priv_dir, - cmd_include_dir]}, + cmd_include_dir, + cmd_sys_config]}, {cover, [], [cover_compiled]}]. init_per_group(basic_app, Config) -> @@ -726,7 +729,7 @@ suite_at_app_root(Config) -> data_dir_correct(Config) -> DataDir = ?config(data_dir, Config), Parts = filename:split(DataDir), - ["rebar_ct_SUITE_data","test","rebar","lib","test","_build"|_] = lists:reverse(Parts). + ["rebar_ct_SUITE_data","test","rebar","lib",_,"_build"|_] = lists:reverse(Parts). cmd_label(Config) -> State = ?config(result, Config), @@ -1020,7 +1023,41 @@ cmd_include_dir(Config) -> CompileOpts = proplists:get_value(options, Info), true = lists:member({i, "foo/bar/baz"}, CompileOpts), true = lists:member({i, "qux"}, CompileOpts). - + +cmd_sys_config(Config) -> + State = ?config(result, Config), + AppDir = ?config(apps, Config), + Name = ?config(name, Config), + AppName = list_to_atom(Name), + + {ok, _} = rebar_prv_common_test:prepare_tests(State), + ?assertEqual(undefined, application:get_env(AppName, key)), + + CfgFile = filename:join([AppDir, "config", "cfg_sys.config"]), + ok = filelib:ensure_dir(CfgFile), + ok = file:write_file(CfgFile, cfg_sys_config_file(AppName)), + RebarConfig = [{ct_opts, [{sys_config, CfgFile}]}], + {ok, State1} = rebar_test_utils:run_and_check(Config, RebarConfig, ["as", "test", "lock"], return), + + {ok, _} = rebar_prv_common_test:prepare_tests(State1), + ?assertEqual({ok, cfg_value}, application:get_env(AppName, key)), + + Providers = rebar_state:providers(State1), + Namespace = rebar_state:namespace(State1), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + + CmdFile = filename:join([AppDir, "config", "cmd_sys.config"]), + ok = filelib:ensure_dir(CmdFile), + ok = file:write_file(CmdFile, cmd_sys_config_file(AppName)), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--sys_config="++CmdFile]), + State2 = rebar_state:command_parsed_args(State1, GetOptResult), + + {ok, _} = rebar_prv_common_test:prepare_tests(State2), + + ?assertEqual({ok ,cmd_value}, application:get_env(AppName, key)). + + cfg_opts(Config) -> C = rebar_test_utils:init_rebar_state(Config, "ct_cfg_opts_"), @@ -1181,10 +1218,15 @@ misspecified_ct_first_files(Config) -> {badconfig, {"Value `~p' of option `~p' must be a list", {some_file, ct_first_files}}} = Error. - %% helper for generating test data test_suite(Name) -> io_lib:format("-module(~ts_SUITE).\n" "-compile(export_all).\n" "all() -> [some_test].\n" "some_test(_) -> ok.\n", [Name]). + +cmd_sys_config_file(AppName) -> + io_lib:format("[{~s, [{key, cmd_value}]}].", [AppName]). + +cfg_sys_config_file(AppName) -> + io_lib:format("[{~s, [{key, cfg_value}]}].", [AppName]). diff --git a/test/rebar_deps_SUITE.erl b/test/rebar_deps_SUITE.erl index c95854a..aa2e7b2 100644 --- a/test/rebar_deps_SUITE.erl +++ b/test/rebar_deps_SUITE.erl @@ -3,7 +3,11 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). -all() -> [sub_app_deps, newly_added_dep, newly_added_after_empty_lock, http_proxy_settings, https_proxy_settings, semver_matching_lt, semver_matching_lte, semver_matching_gt, valid_version, {group, git}, {group, pkg}]. +all() -> [sub_app_deps, newly_added_dep, newly_added_after_empty_lock, + http_proxy_settings, https_proxy_settings, + http_os_proxy_settings, https_os_proxy_settings, + semver_matching_lt, semver_matching_lte, semver_matching_gt, + valid_version, {group, git}, {group, pkg}]. groups() -> [{all, [], [flat, pick_highest_left, pick_highest_right, @@ -82,6 +86,47 @@ init_per_testcase(https_proxy_settings, Config) -> rebar_test_utils:create_config(GlobalConfigDir, [{https_proxy, "http://localhost:1234"} ]), + %% Add a bad value by default to show config overtakes default + os:putenv("https_proxy", "unparseable-garbage"), + rebar_test_utils:init_rebar_state(Config) + end; +init_per_testcase(http_os_proxy_settings, Config) -> + %% Create private rebar.config + Priv = ?config(priv_dir, Config), + GlobalDir = filename:join(Priv, "global"), + GlobalConfigDir = filename:join([GlobalDir, ".config", "rebar3"]), + GlobalConfig = filename:join([GlobalDir, ".config", "rebar3", "rebar.config"]), + + meck:new(rebar_dir, [passthrough]), + meck:expect(rebar_dir, global_config, fun() -> GlobalConfig end), + meck:expect(rebar_dir, global_cache_dir, fun(_) -> GlobalDir end), + + %% Insert proxy variables into os env, but not config + os:putenv("http_proxy", "http://localhost:1234"), + rebar_test_utils:create_config(GlobalConfigDir, []), + rebar_test_utils:init_rebar_state(Config); +init_per_testcase(https_os_proxy_settings, Config) -> + SupportsHttpsProxy = case erlang:system_info(otp_release) of + "R16"++_ -> true; + "R"++_ -> false; + _ -> true % 17 and up don't have a "R" in the version + end, + if not SupportsHttpsProxy -> + {skip, https_proxy_unsupported_before_R16}; + SupportsHttpsProxy -> + %% Create private rebar.config + Priv = ?config(priv_dir, Config), + GlobalDir = filename:join(Priv, "global"), + GlobalConfigDir = filename:join([GlobalDir, ".config", "rebar3"]), + GlobalConfig = filename:join([GlobalDir, ".config", "rebar3", "rebar.config"]), + + meck:new(rebar_dir, [passthrough]), + meck:expect(rebar_dir, global_config, fun() -> GlobalConfig end), + meck:expect(rebar_dir, global_cache_dir, fun(_) -> GlobalDir end), + + %% Insert proxy variables into os env, not in config + os:putenv("https_proxy", "http://localhost:1234"), + rebar_test_utils:create_config(GlobalConfigDir, []), rebar_test_utils:init_rebar_state(Config) end; init_per_testcase(Case, Config) -> @@ -97,11 +142,20 @@ init_per_testcase(Case, Config) -> | setup_project(Case, Config, rebar_test_utils:expand_deps(DepsType, Deps))]. end_per_testcase(https_proxy_settings, Config) -> + os:putenv("https_proxy", ""), meck:unload(rebar_dir), Config; end_per_testcase(http_proxy_settings, Config) -> meck:unload(rebar_dir), Config; +end_per_testcase(http_os_proxy_settings, Config) -> + os:putenv("http_proxy", ""), + meck:unload(rebar_dir), + Config; +end_per_testcase(https_os_proxy_settings, Config) -> + os:putenv("https_proxy", ""), + meck:unload(rebar_dir), + Config; end_per_testcase(_, Config) -> meck:unload(), Config. @@ -311,6 +365,23 @@ https_proxy_settings(_Config) -> ?assertEqual({ok,{{"localhost", 1234}, []}}, httpc:get_option(https_proxy, rebar)). +http_os_proxy_settings(_Config) -> + %% Load config + rebar_utils:set_httpc_options(), + rebar3:init_config(), + + %% Assert variable is right + ?assertEqual({ok,{{"localhost", 1234}, []}}, + httpc:get_option(proxy, rebar)). + +https_os_proxy_settings(_Config) -> + %% Load config + rebar_utils:set_httpc_options(), + rebar3:init_config(), + + %% Assert variable is right + ?assertEqual({ok,{{"localhost", 1234}, []}}, + httpc:get_option(https_proxy, rebar)). semver_matching_lt(_Config) -> Dep = <<"test">>, diff --git a/test/rebar_upgrade_SUITE.erl b/test/rebar_upgrade_SUITE.erl index cfe1d8a..e7651a1 100644 --- a/test/rebar_upgrade_SUITE.erl +++ b/test/rebar_upgrade_SUITE.erl @@ -11,7 +11,7 @@ groups() -> triplet_a, triplet_b, triplet_c, tree_a, tree_b, tree_c, tree_c2, tree_cj, tree_ac, tree_all, delete_d, promote, stable_lock, fwd_lock, - compile_upgrade_parity]}, + compile_upgrade_parity, umbrella_config]}, {git, [], [{group, all}]}, {pkg, [], [{group, all}]}]. @@ -66,6 +66,18 @@ end_per_testcase(_, Config) -> meck:unload(), Config. +setup_project(Case=umbrella_config, Config0, Deps, UpDeps) -> + DepsType = ?config(deps_type, Config0), + NameRoot = atom_to_list(Case)++"_"++atom_to_list(DepsType), + Config = rebar_test_utils:init_rebar_state(Config0, NameRoot++"_"), + AppDir = filename:join([?config(apps, Config), "apps", NameRoot]), + rebar_test_utils:create_app(AppDir, "Root", "0.0.0", [kernel, stdlib]), + TopDeps = rebar_test_utils:top_level_deps(Deps), + TopConf = rebar_test_utils:create_config(AppDir, [{deps, []}]), + RebarConf = rebar_test_utils:create_config(AppDir, [{deps, TopDeps}]), + [{rebarconfig, TopConf}, + {rebarumbrella, RebarConf}, + {next_top_deps, rebar_test_utils:top_level_deps(UpDeps)} | Config]; setup_project(Case, Config0, Deps, UpDeps) -> DepsType = ?config(deps_type, Config0), Config = rebar_test_utils:init_rebar_state( @@ -437,7 +449,12 @@ upgrades(compile_upgrade_parity) -> [], {"", [{"A","1"}, "D", "J", "E", {"I","1"}, {"B","1"}, "F", "G", - {"C","1"}, "H"]}}. + {"C","1"}, "H"]}}; +upgrades(umbrella_config) -> + {[{"A", "1", []}], + [{"A", "2", []}], + ["A"], + {"A", [{"A","2"}]}}. %% TODO: add a test that verifies that unlocking files and then %% running the upgrade code is enough to properly upgrade things. @@ -570,9 +587,36 @@ compile_upgrade_parity(Config) -> ?assertEqual(CompileLockData1, CompileLockData2), ?assertEqual(CompileLockData1, UpgradeLockData). +umbrella_config(Config) -> + apply(?config(mock, Config), []), + {ok, TopConfig} = file:consult(?config(rebarconfig, Config)), + %% Install dependencies before re-mocking for an upgrade + rebar_test_utils:run_and_check(Config, TopConfig, ["lock"], {ok, []}), + {App, Unlocks} = ?config(expected, Config), + ct:pal("Upgrades: ~p -> ~p", [App, Unlocks]), + Expectation = case Unlocks of + {error, Term} -> {error, Term}; + _ -> {ok, Unlocks} + end, + + meck:new(rebar_prv_upgrade, [passthrough]), + meck:expect(rebar_prv_upgrade, do, fun(S) -> + apply(?config(mock_update, Config), []), + meck:passthrough([S]) + end), + _NewRebarConf = rebar_test_utils:create_config(filename:dirname(?config(rebarumbrella, Config)), + [{deps, ?config(next_top_deps, Config)}]), + %% re-run from the top-level with the old config still in place; + %% detection must happen when going for umbrella apps! + rebar_test_utils:run_and_check( + Config, TopConfig, ["upgrade", App], Expectation + ), + meck:unload(rebar_prv_upgrade). + run(Config) -> apply(?config(mock, Config), []), - {ok, RebarConfig} = file:consult(?config(rebarconfig, Config)), + ConfigPath = ?config(rebarconfig, Config), + {ok, RebarConfig} = file:consult(ConfigPath), %% Install dependencies before re-mocking for an upgrade rebar_test_utils:run_and_check(Config, RebarConfig, ["lock"], {ok, []}), {App, Unlocks} = ?config(expected, Config), @@ -587,7 +631,7 @@ run(Config) -> apply(?config(mock_update, Config), []), meck:passthrough([S]) end), - NewRebarConf = rebar_test_utils:create_config(?config(apps, Config), + NewRebarConf = rebar_test_utils:create_config(filename:dirname(ConfigPath), [{deps, ?config(next_top_deps, Config)}]), {ok, NewRebarConfig} = file:consult(NewRebarConf), rebar_test_utils:run_and_check( |