From 4ff52d991eb1240acbe82859d643295076f90727 Mon Sep 17 00:00:00 2001 From: Louis-Philippe Gauthier Date: Wed, 15 Jun 2016 08:05:21 -0400 Subject: Add support for cover_excl_mods --- src/rebar_prv_cover.erl | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_cover.erl b/src/rebar_prv_cover.erl index e555a9b..b04b1c1 100644 --- a/src/rebar_prv_cover.erl +++ b/src/rebar_prv_cover.erl @@ -302,18 +302,30 @@ cover_compile(State, Dirs) -> {ok, CoverPid} = start_cover(), %% redirect cover output true = redirect_cover_output(State, CoverPid), + ExclMods = rebar_state:get(State, cover_excl_mods, []), + lists:foreach(fun(Dir) -> - ?DEBUG("cover compiling ~p", [Dir]), - case catch(cover:compile_beam_directory(Dir)) of - {error, eacces} -> - ?WARN("Directory ~p not readable, modules will not be included in coverage", [Dir]); - {error, enoent} -> - ?WARN("Directory ~p not found", [Dir]); - {'EXIT', {Reason, _}} -> - ?WARN("Cover compilation for directory ~p failed: ~p", [Dir, Reason]); - Results -> - %% print any warnings about modules that failed to cover compile - lists:foreach(fun print_cover_warnings/1, lists:flatten(Results)) + case file:list_dir(Dir) of + {ok, Files} -> + Files2 = [F || F <- Files, filename:extension(F) == ".beam"], + lists:foreach(fun(File) -> + case lists:any(fun (Excl) -> + File =:= (atom_to_list(Excl) ++ ".beam") + end, ExclMods) of + true -> + ?DEBUG("cover ignoring ~p ~p", [Dir, File]); + _ -> + ?DEBUG("cover compiling ~p ~p", [Dir, File]), + case catch(cover:compile_beam(filename:join(Dir, File))) of + {error, Reason} -> + ?WARN("Cover compilation failed: ~p", [Reason]); + {ok, _} -> + ok + end + end + end, Files2); + {error, Reason} -> + ?WARN("Directory ~p error ~p", [Dir, Reason]) end end, Dirs). @@ -346,10 +358,6 @@ redirect_cover_output(State, CoverPid) -> [append]), group_leader(F, CoverPid). -print_cover_warnings({ok, _}) -> ok; -print_cover_warnings({error, Error}) -> - ?WARN("Cover compilation failed: ~p", [Error]). - write_coverdata(State, Task) -> DataDir = cover_dir(State), ok = filelib:ensure_dir(filename:join([DataDir, "dummy.log"])), -- cgit v1.1 From b75f95e063f493ff19dca9aef54ce42b006e9611 Mon Sep 17 00:00:00 2001 From: Ilya Khaprov Date: Fri, 15 Jul 2016 14:12:30 +0300 Subject: ssl_verify_hostname was renamed to ssl_verify_fun --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index bd0f871..5b735cf 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -23,7 +23,7 @@ erlware_commons, providers, bbmustache, - ssl_verify_hostname, + ssl_verify_fun, certifi, cth_readable, relx, -- cgit v1.1 From ffe5e42521aef4315be97f7baff2c7819fd79237 Mon Sep 17 00:00:00 2001 From: Guillaume Bour Date: Fri, 22 Jul 2016 10:43:16 +0200 Subject: Fix support for `not_valid` dialyzer error --- src/rebar_prv_dialyzer.erl | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src') diff --git a/src/rebar_prv_dialyzer.erl b/src/rebar_prv_dialyzer.erl index 82d2d07..0376918 100644 --- a/src/rebar_prv_dialyzer.erl +++ b/src/rebar_prv_dialyzer.erl @@ -260,6 +260,9 @@ read_plt(_State, Plt) -> Result; {error, no_such_file} -> error; + {error, not_valid} -> + Error = io_lib:format("Could not read the PLT file ~p", [Plt]), + throw({dialyzer_error, Error}); {error, read_error} -> Error = io_lib:format("Could not read the PLT file ~p", [Plt]), throw({dialyzer_error, Error}) -- cgit v1.1 From 2c7296babf5849dba383b52894490e397acb64d2 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 22 Jul 2016 08:12:46 -0400 Subject: Fix opts check when compiler called with dict opts rebar_base_compiler allows to be called with two types of options: a dictionary, or a rebar_state record. In the later case, the options are taken out with a call from rebar_opts, which fetches options that have been inserted in the application via rebar_app_info as part of the app_discovery phase, and are a list. This yields a possibility that options used when formatting warnings can either be a list of a dict, and we only used lists when making checks. This ended up breaking 3rd party compiler users (i.e. LFE compile plugin) since they were calling us with a dict rather than our own internal records. This patch supports both types of lookups to avoid issues. --- src/rebar_base_compiler.erl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_base_compiler.erl b/src/rebar_base_compiler.erl index 6b8c7ca..5d54057 100644 --- a/src/rebar_base_compiler.erl +++ b/src/rebar_base_compiler.erl @@ -155,7 +155,13 @@ format_warnings(Source, Warnings) -> format_warnings(Source, Warnings, []). format_warnings(Source, Warnings, Opts) -> - Prefix = case lists:member(warnings_as_errors, Opts) of + %% `Opts' can be passed in both as a list or a dictionary depending + %% on whether the first call to rebar_erlc_compiler was done with + %% the type `rebar_dict()' or `rebar_state:t()'. + LookupFn = if is_list(Opts) -> fun lists:member/2 + ; true -> fun dict:is_key/2 + end, + Prefix = case LookupFn(warnings_as_errors, Opts) of true -> ""; false -> "Warning: " end, -- cgit v1.1 From c8ec1546e5aa0a5b5aafad7d45d754b6464c2d04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Serre?= Date: Sat, 23 Jul 2016 19:45:46 +0200 Subject: template_dir option was forgotten in documentation plus recurse in directory added --- src/rebar_templater.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_templater.erl b/src/rebar_templater.erl index 2f33bfc..299b957 100644 --- a/src/rebar_templater.erl +++ b/src/rebar_templater.erl @@ -326,7 +326,7 @@ find_other_templates(State) -> undefined -> []; TemplateDir -> - rebar_utils:find_files(TemplateDir, ?TEMPLATE_RE) + rebar_utils:find_files(TemplateDir, ?TEMPLATE_RE, true) % recursive end. %% Fetch template indexes that sit on disk in plugins -- cgit v1.1 From b668329a9e7929f07f00d9a0647714690da4b4bd Mon Sep 17 00:00:00 2001 From: Guillaume Bour Date: Sat, 30 Jul 2016 13:42:00 +0200 Subject: Fix return error on `not_valid` to force .plt file regeneration --- src/rebar_prv_dialyzer.erl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_dialyzer.erl b/src/rebar_prv_dialyzer.erl index 0376918..fc13de1 100644 --- a/src/rebar_prv_dialyzer.erl +++ b/src/rebar_prv_dialyzer.erl @@ -261,8 +261,7 @@ read_plt(_State, Plt) -> {error, no_such_file} -> error; {error, not_valid} -> - Error = io_lib:format("Could not read the PLT file ~p", [Plt]), - throw({dialyzer_error, Error}); + error; {error, read_error} -> Error = io_lib:format("Could not read the PLT file ~p", [Plt]), throw({dialyzer_error, Error}) -- cgit v1.1 From a5bdf11770a8b57f7db2ca28f14dbf66c43673db Mon Sep 17 00:00:00 2001 From: Alexander Sedov Date: Thu, 4 Aug 2016 15:06:47 +0300 Subject: Typo fix. --- src/rebar_prv_common_test.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index fbd0e89..1e0632e 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -221,7 +221,7 @@ select_tests(State, ProjectApps, CmdOpts, CfgOpts) -> Configs = lists:flatmap(fun(Filename) -> rebar_file_utils:consult_config(State, Filename) end, SysConfigs), - [application:load(Application) || Config <- SysConfigs, {Application, _} <- Config], + [application:load(Application) || Config <- Configs, {Application, _} <- Config], rebar_utils:reread_config(Configs), Merged = lists:ukeymerge(1, -- cgit v1.1 From f9576c8598c4d44a3bd8adf36d62f5eb9b8c29a3 Mon Sep 17 00:00:00 2001 From: Nathaniel Waisbrot Date: Thu, 4 Aug 2016 14:36:46 -0400 Subject: Handle `escriptize` when the specified app is missing When rebar.config contains a `escript_main_app` option, but the specified app doesn't exist in the build directory, print an error. --- src/rebar_prv_escriptize.erl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_escriptize.erl b/src/rebar_prv_escriptize.erl index d8704f6..6e10947 100644 --- a/src/rebar_prv_escriptize.erl +++ b/src/rebar_prv_escriptize.erl @@ -72,8 +72,12 @@ do(State) -> end; Name -> AllApps = rebar_state:all_deps(State)++rebar_state:project_apps(State), - {ok, AppInfo} = rebar_app_utils:find(ec_cnv:to_binary(Name), AllApps), - escriptize(State, AppInfo) + case rebar_app_utils:find(ec_cnv:to_binary(Name), AllApps) of + {ok, AppInfo} -> + escriptize(State, AppInfo); + _ -> + ?PRV_ERROR({bad_name, Name}) + end end. escriptize(State0, App) -> -- cgit v1.1 From ea7942d9477bf2b0e3fffb905c04d330536f6c53 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 10 Aug 2016 13:10:55 -0400 Subject: Fix filtering of system libraries in escriptize https://github.com/erlang/rebar3/pull/1249 introduced a mechanism by which escript dependencies of applications only would be included; this required adding a filter to skip system libraries in the OTP root because that tends to break escripts in very nasty ways. However, the problem came that some libraries are just not in the escript path but may still be included; for these libraries the path prefix check failed as they return `{error, bad_name}` from `code:lib_dir(Dep)` rather than just the path they're in -- specifically, this happens with top level apps. The issue was reported in https://github.com/erlang/rebar3/issues/1294 and the current patch fixes it by accepting a `bad_name` dep as valid, since it is obviously not in the root path. --- src/rebar_prv_escriptize.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_escriptize.erl b/src/rebar_prv_escriptize.erl index d8704f6..e92c80d 100644 --- a/src/rebar_prv_escriptize.erl +++ b/src/rebar_prv_escriptize.erl @@ -232,7 +232,9 @@ find_deps_of_deps([Name|Names], Apps, Acc) -> DepNames = proplists:get_value(applications, rebar_app_info:app_details(App), []), BinDepNames = [ec_cnv:to_binary(Dep) || Dep <- DepNames, %% ignore system libs; shouldn't include them. - not lists:prefix(code:root_dir(), code:lib_dir(Dep))] + DepDir <- [code:lib_dir(Dep)], + DepDir =:= {error, bad_name} orelse % those are all local + not lists:prefix(code:root_dir(), DepDir)] -- ([Name|Names]++Acc), % avoid already seen deps ?DEBUG("new deps of ~p found to be ~p", [Name, BinDepNames]), find_deps_of_deps(BinDepNames ++ Names, Apps, BinDepNames ++ Acc). -- cgit v1.1 From ef3e9e7aa2bcc103e064165d24881fd5d961ed70 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sat, 13 Aug 2016 17:58:15 -0400 Subject: Make the escriptize provider hookable This will allow to move and modify the generated files --- src/rebar_prv_escriptize.erl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_escriptize.erl b/src/rebar_prv_escriptize.erl index 06b54ed..7ee20c2 100644 --- a/src/rebar_prv_escriptize.erl +++ b/src/rebar_prv_escriptize.erl @@ -61,8 +61,11 @@ desc() -> "the project's and its dependencies' BEAM files.". do(State) -> + Providers = rebar_state:providers(State), + Cwd = rebar_state:dir(State), + rebar_hooks:run_project_and_app_hooks(Cwd, pre, ?PROVIDER, Providers, State), ?INFO("Building escript...", []), - case rebar_state:get(State, escript_main_app, undefined) of + Res = case rebar_state:get(State, escript_main_app, undefined) of undefined -> case rebar_state:project_apps(State) of [App] -> @@ -78,7 +81,9 @@ do(State) -> _ -> ?PRV_ERROR({bad_name, Name}) end - end. + end, + rebar_hooks:run_project_and_app_hooks(Cwd, post, ?PROVIDER, Providers, State), + Res. escriptize(State0, App) -> AppName = rebar_app_info:name(App), -- cgit v1.1 From 4c2873d213ccdb4182f21c7179038897a66210b2 Mon Sep 17 00:00:00 2001 From: Rasmus Svensson Date: Mon, 15 Aug 2016 21:20:14 +0200 Subject: Print error on too many help arguments Previously the help task would crash when given more than two arguments. After this change it instead print a message: Too many arguments given. Usage: rebar3 help [] --- src/rebar_prv_help.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_help.erl b/src/rebar_prv_help.erl index c028264..75cc609 100644 --- a/src/rebar_prv_help.erl +++ b/src/rebar_prv_help.erl @@ -41,7 +41,10 @@ do(State) -> [Name] -> % default namespace task_help(default, list_to_atom(Name), State); [Namespace, Name] -> - task_help(list_to_atom(Namespace), list_to_atom(Name), State) + task_help(list_to_atom(Namespace), list_to_atom(Name), State); + _ -> + {error, "Too many arguments given. " ++ + "Usage: rebar3 help [] "} end. -spec format_error(any()) -> iolist(). -- cgit v1.1 From 5d475eefa27ddbb7876d74a9a3f6f45267333d57 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 25 Aug 2016 08:56:44 -0400 Subject: Make Edoc carry paths of pre-built libraries Given the topological sort applied to top-level apps, we should be able to carry the edoc values for paths configured when more than one app exists. This allows multiple disjoint app to have defined cross-linking in the documentation. Tests pending. --- src/rebar_prv_edoc.erl | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_edoc.erl b/src/rebar_prv_edoc.erl index 6cefe14..465fc34 100644 --- a/src/rebar_prv_edoc.erl +++ b/src/rebar_prv_edoc.erl @@ -33,17 +33,25 @@ do(State) -> code:add_pathsa(rebar_state:code_paths(State, all_deps)), ProjectApps = rebar_state:project_apps(State), Providers = rebar_state:providers(State), - EDocOpts = rebar_state:get(State, edoc_opts, []), + EdocOpts = rebar_state:get(State, edoc_opts, []), + ShouldAccPaths = not has_configured_paths(EdocOpts), Cwd = rebar_state:dir(State), rebar_hooks:run_all_hooks(Cwd, pre, ?PROVIDER, Providers, State), - lists:foreach(fun(AppInfo) -> - rebar_hooks:run_all_hooks(Cwd, pre, ?PROVIDER, Providers, AppInfo, State), - AppName = ec_cnv:to_list(rebar_app_info:name(AppInfo)), - ?INFO("Running edoc for ~s", [AppName]), - AppDir = rebar_app_info:dir(AppInfo), - ok = edoc:application(list_to_atom(AppName), AppDir, EDocOpts), - rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, AppInfo, State) - end, ProjectApps), + lists:foldl(fun(AppInfo, EdocOptsAcc) -> + rebar_hooks:run_all_hooks(Cwd, pre, ?PROVIDER, Providers, AppInfo, State), + AppName = ec_cnv:to_list(rebar_app_info:name(AppInfo)), + ?INFO("Running edoc for ~s", [AppName]), + AppDir = rebar_app_info:dir(AppInfo), + ok = edoc:application(list_to_atom(AppName), AppDir, EdocOptsAcc), + rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, AppInfo, State), + case ShouldAccPaths of + true -> + %% edoc wants / on all OSes + add_to_paths(EdocOptsAcc, AppDir++"/doc"); + false -> + EdocOptsAcc + end + end, EdocOpts, ProjectApps), rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State), rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), {ok, State}. @@ -55,3 +63,12 @@ format_error(Reason) -> %% =================================================================== %% Internal functions %% =================================================================== +has_configured_paths(EdocOpts) -> + proplists:get_value(dir, EdocOpts) =/= undefined. + +add_to_paths([], Path) -> + [{doc_path, [Path]}]; +add_to_paths([{doc_path, Paths}|T], Path) -> + [{doc_path, [Path | Paths]} | T]; +add_to_paths([H|T], Path) -> + [H | add_to_paths(Path, T)]. -- cgit v1.1 From 4f32da2ab7e0a7cf2f2e2de911c0c8ea0a7d8848 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 25 Aug 2016 22:21:07 -0400 Subject: Equivalent trim_all in bin split for <17.x The trim_all option used in binary:split/3 is not supported in 17.x. This patch makes an equivalent operation by eliminating empty split fragments. From the docs: trim Removes trailing empty parts of the result (as does trim in re:split/3. trim_all Removes all empty parts of the result. The new expression is therefore equivalent to the old one, but with the added benefit of compatibility. Fixes #1275 --- src/rebar_app_utils.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index d256cac..50c6314 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -182,7 +182,7 @@ update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) -> undefined -> get_package(PkgName, "0", State); <<"~>", Vsn/binary>> -> - [Vsn1] = binary:split(Vsn, [<<" ">>], [trim_all, global]), + [Vsn1] = [X || X <- binary:split(Vsn, [<<" ">>], [global]), X =/= <<>>], get_package(PkgName, Vsn1, State); _ -> {PkgName, PkgVsn} -- cgit v1.1 From ff4fd9f6f0c9989d1ff851c6201928f2a3f18e70 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 26 Aug 2016 13:01:22 -0400 Subject: Bumping to 3.3.0 --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 5b735cf..c844475 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.3.0"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 8ee9cc89969afd79bf8687dcdf978f1a09e01d48 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 26 Aug 2016 13:01:58 -0400 Subject: Returning to git-based tagging --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index c844475..5b735cf 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.3.0"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 4b0a07221de5e338cd71d774deff249f0528fa36 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sat, 27 Aug 2016 07:48:06 -0400 Subject: Re-format cover exclusion code - brings back former error handling and debug messages - keeps the filtering of excluded mods and debug messages - breaks up code into multiple functions and removes nesting --- src/rebar_prv_cover.erl | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_cover.erl b/src/rebar_prv_cover.erl index 9772e96..7c2597b 100644 --- a/src/rebar_prv_cover.erl +++ b/src/rebar_prv_cover.erl @@ -308,23 +308,16 @@ cover_compile(State, Dirs) -> lists:foreach(fun(Dir) -> case file:list_dir(Dir) of {ok, Files} -> - Files2 = [F || F <- Files, filename:extension(F) == ".beam"], - lists:foreach(fun(File) -> - case lists:any(fun (Excl) -> - File =:= (atom_to_list(Excl) ++ ".beam") - end, ExclMods) of - true -> - ?DEBUG("cover ignoring ~p ~p", [Dir, File]); - _ -> - ?DEBUG("cover compiling ~p ~p", [Dir, File]), - case catch(cover:compile_beam(filename:join(Dir, File))) of - {error, Reason} -> - ?WARN("Cover compilation failed: ~p", [Reason]); - {ok, _} -> - ok - end - end - end, Files2); + ?DEBUG("cover compiling ~p", [Dir]), + [cover_compile_file(filename:join(Dir, File)) + || File <- Files, + filename:extension(File) == ".beam", + not is_ignored(Dir, File, ExclMods)], + ok; + {error, eacces} -> + ?WARN("Directory ~p not readable, modules will not be included in coverage", [Dir]); + {error, enoent} -> + ?WARN("Directory ~p not found", [Dir]); {error, Reason} -> ?WARN("Directory ~p error ~p", [Dir, Reason]) end @@ -332,6 +325,22 @@ cover_compile(State, Dirs) -> rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), ok. +is_ignored(Dir, File, ExclMods) -> + Ignored = lists:any(fun(Excl) -> + File =:= atom_to_list(Excl) ++ ".beam" + end, + ExclMods), + Ignored andalso ?DEBUG("cover ignoring ~p ~p", [Dir, File]), + Ignored. + +cover_compile_file(FileName) -> + case catch(cover:compile_beam(FileName)) of + {error, Reason} -> + ?WARN("Cover compilation failed: ~p", [Reason]); + {ok, _} -> + ok + end. + app_dirs(Apps) -> lists:foldl(fun app_ebin_dirs/2, [], Apps). -- cgit v1.1 From 6bd8cda77aaa089f9eaa87dd696cfa63af5f2292 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sat, 27 Aug 2016 15:56:01 -0400 Subject: Fix crash when doing hash check with missing index Specifically, when fetching an application where the expected hash is unknown, the hash is validated from the hex index; when the index is available, the hash is fetched fine and later inserted in the lock file. However, if the index is not available, the call would simply crash. This patch fixes thing so that instead, the index is refreshed before giving up and failing. --- src/rebar_app_utils.erl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index 50c6314..847b0a5 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -190,7 +190,7 @@ update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) -> %% store the expected hash for the dependency Hash1 = case Hash of undefined -> % unknown, define the hash since we know the dep - rebar_packages:registry_checksum({pkg, PkgName1, PkgVsn1, Hash}, State); + fetch_checksum(PkgName1, PkgVsn1, Hash, State); _ -> % keep as is Hash end, @@ -203,6 +203,15 @@ update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) -> update_source(AppInfo, Source, _State) -> rebar_app_info:source(AppInfo, Source). +fetch_checksum(PkgName, PkgVsn, Hash, State) -> + try + rebar_packages:registry_checksum({pkg, PkgName, PkgVsn, Hash}, State) + catch + _:_ -> + ?INFO("Package ~s-~s not found. Fetching registry updates and trying again...", [PkgName, PkgVsn]), + {ok, _} = rebar_prv_update:do(State), + rebar_packages:registry_checksum({pkg, PkgName, PkgVsn, Hash}, State) + end. format_error({missing_package, Package}) -> io_lib:format("Package not found in registry: ~s", [Package]); -- cgit v1.1 From 7e554d06978ac52a4e56b48f67819af356a3279b Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Sat, 27 Aug 2016 13:22:41 -0700 Subject: reset accumulated coverdata on writing out to disk. this prevents provider chains like `eunit, ct, proper` from misreporting cover stats from providers later in the sequence --- src/rebar_prv_cover.erl | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src') diff --git a/src/rebar_prv_cover.erl b/src/rebar_prv_cover.erl index 7c2597b..401c331 100644 --- a/src/rebar_prv_cover.erl +++ b/src/rebar_prv_cover.erl @@ -376,6 +376,8 @@ write_coverdata(State, Task) -> ExportFile = filename:join([DataDir, atom_to_list(Task) ++ ".coverdata"]), case cover:export(ExportFile) of ok -> + %% dump accumulated coverdata after writing + ok = cover:reset(), ?DEBUG("Cover data written to ~p.", [ExportFile]); {error, Reason} -> ?WARN("Cover data export failed: ~p", [Reason]) -- cgit v1.1 From d4b76391475ab5ddda80f7799bda3ee1745e8244 Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Tue, 30 Aug 2016 12:17:52 +0200 Subject: Print stacktrace in a more conventional way Insert a newline before printing the stacktrace so that the term is easier to read and copy. This is a more conventional way to print traces, and is, for instance, the way it's done by make and python. --- src/rebar3.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar3.erl b/src/rebar3.erl index d3ea15f..c665f20 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -292,7 +292,7 @@ handle_error(Error) -> case erlang:get_stacktrace() of [] -> ok; Trace -> - ?DEBUG("Stack trace to the error location: ~p", [Trace]) + ?DEBUG("Stack trace to the error location:~n~p", [Trace]) end, ?INFO("When submitting a bug report, please include the output of `rebar3 report \"your command\"`", []), erlang:halt(1). -- cgit v1.1 From 4b69622d72046f26e17a9e3500a924219685032e Mon Sep 17 00:00:00 2001 From: vans163 Date: Thu, 1 Sep 2016 12:06:10 -0400 Subject: Update rebar_agent.erl https://github.com/erlang/rebar3/pull/1317 In reference to with support to load erlang code atomically but load nifs non-atomically. --- src/rebar_agent.erl | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_agent.erl b/src/rebar_agent.erl index 95818d8..5d6b050 100644 --- a/src/rebar_agent.erl +++ b/src/rebar_agent.erl @@ -109,8 +109,7 @@ refresh_paths(RState) -> {X,X} -> ?DEBUG("reloading ~p from ~s", [Modules, Path]), code:replace_path(App, Path), - [begin code:purge(M), code:delete(M), code:load_file(M) end - || M <- Modules]; + reload_modules(Modules); {_,_} -> ?DEBUG("skipping app ~p, stable copy required", [App]) end @@ -123,3 +122,31 @@ refresh_state(RState, _Dir) -> rebar3:init_config(), [fun(S) -> rebar_state:apply_profiles(S, rebar_state:current_profiles(RState)) end] ). + +reload_modules([]) -> noop; +reload_modules(Modules) -> + reload_modules(Modules, erlang:function_exported(code, prepare_loading, 1)). + +%% OTP 19 and later -- use atomic loading and ignore unloadable mods +reload_modules(Modules, true) -> + case code:prepare_loading(Modules) of + {ok, Prepared} -> + [code:purge(M) || M <- Modules], + code:finish_loading(Prepared); + + {error, ModRsns} -> + Blacklist = + (fun Error([], Acc) -> Acc; + Error([ {ModError, on_load_not_allowed} |T], Acc) -> + reload_modules([ModError], false), + Error(T, Acc); + Error([ {ModError, _} |T], Acc) -> + Error(T, [ModError|Acc]) + end)(ModRsns, []), + reload_modules(Modules -- Blacklist, true) + end; + +%% Older versions, use a more ad-hoc mechanism. Specifically has +%% a harder time dealing with NIFs. +reload_modules(Modules, false) -> + [begin code:purge(M), code:load_file(M) end || M <- Modules]. -- cgit v1.1 From d74cd14065bacf786bf4a463aa47715e80b412ec Mon Sep 17 00:00:00 2001 From: vans163 Date: Thu, 1 Sep 2016 13:31:23 -0400 Subject: Update rebar_agent.erl opps. Infinite loop fixed. --- src/rebar_agent.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rebar_agent.erl b/src/rebar_agent.erl index 5d6b050..72bbe44 100644 --- a/src/rebar_agent.erl +++ b/src/rebar_agent.erl @@ -137,9 +137,9 @@ reload_modules(Modules, true) -> {error, ModRsns} -> Blacklist = (fun Error([], Acc) -> Acc; - Error([ {ModError, on_load_not_allowed} |T], Acc) -> - reload_modules([ModError], false), - Error(T, Acc); + Error([ {ModNif, on_load_not_allowed} |T], Acc) -> + reload_modules([ModNif], false), + Error(T, [ModNif|Acc]); Error([ {ModError, _} |T], Acc) -> Error(T, [ModError|Acc]) end)(ModRsns, []), -- cgit v1.1 From 9b64c60f8bbbac7686d0f8d1f56219c847d2c811 Mon Sep 17 00:00:00 2001 From: vans163 Date: Thu, 1 Sep 2016 14:20:48 -0400 Subject: comment and spacing fixup --- src/rebar_agent.erl | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/rebar_agent.erl b/src/rebar_agent.erl index 72bbe44..818401d 100644 --- a/src/rebar_agent.erl +++ b/src/rebar_agent.erl @@ -125,7 +125,7 @@ refresh_state(RState, _Dir) -> reload_modules([]) -> noop; reload_modules(Modules) -> - reload_modules(Modules, erlang:function_exported(code, prepare_loading, 1)). + reload_modules(Modules, erlang:function_exported(code, prepare_loading, 1)). %% OTP 19 and later -- use atomic loading and ignore unloadable mods reload_modules(Modules, true) -> @@ -137,16 +137,15 @@ reload_modules(Modules, true) -> {error, ModRsns} -> Blacklist = (fun Error([], Acc) -> Acc; - Error([ {ModNif, on_load_not_allowed} |T], Acc) -> + Error([ {ModNif, on_load_not_allowed} |T], Acc) -> reload_modules([ModNif], false), Error(T, [ModNif|Acc]); - Error([ {ModError, _} |T], Acc) -> + Error([ {ModError, _} |T], Acc) -> Error(T, [ModError|Acc]) end)(ModRsns, []), reload_modules(Modules -- Blacklist, true) end; -%% Older versions, use a more ad-hoc mechanism. Specifically has -%% a harder time dealing with NIFs. +%% Older versions, use a more ad-hoc mechanism. reload_modules(Modules, false) -> [begin code:purge(M), code:load_file(M) end || M <- Modules]. -- cgit v1.1 From 6180ce0c64041e58feab600aa71ca31978b5d131 Mon Sep 17 00:00:00 2001 From: vans163 Date: Thu, 1 Sep 2016 14:44:38 -0400 Subject: Make less than R17 compatible --- src/rebar_agent.erl | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/rebar_agent.erl b/src/rebar_agent.erl index 818401d..6b4526e 100644 --- a/src/rebar_agent.erl +++ b/src/rebar_agent.erl @@ -136,13 +136,17 @@ reload_modules(Modules, true) -> {error, ModRsns} -> Blacklist = - (fun Error([], Acc) -> Acc; - Error([ {ModNif, on_load_not_allowed} |T], Acc) -> - reload_modules([ModNif], false), - Error(T, [ModNif|Acc]); - Error([ {ModError, _} |T], Acc) -> - Error(T, [ModError|Acc]) - end)(ModRsns, []), + lists:foldr(fun({ModError, Error}, Acc) -> + case Error of + %perhaps cover other cases of failure? + on_load_not_allowed -> + reload_modules([ModError], false), + [ModError|Acc]; + _ -> [ModError|Acc] + end + end, + [], ModRsns + ), reload_modules(Modules -- Blacklist, true) end; -- cgit v1.1 From 0f524bb6c2efc687956af85150b322b3ed883b03 Mon Sep 17 00:00:00 2001 From: vans163 Date: Fri, 2 Sep 2016 10:12:39 -0400 Subject: delete purge load_file specific order --- src/rebar_agent.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_agent.erl b/src/rebar_agent.erl index 6b4526e..8e1c8f6 100644 --- a/src/rebar_agent.erl +++ b/src/rebar_agent.erl @@ -152,4 +152,4 @@ reload_modules(Modules, true) -> %% Older versions, use a more ad-hoc mechanism. reload_modules(Modules, false) -> - [begin code:purge(M), code:load_file(M) end || M <- Modules]. + [begin code:delete(M), code:purge(M), code:load_file(M) end || M <- Modules]. -- cgit v1.1 From ff752834ee292e05c67fee7e53a5e095afe0ce9f Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 2 Sep 2016 20:33:44 -0400 Subject: Bump to 3.3.1 --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 5b735cf..d60df67 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.3.1"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 0e2b1a11b23658f06fb88076234eaad31605ff4c Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 2 Sep 2016 20:34:30 -0400 Subject: Back to semver version post release --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index d60df67..5b735cf 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.3.1"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 04589cf963028c75abd531892b1dcc54dae621b3 Mon Sep 17 00:00:00 2001 From: vans163 Date: Sat, 3 Sep 2016 10:35:26 -0400 Subject: log error if a module failed to load_file, the user should be aware --- src/rebar_agent.erl | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_agent.erl b/src/rebar_agent.erl index 8e1c8f6..2d83683 100644 --- a/src/rebar_agent.erl +++ b/src/rebar_agent.erl @@ -142,7 +142,9 @@ reload_modules(Modules, true) -> on_load_not_allowed -> reload_modules([ModError], false), [ModError|Acc]; - _ -> [ModError|Acc] + _ -> + ?ERROR("Module ~p failed to atomic load because ~p", [ModError, Error]), + [ModError|Acc] end end, [], ModRsns @@ -152,4 +154,13 @@ reload_modules(Modules, true) -> %% Older versions, use a more ad-hoc mechanism. reload_modules(Modules, false) -> - [begin code:delete(M), code:purge(M), code:load_file(M) end || M <- Modules]. + lists:foreach(fun(M) -> + code:delete(M), + code:purge(M), + case code:load_file(M) of + {module, M} -> ok; + {error, Error} -> + ?ERROR("Module ~p failed to load because ~p", [M, Error]) + end + end, Modules + ). \ No newline at end of file -- cgit v1.1 From 9ab25aaf1508e4286d755a0a60d6d22728a46dbf Mon Sep 17 00:00:00 2001 From: vans163 Date: Sat, 3 Sep 2016 15:56:38 -0400 Subject: replace error with debug --- src/rebar_agent.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_agent.erl b/src/rebar_agent.erl index 2d83683..4b0fc5f 100644 --- a/src/rebar_agent.erl +++ b/src/rebar_agent.erl @@ -143,7 +143,7 @@ reload_modules(Modules, true) -> reload_modules([ModError], false), [ModError|Acc]; _ -> - ?ERROR("Module ~p failed to atomic load because ~p", [ModError, Error]), + ?DEBUG("Module ~p failed to atomic load because ~p", [ModError, Error]), [ModError|Acc] end end, @@ -160,7 +160,7 @@ reload_modules(Modules, false) -> case code:load_file(M) of {module, M} -> ok; {error, Error} -> - ?ERROR("Module ~p failed to load because ~p", [M, Error]) + ?DEBUG("Module ~p failed to load because ~p", [M, Error]) end end, Modules ). \ No newline at end of file -- cgit v1.1 From 2b6fa7a25e0acdee4ce2a452152b688f2df27dea Mon Sep 17 00:00:00 2001 From: David de Boer Date: Wed, 14 Sep 2016 16:32:16 +0200 Subject: Ignore mv warnings In some cases, mv will throw a warning, while still moving the files correctly and returning a 0 return code: "mv: can't preserve ownership of ... Permission denied". --- src/rebar_file_utils.erl | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 104c047..437780d 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -171,9 +171,14 @@ mv(Source, Dest) -> {unix, _} -> EscSource = rebar_utils:escape_chars(Source), EscDest = rebar_utils:escape_chars(Dest), - {ok, []} = rebar_utils:sh(?FMT("mv ~s ~s", [EscSource, EscDest]), - [{use_stdout, false}, abort_on_error]), - ok; + case rebar_utils:sh(?FMT("mv ~s ~s", [EscSource, EscDest]), + [{use_stdout, false}, abort_on_error]) of + {ok, []} -> + ok; + {ok, Warning} -> + ?WARN("mv: ~p", [Warning]), + ok + end; {win32, _} -> Cmd = case filelib:is_dir(Source) of true -> -- cgit v1.1 From c0a903bbf9b74df01fad71bbf9d267d1aa7cff4c Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 20 Sep 2016 00:35:46 -0700 Subject: cover compile prior to calculating coverage fixes #1327 --- src/rebar_prv_cover.erl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_cover.erl b/src/rebar_prv_cover.erl index 401c331..968a632 100644 --- a/src/rebar_prv_cover.erl +++ b/src/rebar_prv_cover.erl @@ -15,7 +15,7 @@ -include("rebar.hrl"). -define(PROVIDER, cover). --define(DEPS, [app_discovery]). +-define(DEPS, [compile]). %% =================================================================== %% Public API @@ -84,6 +84,11 @@ reset(State) -> {ok, State}. analyze(State) -> + %% modules have to be cover compiled in order for + %% cover data to be reloaded + %% this maybe breaks if modules have been deleted + %% since code coverage was collected? + ok = cover_compile(State, apps), ?INFO("Performing cover analysis...", []), %% figure out what coverdata we have CoverDir = cover_dir(State), -- cgit v1.1 From f653d4dffec599f0d488aef30430e9761fc8955b Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 20 Sep 2016 00:47:16 -0700 Subject: only compile/cover compile when generating analysis don't compile when resetting coverdata --- src/rebar_prv_cover.erl | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_cover.erl b/src/rebar_prv_cover.erl index 968a632..b62a796 100644 --- a/src/rebar_prv_cover.erl +++ b/src/rebar_prv_cover.erl @@ -15,7 +15,7 @@ -include("rebar.hrl"). -define(PROVIDER, cover). --define(DEPS, [compile]). +-define(DEPS, [lock]). %% =================================================================== %% Public API @@ -84,11 +84,20 @@ reset(State) -> {ok, State}. analyze(State) -> - %% modules have to be cover compiled in order for - %% cover data to be reloaded + %% modules have to be compiled and then cover compiled + %% in order for cover data to be reloaded %% this maybe breaks if modules have been deleted %% since code coverage was collected? - ok = cover_compile(State, apps), + case rebar_prv_compile:do(State) of + %% successfully compiled apps + {ok, S} -> + ok = cover_compile(S, apps), + do_analyze(State); + %% this should look like a compiler error, not a cover error + Error -> Error + end. + +do_analyze(State) -> ?INFO("Performing cover analysis...", []), %% figure out what coverdata we have CoverDir = cover_dir(State), -- cgit v1.1 From dad7900d31fc2744c9d6b877f1a859f1f940cc39 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 20 Sep 2016 02:27:41 -0700 Subject: recompile all files when a parse transform given as an opt needs updating there's no way to detect which files actually rely on a parse transform passed to the compiler via the options (as opposed to `-compile(..)` so if any parse transforms are in modules that need recompiling just recompile the world fixes #1328 --- src/rebar_erlc_compiler.erl | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index 167f2bb..b148172 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -202,7 +202,13 @@ compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, Opts) -> G = init_erlcinfo(include_abs_dirs(ErlOpts, BaseDir), AllErlFiles, BaseDir, OutDir), - NeededErlFiles = needed_files(G, ErlOpts, BaseDir, OutDir, AllErlFiles), + {ParseTransforms, Rest} = split_source_files(AllErlFiles, ErlOpts), + NeededErlFiles = case needed_files(G, ErlOpts, BaseDir, OutDir, ParseTransforms) of + [] -> needed_files(G, ErlOpts, BaseDir, OutDir, Rest); + %% at least one parse transform in the opts needs updating, so recompile all + _ -> AllErlFiles + end, + {ErlFirstFiles, ErlOptsFirst} = erl_first_files(RebarOpts, ErlOpts, BaseDir, NeededErlFiles), {DepErls, OtherErls} = lists:partition( fun(Source) -> digraph:in_degree(G, Source) > 0 end, @@ -296,6 +302,14 @@ erl_first_files(Opts, ErlOpts, Dir, NeededErlFiles) -> end, ErlOpts), {ErlFirstFiles ++ ParseTransformsErls, ErlOptsFirst}. +split_source_files(SourceFiles, ErlOpts) -> + ParseTransforms = proplists:get_all_values(parse_transform, ErlOpts), + lists:partition(fun(Source) -> + lists:member(filename_to_atom(Source), ParseTransforms) + end, SourceFiles). + +filename_to_atom(F) -> list_to_atom(filename:rootname(filename:basename(F))). + %% Get subset of SourceFiles which need to be recompiled, respecting %% dependencies induced by given graph G. needed_files(G, ErlOpts, Dir, OutDir, SourceFiles) -> -- cgit v1.1 From b79e5da2363114de34ce612f32a109d37e7d01ac Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Tue, 20 Sep 2016 21:28:01 -0700 Subject: allow using an alternate regex to locate test modules during eunit runs {`eunit_test_regex`, Regex}` will use the supplied `Regex` instead of the default to locate tests in test dirs. note this matches only the filename, not the path. the regex is applied to all test dirs, recursively fixes #1331 --- src/rebar_prv_eunit.erl | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_eunit.erl b/src/rebar_prv_eunit.erl index 942fd10..82e2458 100644 --- a/src/rebar_prv_eunit.erl +++ b/src/rebar_prv_eunit.erl @@ -18,6 +18,8 @@ %% we need to modify app_info state before compile -define(DEPS, [lock]). +-define(DEFAULT_TEST_REGEX, "^[^._].*\\.erl\$"). + %% =================================================================== %% Public API %% =================================================================== @@ -174,21 +176,24 @@ set_apps([App|Rest], Acc) -> set_modules(Apps, State) -> set_modules(Apps, State, {[], []}). set_modules([], State, {AppAcc, TestAcc}) -> - TestSrc = gather_src([filename:join([rebar_state:dir(State), "test"])]), + Regex = rebar_state:get(State, eunit_test_regex, ?DEFAULT_TEST_REGEX), + BareTestDir = [filename:join([rebar_state:dir(State), "test"])], + TestSrc = gather_src(BareTestDir, Regex), dedupe_tests({AppAcc, TestAcc ++ TestSrc}); set_modules([App|Rest], State, {AppAcc, TestAcc}) -> F = fun(Dir) -> filename:join([rebar_app_info:dir(App), Dir]) end, AppDirs = lists:map(F, rebar_dir:src_dirs(rebar_app_info:opts(App), ["src"])), - AppSrc = gather_src(AppDirs), + Regex = rebar_state:get(State, eunit_test_regex, ?DEFAULT_TEST_REGEX), + AppSrc = gather_src(AppDirs, Regex), TestDirs = [filename:join([rebar_app_info:dir(App), "test"])], - TestSrc = gather_src(TestDirs), + TestSrc = gather_src(TestDirs, Regex), set_modules(Rest, State, {AppSrc ++ AppAcc, TestSrc ++ TestAcc}). -gather_src(Dirs) -> gather_src(Dirs, []). +gather_src(Dirs, Regex) -> gather_src(Dirs, Regex, []). -gather_src([], Srcs) -> Srcs; -gather_src([Dir|Rest], Srcs) -> - gather_src(Rest, Srcs ++ rebar_utils:find_files(Dir, "^[^._].*\\.erl\$", true)). +gather_src([], _Regex, Srcs) -> Srcs; +gather_src([Dir|Rest], Regex, Srcs) -> + gather_src(Rest, Regex, Srcs ++ rebar_utils:find_files(Dir, Regex, true)). dedupe_tests({AppMods, TestMods}) -> %% for each modules in TestMods create a test if there is not a module -- cgit v1.1 From 384e9e58dba9ff8cf801bfdbe3c2ec406453aa5f Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Tue, 27 Sep 2016 10:23:47 -0400 Subject: Properly support package aliasing and alt names Aliasing only had a bit of ad-hoc support in rebar3, and various issues have encountered problems related to the package names not mapping properly with the application name. One such issue is https://github.com/erlang/rebar3/issues/1290 The problem has been hard to find because it only impacts transitive dependencies (not top-level ones) of other packages. The root cause for this is that the application name was not being tracked by rebar3's internal index, only the package name and its version were. When a given application was a package app, the data for the application name would be reconstructed from the lock file, but only if it were a top-level app or a dependency of a source application where parsing the lock file is necessary to know what comes next. When a transitive dependency of a package dependency was fetched, we instead read its dependencies directly from the in-memory package index within rebar3. This caused us to only read the package name and version, and lost all information regarding application name. This worked fine for most cases since for the vast majority of packages, the package name matches the app name, but failed for all aliases, which would then be moved to directories that wouldn't match the app name. This in turn broke some aspects of code analysis (in Dialyzer), or other functionality relying on static paths, such as including .hrl files from dependencies. This patch reformats the internal storage format of dependencies to align with the internal one used by rebar3, so that the app name can be carried along with the package name and its version. The fix can only work once `rebar3 update` is called so the index is rebuilt internally, and will the file cached on disk will be incompatible with older rebar3 versions. Currently, the following is not covered: - Tests - Including the package hashes of dependencies so they may match what is in a lock file -- they're being `undefined` instead, which may break some lookups. The previous format did not lend itself to hashing in the same way, and it is possible transitive deps were not being tracked properly, or worked by respecting the current package hierarchy. This will require further analysis For now this commit can allow reviewing and discussion. --- src/rebar_prv_update.erl | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index 54b135e..ff7194a 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -142,8 +142,8 @@ hex_to_index(State) -> end. update_deps_list(Pkg, PkgVsn, Deps, HexRegistry, State) -> - lists:foldl(fun([Dep, DepVsn, false, _AppName | _], DepsListAcc) -> - Dep1 = {Pkg, PkgVsn, Dep}, + lists:foldl(fun([Dep, DepVsn, false, AppName | _], DepsListAcc) -> + Dep1 = {Pkg, PkgVsn, Dep, AppName}, case {valid_vsn(DepVsn), DepVsn} of %% Those are all not perfectly implemented! %% and doubled since spaces seem not to be @@ -168,9 +168,9 @@ update_deps_list(Pkg, PkgVsn, Deps, HexRegistry, State) -> cmpl(Dep1, rm_ws(Vsn), HexRegistry, State, DepsListAcc, fun ec_semver:lt/2); {_, <<"==", Vsn/binary>>} -> - [{Dep, Vsn} | DepsListAcc]; + [{AppName, {pkg, Dep, Vsn, undefined}} | DepsListAcc]; {_, Vsn} -> - [{Dep, Vsn} | DepsListAcc] + [{AppName, {pkg, Dep, Vsn, undefined}} | DepsListAcc] end; ([_Dep, _DepVsn, true, _AppName | _], DepsListAcc) -> DepsListAcc @@ -188,27 +188,27 @@ valid_vsn(Vsn) -> SupportedVersions = "^(>=?|<=?|~>|==)?\\s*" ++ SemVerRegExp ++ "$", re:run(Vsn, SupportedVersions) =/= nomatch. -highest_matching({Pkg, PkgVsn, Dep}, Vsn, HexRegistry, State, DepsListAcc) -> +highest_matching({Pkg, PkgVsn, Dep, App}, Vsn, HexRegistry, State, DepsListAcc) -> case rebar_packages:find_highest_matching(Pkg, PkgVsn, Dep, Vsn, HexRegistry, State) of {ok, HighestDepVsn} -> - [{Dep, HighestDepVsn} | DepsListAcc]; + [{App, {pkg, Dep, HighestDepVsn, undefined}} | DepsListAcc]; none -> ?WARN("[~s:~s] Missing registry entry for package ~s. Try to fix with `rebar3 update`", [Pkg, PkgVsn, Dep]), DepsListAcc end. -cmp({_Pkg, _PkgVsn, Dep} = Dep1, Vsn, HexRegistry, State, DepsListAcc, CmpFun) -> +cmp({_Pkg, _PkgVsn, Dep, _App} = Dep1, Vsn, HexRegistry, State, DepsListAcc, CmpFun) -> {ok, Vsns} = rebar_packages:find_all(Dep, HexRegistry, State), cmp_(undefined, Vsn, Vsns, DepsListAcc, Dep1, CmpFun). -cmp_(undefined, _MinVsn, [], DepsListAcc, {Pkg, PkgVsn, Dep}, _CmpFun) -> +cmp_(undefined, _MinVsn, [], DepsListAcc, {Pkg, PkgVsn, Dep, _App}, _CmpFun) -> ?WARN("[~s:~s] Missing registry entry for package ~s. Try to fix with `rebar3 update`", [Pkg, PkgVsn, Dep]), DepsListAcc; -cmp_(HighestDepVsn, _MinVsn, [], DepsListAcc, {_Pkg, _PkgVsn, Dep}, _CmpFun) -> - [{Dep, HighestDepVsn} | DepsListAcc]; +cmp_(HighestDepVsn, _MinVsn, [], DepsListAcc, {_Pkg, _PkgVsn, Dep, App}, _CmpFun) -> + [{App, {pkg, Dep, HighestDepVsn, undefined}} | DepsListAcc]; cmp_(BestMatch, MinVsn, [Vsn | R], DepsListAcc, Dep, CmpFun) -> case CmpFun(Vsn, MinVsn) of @@ -224,13 +224,13 @@ cmpl({_Pkg, _PkgVsn, Dep} = Dep1, Vsn, HexRegistry, State, DepsListAcc, CmpFun) {ok, Vsns} = rebar_packages:find_all(Dep, HexRegistry, State), cmpl_(undefined, Vsn, Vsns, DepsListAcc, Dep1, CmpFun). -cmpl_(undefined, _MaxVsn, [], DepsListAcc, {Pkg, PkgVsn, Dep}, _CmpFun) -> +cmpl_(undefined, _MaxVsn, [], DepsListAcc, {Pkg, PkgVsn, Dep, _App}, _CmpFun) -> ?WARN("[~s:~s] Missing registry entry for package ~s. Try to fix with `rebar3 update`", [Pkg, PkgVsn, Dep]), DepsListAcc; -cmpl_(HighestDepVsn, _MaxVsn, [], DepsListAcc, {_Pkg, _PkgVsn, Dep}, _CmpFun) -> - [{Dep, HighestDepVsn} | DepsListAcc]; +cmpl_(HighestDepVsn, _MaxVsn, [], DepsListAcc, {_Pkg, _PkgVsn, Dep, App}, _CmpFun) -> + [{App, {pkg, Dep, HighestDepVsn, undefined}} | DepsListAcc]; cmpl_(undefined, MaxVsn, [Vsn | R], DepsListAcc, Dep, CmpFun) -> case CmpFun(Vsn, MaxVsn) of -- cgit v1.1 From 3f0e56d9c8c6da6c304543e47ff83e101aabf847 Mon Sep 17 00:00:00 2001 From: Brujo Benavides Date: Wed, 28 Sep 2016 19:32:22 -0300 Subject: Add support for behaviors, and not just behaviours --- src/rebar_erlc_compiler.erl | 2 ++ src/rebar_prv_xref.erl | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index b148172..e6f2b71 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -668,6 +668,8 @@ process_attr(include_lib, Form, Includes, Dir) -> [FileNode] = erl_syntax:attribute_arguments(Form), RawFile = erl_syntax:string_value(FileNode), maybe_expand_include_lib_path(RawFile, Dir) ++ Includes; +process_attr(behavior, Form, Includes, _Dir) -> + process_attr(behaviour, Form, Includes, _Dir); process_attr(behaviour, Form, Includes, _Dir) -> [FileNode] = erl_syntax:attribute_arguments(Form), File = module_to_erl(erl_syntax:atom_value(FileNode)), diff --git a/src/rebar_prv_xref.erl b/src/rebar_prv_xref.erl index 45badd3..3d74c9a 100644 --- a/src/rebar_prv_xref.erl +++ b/src/rebar_prv_xref.erl @@ -165,7 +165,8 @@ keyall(Key, List) -> lists:flatmap(fun({K, L}) when Key =:= K -> L; (_) -> [] end, List). get_behaviour_callbacks(exports_not_used, Attributes) -> - [B:behaviour_info(callbacks) || B <- keyall(behaviour, Attributes)]; + [B:behaviour_info(callbacks) || B <- keyall(behaviour, Attributes) ++ + keyall(behavior, Attributes)]; get_behaviour_callbacks(_XrefCheck, _Attributes) -> []. -- cgit v1.1 From 787cd967b632bef4534ade58ab64a51eda838df1 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 30 Sep 2016 09:27:40 -0400 Subject: Fix private includes when compiling in test profile When an include file is set in a private path (i.e. src/), the rebar3 compiler would not add them to the {i, Path} params -- only include/ and the project root were being added. This meant that when some extra source directories were added to the compile job, such as test/ when running under the test profile, the private include paths could not be shared with the test module. This patch fixes the issues (and adds tests) for such a specific case by adding all the configured include paths to the {i, Path} erl_opts arguments, yielding successful compile runs. --- src/rebar_erlc_compiler.erl | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index e6f2b71..bc29939 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -203,8 +203,8 @@ compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, Opts) -> G = init_erlcinfo(include_abs_dirs(ErlOpts, BaseDir), AllErlFiles, BaseDir, OutDir), {ParseTransforms, Rest} = split_source_files(AllErlFiles, ErlOpts), - NeededErlFiles = case needed_files(G, ErlOpts, BaseDir, OutDir, ParseTransforms) of - [] -> needed_files(G, ErlOpts, BaseDir, OutDir, Rest); + NeededErlFiles = case needed_files(G, ErlOpts, RebarOpts, BaseDir, OutDir, ParseTransforms) of + [] -> needed_files(G, ErlOpts, RebarOpts, BaseDir, OutDir, Rest); %% at least one parse transform in the opts needs updating, so recompile all _ -> AllErlFiles end, @@ -224,7 +224,7 @@ compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, Opts) -> true -> ErlOptsFirst; false -> ErlOpts end, - internal_erl_compile(C, BaseDir, S, OutDir, ErlOpts1) + internal_erl_compile(C, BaseDir, S, OutDir, ErlOpts1, RebarOpts) end) after true = digraph:delete(SubGraph), @@ -312,13 +312,15 @@ filename_to_atom(F) -> list_to_atom(filename:rootname(filename:basename(F))). %% Get subset of SourceFiles which need to be recompiled, respecting %% dependencies induced by given graph G. -needed_files(G, ErlOpts, Dir, OutDir, SourceFiles) -> +needed_files(G, ErlOpts, RebarOpts, Dir, OutDir, SourceFiles) -> lists:filter(fun(Source) -> TargetBase = target_base(OutDir, Source), Target = TargetBase ++ ".beam", + PrivIncludes = [{i, filename:join(Dir, Src)} + || Src <- rebar_dir:src_dirs(RebarOpts, ["src"])], AllOpts = [{outdir, filename:dirname(Target)} ,{i, filename:join(Dir, "include")} - ,{i, Dir}] ++ ErlOpts, + ,{i, Dir}] ++ PrivIncludes ++ ErlOpts, digraph:vertex(G, Source) > {Source, filelib:last_modified(Target)} orelse opts_changed(AllOpts, TargetBase) orelse erl_compiler_opts_set() @@ -518,12 +520,15 @@ expand_file_names(Files, Dirs) -> end, Files). -spec internal_erl_compile(rebar_dict(), file:filename(), file:filename(), - file:filename(), list()) -> ok | {ok, any()} | {error, any(), any()}. -internal_erl_compile(Opts, Dir, Module, OutDir, ErlOpts) -> + file:filename(), list(), rebar_dict()) -> + ok | {ok, any()} | {error, any(), any()}. +internal_erl_compile(Opts, Dir, Module, OutDir, ErlOpts, RebarOpts) -> Target = target_base(OutDir, Module) ++ ".beam", ok = filelib:ensure_dir(Target), - AllOpts = [{outdir, filename:dirname(Target)}] ++ ErlOpts ++ - [{i, filename:join(Dir, "include")}, {i, Dir}, return], + PrivIncludes = [{i, filename:join(Dir, Src)} + || Src <- rebar_dir:src_dirs(RebarOpts, ["src"])], + AllOpts = [{outdir, filename:dirname(Target)}] ++ ErlOpts ++ PrivIncludes ++ + [{i, filename:join(Dir, "include")}, {i, Dir}, return], case compile:file(Module, AllOpts) of {ok, _Mod} -> ok; -- cgit v1.1 From 82e7616745bdb05143b78e1b3d3c15db709a7f0a Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 30 Sep 2016 13:28:10 -0400 Subject: Use all_src_dirs for include paths Helps cover extra cases. --- src/rebar_erlc_compiler.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index bc29939..36a247e 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -317,7 +317,7 @@ needed_files(G, ErlOpts, RebarOpts, Dir, OutDir, SourceFiles) -> TargetBase = target_base(OutDir, Source), Target = TargetBase ++ ".beam", PrivIncludes = [{i, filename:join(Dir, Src)} - || Src <- rebar_dir:src_dirs(RebarOpts, ["src"])], + || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])], AllOpts = [{outdir, filename:dirname(Target)} ,{i, filename:join(Dir, "include")} ,{i, Dir}] ++ PrivIncludes ++ ErlOpts, @@ -526,7 +526,7 @@ internal_erl_compile(Opts, Dir, Module, OutDir, ErlOpts, RebarOpts) -> Target = target_base(OutDir, Module) ++ ".beam", ok = filelib:ensure_dir(Target), PrivIncludes = [{i, filename:join(Dir, Src)} - || Src <- rebar_dir:src_dirs(RebarOpts, ["src"])], + || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])], AllOpts = [{outdir, filename:dirname(Target)}] ++ ErlOpts ++ PrivIncludes ++ [{i, filename:join(Dir, "include")}, {i, Dir}, return], case compile:file(Module, AllOpts) of -- cgit v1.1 From 1f86ed1aed990438103c5f668c4ec930ab637fc9 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Tue, 4 Oct 2016 22:34:22 -0400 Subject: Track package hash in memory index, add hash test This adds tracking of package hash in the in-memory index rather than the current `undefined' values. According to the test added, this is not necessary for transitive package dep hash chcking, but does result in a more complete index search result when doing app lookups, and could yield some optimizations on hash checks by checking from the index structure before fetching a package. --- src/rebar_prv_update.erl | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index ff7194a..75c609e 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -104,7 +104,8 @@ hex_to_index(State) -> case lists:any(fun is_supported/1, BuildTools) of true -> DepsList = update_deps_list(Pkg, PkgVsn, Deps, Registry, State), - ets:insert(?PACKAGE_TABLE, {{Pkg, PkgVsn}, DepsList, Checksum}); + HashedDeps = update_deps_hashes(DepsList), + ets:insert(?PACKAGE_TABLE, {{Pkg, PkgVsn}, HashedDeps, Checksum}); false -> true end; @@ -176,6 +177,21 @@ update_deps_list(Pkg, PkgVsn, Deps, HexRegistry, State) -> DepsListAcc end, [], Deps). +update_deps_hashes(List) -> + [{Name, {pkg, PkgName, Vsn, lookup_hash(PkgName, Vsn, Hash)}} + || {Name, {pkg, PkgName, Vsn, Hash}} <- List]. + +lookup_hash(Name, Vsn, undefined) -> + try + ets:lookup_element(?PACKAGE_TABLE, {Name, Vsn}, 3) + catch + _:_ -> + undefined + end; +lookup_hash(_, _, Hash) -> + Hash. + + rm_ws(<<" ", R/binary>>) -> rm_ws(R); rm_ws(R) -> -- cgit v1.1 From 3966610e532f18f71c6e4d04c3a96d6372ddc390 Mon Sep 17 00:00:00 2001 From: Alexander Sedov Date: Tue, 11 Oct 2016 10:31:57 +0300 Subject: Made Common Test load the user's applications before slurping config. --- src/rebar_prv_common_test.erl | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index 1e0632e..8bfa192 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -221,6 +221,7 @@ select_tests(State, ProjectApps, CmdOpts, CfgOpts) -> Configs = lists:flatmap(fun(Filename) -> rebar_file_utils:consult_config(State, Filename) end, SysConfigs), + code:add_pathsa(rebar_state:code_paths(State, all_deps)), [application:load(Application) || Config <- Configs, {Application, _} <- Config], rebar_utils:reread_config(Configs), -- cgit v1.1 From 6e9503c4f0232d6e8152b56a7037f22843e53959 Mon Sep 17 00:00:00 2001 From: Alexander Sedov Date: Tue, 11 Oct 2016 10:33:14 +0300 Subject: Rereading system configuration sets up persistent options if possible. --- src/rebar_utils.erl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index aa9e268..dc0bb42 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -414,8 +414,14 @@ user_agent() -> ?FMT("Rebar/~s (OTP/~s)", [Vsn, otp_release()]). reread_config(ConfigList) -> + SetEnv = case version_tuple(?MODULE:otp_release()) of + {X, _, _} when X =< 17 -> + fun application:set_env/3; + _ -> + fun (App, Key, Val) -> application:set_env(App, Key, Val, [{persistent, true}]) end + end, try - [application:set_env(Application, Key, Val) + [SetEnv(Application, Key, Val) || Config <- ConfigList, {Application, Items} <- Config, {Key, Val} <- Items] -- cgit v1.1 From 0afb4c4d477b1c1fba54ad3ef086d0697a5faeaa Mon Sep 17 00:00:00 2001 From: Alexander Sedov Date: Tue, 11 Oct 2016 10:36:12 +0300 Subject: Made reading sys.configs consistent with OTP specification. --- src/rebar_file_utils.erl | 13 ++++++++----- src/rebar_prv_common_test.erl | 6 +++--- src/rebar_prv_shell.erl | 4 ++-- 3 files changed, 13 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 437780d..28cc516 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -72,11 +72,14 @@ consult_config(State, Filename) -> [T] -> T; [] -> [] end, - SubConfigs = [consult_config(State, Entry ++ ".config") || - Entry <- Config, is_list(Entry) - ], - - [Config | lists:merge(SubConfigs)]. + lists:flatmap( + fun (SubConfig) when is_list(SubConfig) -> + case consult_config(State, SubConfig) of + [] -> consult_config(State, SubConfig ++ ".config"); + X -> X + end; + (Entry) -> [Entry] + end, Config). format_error({bad_term_file, AppFile, Reason}) -> io_lib:format("Error reading file ~s: ~s", [AppFile, file:format_error(Reason)]). diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index 8bfa192..e69d33e 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -218,9 +218,9 @@ select_tests(_, _, _, {error, _} = Error) -> Error; select_tests(State, ProjectApps, CmdOpts, CfgOpts) -> %% set application env if sys_config argument is provided SysConfigs = sys_config_list(CmdOpts, CfgOpts), - Configs = lists:flatmap(fun(Filename) -> - rebar_file_utils:consult_config(State, Filename) - end, SysConfigs), + Configs = lists:map(fun(Filename) -> + rebar_file_utils:consult_config(State, Filename) + end, SysConfigs), code:add_pathsa(rebar_state:code_paths(State, all_deps)), [application:load(Application) || Config <- Configs, {Application, _} <- Config], rebar_utils:reread_config(Configs), diff --git a/src/rebar_prv_shell.erl b/src/rebar_prv_shell.erl index b7febf8..e14ae3a 100644 --- a/src/rebar_prv_shell.erl +++ b/src/rebar_prv_shell.erl @@ -331,8 +331,8 @@ reread_config(State) -> case find_config(State) of no_config -> ok; - ConfigList -> - _ = rebar_utils:reread_config(ConfigList), + Config -> + _ = rebar_utils:reread_config([Config]), ok end. -- cgit v1.1 From 4ce21e4c800108030d5c646b226032f460e68065 Mon Sep 17 00:00:00 2001 From: Alexander Sedov Date: Tue, 11 Oct 2016 16:30:27 +0300 Subject: Avoid backward-compatibility-breaking changes. --- src/rebar_file_utils.erl | 18 ++++++++++-------- src/rebar_prv_common_test.erl | 6 +++--- src/rebar_prv_shell.erl | 4 ++-- 3 files changed, 15 insertions(+), 13 deletions(-) (limited to 'src') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 28cc516..6721b5a 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -72,14 +72,16 @@ consult_config(State, Filename) -> [T] -> T; [] -> [] end, - lists:flatmap( - fun (SubConfig) when is_list(SubConfig) -> - case consult_config(State, SubConfig) of - [] -> consult_config(State, SubConfig ++ ".config"); - X -> X - end; - (Entry) -> [Entry] - end, Config). + JoinedConfig = lists:flatmap( + fun (SubConfig) when is_list(SubConfig) -> + case lists:suffix(".config", SubConfig) of + false -> consult_config(State, SubConfig ++ ".config"); + true -> consult_config(State, SubConfig) + end; + (Entry) -> [Entry] + end, Config), + %% Backwards compatibility + [JoinedConfig]. format_error({bad_term_file, AppFile, Reason}) -> io_lib:format("Error reading file ~s: ~s", [AppFile, file:format_error(Reason)]). diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index e69d33e..8bfa192 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -218,9 +218,9 @@ select_tests(_, _, _, {error, _} = Error) -> Error; select_tests(State, ProjectApps, CmdOpts, CfgOpts) -> %% set application env if sys_config argument is provided SysConfigs = sys_config_list(CmdOpts, CfgOpts), - Configs = lists:map(fun(Filename) -> - rebar_file_utils:consult_config(State, Filename) - end, SysConfigs), + Configs = lists:flatmap(fun(Filename) -> + rebar_file_utils:consult_config(State, Filename) + end, SysConfigs), code:add_pathsa(rebar_state:code_paths(State, all_deps)), [application:load(Application) || Config <- Configs, {Application, _} <- Config], rebar_utils:reread_config(Configs), diff --git a/src/rebar_prv_shell.erl b/src/rebar_prv_shell.erl index e14ae3a..b7febf8 100644 --- a/src/rebar_prv_shell.erl +++ b/src/rebar_prv_shell.erl @@ -331,8 +331,8 @@ reread_config(State) -> case find_config(State) of no_config -> ok; - Config -> - _ = rebar_utils:reread_config([Config]), + ConfigList -> + _ = rebar_utils:reread_config(ConfigList), ok end. -- cgit v1.1 From e71b80752f905d9f4bb5dbf78c16693dbc6133f4 Mon Sep 17 00:00:00 2001 From: Alexander Sedov Date: Tue, 11 Oct 2016 16:34:21 +0300 Subject: Some post-review changes: - restore path after loading applications, - helpful comments. --- src/rebar_prv_common_test.erl | 4 ++++ src/rebar_utils.erl | 2 ++ 2 files changed, 6 insertions(+) (limited to 'src') diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index 8bfa192..46bd1a7 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -221,9 +221,13 @@ select_tests(State, ProjectApps, CmdOpts, CfgOpts) -> Configs = lists:flatmap(fun(Filename) -> rebar_file_utils:consult_config(State, Filename) end, SysConfigs), + %% NB: load the applications (from user directories too) to support OTP < 17 + %% to our best ability. + OldPath = code:get_path(), code:add_pathsa(rebar_state:code_paths(State, all_deps)), [application:load(Application) || Config <- Configs, {Application, _} <- Config], rebar_utils:reread_config(Configs), + code:set_path(OldPath), Merged = lists:ukeymerge(1, lists:ukeysort(1, CmdOpts), diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index dc0bb42..f55f40f 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -414,6 +414,8 @@ user_agent() -> ?FMT("Rebar/~s (OTP/~s)", [Vsn, otp_release()]). reread_config(ConfigList) -> + %% NB: we attempt to mimic -config here, which survives app reload, + %% hence {persistent, true}. SetEnv = case version_tuple(?MODULE:otp_release()) of {X, _, _} when X =< 17 -> fun application:set_env/3; -- cgit v1.1 From 294fd72f4d8f3beb2e856f085e634549d978b7fd Mon Sep 17 00:00:00 2001 From: Alexander Sedov Date: Thu, 13 Oct 2016 12:46:18 +0300 Subject: Get stacktrace only once and as soon as possible. Fixes #1347. It turns out that pretty-printing uses throw-catch and thus garbages out the stack trace sometimes, so we should get it only once. --- src/rebar3.erl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar3.erl b/src/rebar3.erl index c665f20..47dc25a 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -286,10 +286,11 @@ handle_error({error, Error}) when is_list(Error) -> handle_error(Error) -> %% Nothing should percolate up from rebar_core; %% Dump this error to console - ?CRASHDUMP("Error: ~p~n~p~n~n", [Error, erlang:get_stacktrace()]), + StackTrace = erlang:get_stacktrace(), + ?CRASHDUMP("Error: ~p~n~p~n~n", [Error, StackTrace]), ?ERROR("Uncaught error in rebar_core. Run with DEBUG=1 to see stacktrace or consult rebar3.crashdump", []), ?DEBUG("Uncaught error: ~p", [Error]), - case erlang:get_stacktrace() of + case StackTrace of [] -> ok; Trace -> ?DEBUG("Stack trace to the error location:~n~p", [Trace]) -- cgit v1.1 From f0d1a8404c9488135a120e4640c76ff3ff1eda66 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 14 Oct 2016 19:12:43 -0400 Subject: Bump version to 3.3.2 --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 5b735cf..bf520aa 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.3.2"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 94f87477d5db0243992ca222c79486acbfb9fcee Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 14 Oct 2016 19:13:38 -0400 Subject: Return to git-based vsn --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index bf520aa..5b735cf 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.3.2"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 504431473b9cc7f9c1641f640fc4d8a02c9aa079 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 17 Oct 2016 08:41:25 -0400 Subject: Prevent crashes in `rebar3 as` with no tasks checks on hd(...) and so on could not handle empty lists --- src/rebar_prv_as.erl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_as.erl b/src/rebar_prv_as.erl index b4f7ac4..e7c6d68 100644 --- a/src/rebar_prv_as.erl +++ b/src/rebar_prv_as.erl @@ -33,9 +33,11 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> {Profiles, Tasks} = args_to_profiles_and_tasks(rebar_state:command_args(State)), - case Profiles of - [] -> + case {Profiles, Tasks} of + {[], _} -> {error, "At least one profile must be specified when using `as`"}; + {_, []} -> + {error, "At least one task must be specified when using `as`"}; _ -> warn_on_empty_profile(Profiles, State), State1 = rebar_state:apply_profiles(State, [list_to_atom(X) || X <- Profiles]), -- cgit v1.1 From 203e5c15bd1935d66ca29fb071a8bc7ba8dad162 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Mon, 17 Oct 2016 17:10:41 -0700 Subject: allow test specifications to be passed via the command line `rebar3 ct --spec foo.spec,bar.spec,baz.spec` now works also added support for the `join_specs` flag on the command line --- src/rebar_prv_common_test.erl | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index 46bd1a7..189a26f 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -139,6 +139,8 @@ transform_opts([{testcase, Cases}|Rest], Acc) -> transform_opts(Rest, [{testcase, split_string(Cases)}|Acc]); transform_opts([{config, Configs}|Rest], Acc) -> transform_opts(Rest, [{config, split_string(Configs)}|Acc]); +transform_opts([{spec, Specs}|Rest], Acc) -> + transform_opts(Rest, [{spec, split_string(Specs)}|Acc]); transform_opts([{include, Includes}|Rest], Acc) -> transform_opts(Rest, [{include, split_string(Includes)}|Acc]); transform_opts([{logopts, LogOpts}|Rest], Acc) -> @@ -174,9 +176,6 @@ cfgopts(State) -> end. ensure_opts([], Acc) -> lists:reverse(Acc); -ensure_opts([{test_spec, _}|Rest], Acc) -> - ?WARN("Test specs not supported. See http://www.rebar3.org/docs/running-tests#common-test", []), - ensure_opts(Rest, Acc); ensure_opts([{cover, _}|Rest], Acc) -> ?WARN("Cover specs not supported. See http://www.rebar3.org/docs/running-tests#common-test", []), ensure_opts(Rest, Acc); @@ -650,6 +649,8 @@ ct_opts(_State) -> {testcase, undefined, "case", string, help(testcase)}, %% comma-seperated list {label, undefined, "label", string, help(label)}, %% String {config, undefined, "config", string, help(config)}, %% comma-seperated list + {spec, undefined, "spec", string, help(spec)}, %% common-seperated list + {join_specs, undefined, "join_specs", boolean, help(join_specs)}, {allow_user_terms, undefined, "allow_user_terms", boolean, help(allow_user_terms)}, %% Bool {logdir, undefined, "logdir", string, help(logdir)}, %% dir {logopts, undefined, "logopts", string, help(logopts)}, %% comma seperated list @@ -688,6 +689,10 @@ help(label) -> "Test label"; help(config) -> "List of config files"; +help(spec) -> + "List of test specifications"; +help(join_specs) -> + "Merge all test specifications and perform a single test run"; help(sys_config) -> "List of application config files"; help(allow_user_terms) -> -- cgit v1.1 From 14956eaf168dd20b0b1da1c05a82d66da77ec37c Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Mon, 17 Oct 2016 18:52:09 -0700 Subject: fix "helpful" compiler spelling correction --- src/rebar_prv_common_test.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index 189a26f..b91c90f 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -649,7 +649,7 @@ ct_opts(_State) -> {testcase, undefined, "case", string, help(testcase)}, %% comma-seperated list {label, undefined, "label", string, help(label)}, %% String {config, undefined, "config", string, help(config)}, %% comma-seperated list - {spec, undefined, "spec", string, help(spec)}, %% common-seperated list + {spec, undefined, "spec", string, help(spec)}, %% comma-seperated list {join_specs, undefined, "join_specs", boolean, help(join_specs)}, {allow_user_terms, undefined, "allow_user_terms", boolean, help(allow_user_terms)}, %% Bool {logdir, undefined, "logdir", string, help(logdir)}, %% dir -- cgit v1.1 From bcfd8d6f80f551e1b2a0e23eaa6b5ff2c7da5b15 Mon Sep 17 00:00:00 2001 From: James Fish Date: Thu, 3 Nov 2016 15:45:41 +0000 Subject: Add exclude_apps/mods, plt_extra_mods, base_plt_mods config * exclude_apps - never use applications for PLT/analysis * base_plt_mods - add modules to base PLT (overrules exclude_apps) * plt_extra_mods - add modules to PLT (overrules exclude_apps) * exclude_mods - never use modules for PLT/analysis (overrules all) --- src/rebar_prv_dialyzer.erl | 162 +++++++++++++++++++++++++++++---------------- 1 file changed, 105 insertions(+), 57 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_dialyzer.erl b/src/rebar_prv_dialyzer.erl index fc13de1..44dc0d2 100644 --- a/src/rebar_prv_dialyzer.erl +++ b/src/rebar_prv_dialyzer.erl @@ -47,26 +47,33 @@ desc() -> "`plt_apps` - the strategy for determining the applications which included " "in the PLT file, `top_level_deps` to include just the direct dependencies " "or `all_deps` to include all nested dependencies*\n" - "`plt_extra_apps` - a list of applications to include in the PLT file**\n" + "`plt_extra_apps` - a list of extra applications to include in the PLT " + "file\n" + "`plt_extra_mods` - a list of extra modules to includes in the PLT file\n" "`plt_location` - the location of the PLT file, `local` to store in the " "profile's base directory (default) or a custom directory.\n" - "`plt_prefix` - the prefix to the PLT file, defaults to \"rebar3\"***\n" + "`plt_prefix` - the prefix to the PLT file, defaults to \"rebar3\"**\n" "`base_plt_apps` - a list of applications to include in the base " - "PLT file****\n" + "PLT file***\n" + "`base_plt_mods` - a list of modules to include in the base " + "PLT file***\n" "`base_plt_location` - the location of base PLT file, `global` to store in " - "$HOME/.cache/rebar3 (default) or a custom directory****\n" + "$HOME/.cache/rebar3 (default) or a custom directory***\n" "`base_plt_prefix` - the prefix to the base PLT file, defaults to " - "\"rebar3\"*** ****\n" + "\"rebar3\"** ***\n" + "`exclude_apps` - a list of applications to exclude from PLT files and " + "success typing analysis, `plt_extra_mods` and `base_plt_mods` can add " + "modules from excluded applications\n" + "`exclude_mods` - a list of modules to exclude from PLT files and " + "success typing analysis\n" "\n" "For example, to warn on unmatched returns: \n" "{dialyzer, [{warnings, [unmatched_returns]}]}.\n" "\n" "*The direct dependent applications are listed in `applications` and " "`included_applications` of their .app files.\n" - "**The applications in `base_plt_apps` will be added to the " - "list. \n" - "***PLT files are named \"__plt\".\n" - "****The base PLT is a PLT containing the core applications often required " + "**PLT files are named \"__plt\".\n" + "***The base PLT is a PLT containing the core applications often required " "for a project's PLT. One base PLT is created per OTP version and " "stored in `base_plt_location`. A base PLT is used to build project PLTs." "\n". @@ -90,6 +97,10 @@ do(State) -> ?PRV_ERROR({dialyzer_warnings, Warnings}); throw:{unknown_application, _} = Error -> ?PRV_ERROR(Error); + throw:{unknown_module, _} = Error -> + ?PRV_ERROR(Error); + throw:{duplicate_module, _, _, _} = Error -> + ?PRV_ERROR(Error); throw:{output_file_error, _, _} = Error -> ?PRV_ERROR(Error) after @@ -110,6 +121,10 @@ format_error({dialyzer_warnings, Warnings}) -> io_lib:format("Warnings occured running dialyzer: ~b", [Warnings]); format_error({unknown_application, App}) -> io_lib:format("Could not find application: ~s", [App]); +format_error({unknown_module, Mod}) -> + io_lib:format("Could not find module: ~s", [Mod]); +format_error({duplicate_module, Mod, File1, File2}) -> + io_lib:format("Duplicates of module ~s: ~s ~s", [Mod, File1, File2]); format_error({output_file_error, File, Error}) -> Error1 = file:format_error(Error), io_lib:format("Failed to write to ~s: ~s", [File, Error1]); @@ -178,45 +193,45 @@ do_update_proj_plt(State, Plt, Output) -> end. proj_plt_files(State) -> - BasePltApps = get_config(State, base_plt_apps, default_plt_apps()), - PltApps = get_config(State, plt_extra_apps, []), + BasePltApps = base_plt_apps(State), + PltApps = get_config(State, plt_extra_apps, []) ++ BasePltApps, + BasePltMods = get_config(State, base_plt_mods, []), + PltMods = get_config(State, plt_extra_mods, []) ++ BasePltMods, + Apps = proj_apps(State), + DepApps = proj_deps(State), + get_files(State, DepApps ++ PltApps, Apps -- PltApps, PltMods, []). + +proj_apps(State) -> + [ec_cnv:to_atom(rebar_app_info:name(App)) || + App <- rebar_state:project_apps(State)]. + +proj_deps(State) -> Apps = rebar_state:project_apps(State), DepApps = lists:flatmap(fun rebar_app_info:applications/1, Apps), - DepApps1 = - case get_config(State, plt_apps, top_level_deps) of - top_level_deps -> DepApps; - all_deps -> collect_nested_dependent_apps(DepApps) - end, - get_plt_files(BasePltApps ++ PltApps ++ DepApps1, Apps). - -default_plt_apps() -> - [erts, - crypto, - kernel, - stdlib]. - -get_plt_files(DepApps, Apps) -> + case get_config(State, plt_apps, top_level_deps) of + top_level_deps -> DepApps; + all_deps -> collect_nested_dependent_apps(DepApps) + end. + +get_files(State, Apps, SkipApps, Mods, SkipMods) -> ?INFO("Resolving files...", []), - get_plt_files(DepApps, Apps, [], []). + ExcludeApps = get_config(State, exclude_apps, []), + Files = apps_files(Apps, ExcludeApps ++ SkipApps, dict:new()), + ExcludeMods = get_config(State, exclude_mods, []), + Files2 = mods_files(Mods, ExcludeMods ++ SkipMods, Files), + dict:fold(fun(_, File, Acc) -> [File | Acc] end, [], Files2). -get_plt_files([], _, _, Files) -> +apps_files([], _, Files) -> Files; -get_plt_files([AppName | DepApps], Apps, PltApps, Files) -> - case lists:member(AppName, PltApps) orelse app_member(AppName, Apps) of +apps_files([AppName | DepApps], SkipApps, Files) -> + case lists:member(AppName, SkipApps) of true -> - get_plt_files(DepApps, Apps, PltApps, Files); + apps_files(DepApps, SkipApps, Files); false -> - Files2 = app_files(AppName), - ?DEBUG("~s files: ~p", [AppName, Files2]), - get_plt_files(DepApps, Apps, [AppName | PltApps], Files2 ++ Files) - end. - -app_member(AppName, Apps) -> - case rebar_app_utils:find(ec_cnv:to_binary(AppName), Apps) of - {ok, _App} -> - true; - error -> - false + AppFiles = app_files(AppName), + ?DEBUG("~s modules: ~p", [AppName, dict:fetch_keys(AppFiles)]), + Files2 = merge_files(Files, AppFiles), + apps_files(DepApps, [AppName | SkipApps], Files2) end. app_files(AppName) -> @@ -244,9 +259,41 @@ check_ebin(EbinDir) -> end. ebin_files(EbinDir) -> - Wildcard = "*" ++ code:objfile_extension(), - [filename:join(EbinDir, File) || - File <- filelib:wildcard(Wildcard, EbinDir)]. + Ext = code:objfile_extension(), + Wildcard = "*" ++ Ext, + Files = filelib:wildcard(Wildcard, EbinDir), + Store = fun(File, Mods) -> + Mod = list_to_atom(filename:basename(File, Ext)), + Absname = filename:join(EbinDir, File), + dict:store(Mod, Absname, Mods) + end, + lists:foldl(Store, dict:new(), Files). + +merge_files(Files1, Files2) -> + Duplicate = fun(Mod, File1, File2) -> + throw({duplicate_module, Mod, File1, File2}) + end, + dict:merge(Duplicate, Files1, Files2). + +mods_files(Mods, SkipMods, Files) -> + Keep = fun(File) -> File end, + Ensure = fun(Mod, Acc) -> + case lists:member(Mod, SkipMods) of + true -> + Acc; + false -> + dict:update(Mod, Keep, mod_file(Mod), Acc) + end + end, + Files2 = lists:foldl(Ensure, Files, Mods), + lists:foldl(fun dict:erase/2, Files2, SkipMods). + +mod_file(Mod) -> + File = atom_to_list(Mod) ++ code:objfile_extension(), + case code:where_is_file(File) of + non_existing -> throw({unknown_module, Mod}); + Absname -> Absname + end. read_plt(_State, Plt) -> Vsn = dialyzer_version(), @@ -355,9 +402,12 @@ get_base_plt(State) -> end. base_plt_files(State) -> - BasePltApps = get_config(State, base_plt_apps, default_plt_apps()), - Apps = rebar_state:project_apps(State), - get_plt_files(BasePltApps, Apps). + BasePltApps = base_plt_apps(State), + BasePltMods = get_config(State, base_plt_mods, []), + get_files(State, BasePltApps, [], BasePltMods, []). + +base_plt_apps(State) -> + get_config(State, base_plt_apps, [erts, crypto, kernel, stdlib]). update_base_plt(State, BasePlt, Output, BaseFiles) -> case read_plt(State, BasePlt) of @@ -394,9 +444,8 @@ succ_typings(State, Plt, Output) -> false -> {0, State}; _ -> - Apps = rebar_state:project_apps(State), ?INFO("Doing success typing analysis...", []), - Files = apps_to_files(Apps), + Files = proj_files(State), succ_typings(State, Plt, Output, Files) end. @@ -412,14 +461,13 @@ succ_typings(State, Plt, Output, Files) -> {init_plt, Plt}], run_dialyzer(State, Opts, Output). -apps_to_files(Apps) -> - ?INFO("Resolving files...", []), - [File || App <- Apps, - File <- app_to_files(App)]. - -app_to_files(App) -> - AppName = ec_cnv:to_atom(rebar_app_info:name(App)), - app_files(AppName). +proj_files(State) -> + Apps = proj_apps(State), + BasePltApps = get_config(State, base_plt_apps, []), + PltApps = get_config(State, plt_extra_apps, []) ++ BasePltApps, + BasePltMods = get_config(State, base_plt_mods, []), + PltMods = get_config(State, plt_extra_mods, []) ++ BasePltMods, + get_files(State, Apps, PltApps, [], PltMods). run_dialyzer(State, Opts, Output) -> %% dialyzer may return callgraph warnings when get_warnings is false -- cgit v1.1 From 95844531946ab7eabe3bd40df7133b200b292646 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Thu, 3 Nov 2016 17:02:38 +0100 Subject: Fix usage decription of 'rebar3' --- src/rebar_prv_help.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_help.erl b/src/rebar_prv_help.erl index 75cc609..f34c755 100644 --- a/src/rebar_prv_help.erl +++ b/src/rebar_prv_help.erl @@ -57,7 +57,7 @@ format_error(Reason) -> help(State) -> ?CONSOLE("Rebar3 is a tool for working with Erlang projects.~n~n", []), OptSpecList = rebar3:global_option_spec_list(), - getopt:usage(OptSpecList, "rebar", "", []), + getopt:usage(OptSpecList, "rebar3", "", []), ?CONSOLE("~nSeveral tasks are available:~n", []), providers:help(rebar_state:providers(State)), -- cgit v1.1 From e08145b498a59c1c0c100323646471ea1aa37c41 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 17 Nov 2016 08:57:16 -0500 Subject: Allow rebar3 to edoc itself --- src/rebar_erlc_compiler.erl | 12 +++--------- src/rebar_file_utils.erl | 4 ++-- 2 files changed, 5 insertions(+), 11 deletions(-) (limited to 'src') diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index 36a247e..3711fe9 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -88,14 +88,12 @@ %% 'old_inets'}]}. %% -%% @equiv compile(AppInfo, []). - +%% @equiv compile(AppInfo, []) -spec compile(rebar_app_info:t()) -> ok. compile(AppInfo) when element(1, AppInfo) == app_info_t -> compile(AppInfo, []). %% @doc compile an individual application. - -spec compile(rebar_app_info:t(), compile_opts()) -> ok. compile(AppInfo, CompileOpts) when element(1, AppInfo) == app_info_t -> Dir = ec_cnv:to_list(rebar_app_info:out_dir(AppInfo)), @@ -162,16 +160,14 @@ compile(RebarOpts, BaseDir, OutDir, CompileOpts) -> end, lists:foreach(F, lists:map(fun(SrcDir) -> filename:join(BaseDir, SrcDir) end, ExtraDirs)). -%% @equiv compile_dirs(Context, BaseDir, [Dir], Dir, [{recursive, false}]). - +%% @equiv compile_dirs(Context, BaseDir, [Dir], Dir, [{recursive, false}]) -spec compile_dir(rebar_dict() | rebar_state:t(), file:name(), file:name()) -> ok. compile_dir(State, BaseDir, Dir) when element(1, State) == state_t -> compile_dir(rebar_state:opts(State), BaseDir, Dir, [{recursive, false}]); compile_dir(RebarOpts, BaseDir, Dir) -> compile_dir(RebarOpts, BaseDir, Dir, [{recursive, false}]). -%% @equiv compile_dirs(Context, BaseDir, [Dir], Dir, Opts). - +%% @equiv compile_dirs(Context, BaseDir, [Dir], Dir, Opts) -spec compile_dir(rebar_dict() | rebar_state:t(), file:name(), file:name(), compile_opts()) -> ok. compile_dir(State, BaseDir, Dir, Opts) when element(1, State) == state_t -> compile_dirs(rebar_state:opts(State), BaseDir, [Dir], Dir, Opts); @@ -179,7 +175,6 @@ compile_dir(RebarOpts, BaseDir, Dir, Opts) -> compile_dirs(RebarOpts, BaseDir, [Dir], Dir, Opts). %% @doc compile a list of directories with the given opts. - -spec compile_dirs(rebar_dict() | rebar_state:t(), file:filename(), [file:filename()], @@ -233,7 +228,6 @@ compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, Opts) -> ok. %% @doc remove compiled artifacts from an AppDir. - -spec clean(rebar_app_info:t()) -> 'ok'. clean(AppInfo) -> AppDir = rebar_app_info:out_dir(AppInfo), diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 6721b5a..f2467c5 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -237,10 +237,10 @@ write_file_if_contents_differ(Filename, Bytes) -> %% returns an os appropriate tmpdir given a path -spec system_tmpdir() -> file:filename(). +system_tmpdir() -> system_tmpdir([]). + -spec system_tmpdir(PathComponents) -> file:filename() when PathComponents :: [file:name()]. - -system_tmpdir() -> system_tmpdir([]). system_tmpdir(PathComponents) -> Tmp = case erlang:system_info(system_architecture) of "win32" -> -- cgit v1.1 From e85cf555e24b2a4b9b8681b9028c87826a2c0ea6 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sat, 19 Nov 2016 16:15:43 -0500 Subject: Fix rebar3 dialyzer warnings Some tricky changes in there but should be okay --- src/rebar_app_info.erl | 6 +++--- src/rebar_config.erl | 4 +--- src/rebar_erlc_compiler.erl | 6 ++++-- src/rebar_prv_common_test.erl | 15 ++++----------- src/rebar_prv_cover.erl | 11 +++-------- src/rebar_prv_deps.erl | 2 +- src/rebar_prv_eunit.erl | 14 ++++---------- src/rebar_prv_shell.erl | 2 +- src/rebar_prv_update.erl | 2 +- src/rebar_prv_xref.erl | 2 +- src/rebar_state.erl | 31 +++++++++++++++++++------------ 11 files changed, 42 insertions(+), 53 deletions(-) (limited to 'src') diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index cf3b82e..4f19d77 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -68,7 +68,7 @@ -export_type([t/0]). --record(app_info_t, {name :: binary(), +-record(app_info_t, {name :: binary() | undefined, app_file_src :: file:filename_all() | undefined, app_file_src_script:: file:filename_all() | undefined, app_file :: file:filename_all() | undefined, @@ -83,11 +83,11 @@ dep_level=0 :: integer(), dir :: file:name(), out_dir :: file:name(), - resource_type :: pkg | src, + resource_type :: pkg | src | undefined, source :: string() | tuple() | checkout | undefined, is_lock=false :: boolean(), is_checkout=false :: boolean(), - valid :: boolean()}). + valid :: boolean() | undefined}). %%============================================================================ %% types diff --git a/src/rebar_config.erl b/src/rebar_config.erl index b50c030..72bc6e9 100644 --- a/src/rebar_config.erl +++ b/src/rebar_config.erl @@ -106,9 +106,7 @@ write_lock_file(LockFile, Locks) -> format_attrs([]) -> []; format_attrs([{pkg_hash, Vals}|T]) -> [io_lib:format("{pkg_hash,[~n",[]), format_hashes(Vals), "]}", - maybe_comma(T) | format_attrs(T)]; -format_attrs([H|T]) -> - [io_lib:format("~p~s", [H, maybe_comma(T)]) | format_attrs(T)]. + maybe_comma(T) | format_attrs(T)]. format_hashes([]) -> []; format_hashes([{Pkg,Hash}|T]) -> diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index 36a247e..d580792 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -573,9 +573,11 @@ compile_mib(AppInfo) -> MibToHrlOpts = case proplists:get_value(verbosity, AllOpts, undefined) of undefined -> - #options{specific = []}; + #options{specific = [], + cwd = rebar_dir:get_cwd()}; Verbosity -> - #options{specific = [{verbosity, Verbosity}]} + #options{specific = [{verbosity, Verbosity}], + cwd = rebar_dir:get_cwd()} end, ok = snmpc:mib_to_hrl(Mib, Mib, MibToHrlOpts), rebar_file_utils:mv(HrlFilename, AppInclude), diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index b91c90f..be31e8c 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -212,7 +212,6 @@ add_hooks(Opts, State) -> lists:keyreplace(ct_hooks, 1, Opts, {ct_hooks, NewHooks}) end. -select_tests(_, _, {error, _} = Error, _) -> Error; select_tests(_, _, _, {error, _} = Error) -> Error; select_tests(State, ProjectApps, CmdOpts, CfgOpts) -> %% set application env if sys_config argument is provided @@ -293,14 +292,9 @@ compile(State, {ok, _} = Tests) -> compile(_State, Error) -> Error. do_compile(State) -> - case rebar_prv_compile:do(State) of - %% successfully compiled apps - {ok, S} -> - ok = maybe_cover_compile(S), - {ok, S}; - %% this should look like a compiler error, not an eunit error - Error -> Error - end. + {ok, S} = rebar_prv_compile:do(State), + ok = maybe_cover_compile(S), + {ok, S}. inject_ct_state(State, {ok, Tests}) -> Apps = rebar_state:project_apps(State), @@ -308,8 +302,7 @@ inject_ct_state(State, {ok, Tests}) -> {ok, {NewState, ModdedApps}} -> test_dirs(NewState, ModdedApps, Tests); {error, _} = Error -> Error - end; -inject_ct_state(_State, Error) -> Error. + end. inject_ct_state(State, Tests, [App|Rest], Acc) -> case inject(rebar_app_info:opts(App), State, Tests) of diff --git a/src/rebar_prv_cover.erl b/src/rebar_prv_cover.erl index b62a796..e53a687 100644 --- a/src/rebar_prv_cover.erl +++ b/src/rebar_prv_cover.erl @@ -88,14 +88,9 @@ analyze(State) -> %% in order for cover data to be reloaded %% this maybe breaks if modules have been deleted %% since code coverage was collected? - case rebar_prv_compile:do(State) of - %% successfully compiled apps - {ok, S} -> - ok = cover_compile(S, apps), - do_analyze(State); - %% this should look like a compiler error, not a cover error - Error -> Error - end. + {ok, S} = rebar_prv_compile:do(State), + ok = cover_compile(S, apps), + do_analyze(State). do_analyze(State) -> ?INFO("Performing cover analysis...", []), diff --git a/src/rebar_prv_deps.erl b/src/rebar_prv_deps.erl index c865276..9ff2bfa 100644 --- a/src/rebar_prv_deps.erl +++ b/src/rebar_prv_deps.erl @@ -97,7 +97,7 @@ display_dep(_State, {Name, _Vsn, Source}) when is_tuple(Source) -> display_dep(_State, {Name, _Vsn, Source, _Opts}) when is_tuple(Source) -> ?CONSOLE("~s* (~s source)", [ec_cnv:to_binary(Name), type(Source)]); %% Locked -display_dep(State, {Name, Source={pkg, _, Vsn, _}, Level}) when is_integer(Level) -> +display_dep(State, {Name, Source={pkg, _, Vsn}, Level}) when is_integer(Level) -> DepsDir = rebar_dir:deps_dir(State), AppDir = filename:join([DepsDir, ec_cnv:to_binary(Name)]), NeedsUpdate = case rebar_fetch:needs_update(AppDir, Source, State) of diff --git a/src/rebar_prv_eunit.erl b/src/rebar_prv_eunit.erl index 82e2458..e85ab4c 100644 --- a/src/rebar_prv_eunit.erl +++ b/src/rebar_prv_eunit.erl @@ -153,7 +153,6 @@ cfg_tests(State) -> ?PRV_ERROR({badconfig, {"Value `~p' of option `~p' must be a list", {Wrong, eunit_tests}}}) end. -select_tests(_State, _ProjectApps, {error, _} = Error, _) -> Error; select_tests(_State, _ProjectApps, _, {error, _} = Error) -> Error; select_tests(State, ProjectApps, [], []) -> {ok, default_tests(State, ProjectApps)}; select_tests(_State, _ProjectApps, [], Tests) -> {ok, Tests}; @@ -316,14 +315,9 @@ inject_test_dir(Opts, Dir) -> compile({error, _} = Error) -> Error; compile(State) -> - case rebar_prv_compile:do(State) of - %% successfully compiled apps - {ok, S} -> - ok = maybe_cover_compile(S), - {ok, S}; - %% this should look like a compiler error, not an eunit error - Error -> Error - end. + {ok, S} = rebar_prv_compile:do(State), + ok = maybe_cover_compile(S), + {ok, S}. validate_tests(State, {ok, Tests}) -> gather_tests(fun(Elem) -> validate(State, Elem) end, Tests, []); @@ -453,7 +447,7 @@ translate(State, [], {dir, Dir}) -> translate(State, [], {file, FilePath}) -> Dir = filename:dirname(FilePath), File = filename:basename(FilePath), - case rebar_file_utils:path_from_ancestor(Dir, rebar_app_info:dir(State)) of + case rebar_file_utils:path_from_ancestor(Dir, rebar_state:dir(State)) of {ok, Path} -> {file, filename:join([rebar_dir:base_dir(State), "extras", Path, File])}; %% not relative, leave as is {error, badparent} -> {file, FilePath} diff --git a/src/rebar_prv_shell.erl b/src/rebar_prv_shell.erl index b7febf8..31b2e17 100644 --- a/src/rebar_prv_shell.erl +++ b/src/rebar_prv_shell.erl @@ -120,7 +120,7 @@ info() -> setup_shell() -> OldUser = kill_old_user(), %% Test for support here - NewUser = try erlang:open_port({spawn,'tty_sl -c -e'}, []) of + NewUser = try erlang:open_port({spawn,"tty_sl -c -e"}, []) of Port when is_port(Port) -> true = port_close(Port), setup_new_shell() diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index 75c609e..54f1796 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -236,7 +236,7 @@ cmp_(BestMatch, MinVsn, [Vsn | R], DepsListAcc, Dep, CmpFun) -> %% We need to treat this differently since we want a version that is LOWER but %% the higest possible one. -cmpl({_Pkg, _PkgVsn, Dep} = Dep1, Vsn, HexRegistry, State, DepsListAcc, CmpFun) -> +cmpl({_Pkg, _PkgVsn, Dep, _App} = Dep1, Vsn, HexRegistry, State, DepsListAcc, CmpFun) -> {ok, Vsns} = rebar_packages:find_all(Dep, HexRegistry, State), cmpl_(undefined, Vsn, Vsns, DepsListAcc, Dep1, CmpFun). diff --git a/src/rebar_prv_xref.erl b/src/rebar_prv_xref.erl index 3d74c9a..c4e72e7 100644 --- a/src/rebar_prv_xref.erl +++ b/src/rebar_prv_xref.erl @@ -273,7 +273,7 @@ find_function_source(M, F, A, Bin) -> AbstractCode = beam_lib:chunks(Bin, [abstract_code]), {ok, {M, [{abstract_code, {raw_abstract_v1, Code}}]}} = AbstractCode, %% Extract the original source filename from the abstract code - [{attribute, 1, file, {Source, _}} | _] = Code, + [{attribute, _, file, {Source, _}} | _] = Code, %% Extract the line number for a given function def Fn = [E || E <- Code, safe_element(1, E) == function, diff --git a/src/rebar_state.erl b/src/rebar_state.erl index bdd4aeb..9683709 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -59,7 +59,7 @@ command_args = [], command_parsed_args = {[], []}, - current_app :: rebar_app_info:t(), + current_app :: undefined | rebar_app_info:t(), project_apps = [] :: [rebar_app_info:t()], deps_to_build = [] :: [rebar_app_info:t()], all_plugin_deps = [] :: [rebar_app_info:t()], @@ -424,17 +424,24 @@ create_logic_providers(ProviderModules, State0) -> to_list(#state_t{} = State) -> Fields = record_info(fields, state_t), Values = tl(tuple_to_list(State)), - DictSz = tuple_size(dict:new()), - lists:zip(Fields, [reformat(I, DictSz) || I <- Values]). - -reformat({K,V}, DSz) when is_list(V) -> - {K, [reformat(I, DSz) || I <- V]}; -reformat(V, DSz) when is_tuple(V), element(1,V) =:= dict, tuple_size(V) =:= DSz -> - [reformat(I, DSz) || I <- dict:to_list(V)]; -reformat({K,V}, DSz) when is_tuple(V), element(1,V) =:= dict, tuple_size(V) =:= DSz -> - {K, [reformat(I, DSz) || I <- dict:to_list(V)]}; -reformat(Other, _DSz) -> - Other. + lists:zip(Fields, [reformat(I) || I <- Values]). + +reformat({K,V}) when is_list(V) -> + {K, [reformat(I) || I <- V]}; +reformat({K,V}) -> + try + {K, [reformat(I) || I <- dict:to_list(V)]} + catch + error:{badrecord,dict} -> + {K,V} + end; +reformat(V) -> + try + [reformat(I) || I <- dict:to_list(V)] + catch + error:{badrecord,dict} -> + V + end. %% =================================================================== %% Internal functions -- cgit v1.1 From 44fabbbf9173bc97365ccb3c7d28d1bc8503c28d Mon Sep 17 00:00:00 2001 From: Siri Hansen Date: Fri, 18 Nov 2016 17:42:39 +0100 Subject: Add 'recursive' option The option {recursive,boolean()} can now be set pr directory in 'src_dirs' and 'extra_src_dirs', and on top level in the new 'erlc_compiler' option. Example config: {erlc_compiler,[{recursive,false}]}. {src_dirs,[{"src",[{recursive,true}]}]}. This will cause recursive compilation within the "src" directory, but not in any other directoires. --- src/rebar_dir.erl | 60 ++++++++++++++++++++++++++++++++++++++------- src/rebar_erlc_compiler.erl | 53 ++++++++++++++++++++++----------------- src/rebar_prv_eunit.erl | 2 +- 3 files changed, 82 insertions(+), 33 deletions(-) (limited to 'src') diff --git a/src/rebar_dir.erl b/src/rebar_dir.erl index 1ec58d4..79a1c7f 100644 --- a/src/rebar_dir.erl +++ b/src/rebar_dir.erl @@ -22,6 +22,7 @@ processing_base_dir/2, make_relative_path/2, src_dirs/1, src_dirs/2, + src_dir_opts/2, recursive/2, extra_src_dirs/1, extra_src_dirs/2, all_src_dirs/1, all_src_dirs/3, retarget_path/2]). @@ -160,28 +161,43 @@ do_make_relative_path(Source, Target) -> Base = lists:duplicate(max(length(Target) - 1, 0), ".."), filename:join(Base ++ Source). +%%%----------------------------------------------------------------- +%%% 'src_dirs' and 'extra_src_dirs' can be configured with options +%%% like this: +%%% +%%% {src_dirs,[{"foo",[{recursive,false}]}]} +%%% {extra_src_dirs,[{"bar",[recursive]}]} (equivalent to {recursive,true}) +%%% +%%% src_dirs/1,2 and extra_src_dirs/1,2 return only the list of +%%% directories for the 'src_dirs' and 'extra_src_dirs' options +%%% respectively, while src_dirs_opts/2 return the options list for +%%% the given directory, no matter if it is configured as 'src_dirs' or +%%% 'extra_src_dirs'. +%%% -spec src_dirs(rebar_dict()) -> list(file:filename_all()). src_dirs(Opts) -> src_dirs(Opts, []). -spec src_dirs(rebar_dict(), list(file:filename_all())) -> list(file:filename_all()). src_dirs(Opts, Default) -> - ErlOpts = rebar_opts:erl_opts(Opts), - Vs = proplists:get_all_values(src_dirs, ErlOpts), - case lists:append([rebar_opts:get(Opts, src_dirs, []) | Vs]) of - [] -> Default; - Dirs -> lists:usort(Dirs) - end. + src_dirs(src_dirs, Opts, Default). -spec extra_src_dirs(rebar_dict()) -> list(file:filename_all()). extra_src_dirs(Opts) -> extra_src_dirs(Opts, []). -spec extra_src_dirs(rebar_dict(), list(file:filename_all())) -> list(file:filename_all()). extra_src_dirs(Opts, Default) -> + src_dirs(extra_src_dirs, Opts, Default). + +src_dirs(Type, Opts, Default) -> + lists:usort([case D0 of {D,_} -> D; _ -> D0 end || + D0 <- raw_src_dirs(Type,Opts,Default)]). + +raw_src_dirs(Type, Opts, Default) -> ErlOpts = rebar_opts:erl_opts(Opts), - Vs = proplists:get_all_values(extra_src_dirs, ErlOpts), - case lists:append([rebar_opts:get(Opts, extra_src_dirs, []) | Vs]) of + Vs = proplists:get_all_values(Type, ErlOpts), + case lists:append([rebar_opts:get(Opts, Type, []) | Vs]) of [] -> Default; - Dirs -> lists:usort(Dirs) + Dirs -> Dirs end. -spec all_src_dirs(rebar_dict()) -> list(file:filename_all()). @@ -192,6 +208,32 @@ all_src_dirs(Opts) -> all_src_dirs(Opts, [], []). all_src_dirs(Opts, SrcDefault, ExtraDefault) -> lists:usort(src_dirs(Opts, SrcDefault) ++ extra_src_dirs(Opts, ExtraDefault)). +%%%----------------------------------------------------------------- +%%% Return the list of options for the given src directory +%%% If the same option is given multiple times for a directory in the +%%% config, the priority order is: first occurence of 'src_dirs' +%%% followed by first occurence of 'extra_src_dirs'. +-spec src_dir_opts(rebar_dict(), file:filename_all()) -> [{atom(),term()}]. +src_dir_opts(Opts, Dir) -> + RawSrcDirs = raw_src_dirs(src_dirs, Opts, []), + RawExtraSrcDirs = raw_src_dirs(extra_src_dirs, Opts, []), + AllOpts = [Opt || {D,Opt} <- RawSrcDirs++RawExtraSrcDirs, + D==Dir], + lists:ukeysort(1,proplists:unfold(lists:append(AllOpts))). + +%%%----------------------------------------------------------------- +%%% Return the value of the 'recursive' option for the given directory. +%%% If not given, the value of 'recursive' in the 'erlc_compiler' +%%% options is used, and finally the default is 'true'. +-spec recursive(rebar_dict(), file:filename_all()) -> boolean(). +recursive(Opts, Dir) -> + DirOpts = src_dir_opts(Opts, Dir), + Default = proplists:get_value(recursive, + rebar_opts:get(Opts, erlc_compiler, []), + true), + R = proplists:get_value(recursive, DirOpts, Default), + R. + %% given a path if that path is an ancestor of an app dir return the path relative to that %% apps outdir. if the path is not an ancestor to any app dirs but is an ancestor of the %% project root return the path relative to the project base_dir. if it is not an ancestor diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index 97235aa..325bb4f 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -47,10 +47,6 @@ -type compile_opts() :: [compile_opt()]. -type compile_opt() :: {recursive, boolean()}. --record(compile_opts, { - recursive = true -}). - -define(DEFAULT_OUTDIR, "ebin"). -define(RE_PREFIX, "^[^._]"). @@ -99,21 +95,25 @@ compile(AppInfo, CompileOpts) when element(1, AppInfo) == app_info_t -> Dir = ec_cnv:to_list(rebar_app_info:out_dir(AppInfo)), RebarOpts = rebar_app_info:opts(AppInfo), + SrcOpts = [check_last_mod, + {recursive, dir_recursive(RebarOpts, "src", CompileOpts)}], + MibsOpts = [check_last_mod, + {recursive, dir_recursive(RebarOpts, "mibs", CompileOpts)}], rebar_base_compiler:run(RebarOpts, check_files([filename:join(Dir, File) || File <- rebar_opts:get(RebarOpts, xrl_first_files, [])]), filename:join(Dir, "src"), ".xrl", filename:join(Dir, "src"), ".erl", - fun compile_xrl/3), + fun compile_xrl/3, SrcOpts), rebar_base_compiler:run(RebarOpts, check_files([filename:join(Dir, File) || File <- rebar_opts:get(RebarOpts, yrl_first_files, [])]), filename:join(Dir, "src"), ".yrl", filename:join(Dir, "src"), ".erl", - fun compile_yrl/3), + fun compile_yrl/3, SrcOpts), rebar_base_compiler:run(RebarOpts, check_files([filename:join(Dir, File) || File <- rebar_opts:get(RebarOpts, mib_first_files, [])]), filename:join(Dir, "mibs"), ".mib", filename:join([Dir, "priv", "mibs"]), ".bin", - compile_mib(AppInfo)), + compile_mib(AppInfo), MibsOpts), SrcDirs = lists:map(fun(SrcDir) -> filename:join(Dir, SrcDir) end, rebar_dir:src_dirs(RebarOpts, ["src"])), @@ -182,13 +182,10 @@ compile_dir(RebarOpts, BaseDir, Dir, Opts) -> compile_opts()) -> ok. compile_dirs(State, BaseDir, Dirs, OutDir, CompileOpts) when element(1, State) == state_t -> compile_dirs(rebar_state:opts(State), BaseDir, Dirs, OutDir, CompileOpts); -compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, Opts) -> - CompileOpts = parse_opts(Opts), - +compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, CompileOpts) -> ErlOpts = rebar_opts:erl_opts(RebarOpts), ?DEBUG("erlopts ~p", [ErlOpts]), - Recursive = CompileOpts#compile_opts.recursive, - AllErlFiles = gather_src(SrcDirs, Recursive), + AllErlFiles = gather_src(RebarOpts, BaseDir, SrcDirs, CompileOpts), ?DEBUG("files to compile ~p", [AllErlFiles]), %% Make sure that outdir is on the path @@ -266,12 +263,22 @@ clean_dirs(AppDir, [Dir|Rest]) -> %% Internal functions %% =================================================================== -gather_src(Dirs, Recursive) -> - gather_src(Dirs, [], Recursive). - -gather_src([], Srcs, _Recursive) -> Srcs; -gather_src([Dir|Rest], Srcs, Recursive) -> - gather_src(Rest, Srcs ++ rebar_utils:find_files(Dir, ?RE_PREFIX".*\\.erl\$", Recursive), Recursive). +gather_src(Opts, BaseDir, Dirs, CompileOpts) -> + gather_src(Opts, filename:split(BaseDir), Dirs, [], CompileOpts). + +gather_src(_Opts, _BaseDirParts, [], Srcs, _CompileOpts) -> Srcs; +gather_src(Opts, BaseDirParts, [Dir|Rest], Srcs, CompileOpts) -> + DirParts = filename:split(Dir), + RelDir = case lists:prefix(BaseDirParts,DirParts) of + true -> + case lists:nthtail(length(BaseDirParts),DirParts) of + [] -> "."; + RestParts -> filename:join(RestParts) + end; + false -> Dir + end, + DirRecursive = dir_recursive(Opts, RelDir, CompileOpts), + gather_src(Opts, BaseDirParts, Rest, Srcs ++ rebar_utils:find_files(Dir, ?RE_PREFIX".*\\.erl\$", DirRecursive), CompileOpts). %% Get files which need to be compiled first, i.e. those specified in erl_first_files %% and parse_transform options. Also produce specific erl_opts for these first @@ -758,8 +765,8 @@ include_abs_dirs(ErlOpts, BaseDir) -> InclDirs = ["include"|proplists:get_all_values(i, ErlOpts)], lists:map(fun(Incl) -> filename:join([BaseDir, Incl]) end, InclDirs). -parse_opts(Opts) -> parse_opts(Opts, #compile_opts{}). - -parse_opts([], CompileOpts) -> CompileOpts; -parse_opts([{recursive, Recursive}|Rest], CompileOpts) when Recursive == true; Recursive == false -> - parse_opts(Rest, CompileOpts#compile_opts{recursive = Recursive}). +dir_recursive(Opts, Dir, CompileOpts) when is_list(CompileOpts) -> + case proplists:get_value(recursive,CompileOpts) of + undefined -> rebar_dir:recursive(Opts, Dir); + Recursive -> Recursive + end. diff --git a/src/rebar_prv_eunit.erl b/src/rebar_prv_eunit.erl index e85ab4c..0908ec9 100644 --- a/src/rebar_prv_eunit.erl +++ b/src/rebar_prv_eunit.erl @@ -310,7 +310,7 @@ maybe_inject_test_dir(State, AppAcc, [], Dir) -> inject_test_dir(Opts, Dir) -> %% append specified test targets to app defined `extra_src_dirs` - ExtraSrcDirs = rebar_dir:extra_src_dirs(Opts), + ExtraSrcDirs = rebar_opts:get(Opts, extra_src_dirs, []), rebar_opts:set(Opts, extra_src_dirs, ExtraSrcDirs ++ [Dir]). compile({error, _} = Error) -> Error; -- cgit v1.1 From 5323fdc377d6034cdb625547600144881ae1af46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Tue, 22 Nov 2016 22:22:05 +0100 Subject: Always read REBAR_CONFIG env var when loading config --- src/rebar.hrl | 1 - src/rebar3.erl | 7 +------ src/rebar_config.erl | 19 +++++++++++++++++-- 3 files changed, 18 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/rebar.hrl b/src/rebar.hrl index f96ed5e..c94a84a 100644 --- a/src/rebar.hrl +++ b/src/rebar.hrl @@ -22,7 +22,6 @@ -define(DEFAULT_PLUGINS_DIR, "plugins"). -define(DEFAULT_TEST_DEPS_DIR, "test/lib"). -define(DEFAULT_RELEASE_DIR, "rel"). --define(DEFAULT_CONFIG_FILE, "rebar.config"). -define(CONFIG_VERSION, "1.1.0"). -define(DEFAULT_CDN, "https://repo.hex.pm/"). -define(REMOTE_PACKAGE_DIR, "tarballs"). diff --git a/src/rebar3.erl b/src/rebar3.erl index 47dc25a..4e7e284 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -149,12 +149,7 @@ init_config() -> Verbosity = log_level(), ok = rebar_log:init(command_line, Verbosity), - Config = case os:getenv("REBAR_CONFIG") of - false -> - rebar_config:consult_file(?DEFAULT_CONFIG_FILE); - ConfigFile -> - rebar_config:consult_file(ConfigFile) - end, + Config = rebar_config:consult(), Config1 = rebar_config:merge_locks(Config, rebar_config:consult_lock_file(?LOCK_FILE)), %% If $HOME/.config/rebar3/rebar.config exists load and use as global config diff --git a/src/rebar_config.erl b/src/rebar_config.erl index b50c030..b2db868 100644 --- a/src/rebar_config.erl +++ b/src/rebar_config.erl @@ -26,7 +26,8 @@ %% ------------------------------------------------------------------- -module(rebar_config). --export([consult/1 +-export([consult/0 + ,consult/1 ,consult_app_file/1 ,consult_file/1 ,consult_lock_file/1 @@ -39,13 +40,19 @@ -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). +-define(DEFAULT_CONFIG_FILE, "rebar.config"). + %% =================================================================== %% Public API %% =================================================================== +-spec consult() -> [any()]. +consult() -> + consult_file(config_file()). + -spec consult(file:name()) -> [any()]. consult(Dir) -> - consult_file(filename:join(Dir, ?DEFAULT_CONFIG_FILE)). + consult_file(filename:join(Dir, config_file())). consult_app_file(File) -> consult_file_(File). @@ -300,3 +307,11 @@ check_newly_added_(Dep, LockedDeps) when is_atom(Dep) -> end; check_newly_added_(Dep, _) -> throw(?PRV_ERROR({bad_dep_name, Dep})). + +config_file() -> + case os:getenv("REBAR_CONFIG") of + false -> + ?DEFAULT_CONFIG_FILE; + ConfigFile -> + ConfigFile + end. -- cgit v1.1 From 62e1aaf6f5a58ac60871248c193c20d5c1e7bbc5 Mon Sep 17 00:00:00 2001 From: Nathaniel Waisbrot Date: Wed, 23 Nov 2016 09:19:19 -0500 Subject: expect the `missing_package` error to have arity 2 or 3 --- src/rebar_packages.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 8b4611b..4cce5a8 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -216,7 +216,7 @@ handle_single_vsn(Pkg, PkgVsn, Dep, Vsn, Constraint) -> {ok, Vsn} end. -format_error({missing_package, {Name, Vsn}}) -> +format_error({missing_package, Name, Vsn}) -> io_lib:format("Package not found in registry: ~s-~s.", [ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)]); format_error({missing_package, Dep}) -> io_lib:format("Package not found in registry: ~p.", [Dep]). -- cgit v1.1 From 3da4cc222197e01886d3baaeca7a380e02ff3125 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 25 Nov 2016 21:32:43 -0500 Subject: Type specifications and edocs improvements Includes improvments and function documentation for all modules (in alphabetical order) up to rebar_core, and may have included more in other modules as I saw fit to dig and understand more of the internals. --- src/r3.erl | 7 ++- src/rebar3.erl | 76 +++++++++++++++++++----- src/rebar_agent.erl | 59 +++++++++++++++---- src/rebar_api.erl | 38 +++++++++--- src/rebar_app_discover.erl | 81 +++++++++++++++++++++++-- src/rebar_app_info.erl | 131 ++++++++++++++++++++++++++++++++++++----- src/rebar_app_utils.erl | 79 ++++++++++++++++++++++++- src/rebar_base_compiler.erl | 108 +++++++++++++++++++++++++++++++++ src/rebar_config.erl | 82 ++++++++++++++++++++++++-- src/rebar_core.erl | 34 ++++++++++- src/rebar_dir.erl | 20 +++++++ src/rebar_dist_utils.erl | 2 +- src/rebar_file_utils.erl | 6 +- src/rebar_pkg_resource.erl | 6 ++ src/rebar_prv_install_deps.erl | 9 ++- src/rebar_utils.erl | 23 ++++++-- 16 files changed, 686 insertions(+), 75 deletions(-) (limited to 'src') diff --git a/src/r3.erl b/src/r3.erl index 5e8b26d..d0d6c47 100644 --- a/src/r3.erl +++ b/src/r3.erl @@ -1,7 +1,12 @@ -%%% external alias for rebar_agent +%%% @doc external alias for `rebar_agent' for more convenient +%%% calls from a shell. -module(r3). -export([do/1, do/2]). +%% @doc alias for `rebar_agent:do/1' +-spec do(atom()) -> ok | {error, term()}. do(Command) -> rebar_agent:do(Command). +%% @doc alias for `rebar_agent:do/2' +-spec do(atom(), atom()) -> ok | {error, term()}. do(Namespace, Command) -> rebar_agent:do(Namespace, Command). diff --git a/src/rebar3.erl b/src/rebar3.erl index 4e7e284..fa26ab2 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -24,6 +24,16 @@ %% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN %% THE SOFTWARE. %% ------------------------------------------------------------------- +%% +%% @doc Main module for rebar3. Supports two interfaces; one for escripts, +%% and one for usage as a library (although rebar3 makes a lot of +%% assumptions about its environment, making it a bit tricky to use as +%% a lib). +%% +%% This module's job is mostly to set up the root environment for rebar3 +%% and handle global options (mostly all from the ENV) and make them +%% accessible to the rest of the run. +%% @end -module(rebar3). -export([main/0, @@ -43,14 +53,14 @@ %% Public API %% ==================================================================== -%% For running with: +%% @doc For running with: %% erl +sbtu +A0 -noinput -mode minimal -boot start_clean -s rebar3 main -extra "$@" -spec main() -> no_return(). main() -> List = init:get_plain_arguments(), main(List). -%% escript Entry point +%% @doc escript Entry point -spec main(list()) -> no_return(). main(Args) -> try run(Args) of @@ -63,7 +73,8 @@ main(Args) -> handle_error(Error) end. -%% Erlang-API entry point +%% @doc Erlang-API entry point +-spec run(rebar_state:t(), [string()]) -> {ok, rebar_state:t()} | {error, term()}. run(BaseState, Commands) -> start_and_load_apps(api), BaseState1 = rebar_state:set(BaseState, task, Commands), @@ -78,6 +89,10 @@ run(BaseState, Commands) -> %% Internal functions %% ==================================================================== +%% @private sets up the rebar3 environment based on the command line +%% arguments passed, if they have any relevance; used to translate +%% from the escript call-site into a common one with the library +%% usage. run(RawArgs) -> start_and_load_apps(command_line), @@ -95,7 +110,13 @@ run(RawArgs) -> {BaseState2, _Args1} = set_options(BaseState1, {[], []}), run_aux(BaseState2, RawArgs). +%% @private Junction point between the CLI and library entry points. +%% From here on the module's role is a shared path here to finish +%% up setting the environment for the run. +-spec run_aux(rebar_state:t(), [string()]) -> + {ok, rebar_state:t()} | {error, term()}. run_aux(State, RawArgs) -> + %% Profile override; can only support one profile State1 = case os:getenv("REBAR_PROFILE") of false -> State; @@ -108,6 +129,7 @@ run_aux(State, RawArgs) -> rebar_utils:check_min_otp_version(rebar_state:get(State1, minimum_otp_vsn, undefined)), rebar_utils:check_blacklisted_otp_versions(rebar_state:get(State1, blacklisted_otp_vsns, undefined)), + %% Change the default hex CDN State2 = case os:getenv("HEX_CDN") of false -> State1; @@ -144,6 +166,9 @@ run_aux(State, RawArgs) -> rebar_core:init_command(rebar_state:command_args(State10, Args), Task). +%% @doc set up base configuration having to do with verbosity, where +%% to find config files, and so on, and return an internal rebar3 state term. +-spec init_config() -> rebar_state:t(). init_config() -> %% Initialize logging system Verbosity = log_level(), @@ -188,6 +213,17 @@ init_config() -> %% Initialize vsn cache rebar_state:set(State1, vsn_cache, dict:new()). +%% @doc Parse basic rebar3 arguments to find the top-level task +%% to be run; this parsing is only partial from the point of view that +%% runs done with arguments like `as $PROFILE do $TASK' will just +%% return `as', which is then in charge of doing a more dynamic +%% dispatch. +%% If no arguments are given, the `help' task is returned. +%% If special arguments like `-h' or `-v' are translated to `help' +%% and `version' tasks. +%% The unparsed parts of arguments are returned in: +%% `{Task, Rest}'. +-spec parse_args([string()]) -> {atom(), [string()]}. parse_args([]) -> parse_args(["help"]); parse_args([H | Rest]) when H =:= "-h" @@ -199,6 +235,7 @@ parse_args([H | Rest]) when H =:= "-v" parse_args([Task | RawRest]) -> {list_to_atom(Task), RawRest}. +%% @private actually not too sure what this does anymore. set_options(State, {Options, NonOptArgs}) -> GlobalDefines = proplists:get_all_values(defines, Options), @@ -211,9 +248,8 @@ set_options(State, {Options, NonOptArgs}) -> {rebar_state:set(State2, task, Task), NonOptArgs}. -%% -%% get log level based on getopt option -%% +%% @doc get log level based on getopt options and ENV +-spec log_level() -> integer(). log_level() -> case os:getenv("QUIET") of Q when Q == false; Q == "" -> @@ -228,18 +264,16 @@ log_level() -> rebar_log:error_level() end. -%% -%% show version information and halt -%% +%% @doc show version information +-spec version() -> ok. version() -> {ok, Vsn} = application:get_key(rebar, vsn), ?CONSOLE("rebar ~s on Erlang/OTP ~s Erts ~s", [Vsn, erlang:system_info(otp_release), erlang:system_info(version)]). +%% @private set global flag based on getopt option boolean value %% TODO: Actually make it 'global' -%% -%% set global flag based on getopt option boolean value -%% +-spec set_global_flag(rebar_state:t(), list(), term()) -> rebar_state:t(). set_global_flag(State, Options, Flag) -> Value = case proplists:get_bool(Flag, Options) of true -> @@ -249,9 +283,9 @@ set_global_flag(State, Options, Flag) -> end, rebar_state:set(State, Flag, Value). -%% -%% options accepted via getopt -%% + +%% @doc options accepted via getopt +-spec global_option_spec_list() -> [{atom(), char(), string(), atom(), string()}, ...]. global_option_spec_list() -> [ %% {Name, ShortOpt, LongOpt, ArgSpec, HelpMsg} @@ -260,6 +294,9 @@ global_option_spec_list() -> {task, undefined, undefined, string, "Task to run."} ]. +%% @private translate unhandled errors and internal return codes into proper +%% erroneous program exits. +-spec handle_error(term()) -> no_return(). handle_error(rebar_abort) -> erlang:halt(1); handle_error({error, rebar_abort}) -> @@ -293,6 +330,11 @@ handle_error(Error) -> ?INFO("When submitting a bug report, please include the output of `rebar3 report \"your command\"`", []), erlang:halt(1). +%% @private Boot Erlang dependencies; problem is that escripts don't auto-boot +%% stuff the way releases do and we have to do it by hand. +%% This also lets us detect and show nicer errors when a critical lib is +%% not supported +-spec start_and_load_apps(command_line|api) -> term(). start_and_load_apps(Caller) -> _ = application:load(rebar), %% Make sure crypto is running @@ -303,6 +345,9 @@ start_and_load_apps(Caller) -> inets:start(), inets:start(httpc, [{profile, rebar}]). +%% @doc Make sure a required app is running, or display an error message +%% and abort if there's a problem. +-spec ensure_running(atom(), command_line|api) -> ok | no_return(). ensure_running(App, Caller) -> case application:start(App) of ok -> ok; @@ -319,6 +364,7 @@ ensure_running(App, Caller) -> throw(rebar_abort) end. +-spec state_from_global_config([term()], file:filename()) -> rebar_state:t(). state_from_global_config(Config, GlobalConfigFile) -> rebar_utils:set_httpc_options(), GlobalConfigTerms = rebar_config:consult_file(GlobalConfigFile), diff --git a/src/rebar_agent.erl b/src/rebar_agent.erl index 4b0fc5f..ed9e45d 100644 --- a/src/rebar_agent.erl +++ b/src/rebar_agent.erl @@ -1,3 +1,5 @@ +%%% @doc Runs a process that holds a rebar3 state and can be used +%%% to statefully maintain loaded project state into a running VM. -module(rebar_agent). -export([start_link/1, do/1, do/2]). -export([init/1, @@ -10,19 +12,34 @@ cwd, show_warning=true}). +%% @doc boots an agent server; requires a full rebar3 state already. +%% By default (within rebar3), this isn't called; `rebar_prv_shell' +%% enters and transforms into this module +-spec start_link(rebar_state:t()) -> {ok, pid()}. start_link(State) -> gen_server:start_link({local, ?MODULE}, ?MODULE, State, []). +%% @doc runs a given command in the agent's context. +-spec do(atom()) -> ok | {error, term()}. do(Command) when is_atom(Command) -> gen_server:call(?MODULE, {cmd, Command}, infinity). +%% @doc runs a given command in the agent's context, under a given +%% namespace. +-spec do(atom(), atom()) -> ok | {error, term()}. do(Namespace, Command) when is_atom(Namespace), is_atom(Command) -> gen_server:call(?MODULE, {cmd, Namespace, Command}, infinity). +%%%%%%%%%%%%%%%%% +%%% CALLBACKS %%% +%%%%%%%%%%%%%%%%% + +%% @private init(State) -> Cwd = rebar_dir:get_cwd(), {ok, #state{state=State, cwd=Cwd}}. +%% @private handle_call({cmd, Command}, _From, State=#state{state=RState, cwd=Cwd}) -> MidState = maybe_show_warning(State), {Res, NewRState} = run(default, Command, RState, Cwd), @@ -34,18 +51,29 @@ handle_call({cmd, Namespace, Command}, _From, State = #state{state=RState, cwd=C handle_call(_Call, _From, State) -> {noreply, State}. +%% @private handle_cast(_Cast, State) -> {noreply, State}. +%% @private handle_info(_Info, State) -> {noreply, State}. +%% @private code_change(_OldVsn, State, _Extra) -> {ok, State}. +%% @private terminate(_Reason, _State) -> ok. +%%%%%%%%%%%%%%% +%%% PRIVATE %%% +%%%%%%%%%%%%%%% + +%% @private runs the actual command and maintains the state changes +-spec run(atom(), atom(), rebar_state:t(), file:filename()) -> + {ok, rebar_state:t()} | {{error, term()}, rebar_state:t()}. run(Namespace, Command, RState, Cwd) -> try case rebar_dir:get_cwd() of @@ -74,12 +102,17 @@ run(Namespace, Command, RState, Cwd) -> {{error, {Type, Reason}}, RState} end. +%% @private function to display a warning for the feature only once +-spec maybe_show_warning(#state{}) -> #state{}. maybe_show_warning(S=#state{show_warning=true}) -> ?WARN("This feature is experimental and may be modified or removed at any time.", []), S#state{show_warning=false}; maybe_show_warning(State) -> State. +%% @private based on a rebar3 state term, reload paths in a way +%% that makes sense. +-spec refresh_paths(rebar_state:t()) -> ok. refresh_paths(RState) -> ToRefresh = (rebar_state:code_paths(RState, all_deps) ++ [filename:join([rebar_app_info:out_dir(App), "test"]) @@ -116,6 +149,9 @@ refresh_paths(RState) -> end end, ToRefresh). +%% @private from a disk config, reload and reapply with the current +%% profiles; used to find changes in the config from a prior run. +-spec refresh_state(rebar_state:t(), file:filename()) -> rebar_state:t(). refresh_state(RState, _Dir) -> lists:foldl( fun(F, State) -> F(State) end, @@ -123,26 +159,28 @@ refresh_state(RState, _Dir) -> [fun(S) -> rebar_state:apply_profiles(S, rebar_state:current_profiles(RState)) end] ). +%% @private takes a list of modules and reloads them +-spec reload_modules([module()]) -> term(). reload_modules([]) -> noop; -reload_modules(Modules) -> +reload_modules(Modules) -> reload_modules(Modules, erlang:function_exported(code, prepare_loading, 1)). -%% OTP 19 and later -- use atomic loading and ignore unloadable mods +%% @private reloading modules, when there are modules to actually reload reload_modules(Modules, true) -> + %% OTP 19 and later -- use atomic loading and ignore unloadable mods case code:prepare_loading(Modules) of {ok, Prepared} -> [code:purge(M) || M <- Modules], code:finish_loading(Prepared); - {error, ModRsns} -> - Blacklist = + Blacklist = lists:foldr(fun({ModError, Error}, Acc) -> case Error of - %perhaps cover other cases of failure? + % perhaps cover other cases of failure? on_load_not_allowed -> reload_modules([ModError], false), [ModError|Acc]; - _ -> + _ -> ?DEBUG("Module ~p failed to atomic load because ~p", [ModError, Error]), [ModError|Acc] end @@ -151,16 +189,15 @@ reload_modules(Modules, true) -> ), reload_modules(Modules -- Blacklist, true) end; - -%% Older versions, use a more ad-hoc mechanism. reload_modules(Modules, false) -> + %% Older versions, use a more ad-hoc mechanism. lists:foreach(fun(M) -> - code:delete(M), - code:purge(M), + code:delete(M), + code:purge(M), case code:load_file(M) of {module, M} -> ok; {error, Error} -> ?DEBUG("Module ~p failed to load because ~p", [M, Error]) end end, Modules - ). \ No newline at end of file + ). diff --git a/src/rebar_api.erl b/src/rebar_api.erl index 6ebc500..9d9071e 100644 --- a/src/rebar_api.erl +++ b/src/rebar_api.erl @@ -1,4 +1,4 @@ -%%% Packages rebar.hrl features and macros into a more generic API +%%% @doc Packages rebar.hrl features and macros into a more generic API %%% that can be used by plugin builders. -module(rebar_api). -include("rebar.hrl"). @@ -30,42 +30,62 @@ abort() -> ?FAIL. abort(Str, Args) -> ?ABORT(Str, Args). %% @doc Prints to the console, including a newline +-spec console(string(), list()) -> ok. console(Str, Args) -> ?CONSOLE(Str, Args). %% @doc logs with severity `debug' +-spec debug(string(), list()) -> ok. debug(Str, Args) -> ?DEBUG(Str, Args). + %% @doc logs with severity `info' +-spec info(string(), list()) -> ok. info(Str, Args) -> ?INFO(Str, Args). + %% @doc logs with severity `warn' +-spec warn(string(), list()) -> ok. warn(Str, Args) -> ?WARN(Str, Args). + %% @doc logs with severity `error' +-spec error(string(), list()) -> ok. error(Str, Args) -> ?ERROR(Str, Args). -%% -%% Given env. variable FOO we want to expand all references to -%% it in InStr. References can have two forms: $FOO and ${FOO} -%% The end of form $FOO is delimited with whitespace or eol -%% +%% @doc Given env. variable `FOO' we want to expand all references to +%% it in `InStr'. References can have two forms: `$FOO' and `${FOO}' +%% The end of form `$FOO' is delimited with whitespace or EOL +-spec expand_env_variable(string(), string(), term()) -> string(). expand_env_variable(InStr, VarName, RawVarValue) -> rebar_utils:expand_env_variable(InStr, VarName, RawVarValue). +%% @doc returns the sytem architecture, in strings like +%% `"19.0.4-x86_64-unknown-linux-gnu-64"'. +-spec get_arch() -> string(). get_arch() -> rebar_utils:get_arch(). +%% @doc returns the size of a word on the system, as a string +-spec wordsize() -> string(). wordsize() -> rebar_utils:wordsize(). - -%% Add deps to the code path +%% @doc Add deps to the code path +-spec add_deps_to_path(rebar_state:t()) -> ok. add_deps_to_path(State) -> code:add_pathsa(rebar_state:code_paths(State, all_deps)). -%% Revert to only having the beams necessary for running rebar3 and plugins in the path +%% @doc Revert to only having the beams necessary for running rebar3 and +%% plugins in the path +-spec restore_code_path(rebar_state:t()) -> true | {error, term()}. restore_code_path(State) -> rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)). +%% @doc checks if the current working directory is the base directory +%% for the project. +-spec processing_base_dir(rebar_state:t()) -> boolean(). processing_base_dir(State) -> rebar_dir:processing_base_dir(State). +%% @doc returns the SSL options adequate for the project based on +%% its configuration, including for validation of certs. +-spec ssl_opts(string()) -> [term()]. ssl_opts(Url) -> rebar_pkg_resource:ssl_opts(Url). diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index 67acf54..acefdb4 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -1,3 +1,5 @@ +%%% @doc utility functions to do the basic discovery of apps +%%% and layout for the project. -module(rebar_app_discover). -export([do/2, @@ -11,6 +13,8 @@ -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). +%% @doc from the base directory, +-spec do(rebar_state:t(), [file:filename()]) -> rebar_state:t(). do(State, LibDirs) -> BaseDir = rebar_state:dir(State), Dirs = [filename:join(BaseDir, LibDir) || LibDir <- LibDirs], @@ -55,6 +59,10 @@ do(State, LibDirs) -> end end, State1, SortedApps). +%% @doc checks whether there is an app at the top level (and returns its +%% name) or the 'root' atom in case we're in an umbrella project. +-spec define_root_app([rebar_app_info:t()], rebar_state:t()) -> + root | binary(). define_root_app(Apps, State) -> RootDir = rebar_dir:root_dir(State), case ec_lists:find(fun(X) -> @@ -67,11 +75,17 @@ define_root_app(Apps, State) -> root end. +%% @doc formatting errors from the module. +-spec format_error(term()) -> iodata(). format_error({module_list, File}) -> io_lib:format("Error reading module list from ~p~n", [File]); format_error({missing_module, Module}) -> io_lib:format("Module defined in app file missing: ~p~n", [Module]). +%% @doc handles the merging and application of profiles and overrides +%% for a given application, within its own context. +-spec merge_deps(rebar_app_info:t(), rebar_state:t()) -> + {rebar_app_info:t(), rebar_state:t()}. merge_deps(AppInfo, State) -> %% These steps make sure that hooks and artifacts are run in the context of %% the application they are defined at. If an umbrella structure is used and @@ -97,6 +111,10 @@ merge_deps(AppInfo, State) -> {AppInfo2, State2}. +%% @doc Applies a given profile for an app, ensuring the deps +%% match the context it will require. +-spec handle_profile(atom(), binary(), rebar_app_info:t(), rebar_state:t()) -> + rebar_state:t(). handle_profile(Profile, Name, AppInfo, State) -> TopParsedDeps = rebar_state:get(State, {parsed_deps, Profile}, {[], []}), TopLevelProfileDeps = rebar_state:get(State, {deps, Profile}, []), @@ -113,6 +131,12 @@ handle_profile(Profile, Name, AppInfo, State) -> State2 = rebar_state:set(State1, {deps, Profile}, ProfileDeps2), rebar_state:set(State2, {parsed_deps, Profile}, TopParsedDeps++ParsedDeps). +%% @doc parses all the known dependencies for a given profile +-spec parse_profile_deps(Profile, Name, Deps, Opts, rebar_state:t()) -> [rebar_app_info:t()] when + Profile :: atom(), + Name :: binary(), + Deps :: [term()], % TODO: refine types + Opts :: term(). % TODO: refine types parse_profile_deps(Profile, Name, Deps, Opts, State) -> DepsDir = rebar_prv_install_deps:profile_dep_dir(State, Profile), Locks = rebar_state:get(State, {locks, Profile}, []), @@ -123,14 +147,21 @@ parse_profile_deps(Profile, Name, Deps, Opts, State) -> ,Locks ,1). +%% @doc Find the app-level config and return the state updated +%% with the relevant app-level data. +-spec project_app_config(rebar_app_info:t(), rebar_state:t()) -> + {Config, rebar_state:t()} when + Config :: [any()]. project_app_config(AppInfo, State) -> C = rebar_config:consult(rebar_app_info:dir(AppInfo)), Dir = rebar_app_info:dir(AppInfo), Opts = maybe_reset_hooks(Dir, rebar_state:opts(State), State), {C, rebar_state:opts(State, Opts)}. -%% Here we check if the app is at the root of the project. +%% @doc Here we check if the app is at the root of the project. %% If it is, then drop the hooks from the config so they aren't run twice +-spec maybe_reset_hooks(file:filename(), Opts, rebar_state:t()) -> Opts when + Opts :: rebar_dict(). maybe_reset_hooks(Dir, Opts, State) -> case ec_file:real_dir_path(rebar_dir:root_dir(State)) of Dir -> @@ -139,17 +170,22 @@ maybe_reset_hooks(Dir, Opts, State) -> Opts end. +%% @doc make the hooks empty for a given set of options +-spec reset_hooks(Opts) -> Opts when Opts :: rebar_dict(). reset_hooks(Opts) -> lists:foldl(fun(Key, OptsAcc) -> rebar_opts:set(OptsAcc, Key, []) end, Opts, [post_hooks, pre_hooks, provider_hooks, artifacts]). --spec all_app_dirs(list(file:name())) -> list(file:name()). +%% @doc find the directories for all apps +-spec all_app_dirs([file:name()]) -> [file:name()]. all_app_dirs(LibDirs) -> lists:flatmap(fun(LibDir) -> app_dirs(LibDir) end, LibDirs). +%% @doc find the directories based on the library directories +-spec app_dirs([file:name()]) -> [file:name()]. app_dirs(LibDir) -> Path1 = filename:join([LibDir, "src", @@ -168,32 +204,51 @@ app_dirs(LibDir) -> [app_dir(File) || File <- Files] ++ Acc end, [], [Path1, Path2, Path3])). +%% @doc find all apps that haven't been built in a list of directories +-spec find_unbuilt_apps([file:filename_all()]) -> [rebar_app_info:t()]. find_unbuilt_apps(LibDirs) -> find_apps(LibDirs, invalid). +%% @doc for each directory passed, find all apps that are valid. +%% Returns all the related app info records. -spec find_apps([file:filename_all()]) -> [rebar_app_info:t()]. find_apps(LibDirs) -> find_apps(LibDirs, valid). +%% @doc for each directory passed, find all apps according +%% to the validity rule passed in. Returns all the related +%% app info records. -spec find_apps([file:filename_all()], valid | invalid | all) -> [rebar_app_info:t()]. find_apps(LibDirs, Validate) -> rebar_utils:filtermap(fun(AppDir) -> find_app(AppDir, Validate) end, all_app_dirs(LibDirs)). +%% @doc check that a given app in a directory is there, and whether it's +%% valid or not based on the second argument. Returns the related +%% app info record. -spec find_app(file:filename_all(), valid | invalid | all) -> {true, rebar_app_info:t()} | false. find_app(AppDir, Validate) -> find_app(rebar_app_info:new(), AppDir, Validate). +%% @doc check that a given app in a directory is there, and whether it's +%% valid or not based on the second argument. Returns the related +%% app info record. +-spec find_app(rebar_app_info:t(), file:filename_all(), valid | invalid | all) -> + {true, rebar_app_info:t()} | false. find_app(AppInfo, AppDir, Validate) -> AppFile = filelib:wildcard(filename:join([AppDir, "ebin", "*.app"])), AppSrcFile = filelib:wildcard(filename:join([AppDir, "src", "*.app.src"])), AppSrcScriptFile = filelib:wildcard(filename:join([AppDir, "src", "*.app.src.script"])), try_handle_app_file(AppInfo, AppFile, AppDir, AppSrcFile, AppSrcScriptFile, Validate). +%% @doc find the directory that an appfile has +-spec app_dir(file:filename()) -> file:filename(). app_dir(AppFile) -> filename:join(rebar_utils:droplast(filename:split(filename:dirname(AppFile)))). +%% @doc populates an app info record based on an app directory and its +%% app file. -spec create_app_info(rebar_app_info:t(), file:name(), file:name()) -> rebar_app_info:t(). create_app_info(AppInfo, AppDir, AppFile) -> [{application, AppName, AppDetails}] = rebar_config:consult_app_file(AppFile), @@ -215,8 +270,15 @@ create_app_info(AppInfo, AppDir, AppFile) -> end, rebar_app_info:dir(rebar_app_info:valid(AppInfo2, Valid), AppDir). -%% Read in and parse the .app file if it is availabe. Do the same for +%% @doc Read in and parse the .app file if it is availabe. Do the same for %% the .app.src file if it exists. +-spec try_handle_app_file(AppInfo, AppFile, AppDir, AppSrcFile, AppSrcScriptFile, valid | invalid | all) -> + {true, AppInfo} | false when + AppInfo :: rebar_app_info:t(), + AppFile :: file:filename(), + AppDir :: file:filename(), + AppSrcFile :: file:filename(), + AppSrcScriptFile :: file:filename(). try_handle_app_file(AppInfo, [], AppDir, [], AppSrcScriptFile, Validate) -> try_handle_app_src_file(AppInfo, [], AppDir, AppSrcScriptFile, Validate); try_handle_app_file(AppInfo, [], AppDir, AppSrcFile, _, Validate) -> @@ -260,7 +322,14 @@ try_handle_app_file(AppInfo0, [File], AppDir, AppSrcFile, _, Validate) -> try_handle_app_file(_AppInfo, Other, _AppDir, _AppSrcFile, _, _Validate) -> throw({error, {multiple_app_files, Other}}). -%% Read in the .app.src file if we aren't looking for a valid (already built) app +%% @doc Read in the .app.src file if we aren't looking for a valid (already +%% built) app. +-spec try_handle_app_src_file(AppInfo, AppFile, AppDir, AppSrcFile, valid | invalid | all) -> + {true, AppInfo} | false when + AppInfo :: rebar_app_info:t(), + AppFile :: file:filename(), + AppDir :: file:filename(), + AppSrcFile :: file:filename(). try_handle_app_src_file(_AppInfo, _, _AppDir, [], _Validate) -> false; try_handle_app_src_file(_AppInfo, _, _AppDir, _AppSrcFile, valid) -> @@ -277,9 +346,13 @@ try_handle_app_src_file(AppInfo, _, AppDir, [File], Validate) when Validate =:= try_handle_app_src_file(_AppInfo, _, _AppDir, Other, _Validate) -> throw({error, {multiple_app_files, Other}}). +%% @doc checks whether the given app is not blacklisted in the config. +-spec enable(rebar_state:t(), rebar_app_info:t()) -> boolean(). enable(State, AppInfo) -> not lists:member(to_atom(rebar_app_info:name(AppInfo)), rebar_state:get(State, excluded_apps, [])). +%% @private convert a binary to an atom. +-spec to_atom(binary()) -> atom(). to_atom(Bin) -> list_to_atom(binary_to_list(Bin)). diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index 4f19d77..fdaadb8 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -103,11 +103,13 @@ new() -> #app_info_t{}. +%% @doc Build a new app info value with only the app name set. -spec new(atom() | binary() | string()) -> {ok, t()}. new(AppName) -> {ok, #app_info_t{name=ec_cnv:to_binary(AppName)}}. +%% @doc Build a new app info value with only the name and version set. -spec new(atom() | binary() | string(), binary() | string()) -> {ok, t()}. new(AppName, Vsn) -> @@ -144,6 +146,9 @@ new(Parent, AppName, Vsn, Dir, Deps) -> out_dir=ec_cnv:to_list(Dir), deps=Deps}}. +%% @doc update the opts based on the contents of a config +%% file for the app +-spec update_opts(t(), rebar_dict(), [any()]) -> t(). update_opts(AppInfo, Opts, Config) -> LockDeps = case resource_type(AppInfo) of pkg -> @@ -163,6 +168,8 @@ update_opts(AppInfo, Opts, Config) -> AppInfo#app_info_t{opts=NewOpts ,default=NewOpts}. +%% @private extract the deps for an app in `Dir' based on its config file data +-spec deps_from_config(file:filename(), [any()]) -> [{tuple(), any()}, ...]. deps_from_config(Dir, Config) -> case rebar_config:consult_lock_file(filename:join(Dir, ?LOCK_FILE)) of [] -> @@ -184,30 +191,48 @@ discover(Dir) -> not_found end. +%% @doc get the name of the app. -spec name(t()) -> binary(). name(#app_info_t{name=Name}) -> Name. +%% @doc set the name of the app. -spec name(t(), atom() | binary() | string()) -> t(). name(AppInfo=#app_info_t{}, AppName) -> AppInfo#app_info_t{name=ec_cnv:to_binary(AppName)}. +%% @doc get the dictionary of options for the app. +-spec opts(t()) -> rebar_dict(). opts(#app_info_t{opts=Opts}) -> Opts. +%% @doc set the dictionary of options for the app. +-spec opts(t(), rebar_dict()) -> t(). opts(AppInfo, Opts) -> AppInfo#app_info_t{opts=Opts}. +%% @doc get the dictionary of options under the default profile. +%% Represents a root set prior to applying other profiles. +-spec default(t()) -> rebar_dict(). default(#app_info_t{default=Default}) -> Default. +%% @doc set the dictionary of options under the default profile. +%% Useful when re-applying profile. +-spec default(t(), rebar_dict()) -> t(). default(AppInfo, Default) -> AppInfo#app_info_t{default=Default}. +%% @doc look up a value in the dictionary of options; fails if +%% the key for it does not exist. +-spec get(t(), term()) -> term(). get(AppInfo, Key) -> {ok, Value} = dict:find(Key, AppInfo#app_info_t.opts), Value. +%% @doc look up a value in the dictionary of options; returns +%% a `Default' value otherwise. +-spec get(t(), term(), term()) -> term(). get(AppInfo, Key, Default) -> case dict:find(Key, AppInfo#app_info_t.opts) of {ok, Value} -> @@ -216,10 +241,12 @@ get(AppInfo, Key, Default) -> Default end. +%% @doc sets a given value in the dictionary of options for the app. -spec set(t(), any(), any()) -> t(). set(AppInfo=#app_info_t{opts=Opts}, Key, Value) -> AppInfo#app_info_t{opts = dict:store(Key, Value, Opts)}. +%% @doc finds the .app.src file for an app, if any. -spec app_file_src(t()) -> file:filename_all() | undefined. app_file_src(#app_info_t{app_file_src=undefined, dir=Dir, name=Name}) -> AppFileSrc = filename:join([ec_cnv:to_list(Dir), "src", ec_cnv:to_list(Name)++".app.src"]), @@ -232,12 +259,15 @@ app_file_src(#app_info_t{app_file_src=undefined, dir=Dir, name=Name}) -> app_file_src(#app_info_t{app_file_src=AppFileSrc}) -> ec_cnv:to_list(AppFileSrc). +%% @doc sets the .app.src file for an app. An app without such a file +%% can explicitly be set with `undefined'. -spec app_file_src(t(), file:filename_all() | undefined) -> t(). app_file_src(AppInfo=#app_info_t{}, undefined) -> AppInfo#app_info_t{app_file_src=undefined}; app_file_src(AppInfo=#app_info_t{}, AppFileSrc) -> AppInfo#app_info_t{app_file_src=ec_cnv:to_list(AppFileSrc)}. +%% @doc finds the .app.src.script file for an app, if any. -spec app_file_src_script(t()) -> file:filename_all() | undefined. app_file_src_script(#app_info_t{app_file_src_script=undefined, dir=Dir, name=Name}) -> AppFileSrcScript = filename:join([ec_cnv:to_list(Dir), "src", ec_cnv:to_list(Name)++".app.src.script"]), @@ -250,12 +280,15 @@ app_file_src_script(#app_info_t{app_file_src_script=undefined, dir=Dir, name=Nam app_file_src_script(#app_info_t{app_file_src_script=AppFileSrcScript}) -> ec_cnv:to_list(AppFileSrcScript). +%% @doc sets the .app.src.script file for an app. An app without such a file +%% can explicitly be set with `undefined'. -spec app_file_src_script(t(), file:filename_all()) -> t(). app_file_src_script(AppInfo=#app_info_t{}, undefined) -> AppInfo#app_info_t{app_file_src_script=undefined}; app_file_src_script(AppInfo=#app_info_t{}, AppFileSrcScript) -> AppInfo#app_info_t{app_file_src_script=ec_cnv:to_list(AppFileSrcScript)}. +%% @doc finds the .app file for an app, if any. -spec app_file(t()) -> file:filename_all() | undefined. app_file(#app_info_t{app_file=undefined, out_dir=Dir, name=Name}) -> AppFile = filename:join([ec_cnv:to_list(Dir), "ebin", ec_cnv:to_list(Name)++".app"]), @@ -268,10 +301,13 @@ app_file(#app_info_t{app_file=undefined, out_dir=Dir, name=Name}) -> app_file(#app_info_t{app_file=AppFile}) -> AppFile. +%% @doc sets the .app file for an app. -spec app_file(t(), file:filename_all()) -> t(). app_file(AppInfo=#app_info_t{}, AppFile) -> AppInfo#app_info_t{app_file=AppFile}. +%% @doc returns the information stored in the app's app file, +%% or if none, from the .app.src file. -spec app_details(t()) -> list(). app_details(AppInfo=#app_info_t{app_details=[]}) -> case app_file(AppInfo) of @@ -290,59 +326,83 @@ app_details(AppInfo=#app_info_t{app_details=[]}) -> app_details(#app_info_t{app_details=AppDetails}) -> AppDetails. +%% @doc stores the information that would be returned from the +%% app file, when reading from `app_details/1'. -spec app_details(t(), list()) -> t(). app_details(AppInfo=#app_info_t{}, AppDetails) -> AppInfo#app_info_t{app_details=AppDetails}. +%% @doc returns the app's parent in the dep tree. +-spec parent(t()) -> root | binary(). parent(#app_info_t{parent=Parent}) -> Parent. +%% @doc sets the app's parent. -spec parent(t(), binary() | root) -> t(). parent(AppInfo=#app_info_t{}, Parent) -> AppInfo#app_info_t{parent=Parent}. +%% @doc returns the original version of the app (unevaluated if +%% asking for a semver) -spec original_vsn(t()) -> string(). original_vsn(#app_info_t{original_vsn=Vsn}) -> Vsn. +%% @doc stores the original version of the app (unevaluated if +%% asking for a semver) -spec original_vsn(t(), string()) -> t(). original_vsn(AppInfo=#app_info_t{}, Vsn) -> AppInfo#app_info_t{original_vsn=Vsn}. +%% @doc returns the list of applications the app depends on. -spec applications(t()) -> list(). applications(#app_info_t{applications=Applications}) -> Applications. +%% @doc sets the list of applications the app depends on. +%% Should be obtained from the app file. -spec applications(t(), list()) -> t(). applications(AppInfo=#app_info_t{}, Applications) -> AppInfo#app_info_t{applications=Applications}. +%% @doc returns the list of active profiles -spec profiles(t()) -> list(). profiles(#app_info_t{profiles=Profiles}) -> Profiles. +%% @doc sets the list of active profiles -spec profiles(t(), list()) -> t(). profiles(AppInfo=#app_info_t{}, Profiles) -> AppInfo#app_info_t{profiles=Profiles}. +%% @doc returns the list of dependencies -spec deps(t()) -> list(). deps(#app_info_t{deps=Deps}) -> Deps. +%% @doc sets the list of dependencies. -spec deps(t(), list()) -> t(). deps(AppInfo=#app_info_t{}, Deps) -> AppInfo#app_info_t{deps=Deps}. -dep_level(AppInfo=#app_info_t{}, Level) -> - AppInfo#app_info_t{dep_level=Level}. - +%% @doc returns the level the app has in the lock files or in the +%% dep tree. +-spec dep_level(t()) -> non_neg_integer(). dep_level(#app_info_t{dep_level=Level}) -> Level. +%% @doc sets the level the app has in the lock files or in the +%% dep tree. +-spec dep_level(t(), non_neg_integer()) -> t(). +dep_level(AppInfo=#app_info_t{}, Level) -> + AppInfo#app_info_t{dep_level=Level}. + +%% @doc returns the directory that contains the app. -spec dir(t()) -> file:name(). dir(#app_info_t{dir=Dir}) -> Dir. +%% @doc sets the directory that contains the app. -spec dir(t(), file:name()) -> t(). dir(AppInfo=#app_info_t{out_dir=undefined}, Dir) -> AppInfo#app_info_t{dir=ec_cnv:to_list(Dir), @@ -350,54 +410,69 @@ dir(AppInfo=#app_info_t{out_dir=undefined}, Dir) -> dir(AppInfo=#app_info_t{}, Dir) -> AppInfo#app_info_t{dir=ec_cnv:to_list(Dir)}. +%% @doc returns the directory where build artifacts for the app +%% should go -spec out_dir(t()) -> file:name(). out_dir(#app_info_t{out_dir=OutDir}) -> OutDir. +%% @doc sets the directory where build artifacts for the app +%% should go -spec out_dir(t(), file:name()) -> t(). out_dir(AppInfo=#app_info_t{}, OutDir) -> AppInfo#app_info_t{out_dir=ec_cnv:to_list(OutDir)}. +%% @doc gets the directory where ebin files for the app should go -spec ebin_dir(t()) -> file:name(). ebin_dir(#app_info_t{out_dir=OutDir}) -> ec_cnv:to_list(filename:join(OutDir, "ebin")). +%% @doc gets the directory where private files for the app should go -spec priv_dir(t()) -> file:name(). priv_dir(#app_info_t{out_dir=OutDir}) -> ec_cnv:to_list(filename:join(OutDir, "priv")). --spec resource_type(t(), pkg | src) -> t(). -resource_type(AppInfo=#app_info_t{}, Type) -> - AppInfo#app_info_t{resource_type=Type}. - +%% @doc returns whether the app is source app or a package app. -spec resource_type(t()) -> pkg | src. resource_type(#app_info_t{resource_type=ResourceType}) -> ResourceType. --spec source(t(), string() | tuple() | checkout) -> t(). -source(AppInfo=#app_info_t{}, Source) -> - AppInfo#app_info_t{source=Source}. +%% @doc sets whether the app is source app or a package app. +-spec resource_type(t(), pkg | src) -> t(). +resource_type(AppInfo=#app_info_t{}, Type) -> + AppInfo#app_info_t{resource_type=Type}. +%% @doc finds the source specification for the app -spec source(t()) -> string() | tuple(). source(#app_info_t{source=Source}) -> Source. --spec is_lock(t(), boolean()) -> t(). -is_lock(AppInfo=#app_info_t{}, IsLock) -> - AppInfo#app_info_t{is_lock=IsLock}. +%% @doc sets the source specification for the app +-spec source(t(), string() | tuple() | checkout) -> t(). +source(AppInfo=#app_info_t{}, Source) -> + AppInfo#app_info_t{source=Source}. +%% @doc returns the lock status for the app -spec is_lock(t()) -> boolean(). is_lock(#app_info_t{is_lock=IsLock}) -> IsLock. --spec is_checkout(t(), boolean()) -> t(). -is_checkout(AppInfo=#app_info_t{}, IsCheckout) -> - AppInfo#app_info_t{is_checkout=IsCheckout}. +%% @doc sets the lock status for the app +-spec is_lock(t(), boolean()) -> t(). +is_lock(AppInfo=#app_info_t{}, IsLock) -> + AppInfo#app_info_t{is_lock=IsLock}. +%% @doc returns whether the app is a checkout app or not -spec is_checkout(t()) -> boolean(). is_checkout(#app_info_t{is_checkout=IsCheckout}) -> IsCheckout. +%% @doc sets whether the app is a checkout app or not +-spec is_checkout(t(), boolean()) -> t(). +is_checkout(AppInfo=#app_info_t{}, IsCheckout) -> + AppInfo#app_info_t{is_checkout=IsCheckout}. + +%% @doc returns whether the app is valid (built) or not -spec valid(t()) -> boolean(). valid(AppInfo=#app_info_t{valid=undefined}) -> case rebar_app_utils:validate_application_info(AppInfo) =:= true @@ -410,14 +485,22 @@ valid(AppInfo=#app_info_t{valid=undefined}) -> valid(#app_info_t{valid=Valid}) -> Valid. +%% @doc sets whether the app is valid (built) or not. If left unset, +%% rebar3 will do the detection of the status itself. -spec valid(t(), boolean()) -> t(). valid(AppInfo=#app_info_t{}, Valid) -> AppInfo#app_info_t{valid=Valid}. +%% @doc checks whether the app can be built with the current +%% Erlang/OTP version. If the check fails, the function raises +%% an exception and displays an error. +-spec verify_otp_vsn(t()) -> ok | no_return(). verify_otp_vsn(AppInfo) -> rebar_utils:check_min_otp_version(rebar_app_info:get(AppInfo, minimum_otp_vsn, undefined)), rebar_utils:check_blacklisted_otp_versions(rebar_app_info:get(AppInfo, blacklisted_otp_vsns, [])). +%% @doc checks whether all the build artifacts for an app to be considered +%% valid are present. -spec has_all_artifacts(#app_info_t{}) -> true | {false, file:filename()}. has_all_artifacts(AppInfo) -> Artifacts = rebar_app_info:get(AppInfo, artifacts, []), @@ -427,6 +510,10 @@ has_all_artifacts(AppInfo) -> ,{out_dir, OutDir}], all(OutDir, Context, Artifacts). +%% @private checks that all files/artifacts in the directory are found. +%% Template evaluation must happen and a bbmustache context needs to +%% be provided. +-spec all(file:filename(), term(), [string()]) -> true | {false, string()}. all(_, _, []) -> true; all(Dir, Context, [File|Artifacts]) -> @@ -441,15 +528,23 @@ all(Dir, Context, [File|Artifacts]) -> %%%%% +%% @doc given a set of override rules, modify the app info accordingly +-spec apply_overrides(list(), t()) -> t(). apply_overrides(Overrides, AppInfo) -> Name = binary_to_atom(rebar_app_info:name(AppInfo), utf8), Opts = rebar_opts:apply_overrides(opts(AppInfo), Name, Overrides), AppInfo#app_info_t{default=Opts, opts=Opts}. +%% @doc adds a new profile with its own config to the app data +-spec add_to_profile(t(), atom(), [{_,_}]) -> t(). add_to_profile(AppInfo, Profile, KVs) when is_atom(Profile), is_list(KVs) -> Opts = rebar_opts:add_to_profile(opts(AppInfo), Profile, KVs), AppInfo#app_info_t{opts=Opts}. +%% @doc applies and merges the profile configuration in the specified order +%% of profiles (or for a single profile) and returns an app info record +%% with the resulting configuration +-spec apply_profiles(t(), atom() | [atom(),...]) -> t(). apply_profiles(AppInfo, Profile) when not is_list(Profile) -> apply_profiles(AppInfo, [Profile]); apply_profiles(AppInfo, [default]) -> @@ -481,9 +576,13 @@ apply_profiles(AppInfo=#app_info_t{default = Defaults, profiles=CurrentProfiles} end, Defaults, AppliedProfiles), AppInfo#app_info_t{profiles = AppliedProfiles, opts=NewOpts}. +%% @private drops duplicated profile definitions +-spec deduplicate(list()) -> list(). deduplicate(Profiles) -> do_deduplicate(lists:reverse(Profiles), []). +%% @private drops duplicated profile definitions +-spec do_deduplicate(list(), list()) -> list(). do_deduplicate([], Acc) -> Acc; do_deduplicate([Head | Rest], Acc) -> diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index 847b0a5..7154999 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -44,10 +44,14 @@ %% Public API %% =================================================================== +%% @doc finds the proper app info record for a given app name in a list of +%% such records. -spec find(binary(), [rebar_app_info:t()]) -> {ok, rebar_app_info:t()} | error. find(Name, Apps) -> ec_lists:find(fun(App) -> rebar_app_info:name(App) =:= Name end, Apps). +%% @doc finds the proper app info record for a given app name at a given version +%% in a list of such records. -spec find(binary(), binary(), [rebar_app_info:t()]) -> {ok, rebar_app_info:t()} | error. find(Name, Vsn, Apps) -> ec_lists:find(fun(App) -> @@ -55,11 +59,18 @@ find(Name, Vsn, Apps) -> andalso rebar_app_info:original_vsn(App) =:= Vsn end, Apps). +%% @doc checks if a given file is .app.src file is_app_src(Filename) -> %% If removing the extension .app.src yields a shorter name, %% this is an .app.src file. Filename =/= filename:rootname(Filename, ".app.src"). +%% @doc translates the name of the .app.src[.script] file to where +%% its .app counterpart should be stored. +-spec app_src_to_app(OutDir, SrcFilename) -> OutFilename when + OutDir :: file:filename(), + SrcFilename :: file:filename(), + OutFilename :: file:filename(). app_src_to_app(OutDir, Filename) -> AppFile = case lists:suffix(".app.src", Filename) of @@ -72,10 +83,16 @@ app_src_to_app(OutDir, Filename) -> filelib:ensure_dir(AppFile), AppFile. +%% @doc checks whether the .app file has all the required data to be valid, +%% and cross-references it with compiled modules on disk -spec validate_application_info(rebar_app_info:t()) -> boolean(). validate_application_info(AppInfo) -> validate_application_info(AppInfo, rebar_app_info:app_details(AppInfo)). +%% @doc checks whether the .app file has all the required data to be valid +%% and cross-references it with compiled modules on disk. +%% The app info is passed explicitly as a second argument. +-spec validate_application_info(rebar_app_info:t(), list()) -> boolean(). validate_application_info(AppInfo, AppDetail) -> EbinDir = rebar_app_info:ebin_dir(AppInfo), case rebar_app_info:app_file(AppInfo) of @@ -90,13 +107,37 @@ validate_application_info(AppInfo, AppDetail) -> end end. --spec parse_deps(binary(), list(), rebar_state:t(), list(), integer()) -> [rebar_app_info:t()]. +%% @doc parses all dependencies from the root of the project +-spec parse_deps(Dir, Deps, State, Locks, Level) -> [rebar_app_info:t()] when + Dir :: file:filename(), + Deps :: [tuple() | atom() | binary()], % TODO: meta to source() | lock() + State :: rebar_state:t(), + Locks :: [tuple()], % TODO: meta to [lock()] + Level :: non_neg_integer(). parse_deps(DepsDir, Deps, State, Locks, Level) -> parse_deps(root, DepsDir, Deps, State, Locks, Level). +%% @doc runs `parse_dep/6' for a set of dependencies. +-spec parse_deps(Parent, Dir, Deps, State, Locks, Level) -> [rebar_app_info:t()] when + Parent :: root | binary(), + Dir :: file:filename(), + Deps :: [tuple() | atom() | binary()], % TODO: meta to source() | lock() + State :: rebar_state:t(), + Locks :: [tuple()], % TODO: meta to [lock()] + Level :: non_neg_integer(). parse_deps(Parent, DepsDir, Deps, State, Locks, Level) -> [parse_dep(Dep, Parent, DepsDir, State, Locks, Level) || Dep <- Deps]. +%% @doc for a given dep, return its app info record. The function +%% also has to choose whether to define the dep from its immediate spec +%% (if it is a newer thing) or from the locks specified in the lockfile. +-spec parse_dep(Dep, Parent, Dir, State, Locks, Level) -> rebar_app_info:t() when + Dep :: tuple() | atom() | binary(), % TODO: meta to source() | lock() + Parent :: root | binary(), + Dir :: file:filename(), + State :: rebar_state:t(), + Locks :: [tuple()], % TODO: meta to [lock()] + Level :: non_neg_integer(). parse_dep(Dep, Parent, DepsDir, State, Locks, Level) -> Name = case Dep of Dep when is_tuple(Dep) -> @@ -117,6 +158,14 @@ parse_dep(Dep, Parent, DepsDir, State, Locks, Level) -> end end. +%% @doc converts a dependency definition and a location for it on disk +%% into an app info tuple representing it. +-spec parse_dep(Parent, Dep, Dir, IsLock, State) -> rebar_app_info:t() when + Parent :: root | binary(), + Dep :: tuple() | atom() | binary(), % TODO: meta to source() | lock() + Dir :: file:filename(), + IsLock :: boolean(), + State :: rebar_state:t(). parse_dep(Parent, {Name, Vsn, {pkg, PkgName}}, DepsDir, IsLock, State) -> {PkgName1, PkgVsn} = {ec_cnv:to_binary(PkgName), ec_cnv:to_binary(Vsn)}, dep_to_app(Parent, DepsDir, Name, PkgVsn, {pkg, PkgName1, PkgVsn, undefined}, IsLock, State); @@ -152,6 +201,16 @@ parse_dep(Parent, {Name, Source, Level}, DepsDir, IsLock, State) when is_tuple(S parse_dep(_, Dep, _, _, _) -> throw(?PRV_ERROR({parse_dep, Dep})). +%% @doc convert a dependency that has just been fetched into +%% an app info record related to it +-spec dep_to_app(Parent, Dir, Name, Vsn, Source, IsLock, State) -> rebar_app_info:t() when + Parent :: root | binary(), + Dir :: file:filename(), + Name :: binary(), + Vsn :: iodata() | undefined, + Source :: tuple(), + IsLock :: boolean(), + State :: rebar_state:t(). dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), AppInfo = case rebar_app_info:discover(CheckoutsDir) of @@ -166,7 +225,7 @@ dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> not_found -> rebar_app_info:new(Parent, Name, Vsn, Dir, []) end, - update_source(AppInfo0, Source, State) + update_source(AppInfo0, Source, State) end, C = rebar_config:consult(rebar_app_info:dir(AppInfo)), AppInfo1 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), C), @@ -177,6 +236,11 @@ dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> AppInfo5 = rebar_app_info:profiles(AppInfo4, [default]), rebar_app_info:is_lock(AppInfo5, IsLock). +%% @doc sets the source for a given dependency or app along with metadata +%% around version if required. +-spec update_source(rebar_app_info:t(), Source, rebar_state:t()) -> + rebar_app_info:t() when + Source :: tuple() | atom() | binary(). % TODO: meta to source() update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) -> {PkgName1, PkgVsn1} = case PkgVsn of undefined -> @@ -203,6 +267,9 @@ update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) -> update_source(AppInfo, Source, _State) -> rebar_app_info:source(AppInfo, Source). +%% @doc grab the checksum for a given package +-spec fetch_checksum(atom(), string(), iodata() | undefined, rebar_state:t()) -> + iodata() | no_return(). fetch_checksum(PkgName, PkgVsn, Hash, State) -> try rebar_packages:registry_checksum({pkg, PkgName, PkgVsn, Hash}, State) @@ -213,6 +280,8 @@ fetch_checksum(PkgName, PkgVsn, Hash, State) -> rebar_packages:registry_checksum({pkg, PkgName, PkgVsn, Hash}, State) end. +%% @doc convert a given exception's payload into an io description. +-spec format_error(any()) -> iolist(). format_error({missing_package, Package}) -> io_lib:format("Package not found in registry: ~s", [Package]); format_error({parse_dep, Dep}) -> @@ -224,6 +293,10 @@ format_error(Error) -> %% Internal functions %% =================================================================== +%% @private find the correct version of a package based on the version +%% and name passed in. +-spec get_package(binary(), binary() | string(), rebar_state:t()) -> + term() | no_return(). get_package(Dep, Vsn, State) -> case rebar_packages:find_highest_matching(Dep, Vsn, ?PACKAGE_TABLE, State) of {ok, HighestDepVsn} -> @@ -232,6 +305,8 @@ get_package(Dep, Vsn, State) -> throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Dep)})) end. +%% @private checks that all the beam files have been properly +%% created. -spec has_all_beams(file:filename_all(), [module()]) -> true | ?PRV_ERROR({missing_module, module()}). has_all_beams(EbinDir, [Module | ModuleList]) -> diff --git a/src/rebar_base_compiler.erl b/src/rebar_base_compiler.erl index 5d54057..02d567c 100644 --- a/src/rebar_base_compiler.erl +++ b/src/rebar_base_compiler.erl @@ -36,21 +36,65 @@ format_error_source/2]). -define(DEFAULT_COMPILER_SOURCE_FORMAT, relative). +-type desc() :: term(). +-type loc() :: {line(), col()} | line(). +-type line() :: integer(). +-type col() :: integer(). +-type err_or_warn() :: {module(), desc()} | {loc(), module(), desc()}. + +-type compile_fn_ret() :: ok | {ok, [string()]} | skipped | term(). +-type compile_fn() :: fun((file:filename(), [{_,_}] | rebar_dict()) -> compile_fn_ret()). +-type compile_fn3() :: fun((file:filename(), file:filename(), [{_,_}] | rebar_dict()) + -> compile_fn_ret()). +-type error_tuple() :: {error, [string()], [string()]}. +-export_type([compile_fn/0, compile_fn_ret/0, error_tuple/0]). %% =================================================================== %% Public API %% =================================================================== +%% @doc Runs a compile job, applying `compile_fn()' to all files, +%% starting with `First' files, and then `RestFiles'. +-spec run(rebar_dict() | [{_,_}] , [First], [Next], compile_fn()) -> + compile_fn_ret() when + First :: file:filename(), + Next :: file:filename(). run(Config, FirstFiles, RestFiles, CompileFn) -> %% Compile the first files in sequence compile_each(FirstFiles++RestFiles, Config, CompileFn). +%% @doc Runs a compile job, applying `compile_fn3()' to all files, +%% starting with `First' files, and then the other content of `SourceDir'. +%% Files looked for are those ending in `SourceExt'. Results of the +%% compilation are put in `TargetDir' with the base file names +%% postfixed with `SourceExt'. +-spec run(rebar_dict() | [{_,_}] , [First], SourceDir, SourceExt, + TargetDir, TargetExt, compile_fn3()) -> compile_fn_ret() when + First :: file:filename(), + SourceDir :: file:filename(), + TargetDir :: file:filename(), + SourceExt :: string(), + TargetExt :: string(). run(Config, FirstFiles, SourceDir, SourceExt, TargetDir, TargetExt, Compile3Fn) -> run(Config, FirstFiles, SourceDir, SourceExt, TargetDir, TargetExt, Compile3Fn, [check_last_mod]). +%% @doc Runs a compile job, applying `compile_fn3()' to all files, +%% starting with `First' files, and then the other content of `SourceDir'. +%% Files looked for are those ending in `SourceExt'. Results of the +%% compilation are put in `TargetDir' with the base file names +%% postfixed with `SourceExt'. +%% Additional compile options can be passed in the last argument as +%% a proplist. +-spec run(rebar_dict() | [{_,_}] , [First], SourceDir, SourceExt, + TargetDir, TargetExt, compile_fn3(), [term()]) -> compile_fn_ret() when + First :: file:filename(), + SourceDir :: file:filename(), + TargetDir :: file:filename(), + SourceExt :: string(), + TargetExt :: string(). run(Config, FirstFiles, SourceDir, SourceExt, TargetDir, TargetExt, Compile3Fn, Opts) -> %% Convert simple extension to proper regex @@ -73,13 +117,26 @@ run(Config, FirstFiles, SourceDir, SourceExt, TargetDir, TargetExt, simple_compile_wrapper(S, Target, Compile3Fn, C, CheckLastMod) end). +%% @doc Format good compiler results with warnings to work with +%% module internals. Assumes that warnings are not treated as errors. +-spec ok_tuple(file:filename(), [string()]) -> {ok, [string()]}. ok_tuple(Source, Ws) -> {ok, format_warnings(Source, Ws)}. +%% @doc format error and warning strings for a given source file +%% according to user preferences. +-spec error_tuple(file:filename(), [Err], [Warn], rebar_dict() | [{_,_}]) -> + error_tuple() when + Err :: string(), + Warn :: string(). error_tuple(Source, Es, Ws, Opts) -> {error, format_errors(Source, Es), format_warnings(Source, Ws, Opts)}. +%% @doc from a given path, and based on the user-provided options, +%% format the file path according to the preferences. +-spec format_error_source(file:filename(), rebar_dict() | [{_,_}]) -> + file:filename(). format_error_source(Path, Opts) -> Type = case rebar_opts:get(Opts, compiler_source_format, ?DEFAULT_COMPILER_SOURCE_FORMAT) of @@ -98,6 +155,8 @@ format_error_source(Path, Opts) -> rebar_dir:make_relative_path(resolve_linked_source(Path), Cwd) end. +%% @private takes a filename and canonicalizes its path if it is a link. +-spec resolve_linked_source(file:filename()) -> file:filename(). resolve_linked_source(Src) -> {Dir, Base} = rebar_file_utils:split_dirname(Src), filename:join(rebar_file_utils:resolve_link(Dir), Base). @@ -106,6 +165,11 @@ resolve_linked_source(Src) -> %% Internal functions %% =================================================================== +%% @private if a check for last modifications is required, do the verification +%% and possibly skip the compile job. +-spec simple_compile_wrapper(Source, Target, compile_fn3(), [{_,_}] | rebar_dict(), boolean()) -> compile_fn_ret() when + Source :: file:filename(), + Target :: file:filename(). simple_compile_wrapper(Source, Target, Compile3Fn, Config, false) -> Compile3Fn(Source, Target, Config); simple_compile_wrapper(Source, Target, Compile3Fn, Config, true) -> @@ -116,18 +180,42 @@ simple_compile_wrapper(Source, Target, Compile3Fn, Config, true) -> skipped end. +%% @private take a basic source set of file fragments and a target location, +%% create a file path and name for a compile artifact. +-spec target_file(SourceFile, SourceDir, SourceExt, TargetDir, TargetExt) -> File when + SourceFile :: file:filename(), + SourceDir :: file:filename(), + TargetDir :: file:filename(), + SourceExt :: string(), + TargetExt :: string(), + File :: file:filename(). target_file(SourceFile, SourceDir, SourceExt, TargetDir, TargetExt) -> BaseFile = remove_common_path(SourceFile, SourceDir), filename:join([TargetDir, filename:basename(BaseFile, SourceExt) ++ TargetExt]). +%% @private removes the common prefix between two file paths. +%% The remainder of the first file path passed will have its ending returned +%% when either path starts diverging. +-spec remove_common_path(file:filename(), file:filename()) -> file:filename(). remove_common_path(Fname, Path) -> remove_common_path1(filename:split(Fname), filename:split(Path)). +%% @private given two lists of file fragments, discard the identical +%% prefixed sections, and return the final bit of the first operand +%% as a filename. +-spec remove_common_path1([string()], [string()]) -> file:filename(). remove_common_path1([Part | RestFilename], [Part | RestPath]) -> remove_common_path1(RestFilename, RestPath); remove_common_path1(FilenameParts, _) -> filename:join(FilenameParts). +%% @private runs the compile function `CompileFn' on every file +%% passed internally, along with the related project configuration. +%% If any errors are encountered, they're reported to stdout. +-spec compile_each([file:filename()], Config, CompileFn) -> Ret | no_return() when + Config :: [{_,_}] | rebar_dict(), + CompileFn :: compile_fn(), + Ret :: compile_fn_ret(). compile_each([], _Config, _CompileFn) -> ok; compile_each([Source | Rest], Config, CompileFn) -> @@ -148,12 +236,19 @@ compile_each([Source | Rest], Config, CompileFn) -> end, compile_each(Rest, Config, CompileFn). +%% @private Formats and returns errors ready to be output. +-spec format_errors(string(), [err_or_warn()]) -> [string()]. format_errors(Source, Errors) -> format_errors(Source, "", Errors). +%% @private Formats and returns warning strings ready to be output. +-spec format_warnings(string(), [err_or_warn()]) -> [string()]. format_warnings(Source, Warnings) -> format_warnings(Source, Warnings, []). +%% @private Formats and returns warnings; chooses the distinct format they +%% may have based on whether `warnings_as_errors' option is on. +-spec format_warnings(string(), [err_or_warn()], rebar_dict() | [{_,_}]) -> [string()]. format_warnings(Source, Warnings, Opts) -> %% `Opts' can be passed in both as a list or a dictionary depending %% on whether the first call to rebar_erlc_compiler was done with @@ -167,6 +262,11 @@ format_warnings(Source, Warnings, Opts) -> end, format_errors(Source, Prefix, Warnings). +%% @private output compiler errors if they're judged to be reportable. +-spec maybe_report(Reportable | term()) -> ok when + Reportable :: {{error, error_tuple()}, Source} | error_tuple() | ErrProps, + ErrProps :: [{error, string()} | Source, ...], + Source :: {source, string()}. maybe_report({{error, {error, _Es, _Ws}=ErrorsAndWarnings}, {source, _}}) -> maybe_report(ErrorsAndWarnings); maybe_report([{error, E}, {source, S}]) -> @@ -177,15 +277,23 @@ maybe_report({error, Es, Ws}) -> maybe_report(_) -> ok. +%% @private Outputs a bunch of strings, including a newline +-spec report([string()]) -> ok. report(Messages) -> lists:foreach(fun(Msg) -> io:format("~s~n", [Msg]) end, Messages). +%% private format compiler errors into proper outputtable strings +-spec format_errors(_, Extra, [err_or_warn()]) -> [string()] when + Extra :: string(). format_errors(_MainSource, Extra, Errors) -> [begin [format_error(Source, Extra, Desc) || Desc <- Descs] end || {Source, Descs} <- Errors]. +%% @private format compiler errors into proper outputtable strings +-spec format_error(file:filename(), Extra, err_or_warn()) -> string() when + Extra :: string(). format_error(Source, Extra, {{Line, Column}, Mod, Desc}) -> ErrorDesc = Mod:format_error(Desc), ?FMT("~s:~w:~w: ~s~s~n", [Source, Line, Column, Extra, ErrorDesc]); diff --git a/src/rebar_config.erl b/src/rebar_config.erl index 5a35b87..84f40d4 100644 --- a/src/rebar_config.erl +++ b/src/rebar_config.erl @@ -46,17 +46,25 @@ %% Public API %% =================================================================== +%% @doc reads the default config file. -spec consult() -> [any()]. consult() -> consult_file(config_file()). +%% @doc reads the default config file in a given directory. -spec consult(file:name()) -> [any()]. consult(Dir) -> consult_file(filename:join(Dir, config_file())). +%% @doc reads a given app file, including the `.script' variations, +%% if any can be found. +-spec consult_app_file(file:filename()) -> [any()]. consult_app_file(File) -> consult_file_(File). +%% @doc reads the lock file for the project, and re-formats its +%% content to match the internals for rebar3. +-spec consult_lock_file(file:filename()) -> [any()]. % TODO: refine lock() consult_lock_file(File) -> Terms = consult_file_(File), case Terms of @@ -80,6 +88,11 @@ consult_lock_file(File) -> read_attrs(Vsn, Locks, Attrs) end. +%% @private outputs a warning for a newer lockfile format than supported +%% at most once. +%% The warning can also be cancelled by configuring the `warn_config_vsn' +%% OTP env variable. +-spec warn_vsn_once() -> ok. warn_vsn_once() -> Warn = application:get_env(rebar, warn_config_vsn) =/= {ok, false}, application:set_env(rebar, warn_config_vsn, false), @@ -93,6 +106,11 @@ warn_vsn_once() -> end. +%% @doc Converts the internal format for locks into the multi-version +%% compatible one used within rebar3 lock files. +%% @end +%% TODO: refine type for lock() +-spec write_lock_file(file:filename(), [any()]) -> ok | {error, term()}. write_lock_file(LockFile, Locks) -> {NewLocks, Attrs} = write_attrs(Locks), %% Write locks in the beta format, at least until it's been long @@ -107,27 +125,41 @@ write_lock_file(LockFile, Locks) -> format_attrs(Attrs)])) end. -%% Attributes have a special formatting to ensure there's only one per +%% @private Attributes have a special formatting to ensure there's only one per %% line in terms of pkg_hash, so we disturb source diffing as little %% as possible. +-spec format_attrs([term()]) -> iodata(). format_attrs([]) -> []; format_attrs([{pkg_hash, Vals}|T]) -> [io_lib:format("{pkg_hash,[~n",[]), format_hashes(Vals), "]}", maybe_comma(T) | format_attrs(T)]. +%% @private format hashing in order to disturb source diffing as little +%% as possible +-spec format_hashes([term()]) -> iodata(). format_hashes([]) -> []; format_hashes([{Pkg,Hash}|T]) -> [" {", io_lib:format("~p",[Pkg]), ", ", io_lib:format("~p", [Hash]), "}", maybe_comma(T) | format_hashes(T)]. +%% @private add a comma if we're not done with the full list of terms +%% to convert. +-spec maybe_comma([term()]) -> iodata(). maybe_comma([]) -> ""; maybe_comma([_|_]) -> io_lib:format(",~n", []). +%% @private extract attributes from the lock file and integrate them +%% into the full-blow internal lock format +%% @end +%% TODO: refine typings for lock() +-spec read_attrs(_, [any()], [any()]) -> [any()]. read_attrs(_Vsn, Locks, Attrs) -> %% Beta copy does not know how to expand attributes, but %% is ready to support it. expand_locks(Locks, extract_pkg_hashes(Attrs)). +%% @private extract the package hashes from lockfile attributes, if any. +-spec extract_pkg_hashes(list()) -> [binary()]. extract_pkg_hashes(Attrs) -> Props = case Attrs of [First|_] -> First; @@ -135,6 +167,11 @@ extract_pkg_hashes(Attrs) -> end, proplists:get_value(pkg_hash, Props, []). +%% @private extract attributes from the lock file and integrate them +%% into the full-blow internal lock format +%% @end +%% TODO: refine typings for lock() +-spec expand_locks(list(), list()) -> list(). expand_locks([], _Hashes) -> []; expand_locks([{Name, {pkg,PkgName,Vsn}, Lvl} | Locks], Hashes) -> @@ -143,6 +180,9 @@ expand_locks([{Name, {pkg,PkgName,Vsn}, Lvl} | Locks], Hashes) -> expand_locks([Lock|Locks], Hashes) -> [Lock | expand_locks(Locks, Hashes)]. +%% @private split up extra attributes for locks out of the internal lock +%% structure for backwards compatibility reasons +-spec write_attrs(list()) -> {list(), list()}. write_attrs(Locks) -> %% No attribute known that needs to be taken out of the structure, %% just return terms as is. @@ -152,6 +192,9 @@ write_attrs(Locks) -> _ -> {NewLocks, [{pkg_hash, lists:sort(Hashes)}]} end. +%% @private split up extra attributes for locks out of the internal lock +%% structure for backwards compatibility reasons +-spec split_locks(list(), list(), [{_,binary()}]) -> {list(), list()}. split_locks([], Locks, Hashes) -> {lists:reverse(Locks), Hashes}; split_locks([{Name, {pkg,PkgName,Vsn,undefined}, Lvl} | Locks], LAcc, HAcc) -> @@ -161,11 +204,17 @@ split_locks([{Name, {pkg,PkgName,Vsn,Hash}, Lvl} | Locks], LAcc, HAcc) -> split_locks([Lock|Locks], LAcc, HAcc) -> split_locks(Locks, [Lock|LAcc], HAcc). +%% @doc reads a given config file, including the `.script' variations, +%% if any can be found, and asserts that the config format is in +%% a key-value format. +-spec consult_file(file:filename()) -> [{_,_}]. consult_file(File) -> Terms = consult_file_(File), true = verify_config_format(Terms), Terms. +%% @private reads a given file; if the file has a `.script'-postfixed +%% counterpart, it is evaluated along with the original file. -spec consult_file_(file:name()) -> [any()]. consult_file_(File) when is_binary(File) -> consult_file_(binary_to_list(File)); @@ -185,6 +234,9 @@ consult_file_(File) -> end end. +%% @private checks that a list is in a key-value format. +%% Raises an exception in any other case. +-spec verify_config_format([{_,_}]) -> true. verify_config_format([]) -> true; verify_config_format([{_Key, _Value} | T]) -> @@ -192,11 +244,14 @@ verify_config_format([{_Key, _Value} | T]) -> verify_config_format([Term | _]) -> throw(?PRV_ERROR({bad_config_format, Term})). -%% no lockfile +%% @doc takes an existing configuration and the content of a lockfile +%% and merges the locks into the config. +-spec merge_locks([{_,_}], list()) -> [{_,_}]. merge_locks(Config, []) -> +%% no lockfile Config; -%% lockfile with entries merge_locks(Config, Locks) -> + %% lockfile with entries ConfigDeps = proplists:get_value(deps, Config, []), %% We want the top level deps only from the lock file. %% This ensures deterministic overrides for configs. @@ -206,6 +261,8 @@ merge_locks(Config, Locks) -> NewDeps = find_newly_added(ConfigDeps, Locks), [{{locks, default}, Locks}, {{deps, default}, NewDeps++Deps} | Config]. +%% @doc convert a given exception's payload into an io description. +-spec format_error(any()) -> iolist(). format_error({bad_config_format, Term}) -> io_lib:format("Unable to parse config. Term is not in {Key, Value} format:~n~p", [Term]); format_error({bad_dep_name, Dep}) -> @@ -215,6 +272,8 @@ format_error({bad_dep_name, Dep}) -> %% Internal functions %% =================================================================== +%% @private consults a consult file, then executes its related script file +%% with the data returned from the consult. -spec consult_and_eval(File::file:name_all(), Script::file:name_all()) -> {ok, Terms::[term()]} | {error, Reason::term()}. @@ -234,18 +293,26 @@ consult_and_eval(File, Script) -> Error end. +%% @private drops the .script extension from a filename. +-spec remove_script_ext(file:filename()) -> file:filename(). remove_script_ext(F) -> filename:rootname(F, ".script"). +%% @private sets up bindings for evaluations from a KV list. +-spec bs([{_,_}]) -> erl_eval:binding_struct(). bs(Vars) -> lists:foldl(fun({K,V}, Bs) -> erl_eval:add_binding(K, V, Bs) end, erl_eval:new_bindings(), Vars). -%% Find deps that have been added to the config after the lock was created +%% @private Find deps that have been added to the config after the lock was created +-spec find_newly_added(list(), list()) -> list(). find_newly_added(ConfigDeps, LockedDeps) -> [D || {true, D} <- [check_newly_added(Dep, LockedDeps) || Dep <- ConfigDeps]]. +%% @private checks if a given dependency is not within the lock file. +%% TODO: refine types for dependencies +-spec check_newly_added(term(), list()) -> false | {true, term()}. check_newly_added({_, _}=Dep, LockedDeps) -> check_newly_added_(Dep, LockedDeps); check_newly_added({_, _, {pkg, _}}=Dep, LockedDeps) -> @@ -255,6 +322,10 @@ check_newly_added({Name, _, Source}, LockedDeps) -> check_newly_added(Dep, LockedDeps) -> check_newly_added_(Dep, LockedDeps). +%% @private checks if a given dependency is not within the lock file. +%% TODO: refine types for dependencies +%% @end +-spec check_newly_added_(term(), list()) -> false | {true, term()}. %% get [raw] deps out of the way check_newly_added_({Name, Source, Opts}, LockedDeps) when is_tuple(Source), is_list(Opts) -> @@ -306,6 +377,9 @@ check_newly_added_(Dep, LockedDeps) when is_atom(Dep) -> check_newly_added_(Dep, _) -> throw(?PRV_ERROR({bad_dep_name, Dep})). +%% @private returns the name/path of the default config file, or its +%% override from the OS ENV var `REBAR_CONFIG'. +-spec config_file() -> file:filename(). config_file() -> case os:getenv("REBAR_CONFIG") of false -> diff --git a/src/rebar_core.erl b/src/rebar_core.erl index da8c3e6..14c4906 100644 --- a/src/rebar_core.erl +++ b/src/rebar_core.erl @@ -24,6 +24,8 @@ %% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN %% THE SOFTWARE. %% ------------------------------------------------------------------- +%% @doc Module providing core functionality about command dispatch, namespacing, +%% and chaining for rebar3. -module(rebar_core). -export([init_command/2, process_namespace/2, process_command/2, do/2, format_error/1]). @@ -31,6 +33,12 @@ -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). +%% @doc initial command set up; based on the first fragment of the +%% command, dispatch to special environments. The keywords for +%% `do' and `as' are implicitly reserved here, barring them from +%% being used as other commands or namespaces. +-spec init_command(rebar_state:t(), atom()) -> + {ok, rebar_state:t()} | {error, term()}. init_command(State, do) -> process_command(rebar_state:namespace(State, default), do); init_command(State, as) -> @@ -43,6 +51,14 @@ init_command(State, Command) -> {error, Reason} end. +%% @doc parse the commands starting at the namespace level; +%% a namespace is found if the first keyword to match is not +%% belonging to an existing provider, and iff the keyword also +%% matches a registered namespace. +%% The command to run is returned last; for namespaces, some +%% magic is done implicitly calling `do' as an indirect dispatcher. +-spec process_namespace(rebar_state:t(), atom()) -> + {error, term()} | {ok, rebar_state:t(), atom()}. process_namespace(_State, as) -> {error, "Namespace 'as' is forbidden"}; process_namespace(State, Command) -> @@ -61,7 +77,15 @@ process_namespace(State, Command) -> {ok, rebar_state:namespace(State, default), Command} end. --spec process_command(rebar_state:t(), atom()) -> {ok, rebar_state:t()} | {error, string()} | {error, {module(), any()}}. +%% @doc Dispatches a given command based on the current state. +%% This requires mapping a command name to a specific provider. +%% `as' and `do' are still treated as special providers here. +%% Basic profile application may also be run. +%% +%% The function also takes care of expanding a provider to its +%% dependencies in the proper order. +-spec process_command(rebar_state:t(), atom()) -> + {ok, rebar_state:t()} | {error, string()} | {error, {module(), any()}}. process_command(State, Command) -> %% ? rebar_prv_install_deps:setup_env(State), Providers = rebar_state:providers(State), @@ -104,7 +128,11 @@ process_command(State, Command) -> end end. --spec do([{atom(), atom()}], rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()} | {error, {module(), any()}}. +%% @doc execute the selected providers. If a chain of providers +%% has been returned, run them one after the other, while piping +%% the state from the first into the next one. +-spec do([{atom(), atom()}], rebar_state:t()) -> + {ok, rebar_state:t()} | {error, string()} | {error, {module(), any()}}. do([], State) -> {ok, State}; do([ProviderName | Rest], State) -> @@ -142,6 +170,8 @@ do([ProviderName | Rest], State) -> {error, ProviderName} end. +%% @doc convert a given exception's payload into an io description. +-spec format_error(any()) -> iolist(). format_error({bad_provider_namespace, {Namespace, Name}}) -> io_lib:format("Undefined command ~s in namespace ~s", [Name, Namespace]); format_error({bad_provider_namespace, Name}) -> diff --git a/src/rebar_dir.erl b/src/rebar_dir.erl index 79a1c7f..32cb264 100644 --- a/src/rebar_dir.erl +++ b/src/rebar_dir.erl @@ -115,10 +115,16 @@ template_globals(State) -> template_dir(State) -> filename:join([global_config_dir(State), "templates"]). +%% @doc checks if the current working directory is the base directory +%% for the project. +-spec processing_base_dir(rebar_state:t()) -> boolean(). processing_base_dir(State) -> Cwd = get_cwd(), processing_base_dir(State, Cwd). +%% @doc checks if the passed in directory is the base directory for +%% the project. +-spec processing_base_dir(rebar_state:t(), file:filename()) -> boolean(). processing_base_dir(State, Dir) -> AbsDir = filename:absname(Dir), AbsDir =:= rebar_state:get(State, base_dir). @@ -150,11 +156,25 @@ make_normalized_path([H|T], NormalizedPath) -> _ -> make_normalized_path(T, [H|NormalizedPath]) end. +%% @doc take a source and a target path, and relativize the target path +%% onto the source. +%% +%% Example: +%% ``` +%% 1> rebar_dir:make_relative_path("a/b/c/d/file", "a/b/file"). +%% "c/d/file" +%% 2> rebar_dir:make_relative_path("a/b/file", "a/b/c/d/file"). +%% "../../file" +%% ''' +-spec make_relative_path(file:filename(), file:filename()) -> file:filename(). make_relative_path(Source, Target) -> AbsSource = make_normalized_path(Source), AbsTarget = make_normalized_path(Target), do_make_relative_path(filename:split(AbsSource), filename:split(AbsTarget)). +%% @private based on fragments of paths, replace the number of common +%% segments by `../' bits, and add the rest of the source alone after it +-spec do_make_relative_path([string()], [string()]) -> file:filename(). do_make_relative_path([H|T1], [H|T2]) -> do_make_relative_path(T1, T2); do_make_relative_path(Source, Target) -> diff --git a/src/rebar_dist_utils.erl b/src/rebar_dist_utils.erl index f462826..03f4392 100644 --- a/src/rebar_dist_utils.erl +++ b/src/rebar_dist_utils.erl @@ -25,7 +25,7 @@ short(Name, Opts) -> long(Name, Opts) -> start(Name, longnames, Opts). --spec find_options(rebar_state:state()) -> {Long, Short, Opts} when +-spec find_options(rebar_state:t()) -> {Long, Short, Opts} when Long :: atom(), Short :: atom(), Opts :: [{setcookie,term()}]. diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index f2467c5..8645641 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -300,9 +300,8 @@ canonical_path([_|Acc], [".."|Rest]) -> canonical_path(Acc, Rest); canonical_path([], [".."|Rest]) -> canonical_path([], Rest); canonical_path(Acc, [Component|Rest]) -> canonical_path([Component|Acc], Rest). -%% returns canonical target of path if path is a link, otherwise returns path +%% @doc returns canonical target of path if path is a link, otherwise returns path -spec resolve_link(string()) -> string(). - resolve_link(Path) -> case file:read_link(Path) of {ok, Target} -> @@ -310,9 +309,8 @@ resolve_link(Path) -> {error, _} -> Path end. -%% splits a path into dirname and basename +%% @doc splits a path into dirname and basename -spec split_dirname(string()) -> {string(), string()}. - split_dirname(Path) -> {filename:dirname(Path), filename:basename(Path)}. diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 5817817..8a84ce3 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -135,6 +135,9 @@ etag(Path) -> false end. +%% @doc returns the SSL options adequate for the project based on +%% its configuration, including for validation of certs. +-spec ssl_opts(string()) -> [term()]. ssl_opts(Url) -> case get_ssl_config() of ssl_verify_enabled -> @@ -143,6 +146,9 @@ ssl_opts(Url) -> [{verify, verify_none}] end. +%% @doc returns the SSL options adequate for the project based on +%% its configuration, including for validation of certs. +-spec ssl_opts(atom(), string()) -> [term()]. ssl_opts(ssl_verify_enabled, Url) -> case check_ssl_version() of true -> diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index a8a7ea0..9c5e8ac 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -101,6 +101,7 @@ do_(State) -> {error, Reason} end. +%% @doc convert a given exception's payload into an io description. -spec format_error(any()) -> iolist(). format_error({dep_app_not_found, AppDir, AppName}) -> io_lib:format("Dependency failure: Application ~s not found at the top level of directory ~s", [AppName, AppDir]); @@ -125,8 +126,14 @@ format_error({cycles, Cycles}) -> format_error(Reason) -> io_lib:format("~p", [Reason]). -%% Allows other providers to install deps in a given profile +%% @doc Allows other providers to install deps in a given profile %% manually, outside of what is provided by rebar3's deps tuple. +-spec handle_deps_as_profile(Profile, State, Deps, Upgrade) -> {Apps, State} when + Profile :: atom(), + State :: rebar_state:t(), + Deps :: [tuple() | atom() | binary()], % TODO: meta to source() | lock() + Upgrade :: boolean(), + Apps :: [rebar_app_info:t()]. handle_deps_as_profile(Profile, State, Deps, Upgrade) -> Locks = [], Level = 0, diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index f55f40f..d92f119 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -95,6 +95,12 @@ sort_deps(Deps) -> droplast(L) -> lists:reverse(tl(lists:reverse(L))). +%% @doc filtermap takes in a function that is either or both +%% a predicate and a map, and returns the matching and valid elements. +-spec filtermap(F, [In]) -> [Out] when + F :: fun((In) -> boolean() | {true, Out}), + In :: term(), + Out :: term(). filtermap(F, [Hd|Tail]) -> case F(Hd) of true -> @@ -114,11 +120,16 @@ is_arch(ArchRegex) -> false end. +%% @doc returns the sytem architecture, in strings like +%% `"19.0.4-x86_64-unknown-linux-gnu-64"'. +-spec get_arch() -> string(). get_arch() -> Words = wordsize(), otp_release() ++ "-" ++ erlang:system_info(system_architecture) ++ "-" ++ Words. +%% @doc returns the size of a word on the system, as a string +-spec wordsize() -> string(). wordsize() -> try erlang:system_info({wordsize, external}) of Val -> @@ -502,11 +513,10 @@ patch_on_windows(Cmd, Env) -> Cmd end. -%% -%% Given env. variable FOO we want to expand all references to -%% it in InStr. References can have two forms: $FOO and ${FOO} -%% The end of form $FOO is delimited with whitespace or eol -%% +%% @doc Given env. variable `FOO' we want to expand all references to +%% it in `InStr'. References can have two forms: `$FOO' and `${FOO}' +%% The end of form `$FOO' is delimited with whitespace or EOL +-spec expand_env_variable(string(), string(), term()) -> string(). expand_env_variable(InStr, VarName, RawVarValue) -> case string:chr(InStr, $$) of 0 -> @@ -748,6 +758,9 @@ remove_from_code_path(Paths) -> code:del_path(Path) end, Paths). +%% @doc Revert to only having the beams necessary for running rebar3 and +%% plugins in the path +-spec cleanup_code_path([string()]) -> true | {error, term()}. cleanup_code_path(OrigPath) -> CurrentPath = code:get_path(), AddedPaths = CurrentPath -- OrigPath, -- cgit v1.1 From f605b917b4752f77bb349101573b0f72dec28ad8 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 28 Nov 2016 11:37:55 -0500 Subject: Replace unprocessed ~n linebreaks Fixes #1392 --- src/rebar_prv_install_deps.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index a8a7ea0..84a0563 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -119,9 +119,9 @@ format_error({missing_package, Package}) -> format_error({cycles, Cycles}) -> Prints = [["applications: ", [io_lib:format("~s ", [Dep]) || Dep <- Cycle], - "depend on each other~n"] + "depend on each other\n"] || Cycle <- Cycles], - ["Dependency cycle(s) detected:~n", Prints]; + ["Dependency cycle(s) detected:\n", Prints]; format_error(Reason) -> io_lib:format("~p", [Reason]). -- cgit v1.1 From ffc2cf98d390a35cf3f61c7c0bf5f26e3552fd2c Mon Sep 17 00:00:00 2001 From: Artem Pervin Date: Wed, 30 Nov 2016 13:45:06 -0500 Subject: 1394: added fix for rebar_utils, moved setting of http_options into init_config, added unit tests --- src/rebar3.erl | 3 ++- src/rebar_pkg_resource.erl | 4 +++- src/rebar_utils.erl | 24 +++++++++++++++++++++--- 3 files changed, 26 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/rebar3.erl b/src/rebar3.erl index 4e7e284..1059904 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -145,6 +145,8 @@ run_aux(State, RawArgs) -> rebar_core:init_command(rebar_state:command_args(State10, Args), Task). init_config() -> + rebar_utils:set_httpc_options(), + %% Initialize logging system Verbosity = log_level(), ok = rebar_log:init(command_line, Verbosity), @@ -320,7 +322,6 @@ ensure_running(App, Caller) -> end. state_from_global_config(Config, GlobalConfigFile) -> - rebar_utils:set_httpc_options(), GlobalConfigTerms = rebar_config:consult_file(GlobalConfigFile), GlobalConfig = rebar_state:new(GlobalConfigTerms), diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 5817817..8e1713d 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -107,8 +107,10 @@ make_vsn(_) -> {error, "Replacing version of type pkg not supported."}. request(Url, ETag) -> + HttpOptions = [{ssl, ssl_opts(Url)}, {relaxed, true} | rebar_utils:get_proxy_auth()], + case httpc:request(get, {Url, [{"if-none-match", ETag} || ETag =/= false]++[{"User-Agent", rebar_utils:user_agent()}]}, - [{ssl, ssl_opts(Url)}, {relaxed, true}], + HttpOptions, [{body_format, binary}], rebar) of {ok, {{_Version, 200, _Reason}, Headers, Body}} -> diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index f55f40f..4b43911 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -70,7 +70,9 @@ info_useless/2, list_dir/1, user_agent/0, - reread_config/1]). + reread_config/1, + get_proxy_auth/0, + set_proxy_auth/1]). %% for internal use only -export([otp_release/0]). @@ -825,8 +827,9 @@ set_httpc_options(_, []) -> ok; set_httpc_options(Scheme, Proxy) -> - {ok, {_, _, Host, Port, _, _}} = http_uri:parse(Proxy), - httpc:set_options([{Scheme, {{Host, Port}, []}}], rebar). + {ok, {_, UserInfo, Host, Port, _, _}} = http_uri:parse(Proxy), + httpc:set_options([{Scheme, {{Host, Port}, []}}], rebar), + set_proxy_auth(UserInfo). url_append_path(Url, ExtraPath) -> case http_uri:parse(Url) of @@ -865,3 +868,18 @@ list_dir(Dir) -> true -> file:list_dir_all(Dir); false -> file:list_dir(Dir) end. + +set_proxy_auth([]) -> + ok; +set_proxy_auth(UserInfo) -> + Idx = string:chr(UserInfo, $:), + Username = string:sub_string(UserInfo, 1, Idx-1), + Password = string:sub_string(UserInfo, Idx+1), + %% password may contain url encoded characters, need to decode them first + application:set_env(rebar, proxy_auth, [{proxy_auth, {Username, http_uri:decode(Password)}}]). + +get_proxy_auth() -> + case application:get_env(rebar, proxy_auth) of + undefined -> []; + {ok, ProxyAuth} -> ProxyAuth + end. -- cgit v1.1 From c0184eae706f6c2bf7bcf8fc1db03c0569cb73c1 Mon Sep 17 00:00:00 2001 From: Artem Pervin Date: Thu, 1 Dec 2016 10:27:07 -0500 Subject: 1394: refined export list and tests --- src/rebar_utils.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 4b43911..a0a44d1 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -71,8 +71,8 @@ list_dir/1, user_agent/0, reread_config/1, - get_proxy_auth/0, - set_proxy_auth/1]). + get_proxy_auth/0]). + %% for internal use only -export([otp_release/0]). -- cgit v1.1 From 03832379d6e78c3788c305f43728b485410ac5e4 Mon Sep 17 00:00:00 2001 From: Ted Burghart Date: Wed, 30 Nov 2016 17:22:42 -0500 Subject: Addresses https://github.com/erlang/rebar3/issues/1397 Ensures merged compiler options end up in the correct order to maintain profile precedence. Moves the merge functionality from rebar_opts:merge_opts/2 to a standalone function to ease extension and debugging. --- src/rebar_opts.erl | 93 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 60 insertions(+), 33 deletions(-) (limited to 'src') diff --git a/src/rebar_opts.erl b/src/rebar_opts.erl index b02a504..444b760 100644 --- a/src/rebar_opts.erl +++ b/src/rebar_opts.erl @@ -101,43 +101,70 @@ merge_opts(Profile, NewOpts, OldOpts) -> end. merge_opts(NewOpts, OldOpts) -> - dict:merge(fun(deps, _NewValue, OldValue) -> - OldValue; - ({deps, _}, NewValue, _OldValue) -> - NewValue; - (plugins, NewValue, _OldValue) -> - NewValue; - ({plugins, _}, NewValue, _OldValue) -> - NewValue; - (profiles, NewValue, OldValue) -> - dict:to_list(merge_opts(dict:from_list(NewValue), dict:from_list(OldValue))); - (mib_first_files, Value, Value) -> - Value; - (mib_first_files, NewValue, OldValue) -> - OldValue ++ NewValue; - (relx, NewValue, OldValue) -> - rebar_utils:tup_umerge(OldValue, NewValue); - (_Key, NewValue, OldValue) when is_list(NewValue) -> - case io_lib:printable_list(NewValue) of - true when NewValue =:= [] -> - case io_lib:printable_list(OldValue) of - true -> - NewValue; - false -> - OldValue - end; - true -> - NewValue; - false -> - rebar_utils:tup_umerge(NewValue, OldValue) - end; - (_Key, NewValue, _OldValue) -> - NewValue - end, NewOpts, OldOpts). + dict:merge(fun merge_opt/3, NewOpts, OldOpts). %% Internal functions %% +%% Function for dict:merge/3 (in merge_opts/2) to merge options by priority. +%% +merge_opt(deps, _NewValue, OldValue) -> + OldValue; +merge_opt({deps, _}, NewValue, _OldValue) -> + NewValue; +merge_opt(plugins, NewValue, _OldValue) -> + NewValue; +merge_opt({plugins, _}, NewValue, _OldValue) -> + NewValue; +merge_opt(profiles, NewValue, OldValue) -> + dict:to_list(merge_opts(dict:from_list(NewValue), dict:from_list(OldValue))); +merge_opt(mib_first_files, Value, Value) -> + Value; +merge_opt(mib_first_files, NewValue, OldValue) -> + OldValue ++ NewValue; +merge_opt(relx, NewValue, OldValue) -> + rebar_utils:tup_umerge(OldValue, NewValue); +merge_opt(Key, NewValue, OldValue) + when Key == erl_opts; Key == eunit_compile_opts; Key == ct_compile_opts -> + merge_erl_opts(lists:reverse(OldValue), NewValue); +merge_opt(_Key, NewValue, OldValue) when is_list(NewValue) -> + case io_lib:printable_list(NewValue) of + true when NewValue =:= [] -> + case io_lib:printable_list(OldValue) of + true -> + NewValue; + false -> + OldValue + end; + true -> + NewValue; + false -> + rebar_utils:tup_umerge(NewValue, OldValue) + end; +merge_opt(_Key, NewValue, _OldValue) -> + NewValue. + +%% +%% Merge Erlang compiler options such that the result +%% a) Doesn't contain duplicates. +%% b) Resulting options are ordered by increasing precedence as expected by +%% the compiler. +%% The first parameter is the lower precedence options, in reverse order, to +%% be merged with the higher-precedence options in the second parameter. +%% +merge_erl_opts([Opt | Opts], []) -> + merge_erl_opts(Opts, [Opt]); +merge_erl_opts([Opt | Opts], Merged) -> + case lists:member(Opt, Merged) of + true -> + merge_erl_opts(Opts, Merged); + _ -> + merge_erl_opts(Opts, [Opt | Merged]) + end; +merge_erl_opts([], Merged) -> + Merged. + +%% %% Filter a list of erl_opts platform_define options such that only %% those which match the provided architecture regex are returned. %% -- cgit v1.1 From 74d290b9c2be6352813c71ad6036a24ddacfae79 Mon Sep 17 00:00:00 2001 From: Roberto Aloi Date: Tue, 6 Dec 2016 11:31:28 +0100 Subject: Restrict regexp to match on files starting with '._' --- src/rebar_base_compiler.erl | 2 +- src/rebar_erlc_compiler.erl | 2 +- src/rebar_prv_eunit.erl | 2 +- src/rebar_templater.erl | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/rebar_base_compiler.erl b/src/rebar_base_compiler.erl index 5d54057..bfd79a5 100644 --- a/src/rebar_base_compiler.erl +++ b/src/rebar_base_compiler.erl @@ -54,7 +54,7 @@ run(Config, FirstFiles, SourceDir, SourceExt, TargetDir, TargetExt, run(Config, FirstFiles, SourceDir, SourceExt, TargetDir, TargetExt, Compile3Fn, Opts) -> %% Convert simple extension to proper regex - SourceExtRe = "^[^._].*\\" ++ SourceExt ++ [$$], + SourceExtRe = "^(?!._).*\\" ++ SourceExt ++ [$$], Recursive = proplists:get_value(recursive, Opts, true), %% Find all possible source files diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index 325bb4f..d4d257f 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -48,7 +48,7 @@ -type compile_opt() :: {recursive, boolean()}. -define(DEFAULT_OUTDIR, "ebin"). --define(RE_PREFIX, "^[^._]"). +-define(RE_PREFIX, "^(?!._)"). %% =================================================================== %% Public API diff --git a/src/rebar_prv_eunit.erl b/src/rebar_prv_eunit.erl index 0908ec9..6fdf33e 100644 --- a/src/rebar_prv_eunit.erl +++ b/src/rebar_prv_eunit.erl @@ -18,7 +18,7 @@ %% we need to modify app_info state before compile -define(DEPS, [lock]). --define(DEFAULT_TEST_REGEX, "^[^._].*\\.erl\$"). +-define(DEFAULT_TEST_REGEX, "^(?!._).*\\.erl\$"). %% =================================================================== %% Public API diff --git a/src/rebar_templater.erl b/src/rebar_templater.erl index 299b957..7e0aae4 100644 --- a/src/rebar_templater.erl +++ b/src/rebar_templater.erl @@ -33,7 +33,7 @@ -include("rebar.hrl"). --define(TEMPLATE_RE, "^[^._].*\\.template\$"). +-define(TEMPLATE_RE, "^(?!._).*\\.template\$"). %% =================================================================== %% Public API -- cgit v1.1 From 85e00f2a434d3295d7a683838cda002403f5af17 Mon Sep 17 00:00:00 2001 From: Siri Hansen Date: Mon, 28 Nov 2016 11:45:21 +0100 Subject: Improve merge of command line options and config options Bug: option 'spec' is not specifically handled when merging options from the command line with options from rebar.config. Due to this, if the config specifies a 'spec', then this will take precedence over any 'dir' and/or 'suite' on the command line. This commit takes special care of all options that can be used to select tests - meaning that if any of the options 'spec', 'dir', 'suite', 'group' or 'case' are specified on the command line, then all 'spec', 'dir', 'suite', 'group' and 'case' options in rebar.config will be ignored. --- src/rebar_prv_common_test.erl | 77 +++++++++++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 25 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index be31e8c..5fa7ae4 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -227,20 +227,42 @@ select_tests(State, ProjectApps, CmdOpts, CfgOpts) -> rebar_utils:reread_config(Configs), code:set_path(OldPath), - Merged = lists:ukeymerge(1, - lists:ukeysort(1, CmdOpts), - lists:ukeysort(1, CfgOpts)), - %% make sure `dir` and/or `suite` from command line go in as - %% a pair overriding both `dir` and `suite` from config if - %% they exist - Opts = case {proplists:get_value(suite, CmdOpts), proplists:get_value(dir, CmdOpts)} of - {undefined, undefined} -> Merged; - {_Suite, undefined} -> lists:keydelete(dir, 1, Merged); - {undefined, _Dir} -> lists:keydelete(suite, 1, Merged); - {_Suite, _Dir} -> Merged - end, + Opts = merge_opts(CmdOpts,CfgOpts), discover_tests(State, ProjectApps, Opts). +%% Merge the option lists from command line and rebar.config: +%% +%% - Options set on the command line will replace the same options if +%% set in rebar.config. +%% +%% - Special care is taken with options that select which tests to +%% run - ANY such option on the command line will replace ALL such +%% options in the config. +%% +%% Note that if 'spec' is given, common_test will ignore all 'dir', +%% 'suite', 'group' and 'case', so there is no need to explicitly +%% remove any options from the command line. +%% +%% All faulty combinations of options are also handled by +%% common_test and are not taken into account here. +merge_opts(CmdOpts0, CfgOpts0) -> + TestSelectOpts = [spec,dir,suite,group,testcase], + CmdOpts = lists:ukeysort(1, CmdOpts0), + CfgOpts1 = lists:ukeysort(1, CfgOpts0), + CfgOpts = case is_any_defined(TestSelectOpts,CmdOpts) of + false -> + CfgOpts1; + true -> + [Opt || Opt={K,_} <- CfgOpts1, + not lists:member(K,TestSelectOpts)] + end, + lists:ukeymerge(1, CmdOpts, CfgOpts). + +is_any_defined([Key|Keys],Opts) -> + proplists:is_defined(Key,Opts) orelse is_any_defined(Keys,Opts); +is_any_defined([],_Opts) -> + false. + sys_config_list(CmdOpts, CfgOpts) -> CmdSysConfigs = split_string(proplists:get_value(sys_config, CmdOpts, "")), case proplists:get_value(sys_config, CfgOpts, []) of @@ -253,11 +275,10 @@ sys_config_list(CmdOpts, CfgOpts) -> end. discover_tests(State, ProjectApps, Opts) -> - case {proplists:get_value(suite, Opts), proplists:get_value(dir, Opts)} of - %% no dirs or suites defined, try using `$APP/test` and `$ROOT/test` - %% as suites - {undefined, undefined} -> {ok, [default_tests(State, ProjectApps)|Opts]}; - {_, _} -> {ok, Opts} + case is_any_defined([spec,dir,suite],Opts) of + %% no tests defined, try using `$APP/test` and `$ROOT/test` as dirs + false -> {ok, [default_tests(State, ProjectApps)|Opts]}; + true -> {ok, Opts} end. default_tests(State, ProjectApps) -> @@ -397,14 +418,20 @@ readable(State) -> end. test_dirs(State, Apps, Opts) -> - case {proplists:get_value(suite, Opts), proplists:get_value(dir, Opts)} of - {Suites, undefined} -> set_compile_dirs(State, Apps, {suite, Suites}); - {undefined, Dirs} -> set_compile_dirs(State, Apps, {dir, Dirs}); - {Suites, Dir} when is_integer(hd(Dir)) -> - set_compile_dirs(State, Apps, join(Suites, Dir)); - {Suites, [Dir]} when is_integer(hd(Dir)) -> - set_compile_dirs(State, Apps, join(Suites, Dir)); - {_Suites, _Dirs} -> {error, "Only a single directory may be specified when specifying suites"} + case proplists:get_value(spec, Opts) of + undefined -> + case {proplists:get_value(suite, Opts), proplists:get_value(dir, Opts)} of + {Suites, undefined} -> set_compile_dirs(State, Apps, {suite, Suites}); + {undefined, Dirs} -> set_compile_dirs(State, Apps, {dir, Dirs}); + {Suites, Dir} when is_integer(hd(Dir)) -> + set_compile_dirs(State, Apps, join(Suites, Dir)); + {Suites, [Dir]} when is_integer(hd(Dir)) -> + set_compile_dirs(State, Apps, join(Suites, Dir)); + {_Suites, _Dirs} -> {error, "Only a single directory may be specified when specifying suites"} + end; + _Specs -> + %% Currently not adding any directories from the spec files. + {ok, rebar_state:project_apps(State, Apps)} end. join(Suite, Dir) when is_integer(hd(Suite)) -> -- cgit v1.1 From 3fd29af16cd55defaabfd08bf0bafe275a68a7c7 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Wed, 7 Dec 2016 11:03:00 -0800 Subject: add `get-deps` provider a no-op provider that depends on lock that is slightly more discoverable and user friendly --- src/rebar.app.src | 1 + src/rebar_prv_get_deps.erl | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 src/rebar_prv_get_deps.erl (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 5b735cf..74efe97 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -52,6 +52,7 @@ rebar_prv_edoc, rebar_prv_escriptize, rebar_prv_eunit, + rebar_prv_get_deps, rebar_prv_help, rebar_prv_install_deps, rebar_prv_local_install, diff --git a/src/rebar_prv_get_deps.erl b/src/rebar_prv_get_deps.erl new file mode 100644 index 0000000..020e50b --- /dev/null +++ b/src/rebar_prv_get_deps.erl @@ -0,0 +1,37 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et + +-module(rebar_prv_get_deps). + +-behaviour(provider). + +-export([init/1, + do/1, + format_error/1]). + +-define(PROVIDER, 'get-deps'). +-define(DEPS, [lock]). + +%% =================================================================== +%% Public API +%% =================================================================== + +-spec init(rebar_state:t()) -> {ok, rebar_state:t()}. +init(State) -> + Provider = providers:create([{name, ?PROVIDER}, + {module, ?MODULE}, + {deps, ?DEPS}, + {bare, true}, + {example, "rebar3 get-deps"}, + {short_desc, "Fetch dependencies."}, + {desc, "Fetch project dependencies."}, + {opts, []}, + {profiles, []}]), + {ok, rebar_state:add_provider(State, Provider)}. + +-spec do(rebar_state:t()) -> {ok, rebar_state:t()}. +do(State) -> {ok, State}. + +-spec format_error(any()) -> iolist(). +format_error(Reason) -> + io_lib:format("~p", [Reason]). \ No newline at end of file -- cgit v1.1 From 7780933cfb78fde053ec7c69ba8a20ac14535cf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Fri, 2 Dec 2016 14:36:00 +0100 Subject: Use different git commands for different git versions The option --single-branch was introduced in git version 1.7.10 and thus rebar3 cannot fetch git dependencies on systems where earlier git versions are install. This commit will select other git clone commands if an earlier git version is detected. If the git version cannot be determined rebar3 falls back on the previous behavior and uses --single-branch. --- src/rebar_git_resource.erl | 72 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 59 insertions(+), 13 deletions(-) (limited to 'src') diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl index acb9ec0..201b8b6 100644 --- a/src/rebar_git_resource.erl +++ b/src/rebar_git_resource.erl @@ -107,28 +107,50 @@ download(Dir, {git, Url, ""}, State) -> download(Dir, {git, Url, {branch, "master"}}, State); download(Dir, {git, Url, {branch, Branch}}, _State) -> ok = filelib:ensure_dir(Dir), - rebar_utils:sh(?FMT("git clone ~s ~s -b ~s --single-branch", - [rebar_utils:escape_chars(Url), - rebar_utils:escape_chars(filename:basename(Dir)), - rebar_utils:escape_chars(Branch)]), - [{cd, filename:dirname(Dir)}]); + git_clone(branch, git_vsn(), Url, Dir, Branch); download(Dir, {git, Url, {tag, Tag}}, _State) -> ok = filelib:ensure_dir(Dir), - rebar_utils:sh(?FMT("git clone ~s ~s -b ~s --single-branch", - [rebar_utils:escape_chars(Url), - rebar_utils:escape_chars(filename:basename(Dir)), - rebar_utils:escape_chars(Tag)]), - [{cd, filename:dirname(Dir)}]); + git_clone(tag, git_vsn(), Url, Dir, Tag); download(Dir, {git, Url, {ref, Ref}}, _State) -> ok = filelib:ensure_dir(Dir), + git_clone(ref, git_vsn(), Url, Dir, Ref); +download(Dir, {git, Url, Rev}, _State) -> + ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), + ok = filelib:ensure_dir(Dir), + git_clone(rev, git_vsn(), Url, Dir, Rev). + +%% Use different git clone commands depending on git --version +git_clone(branch,Vsn,Url,Dir,Branch) when Vsn >= {1,7,10}; Vsn =:= undefined -> + rebar_utils:sh(?FMT("git clone ~s ~s -b ~s --single-branch", + [rebar_utils:escape_chars(Url), + rebar_utils:escape_chars(filename:basename(Dir)), + rebar_utils:escape_chars(Branch)]), + [{cd, filename:dirname(Dir)}]); +git_clone(branch,_Vsn,Url,Dir,Branch) -> + rebar_utils:sh(?FMT("git clone ~s ~s -b ~s", + [rebar_utils:escape_chars(Url), + rebar_utils:escape_chars(filename:basename(Dir)), + rebar_utils:escape_chars(Branch)]), + [{cd, filename:dirname(Dir)}]); +git_clone(tag,Vsn,Url,Dir,Tag) when Vsn >= {1,7,10}; Vsn =:= undefined -> + rebar_utils:sh(?FMT("git clone ~s ~s -b ~s --single-branch", + [rebar_utils:escape_chars(Url), + rebar_utils:escape_chars(filename:basename(Dir)), + rebar_utils:escape_chars(Tag)]), + [{cd, filename:dirname(Dir)}]); +git_clone(tag,_Vsn,Url,Dir,Tag) -> + rebar_utils:sh(?FMT("git clone ~s ~s -b ~s", + [rebar_utils:escape_chars(Url), + rebar_utils:escape_chars(filename:basename(Dir)), + rebar_utils:escape_chars(Tag)]), + [{cd, filename:dirname(Dir)}]); +git_clone(ref,_Vsn,Url,Dir,Ref) -> rebar_utils:sh(?FMT("git clone -n ~s ~s", [rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]), rebar_utils:sh(?FMT("git checkout -q ~s", [Ref]), [{cd, Dir}]); -download(Dir, {git, Url, Rev}, _State) -> - ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), - ok = filelib:ensure_dir(Dir), +git_clone(rev,_Vsn,Url,Dir,Rev) -> rebar_utils:sh(?FMT("git clone -n ~s ~s", [rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), @@ -136,6 +158,30 @@ download(Dir, {git, Url, Rev}, _State) -> rebar_utils:sh(?FMT("git checkout -q ~s", [rebar_utils:escape_chars(Rev)]), [{cd, Dir}]). +git_vsn() -> + case application:get_env(rebar, git_vsn) of + {ok, Vsn} -> Vsn; + undefined -> + Vsn = git_vsn_fetch(), + application:set_env(rebar, git_vsn, Vsn), + Vsn + end. + +git_vsn_fetch() -> + case rebar_utils:sh("git --version",[]) of + {ok, VsnStr} -> + case re:run(VsnStr, "git version\\h+(\\d)\\.(\\d)\\.(\\d).*",[{capture,[1,2,3],list}]) of + {match,[Maj,Min,Patch]} -> + {list_to_integer(Maj), + list_to_integer(Min), + list_to_integer(Patch)}; + nomatch -> + undefined + end; + {error, _} -> + undefined + end. + make_vsn(Dir) -> case collect_default_refcount(Dir) of Vsn={plain, _} -> -- cgit v1.1 From a0e7ff2eb95a8093f666d310bd5df6395a243bd8 Mon Sep 17 00:00:00 2001 From: Siri Hansen Date: Thu, 8 Dec 2016 15:46:35 +0100 Subject: Add directory of testspec as extra_src_dir This is necessary in order to automatically get the testspec included as an artifact (i.e. copied to the _build dir) in the case when it is stored in another directory than 'test'. --- src/rebar_prv_common_test.erl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index 5fa7ae4..17358a5 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -429,9 +429,9 @@ test_dirs(State, Apps, Opts) -> set_compile_dirs(State, Apps, join(Suites, Dir)); {_Suites, _Dirs} -> {error, "Only a single directory may be specified when specifying suites"} end; - _Specs -> - %% Currently not adding any directories from the spec files. - {ok, rebar_state:project_apps(State, Apps)} + Specs -> + set_compile_dirs(State, Apps, {spec, Specs}) + end. join(Suite, Dir) when is_integer(hd(Suite)) -> @@ -451,15 +451,15 @@ set_compile_dirs(State, Apps, {dir, Dirs}) -> F = fun(Dir, {S, A}) -> maybe_inject_test_dir(S, [], A, Dir) end, {NewState, NewApps} = lists:foldl(F, {State, Apps}, Dirs), {ok, rebar_state:project_apps(NewState, NewApps)}; -set_compile_dirs(State, Apps, {suite, Suites}) -> - %% suites with dir component - Dirs = find_suite_dirs(Suites), +set_compile_dirs(State, Apps, {Type, Files}) when Type==spec; Type==suite -> + %% specs or suites with dir component + Dirs = find_file_dirs(Files), F = fun(Dir, {S, A}) -> maybe_inject_test_dir(S, [], A, Dir) end, {NewState, NewApps} = lists:foldl(F, {State, Apps}, Dirs), {ok, rebar_state:project_apps(NewState, NewApps)}. -find_suite_dirs(Suites) -> - AllDirs = lists:map(fun(S) -> filename:dirname(filename:absname(S)) end, Suites), +find_file_dirs(Files) -> + AllDirs = lists:map(fun(F) -> filename:dirname(filename:absname(F)) end, Files), %% eliminate duplicates lists:usort(AllDirs). -- cgit v1.1 From a58857162c3a17356a3d634ac28f0e15c3a1a6f9 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 8 Dec 2016 12:41:22 -0500 Subject: Bump to 3.3.3 --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 5b735cf..6f5b6a9 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.3.3"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 51c008fb84ca3313234482be72c80d5aebb0b830 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 8 Dec 2016 12:42:24 -0500 Subject: Go back to git-based versioning --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 6f5b6a9..5b735cf 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.3.3"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From e427a835301c41a1403cf6b9e6c363ceae3e781c Mon Sep 17 00:00:00 2001 From: Siri Hansen Date: Fri, 9 Dec 2016 14:33:43 +0100 Subject: Translate path to testspec This is a bugfix. It makes sure that the given path to a testspec is translated so common_test will pick the spec from the _build directory, and not from the source tree. --- src/rebar_prv_common_test.erl | 61 ++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 33 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index 17358a5..cb33492 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -513,44 +513,39 @@ inject_test_dir(Opts, Dir) -> rebar_opts:set(Opts, extra_src_dirs, ExtraSrcDirs ++ [Dir]). translate_paths(State, Opts) -> - case {proplists:get_value(suite, Opts), proplists:get_value(dir, Opts)} of - {_Suites, undefined} -> translate_suites(State, Opts, []); - {undefined, _Dirs} -> translate_dirs(State, Opts, []); - %% both dirs and suites are defined, only translate dir paths - _ -> translate_dirs(State, Opts, []) + case proplists:get_value(spec, Opts) of + undefined -> + case {proplists:get_value(suite, Opts), proplists:get_value(dir, Opts)} of + {_Suites, undefined} -> translate_paths(State, suite, Opts, []); + {undefined, _Dirs} -> translate_paths(State, dir, Opts, []); + %% both dirs and suites are defined, only translate dir paths + _ -> translate_paths(State, dir, Opts, []) + end; + _Specs -> + translate_paths(State, spec, Opts, []) end. -translate_dirs(_State, [], Acc) -> lists:reverse(Acc); -translate_dirs(State, [{dir, Dir}|Rest], Acc) when is_integer(hd(Dir)) -> - %% single dir - Apps = rebar_state:project_apps(State), - translate_dirs(State, Rest, [{dir, translate(State, Apps, Dir)}|Acc]); -translate_dirs(State, [{dir, Dirs}|Rest], Acc) -> - %% multiple dirs - Apps = rebar_state:project_apps(State), - NewDirs = {dir, lists:map(fun(Dir) -> translate(State, Apps, Dir) end, Dirs)}, - translate_dirs(State, Rest, [NewDirs|Acc]); -translate_dirs(State, [Test|Rest], Acc) -> - translate_dirs(State, Rest, [Test|Acc]). - -translate_suites(_State, [], Acc) -> lists:reverse(Acc); -translate_suites(State, [{suite, Suite}|Rest], Acc) when is_integer(hd(Suite)) -> - %% single suite +translate_paths(_State, _Type, [], Acc) -> lists:reverse(Acc); +translate_paths(State, Type, [{Type, Val}|Rest], Acc) when is_integer(hd(Val)) -> + %% single file or dir + translate_paths(State, Type, [{Type, [Val]}|Rest], Acc); +translate_paths(State, dir, [{dir, Dirs}|Rest], Acc) -> Apps = rebar_state:project_apps(State), - translate_suites(State, Rest, [{suite, translate_suite(State, Apps, Suite)}|Acc]); -translate_suites(State, [{suite, Suites}|Rest], Acc) -> - %% multiple suites + New = {dir, lists:map(fun(Dir) -> translate(State, Apps, Dir) end, Dirs)}, + translate_paths(State, dir, Rest, [New|Acc]); +translate_paths(State, Type, [{Type, Files}|Rest], Acc) -> + %% Type = suites | specs Apps = rebar_state:project_apps(State), - NewSuites = {suite, lists:map(fun(Suite) -> translate_suite(State, Apps, Suite) end, Suites)}, - translate_suites(State, Rest, [NewSuites|Acc]); -translate_suites(State, [Test|Rest], Acc) -> - translate_suites(State, Rest, [Test|Acc]). - -translate_suite(State, Apps, Suite) -> - Dirname = filename:dirname(Suite), - Basename = filename:basename(Suite), + New = {Type, lists:map(fun(File) -> translate_file(State, Apps, File) end, Files)}, + translate_paths(State, Type, Rest, [New|Acc]); +translate_paths(State, Type, [Test|Rest], Acc) -> + translate_paths(State, Type, Rest, [Test|Acc]). + +translate_file(State, Apps, File) -> + Dirname = filename:dirname(File), + Basename = filename:basename(File), case Dirname of - "." -> Suite; + "." -> File; _ -> filename:join([translate(State, Apps, Dirname), Basename]) end. -- cgit v1.1 From fc12b06d27d0e1df4719a2e38e241f36637d369e Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 9 Dec 2016 09:16:15 -0500 Subject: Fix regex match for ignored file The regex mistakenly matched too many files (any character followed by an underscore) rather than only files starting in '._' This properly escapes the expressions to work in all cases. --- src/rebar_base_compiler.erl | 2 +- src/rebar_erlc_compiler.erl | 2 +- src/rebar_prv_eunit.erl | 2 +- src/rebar_templater.erl | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/rebar_base_compiler.erl b/src/rebar_base_compiler.erl index bfd79a5..9aa7419 100644 --- a/src/rebar_base_compiler.erl +++ b/src/rebar_base_compiler.erl @@ -54,7 +54,7 @@ run(Config, FirstFiles, SourceDir, SourceExt, TargetDir, TargetExt, run(Config, FirstFiles, SourceDir, SourceExt, TargetDir, TargetExt, Compile3Fn, Opts) -> %% Convert simple extension to proper regex - SourceExtRe = "^(?!._).*\\" ++ SourceExt ++ [$$], + SourceExtRe = "^(?!\\._).*\\" ++ SourceExt ++ [$$], Recursive = proplists:get_value(recursive, Opts, true), %% Find all possible source files diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index d4d257f..95573fd 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -48,7 +48,7 @@ -type compile_opt() :: {recursive, boolean()}. -define(DEFAULT_OUTDIR, "ebin"). --define(RE_PREFIX, "^(?!._)"). +-define(RE_PREFIX, "^(?!\\._)"). %% =================================================================== %% Public API diff --git a/src/rebar_prv_eunit.erl b/src/rebar_prv_eunit.erl index 6fdf33e..a9db12e 100644 --- a/src/rebar_prv_eunit.erl +++ b/src/rebar_prv_eunit.erl @@ -18,7 +18,7 @@ %% we need to modify app_info state before compile -define(DEPS, [lock]). --define(DEFAULT_TEST_REGEX, "^(?!._).*\\.erl\$"). +-define(DEFAULT_TEST_REGEX, "^(?!\\._).*\\.erl\$"). %% =================================================================== %% Public API diff --git a/src/rebar_templater.erl b/src/rebar_templater.erl index 7e0aae4..e64ce71 100644 --- a/src/rebar_templater.erl +++ b/src/rebar_templater.erl @@ -33,7 +33,7 @@ -include("rebar.hrl"). --define(TEMPLATE_RE, "^(?!._).*\\.template\$"). +-define(TEMPLATE_RE, "^(?!\\._).*\\.template\$"). %% =================================================================== %% Public API -- cgit v1.1 From 6466c324493366cc8c01ee27c08a959e272a4c2c Mon Sep 17 00:00:00 2001 From: Luis Rascao Date: Sun, 11 Dec 2016 17:55:05 +0000 Subject: shell: don't crash apps that use release version operators Like for instance: {app, "0.1.0", '='} --- src/rebar_prv_shell.erl | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src') diff --git a/src/rebar_prv_shell.erl b/src/rebar_prv_shell.erl index 31b2e17..0a3e18a 100644 --- a/src/rebar_prv_shell.erl +++ b/src/rebar_prv_shell.erl @@ -352,11 +352,15 @@ boot_apps(Apps) -> normalize_load_apps([]) -> []; normalize_load_apps([{App, _} | T]) -> [App | normalize_load_apps(T)]; normalize_load_apps([{App, _Vsn, load} | T]) -> [App | normalize_load_apps(T)]; +normalize_load_apps([{App, _Vsn, Operator} | T]) when is_atom(Operator) -> + [App | normalize_load_apps(T)]; normalize_load_apps([App | T]) when is_atom(App) -> [App | normalize_load_apps(T)]. normalize_boot_apps([]) -> []; normalize_boot_apps([{_App, load} | T]) -> normalize_boot_apps(T); normalize_boot_apps([{_App, _Vsn, load} | T]) -> normalize_boot_apps(T); +normalize_boot_apps([{App, _Vsn, Operator} | T]) when is_atom(Operator) -> + [App | normalize_boot_apps(T)]; normalize_boot_apps([{App, _Vsn} | T]) -> [App | normalize_boot_apps(T)]; normalize_boot_apps([App | T]) when is_atom(App) -> [App | normalize_boot_apps(T)]. -- cgit v1.1 From a18340c6eed6e62237c4ec63f15daacfe86dc86c Mon Sep 17 00:00:00 2001 From: Siri Hansen Date: Tue, 13 Dec 2016 12:51:01 +0100 Subject: Allow using relative path to suite in project root --- src/rebar_prv_common_test.erl | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index cb33492..9db8106 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -529,26 +529,13 @@ translate_paths(_State, _Type, [], Acc) -> lists:reverse(Acc); translate_paths(State, Type, [{Type, Val}|Rest], Acc) when is_integer(hd(Val)) -> %% single file or dir translate_paths(State, Type, [{Type, [Val]}|Rest], Acc); -translate_paths(State, dir, [{dir, Dirs}|Rest], Acc) -> - Apps = rebar_state:project_apps(State), - New = {dir, lists:map(fun(Dir) -> translate(State, Apps, Dir) end, Dirs)}, - translate_paths(State, dir, Rest, [New|Acc]); translate_paths(State, Type, [{Type, Files}|Rest], Acc) -> - %% Type = suites | specs Apps = rebar_state:project_apps(State), - New = {Type, lists:map(fun(File) -> translate_file(State, Apps, File) end, Files)}, + New = {Type, lists:map(fun(File) -> translate(State, Apps, File) end, Files)}, translate_paths(State, Type, Rest, [New|Acc]); translate_paths(State, Type, [Test|Rest], Acc) -> translate_paths(State, Type, Rest, [Test|Acc]). -translate_file(State, Apps, File) -> - Dirname = filename:dirname(File), - Basename = filename:basename(File), - case Dirname of - "." -> File; - _ -> filename:join([translate(State, Apps, Dirname), Basename]) - end. - translate(State, [App|Rest], Path) -> case rebar_file_utils:path_from_ancestor(Path, rebar_app_info:dir(App)) of {ok, P} -> filename:join([rebar_app_info:out_dir(App), P]); -- cgit v1.1 From b99c15d29755efcb2a573ddbd9eb9cd2d42dc054 Mon Sep 17 00:00:00 2001 From: getong Date: Wed, 14 Dec 2016 20:09:02 +0800 Subject: the releases website has changed --- src/rebar_prv_shell.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_shell.erl b/src/rebar_prv_shell.erl index 31b2e17..7d1ee07 100644 --- a/src/rebar_prv_shell.erl +++ b/src/rebar_prv_shell.erl @@ -339,7 +339,7 @@ reread_config(State) -> boot_apps(Apps) -> ?WARN("The rebar3 shell is a development tool; to deploy " "applications in production, consider using releases " - "(http://www.rebar3.org/v3.0/docs/releases)", []), + "(http://www.rebar3.org/docs/releases)", []), Normalized = normalize_boot_apps(Apps), Res = [application:ensure_all_started(App) || App <- Normalized], _ = [?INFO("Booted ~p", [App]) -- cgit v1.1 From 8ae17c483d86f151d69091546b3577381662e27e Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 16 Dec 2016 21:02:55 -0500 Subject: Fix Alisdair's review, add more types and docs --- src/rebar_app_discover.erl | 6 ++- src/rebar_config.erl | 6 +-- src/rebar_digraph.erl | 22 ++++++++++- src/rebar_dir.erl | 91 +++++++++++++++++++++++++++++++++++++--------- src/rebar_dist_utils.erl | 9 +++++ 5 files changed, 109 insertions(+), 25 deletions(-) (limited to 'src') diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index acefdb4..fd55960 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -13,7 +13,9 @@ -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). -%% @doc from the base directory, +%% @doc from the base directory, find all the applications +%% at the top level and their dependencies based on the configuration +%% and profile information. -spec do(rebar_state:t(), [file:filename()]) -> rebar_state:t(). do(State, LibDirs) -> BaseDir = rebar_state:dir(State), @@ -158,7 +160,7 @@ project_app_config(AppInfo, State) -> Opts = maybe_reset_hooks(Dir, rebar_state:opts(State), State), {C, rebar_state:opts(State, Opts)}. -%% @doc Here we check if the app is at the root of the project. +%% @private Check if the app is at the root of the project. %% If it is, then drop the hooks from the config so they aren't run twice -spec maybe_reset_hooks(file:filename(), Opts, rebar_state:t()) -> Opts when Opts :: rebar_dict(). diff --git a/src/rebar_config.erl b/src/rebar_config.erl index 84f40d4..97e27ab 100644 --- a/src/rebar_config.erl +++ b/src/rebar_config.erl @@ -125,9 +125,9 @@ write_lock_file(LockFile, Locks) -> format_attrs(Attrs)])) end. -%% @private Attributes have a special formatting to ensure there's only one per -%% line in terms of pkg_hash, so we disturb source diffing as little -%% as possible. +%% @private Because attributes for packages are fairly large, there is the need +%% for a special formatting to ensure there's only one entry per lock file +%% line and that diffs are generally stable. -spec format_attrs([term()]) -> iodata(). format_attrs([]) -> []; format_attrs([{pkg_hash, Vals}|T]) -> diff --git a/src/rebar_digraph.erl b/src/rebar_digraph.erl index 363253a..a827735 100644 --- a/src/rebar_digraph.erl +++ b/src/rebar_digraph.erl @@ -1,3 +1,5 @@ +%%% @doc build a digraph of applications in order to figure out dependency +%%% and compile order. -module(rebar_digraph). -export([compile_order/1 @@ -7,7 +9,9 @@ -include("rebar.hrl"). -%% Sort apps with topological sort to get proper build order +%% @doc Sort apps with topological sort to get proper build order +-spec compile_order([rebar_app_info:t()]) -> + {ok, [rebar_app_info:t()]} | {error, no_sort | {cycles, [[binary(),...]]}}. compile_order(Apps) -> Graph = digraph:new(), lists:foreach(fun(App) -> @@ -33,6 +37,11 @@ compile_order(Apps) -> true = digraph:delete(Graph), Order. +%% @private Add a package and its dependencies to an existing digraph +-spec add(digraph:graph(), {PkgName, [Dep]}) -> ok when + PkgName :: binary(), + Dep :: {Name, term()} | Name, + Name :: atom() | iodata(). add(Graph, {PkgName, Deps}) -> case digraph:vertex(Graph, PkgName) of false -> @@ -57,6 +66,8 @@ add(Graph, {PkgName, Deps}) -> digraph:add_edge(Graph, V, V3) end, Deps). +%% @doc based on a list of vertices and edges, build a digraph. +-spec restore_graph({[digraph:vertex()], [digraph:edge()]}) -> digraph:graph(). restore_graph({Vs, Es}) -> Graph = digraph:new(), lists:foreach(fun({V, LastUpdated}) -> @@ -67,6 +78,8 @@ restore_graph({Vs, Es}) -> end, Es), Graph. +%% @doc convert a given exception's payload into an io description. +-spec format_error(any()) -> iolist(). format_error(no_solution) -> io_lib:format("No solution for packages found.", []). @@ -74,22 +87,27 @@ format_error(no_solution) -> %% Internal Functions %%==================================================================== +%% @doc alias for `digraph_utils:subgraph/2'. subgraph(Graph, Vertices) -> digraph_utils:subgraph(Graph, Vertices). +%% @private from a list of app names, fetch the proper app info records +%% for them. -spec names_to_apps([atom()], [rebar_app_info:t()]) -> [rebar_app_info:t()]. names_to_apps(Names, Apps) -> [element(2, App) || App <- [find_app_by_name(Name, Apps) || Name <- Names], App =/= error]. +%% @private fetch the proper app info record for a given app name. -spec find_app_by_name(atom(), [rebar_app_info:t()]) -> {ok, rebar_app_info:t()} | error. find_app_by_name(Name, Apps) -> ec_lists:find(fun(App) -> rebar_app_info:name(App) =:= Name end, Apps). -%% The union of all entries in the applications list for an app and +%% @private The union of all entries in the applications list for an app and %% the deps listed in its rebar.config is all deps that may be needed %% for building the app. +-spec all_apps_deps(rebar_app_info:t()) -> [binary()]. all_apps_deps(App) -> Applications = lists:usort([atom_to_binary(X, utf8) || X <- rebar_app_info:applications(App)]), Deps = lists:usort(lists:map(fun({Name, _}) -> Name; (Name) -> Name end, rebar_app_info:deps(App))), diff --git a/src/rebar_dir.erl b/src/rebar_dir.erl index 32cb264..069d8fd 100644 --- a/src/rebar_dir.erl +++ b/src/rebar_dir.erl @@ -1,3 +1,4 @@ +%%% @doc utility functions for directory and path handling of all kind. -module(rebar_dir). -export([base_dir/1, @@ -29,10 +30,14 @@ -include("rebar.hrl"). +%% @doc returns the directory root for build artifacts +%% for the current profile, such as `_build/default/'. -spec base_dir(rebar_state:t()) -> file:filename_all(). base_dir(State) -> profile_dir(rebar_state:opts(State), rebar_state:current_profiles(State)). +%% @doc returns the directory root for build artifacts for a given set +%% of profiles. -spec profile_dir(rebar_dict(), [atom()]) -> file:filename_all(). profile_dir(Opts, Profiles) -> {BaseDir, ProfilesStrings} = case [ec_cnv:to_list(P) || P <- Profiles] of @@ -46,25 +51,36 @@ profile_dir(Opts, Profiles) -> ProfilesDir = string:join(ProfilesStrings, "+"), filename:join(BaseDir, ProfilesDir). +%% @doc returns the directory where dependencies should be placed +%% given the current profile. -spec deps_dir(rebar_state:t()) -> file:filename_all(). deps_dir(State) -> filename:join(base_dir(State), rebar_state:get(State, deps_dir, ?DEFAULT_DEPS_DIR)). +%% @doc returns the directory where a dependency should be placed +%% given the current profile, based on its app name. Expects to be passed +%% the result of `deps_dir/1' as a first argument. -spec deps_dir(file:filename_all(), file:filename_all()) -> file:filename_all(). deps_dir(DepsDir, App) -> filename:join(DepsDir, App). +%% @doc returns the absolute path for the project root (by default, +%% the current working directory for the currently running escript). root_dir(State) -> filename:absname(rebar_state:get(State, root_dir, ?DEFAULT_ROOT_DIR)). +%% @doc returns the expected location of the `_checkouts' directory. -spec checkouts_dir(rebar_state:t()) -> file:filename_all(). checkouts_dir(State) -> filename:join(root_dir(State), rebar_state:get(State, checkouts_dir, ?DEFAULT_CHECKOUTS_DIR)). +%% @doc returns the expected location of a given app in the checkouts +%% directory for the project. -spec checkouts_dir(rebar_state:t(), file:filename_all()) -> file:filename_all(). checkouts_dir(State, App) -> filename:join(checkouts_dir(State), App). +%% @doc Returns the directory where plugins are located. -spec plugins_dir(rebar_state:t()) -> file:filename_all(). plugins_dir(State) -> case lists:member(global, rebar_state:current_profiles(State)) of @@ -74,33 +90,50 @@ plugins_dir(State) -> filename:join(base_dir(State), rebar_state:get(State, plugins_dir, ?DEFAULT_PLUGINS_DIR)) end. +%% @doc returns the list of relative path where the project applications can +%% be located. -spec lib_dirs(rebar_state:t()) -> file:filename_all(). lib_dirs(State) -> rebar_state:get(State, project_app_dirs, ?DEFAULT_PROJECT_APP_DIRS). +%% @doc returns the user's home directory. +-spec home_dir() -> file:filename_all(). home_dir() -> {ok, [[Home]]} = init:get_argument(home), Home. +%% @doc returns the directory where the global configuration files for rebar3 +%% may be stored. +-spec global_config_dir(rebar_state:t()) -> file:filename_all(). global_config_dir(State) -> Home = home_dir(), rebar_state:get(State, global_rebar_dir, filename:join([Home, ".config", "rebar3"])). +%% @doc returns the path of the global rebar.config file +-spec global_config(rebar_state:t()) -> file:filename_all(). global_config(State) -> filename:join(global_config_dir(State), "rebar.config"). +%% @doc returns the default path of the global rebar.config file +-spec global_config() -> file:filename_all(). global_config() -> Home = home_dir(), filename:join([Home, ".config", "rebar3", "rebar.config"]). +%% @doc returns the location for the global cache directory -spec global_cache_dir(rebar_dict()) -> file:filename_all(). global_cache_dir(Opts) -> Home = home_dir(), rebar_opts:get(Opts, global_rebar_dir, filename:join([Home, ".cache", "rebar3"])). +%% @doc appends the cache directory to the path passed to this function. +-spec local_cache_dir(file:filename_all()) -> file:filename_all(). local_cache_dir(Dir) -> filename:join(Dir, ".rebar3"). +%% @doc returns the current working directory, with some specific +%% conversions and handling done to be cross-platform compatible. +-spec get_cwd() -> file:filename_all(). get_cwd() -> {ok, Dir} = file:get_cwd(), %% On windows cwd may return capital letter for drive, @@ -109,9 +142,14 @@ get_cwd() -> %% cwd as soon as it possible. filename:join([Dir]). +%% @doc returns the file location for the global template +%% configuration variables file. +-spec template_globals(rebar_state:t()) -> file:filename_all(). template_globals(State) -> filename:join([global_config_dir(State), "templates", "globals"]). +%% @doc returns the location for the global template directory +-spec template_dir(rebar_state:t()) -> file:filename_all(). template_dir(State) -> filename:join([global_config_dir(State), "templates"]). @@ -129,6 +167,8 @@ processing_base_dir(State, Dir) -> AbsDir = filename:absname(Dir), AbsDir =:= rebar_state:get(State, base_dir). +%% @doc make a path absolute +-spec make_absolute_path(file:filename()) -> file:filename(). make_absolute_path(Path) -> case filename:pathtype(Path) of absolute -> @@ -142,11 +182,16 @@ make_absolute_path(Path) -> filename:join([Dir, Path]) end. +%% @doc normalizing a path removes all of the `..' and the +%% `.' segments it may contain. +-spec make_normalized_path(file:filename()) -> file:filename(). make_normalized_path(Path) -> AbsPath = make_absolute_path(Path), Components = filename:split(AbsPath), make_normalized_path(Components, []). +%% @private drops path fragments for normalization +-spec make_normalized_path([string()], [string()]) -> file:filename(). make_normalized_path([], NormalizedPath) -> filename:join(lists:reverse(NormalizedPath)); make_normalized_path([H|T], NormalizedPath) -> @@ -181,37 +226,42 @@ do_make_relative_path(Source, Target) -> Base = lists:duplicate(max(length(Target) - 1, 0), ".."), filename:join(Base ++ Source). -%%%----------------------------------------------------------------- -%%% 'src_dirs' and 'extra_src_dirs' can be configured with options +%%% @doc +%%% `src_dirs' and `extra_src_dirs' can be configured with options %%% like this: -%%% +%%% ``` %%% {src_dirs,[{"foo",[{recursive,false}]}]} %%% {extra_src_dirs,[{"bar",[recursive]}]} (equivalent to {recursive,true}) -%%% -%%% src_dirs/1,2 and extra_src_dirs/1,2 return only the list of -%%% directories for the 'src_dirs' and 'extra_src_dirs' options -%%% respectively, while src_dirs_opts/2 return the options list for -%%% the given directory, no matter if it is configured as 'src_dirs' or -%%% 'extra_src_dirs'. -%%% +%%% ''' +%%% `src_dirs/1,2' and `extra_src_dirs/1,2' return only the list of +%%% directories for the `src_dirs' and `extra_src_dirs' options +%%% respectively, while `src_dirs_opts/2' returns the options list for +%%% the given directory, no matter if it is configured as `src_dirs' or +%%% `extra_src_dirs'. -spec src_dirs(rebar_dict()) -> list(file:filename_all()). src_dirs(Opts) -> src_dirs(Opts, []). +%% @doc same as `src_dirs/1', but allows to pass in a list of default options. -spec src_dirs(rebar_dict(), list(file:filename_all())) -> list(file:filename_all()). src_dirs(Opts, Default) -> src_dirs(src_dirs, Opts, Default). +%% @doc same as `src_dirs/1', but for the `extra_src_dirs' options -spec extra_src_dirs(rebar_dict()) -> list(file:filename_all()). extra_src_dirs(Opts) -> extra_src_dirs(Opts, []). +%% @doc same as `src_dirs/2', but for the `extra_src_dirs' options -spec extra_src_dirs(rebar_dict(), list(file:filename_all())) -> list(file:filename_all()). extra_src_dirs(Opts, Default) -> src_dirs(extra_src_dirs, Opts, Default). +%% @private agnostic version of src_dirs and extra_src_dirs. src_dirs(Type, Opts, Default) -> lists:usort([case D0 of {D,_} -> D; _ -> D0 end || D0 <- raw_src_dirs(Type,Opts,Default)]). +%% @private extracts the un-formatted src_dirs or extra_src_dirs +%% options as configured. raw_src_dirs(Type, Opts, Default) -> ErlOpts = rebar_opts:erl_opts(Opts), Vs = proplists:get_all_values(Type, ErlOpts), @@ -220,19 +270,23 @@ raw_src_dirs(Type, Opts, Default) -> Dirs -> Dirs end. +%% @doc returns all the source directories (`src_dirs' and +%% `extra_src_dirs'). -spec all_src_dirs(rebar_dict()) -> list(file:filename_all()). all_src_dirs(Opts) -> all_src_dirs(Opts, [], []). +%% @doc returns all the source directories (`src_dirs' and +%% `extra_src_dirs') while being able to configure defaults for both. -spec all_src_dirs(rebar_dict(), list(file:filename_all()), list(file:filename_all())) -> list(file:filename_all()). all_src_dirs(Opts, SrcDefault, ExtraDefault) -> lists:usort(src_dirs(Opts, SrcDefault) ++ extra_src_dirs(Opts, ExtraDefault)). -%%%----------------------------------------------------------------- +%%% @doc %%% Return the list of options for the given src directory %%% If the same option is given multiple times for a directory in the -%%% config, the priority order is: first occurence of 'src_dirs' -%%% followed by first occurence of 'extra_src_dirs'. +%%% config, the priority order is: first occurence of `src_dirs' +%%% followed by first occurence of `extra_src_dirs'. -spec src_dir_opts(rebar_dict(), file:filename_all()) -> [{atom(),term()}]. src_dir_opts(Opts, Dir) -> RawSrcDirs = raw_src_dirs(src_dirs, Opts, []), @@ -241,7 +295,7 @@ src_dir_opts(Opts, Dir) -> D==Dir], lists:ukeysort(1,proplists:unfold(lists:append(AllOpts))). -%%%----------------------------------------------------------------- +%%% @doc %%% Return the value of the 'recursive' option for the given directory. %%% If not given, the value of 'recursive' in the 'erlc_compiler' %%% options is used, and finally the default is 'true'. @@ -254,16 +308,17 @@ recursive(Opts, Dir) -> R = proplists:get_value(recursive, DirOpts, Default), R. -%% given a path if that path is an ancestor of an app dir return the path relative to that -%% apps outdir. if the path is not an ancestor to any app dirs but is an ancestor of the -%% project root return the path relative to the project base_dir. if it is not an ancestor +%% @doc given a path if that path is an ancestor of an app dir, return the path relative to that +%% apps outdir. If the path is not an ancestor to any app dirs but is an ancestor of the +%% project root, return the path relative to the project base_dir. If it is not an ancestor %% of either return it unmodified -spec retarget_path(rebar_state:t(), string()) -> string(). - retarget_path(State, Path) -> ProjectApps = rebar_state:project_apps(State), retarget_path(State, Path, ProjectApps). +%% @private worker for retarget_path/2 +%% @end %% not relative to any apps in project, check to see it's relative to %% project root retarget_path(State, Path, []) -> diff --git a/src/rebar_dist_utils.erl b/src/rebar_dist_utils.erl index 03f4392..93edf9d 100644 --- a/src/rebar_dist_utils.erl +++ b/src/rebar_dist_utils.erl @@ -7,6 +7,9 @@ %%%%%%%%%%%%%%%%%% %%% PUBLIC API %%% %%%%%%%%%%%%%%%%%% + +%% @doc allows to pick whether to use a short or long name, and +%% starts the distributed mode for it. -spec either(Name::atom(), SName::atom(), Opts::[{setcookie,term()}]) -> atom(). either(undefined, undefined, _) -> 'nonode@nohost'; @@ -19,12 +22,18 @@ either(undefined, SName, Opts) -> either(_, _, _) -> ?ABORT("Cannot have both short and long node names defined", []). +%% @doc starts a node with a short name. +-spec short(SName::atom(), Opts::[{setcookie,term()}]) -> term(). short(Name, Opts) -> start(Name, shortnames, Opts). +%% @doc starts a node with a long name. +-spec long(Name::atom(), Opts::[{setcookie,term()}]) -> term(). long(Name, Opts) -> start(Name, longnames, Opts). +%% @doc utility function to extract all distribution options +%% from a rebar3 state tuple. -spec find_options(rebar_state:t()) -> {Long, Short, Opts} when Long :: atom(), Short :: atom(), -- cgit v1.1 From 4e52ce58e505ba57f2188e34a002d70de1bd6ff5 Mon Sep 17 00:00:00 2001 From: Siri Hansen Date: Fri, 16 Dec 2016 10:34:36 +0100 Subject: Add all dirs from test spec Parse given test specs and add all spec- and suite directories as extra_src_dirs in order to ensure that all these directories are copied to the _build area and the suites are compiled. Specs located in the project- or app root are explicitly copied to the _build area in order to avoid recursive copying of the complete directory tree. --- src/rebar_prv_common_test.erl | 81 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 69 insertions(+), 12 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index 9db8106..4a9ba25 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -429,9 +429,18 @@ test_dirs(State, Apps, Opts) -> set_compile_dirs(State, Apps, join(Suites, Dir)); {_Suites, _Dirs} -> {error, "Only a single directory may be specified when specifying suites"} end; - Specs -> - set_compile_dirs(State, Apps, {spec, Specs}) - + Specs0 -> + case get_dirs_from_specs(Specs0) of + {ok,{Specs,SuiteDirs}} -> + {State1,Apps1} = set_compile_dirs1(State, Apps, + {dir, SuiteDirs}), + {State2,Apps2} = set_compile_dirs1(State1, Apps1, + {spec, Specs}), + [maybe_copy_spec(State2,Apps2,S) || S <- Specs], + {ok, rebar_state:project_apps(State2, Apps2)}; + Error -> + Error + end end. join(Suite, Dir) when is_integer(hd(Suite)) -> @@ -439,24 +448,25 @@ join(Suite, Dir) when is_integer(hd(Suite)) -> join(Suites, Dir) -> {suite, lists:map(fun(S) -> filename:join([Dir, S]) end, Suites)}. -set_compile_dirs(State, Apps, {dir, Dir}) when is_integer(hd(Dir)) -> +set_compile_dirs(State, Apps, What) -> + {NewState,NewApps} = set_compile_dirs1(State, Apps, What), + {ok, rebar_state:project_apps(NewState, NewApps)}. + +set_compile_dirs1(State, Apps, {dir, Dir}) when is_integer(hd(Dir)) -> %% single directory %% insert `Dir` into an app if relative, or the base state if not %% app relative but relative to the root or not at all if outside %% project scope - {NewState, NewApps} = maybe_inject_test_dir(State, [], Apps, Dir), - {ok, rebar_state:project_apps(NewState, NewApps)}; -set_compile_dirs(State, Apps, {dir, Dirs}) -> + maybe_inject_test_dir(State, [], Apps, Dir); +set_compile_dirs1(State, Apps, {dir, Dirs}) -> %% multiple directories F = fun(Dir, {S, A}) -> maybe_inject_test_dir(S, [], A, Dir) end, - {NewState, NewApps} = lists:foldl(F, {State, Apps}, Dirs), - {ok, rebar_state:project_apps(NewState, NewApps)}; -set_compile_dirs(State, Apps, {Type, Files}) when Type==spec; Type==suite -> + lists:foldl(F, {State, Apps}, Dirs); +set_compile_dirs1(State, Apps, {Type, Files}) when Type==spec; Type==suite -> %% specs or suites with dir component Dirs = find_file_dirs(Files), F = fun(Dir, {S, A}) -> maybe_inject_test_dir(S, [], A, Dir) end, - {NewState, NewApps} = lists:foldl(F, {State, Apps}, Dirs), - {ok, rebar_state:project_apps(NewState, NewApps)}. + lists:foldl(F, {State, Apps}, Dirs). find_file_dirs(Files) -> AllDirs = lists:map(fun(F) -> filename:dirname(filename:absname(F)) end, Files), @@ -507,11 +517,58 @@ copy_bare_suites(From, To) -> ok = rebar_file_utils:cp_r(SrcFiles, To), rebar_file_utils:cp_r(DataDirs, To). +maybe_copy_spec(State, [App|Apps], Spec) -> + case rebar_file_utils:path_from_ancestor(filename:dirname(Spec), rebar_app_info:dir(App)) of + {ok, []} -> + ok = rebar_file_utils:cp_r([Spec],rebar_app_info:out_dir(App)); + {ok,_} -> + ok; + {error,badparent} -> + maybe_copy_spec(State, Apps, Spec) + end; +maybe_copy_spec(State, [], Spec) -> + case rebar_file_utils:path_from_ancestor(filename:dirname(Spec), rebar_state:dir(State)) of + {ok, []} -> + ExtrasDir = filename:join([rebar_dir:base_dir(State), "extras"]), + ok = rebar_file_utils:cp_r([Spec],ExtrasDir); + _R -> + ok + end. + inject_test_dir(Opts, Dir) -> %% append specified test targets to app defined `extra_src_dirs` ExtraSrcDirs = rebar_opts:get(Opts, extra_src_dirs, []), rebar_opts:set(Opts, extra_src_dirs, ExtraSrcDirs ++ [Dir]). +get_dirs_from_specs(Specs) -> + case get_tests_from_specs(Specs) of + {ok,Tests} -> + {SpecLists,NodeRunSkipLists} = lists:unzip(Tests), + SpecList = lists:append(SpecLists), + NodeRunSkipList = lists:append(NodeRunSkipLists), + RunList = lists:append([R || {_,R,_} <- NodeRunSkipList]), + DirList = [element(1,R) || R <- RunList], + {ok,{SpecList,DirList}}; + Error -> + Error + end. + +get_tests_from_specs(Specs) -> + case erlang:function_exported(ct_testspec,get_tests,1) of + true -> + ct_testspec:get_tests(Specs); + false -> + case ct_testspec:collect_tests_from_file(Specs,true) of + Tests when is_list(Tests) -> + {ok,[{S,ct_testspec:prepare_tests(R)} || {S,R} <- Tests]}; + R when is_tuple(R), element(1,R)==testspec -> + %% R15 + {ok,[{Specs,ct_testspec:prepare_tests(R)}]}; + Error -> + Error + end + end. + translate_paths(State, Opts) -> case proplists:get_value(spec, Opts) of undefined -> -- cgit v1.1 From d6a34f9d68b962205170c78dd1ef7b97f8475180 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Thu, 22 Dec 2016 00:48:49 -0800 Subject: don't filter eunit test modules based on file extension stops the eunit provider from filtering out test modules based on the file extension. previously, it was hardcoded to expect all test files ended in `.erl`. this change allows for endings like `.lfe` and `.beam` --- src/rebar_prv_eunit.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_eunit.erl b/src/rebar_prv_eunit.erl index a9db12e..f65f700 100644 --- a/src/rebar_prv_eunit.erl +++ b/src/rebar_prv_eunit.erl @@ -198,8 +198,8 @@ dedupe_tests({AppMods, TestMods}) -> %% for each modules in TestMods create a test if there is not a module %% in AppMods that will trigger it F = fun(Mod) -> - M = filename:basename(Mod, ".erl"), - MatchesTest = fun(Dir) -> filename:basename(Dir, ".erl") ++ "_tests" == M end, + M = filename:rootname(filename:basename(Mod)), + MatchesTest = fun(Dir) -> filename:rootname(filename:basename(Dir)) ++ "_tests" == M end, case lists:any(MatchesTest, AppMods) of false -> {true, {module, list_to_atom(M)}}; true -> false -- cgit v1.1 From 998c6756b73ec2c227ed6f7f2e66eb059d1027fc Mon Sep 17 00:00:00 2001 From: Siri Hansen Date: Thu, 22 Dec 2016 20:25:52 +0100 Subject: Make sure ct_testspec is loaded ... before calling erlang:function_exported(ct_testspec,get_tests,1). --- src/rebar_prv_common_test.erl | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index 4a9ba25..30596da 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -554,6 +554,7 @@ get_dirs_from_specs(Specs) -> end. get_tests_from_specs(Specs) -> + _ = ct_testspec:module_info(), % make sure ct_testspec is loaded case erlang:function_exported(ct_testspec,get_tests,1) of true -> ct_testspec:get_tests(Specs); -- cgit v1.1 From 146f2732b96b5db6476e3b86b13e7a3d7b1b2dc5 Mon Sep 17 00:00:00 2001 From: Siri Hansen Date: Thu, 22 Dec 2016 22:14:07 +0100 Subject: Handle errors from ct_testspec --- src/rebar_prv_common_test.erl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index 30596da..2ac8fc7 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -100,7 +100,9 @@ format_error({badconfig, Msg}) -> io_lib:format(Msg, []); format_error({multiple_errors, Errors}) -> io_lib:format(lists:concat(["Error running tests:"] ++ - lists:map(fun(Error) -> "~n " ++ Error end, Errors)), []). + lists:map(fun(Error) -> "~n " ++ Error end, Errors)), []); +format_error({error_reading_testspec, Reason}) -> + io_lib:format("Error reading testspec: ~p", [Reason]). %% =================================================================== %% Internal functions @@ -549,8 +551,8 @@ get_dirs_from_specs(Specs) -> RunList = lists:append([R || {_,R,_} <- NodeRunSkipList]), DirList = [element(1,R) || R <- RunList], {ok,{SpecList,DirList}}; - Error -> - Error + {error,Reason} -> + {error,{?MODULE,{error_reading_testspec,Reason}}} end. get_tests_from_specs(Specs) -> -- cgit v1.1 From 5b3526518689eef8765a9303d43408864200b5d8 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Fri, 23 Dec 2016 10:47:16 -0800 Subject: eunit: remove application modules from the modules eligible to test application modules will be added to the eunit test set automatically, no need to consider them for inclusion in the test set separately --- src/rebar_prv_eunit.erl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_eunit.erl b/src/rebar_prv_eunit.erl index f65f700..7d44137 100644 --- a/src/rebar_prv_eunit.erl +++ b/src/rebar_prv_eunit.erl @@ -195,17 +195,18 @@ gather_src([Dir|Rest], Regex, Srcs) -> gather_src(Rest, Regex, Srcs ++ rebar_utils:find_files(Dir, Regex, true)). dedupe_tests({AppMods, TestMods}) -> + UniqueTestMods = lists:usort(TestMods) -- AppMods, %% for each modules in TestMods create a test if there is not a module %% in AppMods that will trigger it - F = fun(Mod) -> - M = filename:rootname(filename:basename(Mod)), - MatchesTest = fun(Dir) -> filename:rootname(filename:basename(Dir)) ++ "_tests" == M end, + F = fun(TestMod) -> + M = filename:rootname(filename:basename(TestMod)), + MatchesTest = fun(AppMod) -> filename:rootname(filename:basename(AppMod)) ++ "_tests" == M end, case lists:any(MatchesTest, AppMods) of false -> {true, {module, list_to_atom(M)}}; true -> false end end, - lists:usort(rebar_utils:filtermap(F, TestMods)). + rebar_utils:filtermap(F, UniqueTestMods). inject_eunit_state(State, {ok, Tests}) -> Apps = rebar_state:project_apps(State), -- cgit v1.1 From c21a5f7ab2309bedfd430e71ece576cbfaa93114 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 23 Dec 2016 15:27:08 -0500 Subject: Bumping to 3.3.4 --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 74efe97..83c8ecd 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.3.4"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 878cc18f800665b3aa74d9b9905a8a246a05edea Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 23 Dec 2016 15:33:22 -0500 Subject: Back to git-based versions --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 83c8ecd..74efe97 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.3.4"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 27cf4bfea0f7c0c58ae03f71b8db02648dd1ae04 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sat, 31 Dec 2016 16:40:54 -0800 Subject: add option to pass args to user_drv for custom shells --- src/rebar_prv_shell.erl | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_shell.erl b/src/rebar_prv_shell.erl index 72efcf1..c1bf735 100644 --- a/src/rebar_prv_shell.erl +++ b/src/rebar_prv_shell.erl @@ -75,7 +75,10 @@ init(State) -> "A list of apps to boot before starting the " "shell. (E.g. --apps app1,app2,app3) Defaults " "to rebar.config {shell, [{apps, Apps}]} or " - "relx apps if not specified."}]} + "relx apps if not specified."}, + {user_drv_args, undefined, "user_drv_args", string, + "Arguments passed to user_drv start function for " + "creating custom shells."}]} ]) ), {ok, State1}. @@ -99,7 +102,9 @@ format_error(Reason) -> shell(State) -> setup_name(State), setup_paths(State), - setup_shell(), + ShellArgs = debug_get_value(shell_args, rebar_state:get(State, shell, []), undefined, + "Found user_drv args from command line option."), + setup_shell(ShellArgs), maybe_run_script(State), %% apps must be started after the change in shell because otherwise %% their application masters never gets the new group leader (held in @@ -117,13 +122,13 @@ shell(State) -> info() -> "Start a shell with project and deps preloaded similar to~n'erl -pa ebin -pa deps/*/ebin'.~n". -setup_shell() -> +setup_shell(ShellArgs) -> OldUser = kill_old_user(), %% Test for support here NewUser = try erlang:open_port({spawn,"tty_sl -c -e"}, []) of Port when is_port(Port) -> true = port_close(Port), - setup_new_shell() + setup_new_shell(ShellArgs) catch error:_ -> setup_old_shell() @@ -153,11 +158,16 @@ wait_for_port_death(N, P) -> wait_for_port_death(N-10, P) end. -setup_new_shell() -> +setup_new_shell(ShellArgs) -> %% terminate the current user supervision structure, if any _ = supervisor:terminate_child(kernel_sup, user), %% start a new shell (this also starts a new user under the correct group) - _ = user_drv:start(), + case ShellArgs of + undefined -> + _ = user_drv:start(); + _ -> + _ = user_drv:start(ShellArgs) + end, %% wait until user_drv and user have been registered (max 3 seconds) ok = wait_until_user_started(3000), whereis(user). -- cgit v1.1 From f94b7a0d65052cf0ae4b1a5e424af1a0013430dc Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sat, 21 Jan 2017 09:31:19 -0500 Subject: See template local install same builtin as escript When the `new` command is run from a locally installed rebar3 (`rebar3 local install`), the builtin templates would be labelled as custom because of directories. This patch fixes it by splitting off the rebar3 priv dir from the user's configured plugin path for custom ones, and introducing a new internal label for builtins (since handling must remain different from escripts) This fixes issue #819 --- src/rebar_prv_new.erl | 3 +++ src/rebar_templater.erl | 20 +++++++++++++------- 2 files changed, 16 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_new.erl b/src/rebar_prv_new.erl index 064315e..152a56e 100644 --- a/src/rebar_prv_new.erl +++ b/src/rebar_prv_new.erl @@ -132,11 +132,14 @@ show_template({Name, Type, Location, Description, Vars}) -> format_vars(Vars)]). format_type(escript) -> "built-in"; +format_type(builtin) -> "built-in"; format_type(plugin) -> "plugin"; format_type(file) -> "custom". format_type(escript, _) -> "built-in template"; +format_type(builtin, _) -> + "built-in template"; format_type(plugin, Loc) -> io_lib:format("plugin template (~s)", [Loc]); format_type(file, Loc) -> diff --git a/src/rebar_templater.erl b/src/rebar_templater.erl index e64ce71..9b33ec5 100644 --- a/src/rebar_templater.erl +++ b/src/rebar_templater.erl @@ -267,8 +267,8 @@ find_templates(State) -> PluginTemplates = find_plugin_templates(State), {MainTemplates, Files} = case rebar_state:escript_path(State) of - undefined -> - {find_priv_templates(State), []}; + undefined -> % running in local install + {find_localinstall_templates(State), []}; _ -> %% Cache the files since we'll potentially need to walk it several times %% over the course of a run. @@ -307,11 +307,10 @@ find_escript_templates(Files) -> || {Name, _Bin} <- Files, re:run(Name, ?TEMPLATE_RE, [{capture, none}]) == match]. -find_priv_templates(State) -> - OtherTemplates = rebar_utils:find_files(code:priv_dir(rebar), ?TEMPLATE_RE), - HomeFiles = rebar_utils:find_files(rebar_dir:template_dir(State), - ?TEMPLATE_RE, true), % recursive - [{file, F} || F <- OtherTemplates ++ HomeFiles]. +find_localinstall_templates(_State) -> + Templates = rebar_utils:find_files(code:priv_dir(rebar), ?TEMPLATE_RE), + %% Pretend we're still running escripts; should work transparently. + [{builtin, F} || F <- Templates]. %% Fetch template indexes that sit on disk in the user's HOME find_disk_templates(State) -> @@ -354,6 +353,10 @@ prioritize_templates([{Name, Type, File} | Rest], Valid) -> ?DEBUG("Skipping template ~p, due to presence of a built-in " "template with the same name", [Name]), prioritize_templates(Rest, Valid); + {_, builtin, _} -> + ?DEBUG("Skipping template ~p, due to presence of a built-in " + "template with the same name", [Name]), + prioritize_templates(Rest, Valid); {_, plugin, _} -> ?DEBUG("Skipping template ~p, due to presence of a plugin " "template with the same name", [Name]), @@ -369,6 +372,9 @@ prioritize_templates([{Name, Type, File} | Rest], Valid) -> load_file(Files, escript, Name) -> {Name, Bin} = lists:keyfind(Name, 1, Files), Bin; +load_file(_Files, builtin, Name) -> + {ok, Bin} = file:read_file(Name), + Bin; load_file(_Files, plugin, Name) -> {ok, Bin} = file:read_file(Name), Bin; -- cgit v1.1 From ed0614c8691c4a972a9b4c27d04ab01770813b51 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sat, 21 Jan 2017 10:25:01 -0500 Subject: Extract code path formatting out of compiler This allows to reuse the code for any provider that formats source files out to the user. The option to configure it does remain compiler-centric for backwards compatibility --- src/rebar.hrl | 1 + src/rebar_base_compiler.erl | 24 +----------------------- src/rebar_dir.erl | 39 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 40 insertions(+), 24 deletions(-) (limited to 'src') diff --git a/src/rebar.hrl b/src/rebar.hrl index c94a84a..ca44283 100644 --- a/src/rebar.hrl +++ b/src/rebar.hrl @@ -27,6 +27,7 @@ -define(REMOTE_PACKAGE_DIR, "tarballs"). -define(REMOTE_REGISTRY_FILE, "registry.ets.gz"). -define(LOCK_FILE, "rebar.lock"). +-define(DEFAULT_COMPILER_SOURCE_FORMAT, relative). -define(PACKAGE_INDEX_VERSION, 3). -define(PACKAGE_TABLE, package_index). diff --git a/src/rebar_base_compiler.erl b/src/rebar_base_compiler.erl index dcb1975..480e49c 100644 --- a/src/rebar_base_compiler.erl +++ b/src/rebar_base_compiler.erl @@ -35,7 +35,6 @@ error_tuple/4, format_error_source/2]). --define(DEFAULT_COMPILER_SOURCE_FORMAT, relative). -type desc() :: term(). -type loc() :: {line(), col()} | line(). -type line() :: integer(). @@ -138,28 +137,7 @@ error_tuple(Source, Es, Ws, Opts) -> -spec format_error_source(file:filename(), rebar_dict() | [{_,_}]) -> file:filename(). format_error_source(Path, Opts) -> - Type = case rebar_opts:get(Opts, compiler_source_format, - ?DEFAULT_COMPILER_SOURCE_FORMAT) of - V when V == absolute; V == relative; V == build -> - V; - Other -> - ?WARN("Invalid argument ~p for compiler_source_format - " - "assuming ~s~n", [Other, ?DEFAULT_COMPILER_SOURCE_FORMAT]), - ?DEFAULT_COMPILER_SOURCE_FORMAT - end, - case Type of - absolute -> resolve_linked_source(Path); - build -> Path; - relative -> - Cwd = rebar_dir:get_cwd(), - rebar_dir:make_relative_path(resolve_linked_source(Path), Cwd) - end. - -%% @private takes a filename and canonicalizes its path if it is a link. --spec resolve_linked_source(file:filename()) -> file:filename(). -resolve_linked_source(Src) -> - {Dir, Base} = rebar_file_utils:split_dirname(Src), - filename:join(rebar_file_utils:resolve_link(Dir), Base). + rebar_dir:format_source_file_name(Path, Opts). %% =================================================================== %% Internal functions diff --git a/src/rebar_dir.erl b/src/rebar_dir.erl index 069d8fd..b61bfcc 100644 --- a/src/rebar_dir.erl +++ b/src/rebar_dir.erl @@ -26,7 +26,8 @@ src_dir_opts/2, recursive/2, extra_src_dirs/1, extra_src_dirs/2, all_src_dirs/1, all_src_dirs/3, - retarget_path/2]). + retarget_path/2, + format_source_file_name/2]). -include("rebar.hrl"). @@ -334,3 +335,39 @@ retarget_path(State, Path, [App|Rest]) -> {ok, NewPath} -> filename:join([rebar_app_info:out_dir(App), NewPath]); {error, badparent} -> retarget_path(State, Path, Rest) end. + +format_source_file_name(Path, Opts) -> + Type = case rebar_opts:get(Opts, compiler_source_format, + ?DEFAULT_COMPILER_SOURCE_FORMAT) of + V when V == absolute; V == relative; V == build -> + V; + Other -> + warn_source_format_once(Other) + end, + case Type of + absolute -> resolve_linked_source(Path); + build -> Path; + relative -> + Cwd = rebar_dir:get_cwd(), + rebar_dir:make_relative_path(resolve_linked_source(Path), Cwd) + end. + +%% @private displays a warning for the compiler source format option +%% only once +-spec warn_source_format_once(term()) -> ok. +warn_source_format_once(Format) -> + Warn = application:get_env(rebar, warn_source_format) =/= {ok, false}, + application:set_env(rebar, warn_source_format, false), + case Warn of + false -> + ok; + true -> + ?WARN("Invalid argument ~p for compiler_source_format - " + "assuming ~s~n", [Format, ?DEFAULT_COMPILER_SOURCE_FORMAT]) + end. + +%% @private takes a filename and canonicalizes its path if it is a link. +-spec resolve_linked_source(file:filename()) -> file:filename(). +resolve_linked_source(Src) -> + {Dir, Base} = rebar_file_utils:split_dirname(Src), + filename:join(rebar_file_utils:resolve_link(Dir), Base). -- cgit v1.1 From f16fdd35be2d0d6acb0a4b3448b12d7796751b80 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sat, 21 Jan 2017 10:26:01 -0500 Subject: Enable path reformatting for Dialyzer Fixes issue #880 May break backwards compat with projects that manually called the dialyzer formatter, but we never documented or expected this to be exposed. --- src/rebar_dialyzer_format.erl | 16 +++++++++------- src/rebar_prv_dialyzer.erl | 7 ++++--- 2 files changed, 13 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/src/rebar_dialyzer_format.erl b/src/rebar_dialyzer_format.erl index b30c4dc..1d234c3 100644 --- a/src/rebar_dialyzer_format.erl +++ b/src/rebar_dialyzer_format.erl @@ -16,18 +16,19 @@ -include("rebar.hrl"). --export([format_warnings/1]). +-export([format_warnings/2]). %% Formats a list of warnings in a nice per file way. Note that we reverse %% the list at the end to 'undo' the reversal by foldl -format_warnings(Warnings) -> - {_, Res} = lists:foldl(fun format_warning_/2, {undefined, []}, Warnings), +format_warnings(Opts, Warnings) -> + Fold = fun(Warning, Acc) -> format_warning_(Opts, Warning, Acc) end, + {_, Res} = lists:foldl(Fold, {undefined, []}, Warnings), lists:reverse(Res). %% If the last seen file is and the file of this warning are the same %% we skip the file header -format_warning_(Warning = {_Tag, {File, Line}, Msg}, {File, Acc}) -> +format_warning_(_Opts, Warning = {_Tag, {File, Line}, Msg}, {File, Acc}) -> try String = message_to_string(Msg), {File, [lists:flatten(fmt("~!c~4w~!!: ~s", [Line, String])) | Acc]} @@ -39,8 +40,9 @@ format_warning_(Warning = {_Tag, {File, Line}, Msg}, {File, Acc}) -> end; %% With a new file detencted we also write a file header. -format_warning_(Warning = {_Tag, {File, Line}, Msg}, {_LastFile, Acc}) -> +format_warning_(Opts, Warning = {_Tag, {SrcFile, Line}, Msg}, {_LastFile, Acc}) -> try + File = rebar_dir:format_source_file_name(SrcFile, Opts), Base = filename:basename(File), Dir = filename:dirname(File), Root = filename:rootname(Base), @@ -49,12 +51,12 @@ format_warning_(Warning = {_Tag, {File, Line}, Msg}, {_LastFile, Acc}) -> Base1 = fmt("~!_c~s~!!~!__~s", [Root, Ext]), F = fmt("~!__~s", [filename:join(Path, Base1)]), String = message_to_string(Msg), - {File, [lists:flatten(fmt("~n~s~n~!c~4w~!!: ~s", [F, Line, String])) | Acc]} + {SrcFile, [lists:flatten(fmt("~n~s~n~!c~4w~!!: ~s", [F, Line, String])) | Acc]} catch Error:Reason -> ?DEBUG("Failed to pretty format warning: ~p:~p~n~p", [Error, Reason, erlang:get_stacktrace()]), - {File, [dialyzer:format_warning(Warning, fullpath) | Acc]} + {SrcFile, [dialyzer:format_warning(Warning, fullpath) | Acc]} end. fmt(Fmt) -> diff --git a/src/rebar_prv_dialyzer.erl b/src/rebar_prv_dialyzer.erl index 44dc0d2..21d7f5a 100644 --- a/src/rebar_prv_dialyzer.erl +++ b/src/rebar_prv_dialyzer.erl @@ -478,7 +478,8 @@ run_dialyzer(State, Opts, Output) -> {check_plt, false} | Opts], ?DEBUG("Running dialyzer with options: ~p~n", [Opts2]), - Warnings = format_warnings(Output, dialyzer:run(Opts2)), + Warnings = format_warnings(rebar_state:opts(State), + Output, dialyzer:run(Opts2)), {Warnings, State}; false -> Opts2 = [{warnings, no_warnings()}, @@ -497,8 +498,8 @@ legacy_warnings(Warnings) -> Warnings end. -format_warnings(Output, Warnings) -> - Warnings1 = rebar_dialyzer_format:format_warnings(Warnings), +format_warnings(Opts, Output, Warnings) -> + Warnings1 = rebar_dialyzer_format:format_warnings(Opts, Warnings), console_warnings(Warnings1), file_warnings(Output, Warnings), length(Warnings). -- cgit v1.1 From 92e6997cf128382f7596fcc546caceee9d10a428 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 27 Jan 2017 10:56:22 -0500 Subject: Fix mv command on windows Requires changing a bunch of arguments for xerocopy since it does not allow to rename while copying. Lots of tests added --- src/rebar_file_utils.erl | 105 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 87 insertions(+), 18 deletions(-) (limited to 'src') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 8645641..436f2d5 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -185,32 +185,101 @@ mv(Source, Dest) -> ok end; {win32, _} -> - Cmd = case filelib:is_dir(Source) of - true -> - ?FMT("robocopy /move /e \"~s\" \"~s\" 1> nul", - [rebar_utils:escape_double_quotes(filename:nativename(Source)), - rebar_utils:escape_double_quotes(filename:nativename(Dest))]); - false -> - ?FMT("robocopy /move /e \"~s\" \"~s\" \"~s\" 1> nul", - [rebar_utils:escape_double_quotes(filename:nativename(filename:dirname(Source))), - rebar_utils:escape_double_quotes(filename:nativename(Dest)), - rebar_utils:escape_double_quotes(filename:basename(Source))]) - end, - Res = rebar_utils:sh(Cmd, - [{use_stdout, false}, return_on_error]), - case win32_ok(Res) of - true -> ok; + case filelib:is_dir(Source) of + true -> + SrcDir = filename:nativename(Source), + DestDir = filename:nativename(Dest), + robocopy_dir(SrcDir, DestDir); false -> - {error, lists:flatten( - io_lib:format("Failed to move ~s to ~s~n", - [Source, Dest]))} + SrcDir = filename:nativename(filename:dirname(Source)), + SrcName = filename:basename(Source), + DestDir = filename:nativename(filename:dirname(Dest)), + DestName = filename:basename(Dest), + IsDestDir = filelib:is_dir(Dest), + if IsDestDir -> + %% if basename and target name are different because + %% we move to a directory, then just move there. + %% Similarly, if they are the same but we're going to + %% a directory, let's just do that directly. + FullDestDir = filename:nativename(Dest), + robocopy_file(SrcDir, FullDestDir, SrcName) + ; SrcName =:= DestName -> + %% if basename and target name are the same and both are files, + %% we do a regular move with robocopy without rename. + robocopy_file(SrcDir, DestDir, DestName) + ; SrcName =/= DestName-> + %% If we're moving a file and the origin and + %% destination names are different: + %% - mktmp + %% - robocopy source_dir tmp_dir srcname + %% - rename srcname destname (to avoid clobbering) + %% - robocopy tmp_dir dest_dir destname + %% - remove tmp_dir + case ec_file:insecure_mkdtemp() of + {error, _Reason} -> + {error, lists:flatten( + io_lib:format("Failed to move ~s to ~s (tmpdir failed)~n", + [Source, Dest]))}; + TmpPath -> + case robocopy_file(SrcDir, TmpPath, SrcName) of + {error, Reason} -> + {error, Reason}; + ok -> + TmpSrc = filename:join(TmpPath, SrcName), + TmpDst = filename:join(TmpPath, DestName), + case file:rename(TmpSrc, TmpDst) of + {error, _} -> + {error, lists:flatten( + io_lib:format("Failed to move ~s to ~s (via rename)~n", + [Source, Dest]))}; + ok -> + case robocopy_file(TmpPath, DestDir, DestName) of + Err = {error, _} -> Err; + OK -> rm_rf(TmpPath), OK + end + end + end + end + end + end end. +robocopy_file(SrcPath, DestPath, FileName) -> + Cmd = ?FMT("robocopy /move /e \"~s\" \"~s\" \"~s\"", + [rebar_utils:escape_double_quotes(SrcPath), + rebar_utils:escape_double_quotes(DestPath), + rebar_utils:escape_double_quotes(FileName)]), + Res = rebar_utils:sh(Cmd, [{use_stdout, false}, return_on_error]), + case win32_ok(Res) of + false -> + {error, lists:flatten( + io_lib:format("Failed to move ~s to ~s~n", + [filename:join(SrcPath, FileName), + filename:join(DestPath, FileName)]))}; + true -> + ok + end. + +robocopy_dir(Source, Dest) -> + Cmd = ?FMT("robocopy /move /e \"~s\" \"~s\"", + [rebar_utils:escape_double_quotes(Source), + rebar_utils:escape_double_quotes(Dest)]), + Res = rebar_utils:sh(Cmd, + [{use_stdout, false}, return_on_error]), + case win32_ok(Res) of + true -> ok; + false -> + {error, lists:flatten( + io_lib:format("Failed to move ~s to ~s~n", + [Source, Dest]))} + end. + win32_ok({ok, _}) -> true; win32_ok({error, {Rc, _}}) when Rc<9; Rc=:=16 -> true; win32_ok(_) -> false. + delete_each([]) -> ok; delete_each([File | Rest]) -> -- cgit v1.1 From 3f27d877e3ca1c8624bd38de6b2c4d8cf16ef65e Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 27 Jan 2017 14:46:42 -0500 Subject: Re-fix the windows file movement and clean up a bit --- src/rebar_file_utils.erl | 78 ++++++++++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 33 deletions(-) (limited to 'src') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 436f2d5..8158312 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -188,7 +188,16 @@ mv(Source, Dest) -> case filelib:is_dir(Source) of true -> SrcDir = filename:nativename(Source), - DestDir = filename:nativename(Dest), + DestDir = case filelib:is_dir(Dest) of + true -> + %% to simulate unix/posix mv, we have to replicate + %% the same directory movement by moving the whole + %% top-level directory, not just the insides + SrcName = filename:basename(Source), + filename:nativename(filename:join(Dest, SrcName)); + false -> + filename:nativename(Dest) + end, robocopy_dir(SrcDir, DestDir); false -> SrcDir = filename:nativename(filename:dirname(Source)), @@ -208,43 +217,46 @@ mv(Source, Dest) -> %% we do a regular move with robocopy without rename. robocopy_file(SrcDir, DestDir, DestName) ; SrcName =/= DestName-> - %% If we're moving a file and the origin and - %% destination names are different: - %% - mktmp - %% - robocopy source_dir tmp_dir srcname - %% - rename srcname destname (to avoid clobbering) - %% - robocopy tmp_dir dest_dir destname - %% - remove tmp_dir - case ec_file:insecure_mkdtemp() of - {error, _Reason} -> - {error, lists:flatten( - io_lib:format("Failed to move ~s to ~s (tmpdir failed)~n", - [Source, Dest]))}; - TmpPath -> - case robocopy_file(SrcDir, TmpPath, SrcName) of - {error, Reason} -> - {error, Reason}; - ok -> - TmpSrc = filename:join(TmpPath, SrcName), - TmpDst = filename:join(TmpPath, DestName), - case file:rename(TmpSrc, TmpDst) of - {error, _} -> - {error, lists:flatten( - io_lib:format("Failed to move ~s to ~s (via rename)~n", - [Source, Dest]))}; - ok -> - case robocopy_file(TmpPath, DestDir, DestName) of - Err = {error, _} -> Err; - OK -> rm_rf(TmpPath), OK - end - end - end - end + robocopy_mv_and_rename(Source, Dest, SrcDir, SrcName, DestDir, DestName) end end end. +robocopy_mv_and_rename(Source, Dest, SrcDir, SrcName, DestDir, DestName) -> + %% If we're moving a file and the origin and + %% destination names are different: + %% - mktmp + %% - robocopy source_dir tmp_dir srcname + %% - rename srcname destname (to avoid clobbering) + %% - robocopy tmp_dir dest_dir destname + %% - remove tmp_dir + case ec_file:insecure_mkdtemp() of + {error, _Reason} -> + {error, lists:flatten( + io_lib:format("Failed to move ~s to ~s (tmpdir failed)~n", + [Source, Dest]))}; + TmpPath -> + case robocopy_file(SrcDir, TmpPath, SrcName) of + {error, Reason} -> + {error, Reason}; + ok -> + TmpSrc = filename:join(TmpPath, SrcName), + TmpDst = filename:join(TmpPath, DestName), + case file:rename(TmpSrc, TmpDst) of + {error, _} -> + {error, lists:flatten( + io_lib:format("Failed to move ~s to ~s (via rename)~n", + [Source, Dest]))}; + ok -> + case robocopy_file(TmpPath, DestDir, DestName) of + Err = {error, _} -> Err; + OK -> rm_rf(TmpPath), OK + end + end + end + end. + robocopy_file(SrcPath, DestPath, FileName) -> Cmd = ?FMT("robocopy /move /e \"~s\" \"~s\" \"~s\"", [rebar_utils:escape_double_quotes(SrcPath), -- cgit v1.1 From ba954cfb43f0b05e15ee206b13488d62c3537267 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 27 Jan 2017 21:30:18 -0500 Subject: Survive EDoc crashes Instead of a hard crash, display an error message indicating which app failed. We can't report the exact failure; only EDoc does it to stdout itself and we can't capture it. Pre/Post hooks are run properly despite the failure, as per escript and compile providers. --- src/rebar_prv_edoc.erl | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_edoc.erl b/src/rebar_prv_edoc.erl index 465fc34..97f70a9 100644 --- a/src/rebar_prv_edoc.erl +++ b/src/rebar_prv_edoc.erl @@ -7,6 +7,7 @@ format_error/1]). -include("rebar.hrl"). +-include_lib("providers/include/providers.hrl"). -define(PROVIDER, edoc). -define(DEPS, [compile]). @@ -28,7 +29,8 @@ init(State) -> {profiles, [docs]}])), {ok, State1}. --spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. +-spec do(rebar_state:t()) -> + {ok, rebar_state:t()} | {error, string()} | {error, {module(), any()}}. do(State) -> code:add_pathsa(rebar_state:code_paths(State, all_deps)), ProjectApps = rebar_state:project_apps(State), @@ -37,26 +39,42 @@ do(State) -> ShouldAccPaths = not has_configured_paths(EdocOpts), Cwd = rebar_state:dir(State), rebar_hooks:run_all_hooks(Cwd, pre, ?PROVIDER, Providers, State), - lists:foldl(fun(AppInfo, EdocOptsAcc) -> + Res = try + lists:foldl(fun(AppInfo, EdocOptsAcc) -> rebar_hooks:run_all_hooks(Cwd, pre, ?PROVIDER, Providers, AppInfo, State), AppName = ec_cnv:to_list(rebar_app_info:name(AppInfo)), ?INFO("Running edoc for ~s", [AppName]), AppDir = rebar_app_info:dir(AppInfo), - ok = edoc:application(list_to_atom(AppName), AppDir, EdocOptsAcc), + AppRes = (catch edoc:application(list_to_atom(AppName), AppDir, EdocOptsAcc)), rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, AppInfo, State), - case ShouldAccPaths of - true -> + case {AppRes, ShouldAccPaths} of + {ok, true} -> %% edoc wants / on all OSes add_to_paths(EdocOptsAcc, AppDir++"/doc"); - false -> - EdocOptsAcc + {ok, false} -> + EdocOptsAcc; + {{'EXIT', error}, _} -> + %% EDoc is not very descriptive + %% in terms of failures + throw({app_failed, AppName}) end - end, EdocOpts, ProjectApps), + end, EdocOpts, ProjectApps) + catch + {app_failed, AppName} -> + {app_failed, AppName} + end, rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State), rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), - {ok, State}. + case Res of + {app_failed, App} -> + ?PRV_ERROR({app_failed, App}); + _ -> + {ok, State} + end. -spec format_error(any()) -> iolist(). +format_error({app_failed, AppName}) -> + io_lib:format("Failed to generate documentation for app '~s'", [AppName]); format_error(Reason) -> io_lib:format("~p", [Reason]). -- cgit v1.1 From 6c416febc2dfc9811270c20d090f43009d3bfc64 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Sat, 28 Jan 2017 22:24:38 -0800 Subject: consider `ERL_COMPILER_OPTIONS` when recompiling on 19.x forward the compiler should now take into consideration the value of the environment variable `ERL_COMPILER_OPTIONS` when deciding whether or not to recompile a module --- src/rebar_erlc_compiler.erl | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index 95573fd..f7244dc 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -322,6 +322,10 @@ needed_files(G, ErlOpts, RebarOpts, Dir, OutDir, SourceFiles) -> AllOpts = [{outdir, filename:dirname(Target)} ,{i, filename:join(Dir, "include")} ,{i, Dir}] ++ PrivIncludes ++ ErlOpts, + %% necessary for erlang:function_exported/3 to work as expected + %% called here for clarity as it's required by both opts_changed/2 + %% and erl_compiler_opts_set/0 + _ = code:ensure_loaded(compile), digraph:vertex(G, Source) > {Source, filelib:last_modified(Target)} orelse opts_changed(AllOpts, TargetBase) orelse erl_compiler_opts_set() @@ -342,8 +346,12 @@ maybe_rm_beam_and_edge(G, OutDir, Source) -> end. opts_changed(NewOpts, Target) -> + TotalOpts = case erlang:function_exported(compile, env_compiler_options, 0) of + true -> NewOpts ++ compile:env_compiler_options(); + false -> NewOpts + end, case compile_info(Target) of - {ok, Opts} -> lists:sort(Opts) =/= lists:sort(NewOpts); + {ok, Opts} -> lists:sort(Opts) =/= lists:sort(TotalOpts); _ -> true end. @@ -358,10 +366,12 @@ compile_info(Target) -> end. erl_compiler_opts_set() -> - case os:getenv("ERL_COMPILER_OPTIONS") of + EnvSet = case os:getenv("ERL_COMPILER_OPTIONS") of false -> false; _ -> true - end. + end, + %% return false if changed env opts would have been caught in opts_changed/2 + EnvSet andalso not erlang:function_exported(compile, env_compiler_options, 0). erlcinfo_file(Dir) -> filename:join(rebar_dir:local_cache_dir(Dir), ?ERLCINFO_FILE). -- cgit v1.1 From 9b099f41d6100f87dcb334d4acdcfbcdac2c97f1 Mon Sep 17 00:00:00 2001 From: Drew Varner Date: Mon, 30 Jan 2017 02:22:16 -0500 Subject: Allow exclusion of apps in cover Adds a new option `{cover_excl_apps, Apps}` to exclude apps from coverage analysis by name. --- src/rebar_prv_cover.erl | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_cover.erl b/src/rebar_prv_cover.erl index e53a687..865c557 100644 --- a/src/rebar_prv_cover.erl +++ b/src/rebar_prv_cover.erl @@ -303,7 +303,8 @@ strip_coverdir(File) -> 2))). cover_compile(State, apps) -> - Apps = filter_checkouts(rebar_state:project_apps(State)), + ExclApps = [list_to_binary(A) || A <- rebar_state:get(State, cover_excl_apps, [])], + Apps = filter_checkouts_and_excluded(rebar_state:project_apps(State), ExclApps), AppDirs = app_dirs(Apps), cover_compile(State, lists:filter(fun(D) -> ec_file:is_dir(D) end, AppDirs)); cover_compile(State, Dirs) -> @@ -313,7 +314,6 @@ cover_compile(State, Dirs) -> %% redirect cover output true = redirect_cover_output(State, CoverPid), ExclMods = rebar_state:get(State, cover_excl_mods, []), - lists:foreach(fun(Dir) -> case file:list_dir(Dir) of {ok, Files} -> @@ -356,13 +356,14 @@ app_dirs(Apps) -> app_ebin_dirs(App, Acc) -> [rebar_app_info:ebin_dir(App)|Acc]. -filter_checkouts(Apps) -> filter_checkouts(Apps, []). +filter_checkouts_and_excluded(Apps, ExclApps) -> + filter_checkouts_and_excluded(Apps, ExclApps, []). -filter_checkouts([], Acc) -> lists:reverse(Acc); -filter_checkouts([App|Rest], Acc) -> - case rebar_app_info:is_checkout(App) of - true -> filter_checkouts(Rest, Acc); - false -> filter_checkouts(Rest, [App|Acc]) +filter_checkouts_and_excluded([], _ExclApps, Acc) -> lists:reverse(Acc); +filter_checkouts_and_excluded([App|Rest], ExclApps, Acc) -> + case rebar_app_info:is_checkout(App) orelse lists:member(rebar_app_info:name(App), ExclApps) of + true -> filter_checkouts_and_excluded(Rest, ExclApps, Acc); + false -> filter_checkouts_and_excluded(Rest, ExclApps, [App|Acc]) end. start_cover() -> -- cgit v1.1 From eae41ee0a2b13b41c2e542aeb06d118c96f42959 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 3 Feb 2017 08:18:50 -0500 Subject: Fix printing path with unicode names in prv_update --- src/rebar_prv_update.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index 54f1796..046c864 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -59,7 +59,7 @@ do(State) -> {ok, Data} = file:read_file(TmpFile), Unzipped = zlib:gunzip(Data), ok = file:write_file(HexFile, Unzipped), - ?INFO("Writing registry to ~s", [HexFile]), + ?INFO("Writing registry to ~ts", [HexFile]), hex_to_index(State), {ok, State}; _ -> @@ -128,7 +128,7 @@ hex_to_index(State) -> true end, true, Registry), ets:insert(?PACKAGE_TABLE, {package_index_version, ?PACKAGE_INDEX_VERSION}), - ?INFO("Writing index to ~s", [PackageIndex]), + ?INFO("Writing index to ~ts", [PackageIndex]), ets:tab2file(?PACKAGE_TABLE, PackageIndex), true after -- cgit v1.1 From 6fcf464d370a6d18d5c9c95bf21c9162b370b481 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 3 Feb 2017 16:11:06 -0500 Subject: Bump to 3.3.5 --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 74efe97..03d284c 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.3.5"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 95ef84953b3f40b83126f00f0746f7918d607315 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 3 Feb 2017 16:14:33 -0500 Subject: Go back to git-generated version --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 03d284c..74efe97 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.3.5"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 5da09d2d44947952622563ad89e2f2d827af50f4 Mon Sep 17 00:00:00 2001 From: alisdair sullivan Date: Sun, 5 Feb 2017 18:32:33 -0800 Subject: catch badarg exception from eunit in `rebar_prv_eunit` fixes #1469 --- src/rebar_prv_eunit.erl | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_eunit.erl b/src/rebar_prv_eunit.erl index 7d44137..65addc3 100644 --- a/src/rebar_prv_eunit.erl +++ b/src/rebar_prv_eunit.erl @@ -83,13 +83,16 @@ run_tests(State, Tests) -> EUnitOpts = resolve_eunit_opts(State), ?DEBUG("eunit_tests ~p", [T]), ?DEBUG("eunit_opts ~p", [EUnitOpts]), - Result = eunit:test(T, EUnitOpts), - ok = maybe_write_coverdata(State), - case handle_results(Result) of - {error, Reason} -> - ?PRV_ERROR(Reason); - ok -> - {ok, State} + try eunit:test(T, EUnitOpts) of + Result -> + ok = maybe_write_coverdata(State), + case handle_results(Result) of + {error, Reason} -> + ?PRV_ERROR(Reason); + ok -> + {ok, State} + end + catch error:badarg -> ?PRV_ERROR({error, badarg}) end. -spec format_error(any()) -> iolist(). -- cgit v1.1 From 59a7bb51652598c456bf72409083261b988b84aa Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 6 Feb 2017 00:39:10 -0500 Subject: Add support for global plugin-defined templates Did not really know how to automate the testing for this, did it by hand. --- src/rebar_templater.erl | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'src') diff --git a/src/rebar_templater.erl b/src/rebar_templater.erl index 9b33ec5..1c28788 100644 --- a/src/rebar_templater.erl +++ b/src/rebar_templater.erl @@ -334,8 +334,19 @@ find_plugin_templates(State) -> || App <- rebar_state:all_plugin_deps(State), Priv <- [rebar_app_info:priv_dir(App)], Priv =/= undefined, + File <- rebar_utils:find_files(Priv, ?TEMPLATE_RE)] + ++ %% and add global plugins too + [{plugin, File} + || PSource <- rebar_state:get(State, {plugins, global}, []), + Plugin <- [plugin_provider(PSource)], + is_atom(Plugin), + Priv <- [code:priv_dir(Plugin)], + Priv =/= undefined, File <- rebar_utils:find_files(Priv, ?TEMPLATE_RE)]. +plugin_provider(P) when is_atom(P) -> P; +plugin_provider(T) when is_tuple(T) -> element(1, T). + %% Take an existing list of templates and tag them by name the way %% the user would enter it from the CLI tag_names(List) -> -- cgit v1.1 From c64c88f262abaad677042d8eab0c41ff54af4b7b Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sun, 19 Feb 2017 11:11:24 -0500 Subject: Add respect for src_dirs option in app_discover This lets people who use custom directories layout without leaving the .app.src in src/ to still see their app build. --- src/rebar_app_discover.erl | 81 +++++++++++++++++++++++++++++----------------- 1 file changed, 52 insertions(+), 29 deletions(-) (limited to 'src') diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index fd55960..0396be1 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -7,8 +7,10 @@ find_unbuilt_apps/1, find_apps/1, find_apps/2, + find_apps/3, find_app/2, - find_app/3]). + find_app/3, + find_app/4]). -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). @@ -20,7 +22,9 @@ do(State, LibDirs) -> BaseDir = rebar_state:dir(State), Dirs = [filename:join(BaseDir, LibDir) || LibDir <- LibDirs], - Apps = find_apps(Dirs, all), + RebarOpts = rebar_state:opts(State), + SrcDirs = rebar_dir:src_dirs(RebarOpts, ["src"]), + Apps = find_apps(Dirs, SrcDirs, all), ProjectDeps = rebar_state:deps_names(State), DepsDir = rebar_dir:deps_dir(State), CurrentProfiles = rebar_state:current_profiles(State), @@ -179,32 +183,25 @@ reset_hooks(Opts) -> rebar_opts:set(OptsAcc, Key, []) end, Opts, [post_hooks, pre_hooks, provider_hooks, artifacts]). -%% @doc find the directories for all apps --spec all_app_dirs([file:name()]) -> [file:name()]. -all_app_dirs(LibDirs) -> - lists:flatmap(fun(LibDir) -> - app_dirs(LibDir) - end, LibDirs). +%% @doc find the directories for all apps based on their source dirs +-spec all_app_dirs([file:name()], [file:name()]) -> [file:name()]. +all_app_dirs(LibDirs, SrcDirs) -> + lists:flatmap(fun(LibDir) -> app_dirs(LibDir, SrcDirs) end, LibDirs). %% @doc find the directories based on the library directories --spec app_dirs([file:name()]) -> [file:name()]. -app_dirs(LibDir) -> - Path1 = filename:join([LibDir, - "src", - "*.app.src"]), - - Path2 = filename:join([LibDir, - "src", - "*.app.src.script"]), - - Path3 = filename:join([LibDir, - "ebin", - "*.app"]), +-spec app_dirs([file:name()], [file:name()]) -> [file:name()]. +app_dirs(LibDir, SrcDirs) -> + Paths = lists:append([ + [filename:join([LibDir, SrcDir, "*.app.src"]), + filename:join([LibDir, SrcDir, "*.app.src.script"])] + || SrcDir <- SrcDirs + ]), + EbinPath = filename:join([LibDir, "ebin", "*.app"]), lists:usort(lists:foldl(fun(Path, Acc) -> Files = filelib:wildcard(ec_cnv:to_list(Path)), [app_dir(File) || File <- Files] ++ Acc - end, [], [Path1, Path2, Path3])). + end, [], [EbinPath | Paths])). %% @doc find all apps that haven't been built in a list of directories -spec find_unbuilt_apps([file:filename_all()]) -> [rebar_app_info:t()]. @@ -215,23 +212,33 @@ find_unbuilt_apps(LibDirs) -> %% Returns all the related app info records. -spec find_apps([file:filename_all()]) -> [rebar_app_info:t()]. find_apps(LibDirs) -> - find_apps(LibDirs, valid). + find_apps(LibDirs, ["src"], valid). %% @doc for each directory passed, find all apps according %% to the validity rule passed in. Returns all the related %% app info records. -spec find_apps([file:filename_all()], valid | invalid | all) -> [rebar_app_info:t()]. find_apps(LibDirs, Validate) -> - rebar_utils:filtermap(fun(AppDir) -> - find_app(AppDir, Validate) - end, all_app_dirs(LibDirs)). + find_apps(LibDirs, ["src"], Validate). + +%% @doc for each directory passed, with the configured source directories, +%% find all apps according to the validity rule passed in. +%% Returns all the related app info records. +-spec find_apps([file:filename_all()], [file:filename_all()], valid | invalid | all) -> [rebar_app_info:t()]. +find_apps(LibDirs, SrcDirs, Validate) -> + rebar_utils:filtermap( + fun(AppDir) -> + find_app(rebar_app_info:new(), AppDir, SrcDirs, Validate) + end, + all_app_dirs(LibDirs, SrcDirs) + ). %% @doc check that a given app in a directory is there, and whether it's %% valid or not based on the second argument. Returns the related %% app info record. -spec find_app(file:filename_all(), valid | invalid | all) -> {true, rebar_app_info:t()} | false. find_app(AppDir, Validate) -> - find_app(rebar_app_info:new(), AppDir, Validate). + find_app(rebar_app_info:new(), AppDir, ["src"], Validate). %% @doc check that a given app in a directory is there, and whether it's %% valid or not based on the second argument. Returns the related @@ -239,9 +246,25 @@ find_app(AppDir, Validate) -> -spec find_app(rebar_app_info:t(), file:filename_all(), valid | invalid | all) -> {true, rebar_app_info:t()} | false. find_app(AppInfo, AppDir, Validate) -> + find_app(AppInfo, AppDir, ["src"], Validate). + +%% @doc check that a given app in a directory is there, and whether it's +%% valid or not based on the second argument. The third argument includes +%% the directories where source files can be located. Returns the related +%% app info record. +-spec find_app(rebar_app_info:t(), file:filename_all(), + [file:filename_all()], valid | invalid | all) -> + {true, rebar_app_info:t()} | false. +find_app(AppInfo, AppDir, SrcDirs, Validate) -> AppFile = filelib:wildcard(filename:join([AppDir, "ebin", "*.app"])), - AppSrcFile = filelib:wildcard(filename:join([AppDir, "src", "*.app.src"])), - AppSrcScriptFile = filelib:wildcard(filename:join([AppDir, "src", "*.app.src.script"])), + AppSrcFile = lists:append( + [filelib:wildcard(filename:join([AppDir, SrcDir, "*.app.src"])) + || SrcDir <- SrcDirs] + ), + AppSrcScriptFile = lists:append( + [filelib:wildcard(filename:join([AppDir, SrcDir, "*.app.src.script"])) + || SrcDir <- SrcDirs] + ), try_handle_app_file(AppInfo, AppFile, AppDir, AppSrcFile, AppSrcScriptFile, Validate). %% @doc find the directory that an appfile has -- cgit v1.1 From d80c3f4da10857d16315d10ecaa5fe0f2f08c961 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sun, 19 Feb 2017 11:26:35 -0500 Subject: For convenience add appinfo default src_dir lookup This makes rebar_prv_install_deps able to support more src_dir configs --- src/rebar_app_discover.erl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index 0396be1..66319a4 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -246,7 +246,11 @@ find_app(AppDir, Validate) -> -spec find_app(rebar_app_info:t(), file:filename_all(), valid | invalid | all) -> {true, rebar_app_info:t()} | false. find_app(AppInfo, AppDir, Validate) -> - find_app(AppInfo, AppDir, ["src"], Validate). + %% if no src dir is passed, figure it out from the app info, with a default + %% of src/ + AppOpts = rebar_app_info:opts(AppInfo), + SrcDirs = rebar_dir:src_dirs(AppOpts, ["src"]), + find_app(AppInfo, AppDir, SrcDirs, Validate). %% @doc check that a given app in a directory is there, and whether it's %% valid or not based on the second argument. The third argument includes -- cgit v1.1 From e498f603df66be8fa662fb4eafa9e2d6cef2d222 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sun, 19 Feb 2017 11:39:15 -0500 Subject: Fix plugins usage to support src_dirs options --- src/rebar_prv_plugins.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_plugins.erl b/src/rebar_prv_plugins.erl index 7e6b88e..9e04fa8 100644 --- a/src/rebar_prv_plugins.erl +++ b/src/rebar_prv_plugins.erl @@ -34,14 +34,17 @@ do(State) -> GlobalConfigFile = rebar_dir:global_config(), GlobalConfig = rebar_state:new(rebar_config:consult_file(GlobalConfigFile)), GlobalPlugins = rebar_state:get(GlobalConfig, plugins, []), + GlobalSrcDirs = rebar_state:get(GlobalConfig, src_dirs, ["src"]), GlobalPluginsDir = filename:join([rebar_dir:global_cache_dir(rebar_state:opts(State)), "plugins", "*"]), - GlobalApps = rebar_app_discover:find_apps([GlobalPluginsDir], all), + GlobalApps = rebar_app_discover:find_apps([GlobalPluginsDir], GlobalSrcDirs, all), display_plugins("Global plugins", GlobalApps, GlobalPlugins), + RebarOpts = rebar_state:opts(State), + SrcDirs = rebar_dir:src_dirs(RebarOpts, ["src"]), Plugins = rebar_state:get(State, plugins, []), PluginsDir = filename:join(rebar_dir:plugins_dir(State), "*"), CheckoutsDir = filename:join(rebar_dir:checkouts_dir(State), "*"), - Apps = rebar_app_discover:find_apps([CheckoutsDir, PluginsDir], all), + Apps = rebar_app_discover:find_apps([CheckoutsDir, PluginsDir], SrcDirs, all), display_plugins("Local plugins", Apps, Plugins), {ok, State}. -- cgit v1.1 From 1810ae30abdddc95dc93d96f5c9d0f27182f0e62 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 23 Feb 2017 21:48:46 -0500 Subject: Fix detection of src_dirs in unknown apps/deps First find the rebar.config file, and if any data is found, check for src_dirs config. If nothing is there or no app is found, default to src/ --- src/rebar_app_discover.erl | 60 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 49 insertions(+), 11 deletions(-) (limited to 'src') diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index 66319a4..db74cd3 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -183,13 +183,31 @@ reset_hooks(Opts) -> rebar_opts:set(OptsAcc, Key, []) end, Opts, [post_hooks, pre_hooks, provider_hooks, artifacts]). -%% @doc find the directories for all apps based on their source dirs --spec all_app_dirs([file:name()], [file:name()]) -> [file:name()]. +%% @private find the directories for all apps, while detecting their source dirs +%% Returns the app dir with the respective src_dirs for them, in that order, +%% for every app found. +-spec all_app_dirs([file:name()]) -> [{file:name(), [file:name()]}]. +all_app_dirs(LibDirs) -> + lists:flatmap(fun(LibDir) -> + SrcDirs = find_config_src(LibDir, ["src"]), + app_dirs(LibDir, SrcDirs) + end, LibDirs). + +%% @private find the directories for all apps based on their source dirs +%% Returns the app dir with the respective src_dirs for them, in that order, +%% for every app found. +-spec all_app_dirs([file:name()], [file:name()]) -> [{file:name(), [file:name()]}]. all_app_dirs(LibDirs, SrcDirs) -> lists:flatmap(fun(LibDir) -> app_dirs(LibDir, SrcDirs) end, LibDirs). -%% @doc find the directories based on the library directories --spec app_dirs([file:name()], [file:name()]) -> [file:name()]. +%% @private find the directories based on the library directories. +%% Returns the app dir with the respective src_dirs for them, in that order, +%% for every app found. +%% +%% The function returns the src directories since they might have been +%% detected in a top-level loop and we want to skip further detection +%% starting now. +-spec app_dirs([file:name()], [file:name()]) -> [{file:name(), [file:name()]}]. app_dirs(LibDir, SrcDirs) -> Paths = lists:append([ [filename:join([LibDir, SrcDir, "*.app.src"]), @@ -199,8 +217,9 @@ app_dirs(LibDir, SrcDirs) -> EbinPath = filename:join([LibDir, "ebin", "*.app"]), lists:usort(lists:foldl(fun(Path, Acc) -> - Files = filelib:wildcard(ec_cnv:to_list(Path)), - [app_dir(File) || File <- Files] ++ Acc + Files = filelib:wildcard(ec_cnv:to_list(Path)), + [{app_dir(File), SrcDirs} + || File <- Files] ++ Acc end, [], [EbinPath | Paths])). %% @doc find all apps that haven't been built in a list of directories @@ -212,14 +231,19 @@ find_unbuilt_apps(LibDirs) -> %% Returns all the related app info records. -spec find_apps([file:filename_all()]) -> [rebar_app_info:t()]. find_apps(LibDirs) -> - find_apps(LibDirs, ["src"], valid). + find_apps(LibDirs, valid). %% @doc for each directory passed, find all apps according %% to the validity rule passed in. Returns all the related %% app info records. -spec find_apps([file:filename_all()], valid | invalid | all) -> [rebar_app_info:t()]. find_apps(LibDirs, Validate) -> - find_apps(LibDirs, ["src"], Validate). + rebar_utils:filtermap( + fun({AppDir, AppSrcDirs}) -> + find_app(rebar_app_info:new(), AppDir, AppSrcDirs, Validate) + end, + all_app_dirs(LibDirs) + ). %% @doc for each directory passed, with the configured source directories, %% find all apps according to the validity rule passed in. @@ -227,8 +251,8 @@ find_apps(LibDirs, Validate) -> -spec find_apps([file:filename_all()], [file:filename_all()], valid | invalid | all) -> [rebar_app_info:t()]. find_apps(LibDirs, SrcDirs, Validate) -> rebar_utils:filtermap( - fun(AppDir) -> - find_app(rebar_app_info:new(), AppDir, SrcDirs, Validate) + fun({AppDir, AppSrcDirs}) -> + find_app(rebar_app_info:new(), AppDir, AppSrcDirs, Validate) end, all_app_dirs(LibDirs, SrcDirs) ). @@ -238,7 +262,8 @@ find_apps(LibDirs, SrcDirs, Validate) -> %% app info record. -spec find_app(file:filename_all(), valid | invalid | all) -> {true, rebar_app_info:t()} | false. find_app(AppDir, Validate) -> - find_app(rebar_app_info:new(), AppDir, ["src"], Validate). + SrcDirs = find_config_src(AppDir, ["src"]), + find_app(rebar_app_info:new(), AppDir, SrcDirs, Validate). %% @doc check that a given app in a directory is there, and whether it's %% valid or not based on the second argument. Returns the related @@ -385,3 +410,16 @@ enable(State, AppInfo) -> -spec to_atom(binary()) -> atom(). to_atom(Bin) -> list_to_atom(binary_to_list(Bin)). + +%% @private when looking for unknown apps, it's possible they have a +%% rebar.config file specifying non-standard src_dirs. Check for a +%% possible config file and extract src_dirs from it. +find_config_src(AppDir, Default) -> + case rebar_config:consult(AppDir) of + [] -> + Default; + Terms -> + %% TODO: handle profiles I guess, but we don't have that info + proplists:get_value(src_dirs, Terms, Default) + end. + -- cgit v1.1 From ab1f93d6fa712c3b85bce30cab09edb5ddf7f3d9 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 24 Feb 2017 08:08:46 -0500 Subject: Fix default .app.src file for rebar_app_info The finding of the file was done based on an assumed 'src' path which may not be correct. This patch instead replaces the value with a lookup in configured paths and returns the first that matches to an existing file. --- src/rebar_app_info.erl | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index fdaadb8..62ec6dd 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -248,13 +248,12 @@ set(AppInfo=#app_info_t{opts=Opts}, Key, Value) -> %% @doc finds the .app.src file for an app, if any. -spec app_file_src(t()) -> file:filename_all() | undefined. -app_file_src(#app_info_t{app_file_src=undefined, dir=Dir, name=Name}) -> - AppFileSrc = filename:join([ec_cnv:to_list(Dir), "src", ec_cnv:to_list(Name)++".app.src"]), - case filelib:is_file(AppFileSrc) of - true -> - AppFileSrc; - false -> - undefined +app_file_src(#app_info_t{app_file_src=undefined, dir=Dir, name=Name, opts=Opts}) -> + CandidatePaths = [filename:join([ec_cnv:to_list(Dir), Src, ec_cnv:to_list(Name)++".app.src"]) + || Src <- rebar_opts:get(Opts, src_dirs, ["src"])], + case lists:dropwhile(fun(Path) -> not filelib:is_file(Path) end, CandidatePaths) of + [] -> undefined; + [AppFileSrc|_] -> AppFileSrc end; app_file_src(#app_info_t{app_file_src=AppFileSrc}) -> ec_cnv:to_list(AppFileSrc). -- cgit v1.1 From 81b6c0f6b689a17d6b5ea796b59610c7d669b561 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 24 Feb 2017 12:42:18 -0500 Subject: WIP windows escripts get cmd autogenerated --- src/rebar_prv_escriptize.erl | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_escriptize.erl b/src/rebar_prv_escriptize.erl index 7ee20c2..5c0c989 100644 --- a/src/rebar_prv_escriptize.erl +++ b/src/rebar_prv_escriptize.erl @@ -130,9 +130,15 @@ escriptize(State0, App) -> throw(?PRV_ERROR({escript_creation_failed, AppName, EscriptError})) end, - %% Finally, update executable perms for our script - {ok, #file_info{mode = Mode}} = file:read_file_info(Filename), - ok = file:change_mode(Filename, Mode bor 8#00111), + %% Finally, update executable perms for our script on *nix or write out + %% script files on win32 + case os:type() of + {unix, _} -> + {ok, #file_info{mode = Mode}} = file:read_file_info(Filename), + ok = file:change_mode(Filename, Mode bor 8#00111); + {win32, _} -> + write_windows_script(Filename) + end, {ok, State}. -spec format_error(any()) -> iolist(). @@ -258,3 +264,11 @@ def(Rm, State, Key, Default) -> rm_newline(String) -> [C || C <- String, C =/= $\n]. + +write_windows_script(Target) -> + CmdScript= + "@echo off\r\n" + "setlocal\r\n" + "set rebarscript=%~f0\r\n" + "escript.exe \"%rebarscript:.cmd=%\" %*\r\n", + ok = file:write_file(Target ++ ".cmd", CmdScript). -- cgit v1.1 From 0bc0aeba9c19531025967862cde7b9e32d7bc300 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 24 Feb 2017 21:09:43 -0500 Subject: REBAR_CONFIG impacts file project root only --- src/rebar3.erl | 2 +- src/rebar_config.erl | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/rebar3.erl b/src/rebar3.erl index 56bf3e8..5c593cc 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -176,7 +176,7 @@ init_config() -> Verbosity = log_level(), ok = rebar_log:init(command_line, Verbosity), - Config = rebar_config:consult(), + Config = rebar_config:consult_root(), Config1 = rebar_config:merge_locks(Config, rebar_config:consult_lock_file(?LOCK_FILE)), %% If $HOME/.config/rebar3/rebar.config exists load and use as global config diff --git a/src/rebar_config.erl b/src/rebar_config.erl index 97e27ab..82ff0d9 100644 --- a/src/rebar_config.erl +++ b/src/rebar_config.erl @@ -26,7 +26,7 @@ %% ------------------------------------------------------------------- -module(rebar_config). --export([consult/0 +-export([consult_root/0 ,consult/1 ,consult_app_file/1 ,consult_file/1 @@ -46,15 +46,15 @@ %% Public API %% =================================================================== -%% @doc reads the default config file. --spec consult() -> [any()]. -consult() -> +%% @doc reads the default config file at the top of a full project +-spec consult_root() -> [any()]. +consult_root() -> consult_file(config_file()). %% @doc reads the default config file in a given directory. -spec consult(file:name()) -> [any()]. consult(Dir) -> - consult_file(filename:join(Dir, config_file())). + consult_file(filename:join(Dir, ?DEFAULT_CONFIG_FILE)). %% @doc reads a given app file, including the `.script' variations, %% if any can be found. -- cgit v1.1 From 66a24391963f106d9652194d72d751197693d556 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sun, 26 Feb 2017 09:45:41 -0500 Subject: Fix wildcard usage in rebar3 clean The wildcard usage could fail on some OSes by being passed directly and just not finding libs and erroring out --- src/rebar_prv_clean.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_clean.erl b/src/rebar_prv_clean.erl index 8f31fdd..d185f75 100644 --- a/src/rebar_prv_clean.erl +++ b/src/rebar_prv_clean.erl @@ -44,7 +44,8 @@ do(State) -> case All of true -> DepsDir = rebar_dir:deps_dir(State1), - AllApps = rebar_app_discover:find_apps([filename:join(DepsDir, "*")], all), + DepsDirs = filelib:wildcard(filename:join(DepsDir, "*")), + AllApps = rebar_app_discover:find_apps(DepsDirs, all), clean_apps(State1, Providers, AllApps); false -> ProjectApps = rebar_state:project_apps(State1), -- cgit v1.1 From 7c125d95f0583cb200c47c0c6af301c1182ce463 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sun, 26 Feb 2017 12:49:01 -0500 Subject: More fixes for bad calls to find_apps --- src/rebar_prv_plugins.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_plugins.erl b/src/rebar_prv_plugins.erl index 9e04fa8..c76dae1 100644 --- a/src/rebar_prv_plugins.erl +++ b/src/rebar_prv_plugins.erl @@ -42,9 +42,9 @@ do(State) -> RebarOpts = rebar_state:opts(State), SrcDirs = rebar_dir:src_dirs(RebarOpts, ["src"]), Plugins = rebar_state:get(State, plugins, []), - PluginsDir = filename:join(rebar_dir:plugins_dir(State), "*"), - CheckoutsDir = filename:join(rebar_dir:checkouts_dir(State), "*"), - Apps = rebar_app_discover:find_apps([CheckoutsDir, PluginsDir], SrcDirs, all), + PluginsDirs = filelib:wildcard(filename:join(rebar_dir:plugins_dir(State), "*")), + CheckoutsDirs = filelib:wildcard(filename:join(rebar_dir:checkouts_dir(State), "*")), + Apps = rebar_app_discover:find_apps(CheckoutsDirs++PluginsDirs, SrcDirs, all), display_plugins("Local plugins", Apps, Plugins), {ok, State}. -- cgit v1.1 From 9f28bbe3c735c5cc0c670bfe0cb6559ff028680e Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Tue, 28 Feb 2017 07:47:11 -0500 Subject: Path-type sensitive .cmd creation --- src/rebar_prv_escriptize.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_escriptize.erl b/src/rebar_prv_escriptize.erl index 5c0c989..56b0d8a 100644 --- a/src/rebar_prv_escriptize.erl +++ b/src/rebar_prv_escriptize.erl @@ -266,9 +266,12 @@ rm_newline(String) -> [C || C <- String, C =/= $\n]. write_windows_script(Target) -> + CmdPath = if is_binary(Target) -> <>; + is_list(Target) -> Target ++ ".cmd" + end, CmdScript= "@echo off\r\n" "setlocal\r\n" "set rebarscript=%~f0\r\n" "escript.exe \"%rebarscript:.cmd=%\" %*\r\n", - ok = file:write_file(Target ++ ".cmd", CmdScript). + ok = file:write_file(CmdPath, CmdScript). -- cgit v1.1 From 695da1e0c2d9cea1bece49c9e1ef9fdad58464cc Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 2 Mar 2017 19:30:11 -0800 Subject: change package version warning to debug --- src/rebar_packages.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 4cce5a8..5b6ab5c 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -207,10 +207,10 @@ handle_single_vsn(Pkg, PkgVsn, Dep, Vsn, Constraint) -> false -> case {Pkg, PkgVsn} of {undefined, undefined} -> - ?WARN("Only existing version of ~s is ~s which does not match constraint ~~> ~s. " + ?DEBUG("Only existing version of ~s is ~s which does not match constraint ~~> ~s. " "Using anyway, but it is not guaranteed to work.", [Dep, Vsn, Constraint]); _ -> - ?WARN("[~s:~s] Only existing version of ~s is ~s which does not match constraint ~~> ~s. " + ?DEBUG("[~s:~s] Only existing version of ~s is ~s which does not match constraint ~~> ~s. " "Using anyway, but it is not guaranteed to work.", [Pkg, PkgVsn, Dep, Vsn, Constraint]) end, {ok, Vsn} -- cgit v1.1 From 5411a67ee63055ff4e11abd5e8eee95d71aea997 Mon Sep 17 00:00:00 2001 From: Juan Facorro Date: Fri, 3 Mar 2017 11:14:02 +0100 Subject: Fix typo in help text for upgrade command --- src/rebar_prv_upgrade.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl index 18c307b..34631ff 100644 --- a/src/rebar_prv_upgrade.erl +++ b/src/rebar_prv_upgrade.erl @@ -32,7 +32,7 @@ init(State) -> {deps, ?DEPS}, {example, "rebar3 upgrade [cowboy[,ranch]]"}, {short_desc, "Upgrade dependencies."}, - {desc, "Upgrade project dependecies. Mentioning no application " + {desc, "Upgrade project dependencies. Mentioning no application " "will upgrade all dependencies. To upgrade specific dependencies, " "their names can be listed in the command."}, {opts, [ -- cgit v1.1 From c6eea2bd468cad56220e7f7c63c4689fe8328789 Mon Sep 17 00:00:00 2001 From: Vlad Dumitrescu Date: Sat, 4 Mar 2017 17:35:37 +0100 Subject: rebar_utils:escape_chars handles quotes rebar_file_utils:cp_r uses rebar_utils:escape_chars to ensure that the file names are safe to use, but it doesn't escape double and single quotes. If the file name includes those characters, they disappear when the shell processes them and we get "file not found" errors. The main culprit here is eunit, that creates reports whose names are `TEST-file_"myfile.app".xml`, and I wish it didn't but I think escape_chars should still do its job all the way. --- src/rebar_utils.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index c357e94..c684e2d 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -857,7 +857,7 @@ url_append_path(Url, ExtraPath) -> escape_chars(Str) when is_atom(Str) -> escape_chars(atom_to_list(Str)); escape_chars(Str) -> - re:replace(Str, "([ ()?`!$&;])", "\\\\&", [global, {return, list}]). + re:replace(Str, "([ ()?`!$&;\"\'])", "\\\\&", [global, {return, list}]). %% "escape inside these" escape_double_quotes(Str) -> -- cgit v1.1 From 3d9e5dff85fd4193f8aab7a6dbd0cffeeea0c1de Mon Sep 17 00:00:00 2001 From: Carlo Carraro Date: Sun, 5 Mar 2017 11:14:45 +0100 Subject: Properly warn on missing rebar3 deps: add inets --- src/rebar3.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar3.erl b/src/rebar3.erl index 5c593cc..1e1d0b0 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -344,7 +344,7 @@ start_and_load_apps(Caller) -> ensure_running(asn1, Caller), ensure_running(public_key, Caller), ensure_running(ssl, Caller), - inets:start(), + ensure_running(inets, Caller), inets:start(httpc, [{profile, rebar}]). %% @doc Make sure a required app is running, or display an error message -- cgit v1.1 From 64b5d878af4839ac7e2514bdc497cce818b1755e Mon Sep 17 00:00:00 2001 From: Siri Hansen Date: Mon, 6 Mar 2017 11:36:26 +0100 Subject: Allow single test spec in ct_opts The option {spec,Specs} is allowed in ct_opts, but rebar_prv_common_test:test_dirs did not take into account that Specs could also be a string only, i.e. not a list of strings. --- src/rebar_prv_common_test.erl | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index 2ac8fc7..7a060f8 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -431,18 +431,21 @@ test_dirs(State, Apps, Opts) -> set_compile_dirs(State, Apps, join(Suites, Dir)); {_Suites, _Dirs} -> {error, "Only a single directory may be specified when specifying suites"} end; - Specs0 -> - case get_dirs_from_specs(Specs0) of - {ok,{Specs,SuiteDirs}} -> - {State1,Apps1} = set_compile_dirs1(State, Apps, - {dir, SuiteDirs}), - {State2,Apps2} = set_compile_dirs1(State1, Apps1, - {spec, Specs}), - [maybe_copy_spec(State2,Apps2,S) || S <- Specs], - {ok, rebar_state:project_apps(State2, Apps2)}; - Error -> - Error - end + Spec when is_integer(hd(Spec)) -> + spec_test_dirs(State, Apps, [Spec]); + Specs -> + spec_test_dirs(State, Apps, Specs) + end. + +spec_test_dirs(State, Apps, Specs0) -> + case get_dirs_from_specs(Specs0) of + {ok,{Specs,SuiteDirs}} -> + {State1,Apps1} = set_compile_dirs1(State, Apps, {dir, SuiteDirs}), + {State2,Apps2} = set_compile_dirs1(State1, Apps1, {spec, Specs}), + [maybe_copy_spec(State2,Apps2,S) || S <- Specs], + {ok, rebar_state:project_apps(State2, Apps2)}; + Error -> + Error end. join(Suite, Dir) when is_integer(hd(Suite)) -> -- cgit v1.1 From b763a9e14a0d96ad7663ea35c7d6be8b05c84006 Mon Sep 17 00:00:00 2001 From: Vlad Dumitrescu Date: Mon, 6 Mar 2017 17:44:28 +0100 Subject: minor fix: name of internal function --- src/rebar_prv_path.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_path.erl b/src/rebar_prv_path.erl index 4259eec..05fb046 100644 --- a/src/rebar_prv_path.erl +++ b/src/rebar_prv_path.erl @@ -27,7 +27,7 @@ init(State) -> {example, "rebar3 path"}, {short_desc, "Print paths to build dirs in current profile."}, {desc, "Print paths to build dirs in current profile."}, - {opts, eunit_opts(State)}])), + {opts, path_opts(State)}])), {ok, State1}. @@ -107,8 +107,8 @@ normalize(AppName) when is_list(AppName) -> AppName; normalize(AppName) when is_atom(AppName) -> atom_to_list(AppName); normalize(AppName) when is_binary(AppName) -> binary_to_list(AppName). -eunit_opts(_State) -> - [{app, undefined, "app", string, help(app)}, +path_opts(_State) -> + [{app, undefined, "apps", string, help(app)}, {base, undefined, "base", boolean, help(base)}, {bin, undefined, "bin", boolean, help(bin)}, {ebin, undefined, "ebin", boolean, help(ebin)}, -- cgit v1.1 From d4beb8edf15f6e0122498ed6394fe933a0b9c4c7 Mon Sep 17 00:00:00 2001 From: Vlad Dumitrescu Date: Mon, 6 Mar 2017 18:13:35 +0100 Subject: fix "apps" name --- src/rebar_prv_path.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_path.erl b/src/rebar_prv_path.erl index 05fb046..d8e14a4 100644 --- a/src/rebar_prv_path.erl +++ b/src/rebar_prv_path.erl @@ -108,7 +108,7 @@ normalize(AppName) when is_atom(AppName) -> atom_to_list(AppName); normalize(AppName) when is_binary(AppName) -> binary_to_list(AppName). path_opts(_State) -> - [{app, undefined, "apps", string, help(app)}, + [{app, undefined, "app", string, help(app)}, {base, undefined, "base", boolean, help(base)}, {bin, undefined, "bin", boolean, help(bin)}, {ebin, undefined, "ebin", boolean, help(ebin)}, -- cgit v1.1 From 62a737766db4db04b05fa501df6dfd1401ce1f51 Mon Sep 17 00:00:00 2001 From: Alin Popa Date: Sun, 12 Mar 2017 12:31:16 +0000 Subject: Fix git SHAs comparison for update. 86e883b8d8d1d16487e245fff02eba8c83da2cdd always returns the full length SHA, therefore when using a dependency having the short SHA, it'll always consider that the SHAs are different, hence it'll alway return true for . --- src/rebar_git_resource.erl | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl index 201b8b6..f666d17 100644 --- a/src/rebar_git_resource.erl +++ b/src/rebar_git_resource.erl @@ -53,24 +53,22 @@ needs_update(Dir, {git, Url, {branch, Branch}}) -> needs_update(Dir, {git, Url, "master"}) -> needs_update(Dir, {git, Url, {branch, "master"}}); needs_update(Dir, {git, _, Ref}) -> - {ok, Current} = rebar_utils:sh(?FMT("git rev-parse -q HEAD", []), + {ok, Current} = rebar_utils:sh(?FMT("git rev-parse --short=7 -q HEAD", []), [{cd, Dir}]), Current1 = string:strip(string:strip(Current, both, $\n), both, $\r), Ref2 = case Ref of {ref, Ref1} -> Length = length(Current1), - if - Length >= 7 -> - lists:sublist(Ref1, Length); - true -> - Ref1 + case Length >= 7 of + true -> lists:sublist(Ref1, Length); + false -> Ref1 end; - Ref1 -> - Ref1 + _ -> + Ref end, - ?DEBUG("Comparing git ref ~s with ~s", [Ref1, Current1]), + ?DEBUG("Comparing git ref ~s with ~s", [Ref2, Current1]), (Current1 =/= Ref2). compare_url(Dir, Url) -> -- cgit v1.1 From 8142cd739999994e85632c407d272de3d23ba982 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 23 Mar 2017 07:14:36 -0400 Subject: Bump to 3.3.6 - [OSX tests added to CI](https://github.com/erlang/rebar3/pull/1471), joining linux and windows 10 - [upgrade relx to 3.22.3](https://github.com/erlang/rebar3/pull/1518), [fixing multiple issues](https://github.com/erlware/relx/releases/tag/v3.22.3) - [Fixing eunit errors in 19.3 when `init_per_*` isn't paired with `end_per_*`](https://github.com/erlang/rebar3/pull/1516) - [Allow single test spec in ct_opts](https://github.com/erlang/rebar3/pull/1506) - [Properly warn when missing inets at runtime](https://github.com/erlang/rebar3/pull/1505) - [rebar_utils:escape_chars handles quotes in paths](https://github.com/erlang/rebar3/pull/1504) - [Fix typo in help text for upgrade command](https://github.com/erlang/rebar3/pull/1503) - [change package version warnings to debug level](https://github.com/erlang/rebar3/pull/1502) - [Path-type sensitive .cmd creation for escripts](https://github.com/erlang/rebar3/pull/1500) - [Windows escripts get cmd autogenerated](https://github.com/erlang/rebar3/pull/1494) - [REBAR_CONFIG impacts config file for project root only](https://github.com/erlang/rebar3/pull/1497) - [Fix wildcard usage in rebar3 clean](https://github.com/erlang/rebar3/pull/1498) - [Bump cth_readable to 1.2.4](https://github.com/erlang/rebar3/pull/1496) (fixing some erroneous reports when suites get skipped) - [Add respect for src_dirs option in app_discover and app_info](https://github.com/erlang/rebar3/pull/1486) - [Add support for global plugin-defined templates](https://github.com/erlang/rebar3/pull/1473) - [catch badarg exception from eunit in `rebar_prv_eunit`](https://github.com/erlang/rebar3/pull/1472) --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 74efe97..86c71df 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.3.6"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 2cc07d8f1fa0f0f3491720389d9c85b755ec8e59 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 23 Mar 2017 07:30:52 -0400 Subject: Back to git-based versioning --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 86c71df..74efe97 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.3.6"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From e83100bbc85967134fe68dbf691c2cca8d6db10c Mon Sep 17 00:00:00 2001 From: Michiel Beijen Date: Mon, 27 Mar 2017 21:26:23 +0200 Subject: Typo: seperate -> separate --- src/rebar_dialyzer_format.erl | 50 +++++++++++++++++++++---------------------- src/rebar_prv_common_test.erl | 16 +++++++------- src/rebar_prv_cover.erl | 10 ++++----- src/rebar_prv_path.erl | 2 +- 4 files changed, 39 insertions(+), 39 deletions(-) (limited to 'src') diff --git a/src/rebar_dialyzer_format.erl b/src/rebar_dialyzer_format.erl index 1d234c3..be8cc48 100644 --- a/src/rebar_dialyzer_format.erl +++ b/src/rebar_dialyzer_format.erl @@ -370,7 +370,7 @@ good_arg(N, Args) -> colour_arg(N, C, Args) when is_integer(N) -> colour_arg([N], C, Args); colour_arg(Ns, C, Args) -> - {Args1, Rest} =seperate_args(Args), + {Args1, Rest} =separate_args(Args), Args2 = highlight(Ns, 1, C, Args1), join_args(Args2) ++ Rest. @@ -388,43 +388,43 @@ highlight(Ns, N, C, [Arg | Rest]) -> %% Arugments to functions and constraints are passed as %% strings not as data, this function pulls them apart -%% to allow interacting with them seperately and not +%% to allow interacting with them separately and not %% as one bug chunk of data. -seperate_args([$( | S]) -> - seperate_args([], S, "", []). +separate_args([$( | S]) -> + separate_args([], S, "", []). %% We strip this space since dialyzer is inconsistant in adding or not adding %% it .... -seperate_args([], [$,, $\s | R], Arg, Args) -> - seperate_args([], R, [], [lists:reverse(Arg) | Args]); +separate_args([], [$,, $\s | R], Arg, Args) -> + separate_args([], R, [], [lists:reverse(Arg) | Args]); -seperate_args([], [$, | R], Arg, Args) -> - seperate_args([], R, [], [lists:reverse(Arg) | Args]); +separate_args([], [$, | R], Arg, Args) -> + separate_args([], R, [], [lists:reverse(Arg) | Args]); -seperate_args([], [$) | Rest], Arg, Args) -> +separate_args([], [$) | Rest], Arg, Args) -> {lists:reverse([lists:reverse(Arg) | Args]), Rest}; -seperate_args([C | D], [C | R], Arg, Args) -> - seperate_args(D, R, [C | Arg], Args); +separate_args([C | D], [C | R], Arg, Args) -> + separate_args(D, R, [C | Arg], Args); %% Brackets -seperate_args(D, [${ | R], Arg, Args) -> - seperate_args([$}|D], R, [${ | Arg], Args); +separate_args(D, [${ | R], Arg, Args) -> + separate_args([$}|D], R, [${ | Arg], Args); -seperate_args(D, [$( | R], Arg, Args) -> - seperate_args([$)|D], R, [$( | Arg], Args); +separate_args(D, [$( | R], Arg, Args) -> + separate_args([$)|D], R, [$( | Arg], Args); -seperate_args(D, [$[ | R], Arg, Args) -> - seperate_args([$]|D], R, [$[ | Arg], Args); +separate_args(D, [$[ | R], Arg, Args) -> + separate_args([$]|D], R, [$[ | Arg], Args); -seperate_args(D, [$< | R], Arg, Args) -> - seperate_args([$>|D], R, [$< | Arg], Args); +separate_args(D, [$< | R], Arg, Args) -> + separate_args([$>|D], R, [$< | Arg], Args); %% 'strings' -seperate_args(D, [$' | R], Arg, Args) -> - seperate_args([$'|D], R, [$' | Arg], Args); -seperate_args(D, [$" | R], Arg, Args) -> - seperate_args([$"|D], R, [$" | Arg], Args); +separate_args(D, [$' | R], Arg, Args) -> + separate_args([$'|D], R, [$' | Arg], Args); +separate_args(D, [$" | R], Arg, Args) -> + separate_args([$"|D], R, [$" | Arg], Args); -seperate_args(D, [C | R], Arg, Args) -> - seperate_args(D, R, [C | Arg], Args). +separate_args(D, [C | R], Arg, Args) -> + separate_args(D, R, [C | Arg], Args). join_args(Args) -> [$(, string:join(Args, ", "), $)]. diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index 7a060f8..bbd2a11 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -708,17 +708,17 @@ maybe_write_coverdata(State) -> rebar_prv_cover:maybe_write_coverdata(State1, ?PROVIDER). ct_opts(_State) -> - [{dir, undefined, "dir", string, help(dir)}, %% comma-seperated list - {suite, undefined, "suite", string, help(suite)}, %% comma-seperated list - {group, undefined, "group", string, help(group)}, %% comma-seperated list - {testcase, undefined, "case", string, help(testcase)}, %% comma-seperated list + [{dir, undefined, "dir", string, help(dir)}, %% comma-separated list + {suite, undefined, "suite", string, help(suite)}, %% comma-separated list + {group, undefined, "group", string, help(group)}, %% comma-separated list + {testcase, undefined, "case", string, help(testcase)}, %% comma-separated list {label, undefined, "label", string, help(label)}, %% String - {config, undefined, "config", string, help(config)}, %% comma-seperated list - {spec, undefined, "spec", string, help(spec)}, %% comma-seperated list + {config, undefined, "config", string, help(config)}, %% comma-separated list + {spec, undefined, "spec", string, help(spec)}, %% comma-separated list {join_specs, undefined, "join_specs", boolean, help(join_specs)}, {allow_user_terms, undefined, "allow_user_terms", boolean, help(allow_user_terms)}, %% Bool {logdir, undefined, "logdir", string, help(logdir)}, %% dir - {logopts, undefined, "logopts", string, help(logopts)}, %% comma seperated list + {logopts, undefined, "logopts", string, help(logopts)}, %% comma-separated list {verbosity, undefined, "verbosity", integer, help(verbosity)}, %% Integer {cover, $c, "cover", {boolean, false}, help(cover)}, {repeat, undefined, "repeat", integer, help(repeat)}, %% integer @@ -739,7 +739,7 @@ ct_opts(_State) -> {name, undefined, "name", atom, help(name)}, {sname, undefined, "sname", atom, help(sname)}, {setcookie, undefined, "setcookie", atom, help(setcookie)}, - {sys_config, undefined, "sys_config", string, help(sys_config)} %% comma-seperated list + {sys_config, undefined, "sys_config", string, help(sys_config)} %% comma-separated list ]. help(dir) -> diff --git a/src/rebar_prv_cover.erl b/src/rebar_prv_cover.erl index 865c557..bf142c4 100644 --- a/src/rebar_prv_cover.erl +++ b/src/rebar_prv_cover.erl @@ -215,18 +215,18 @@ print_analysis(Analysis, true) -> format_table(Stats, CoverFiles) -> MaxLength = max(lists:foldl(fun max_length/2, 0, Stats), 20), Header = header(MaxLength), - Seperator = seperator(MaxLength), + Separator = separator(MaxLength), TotalLabel = format("total", MaxLength), TotalCov = format(calculate_total(Stats), 8), - [io_lib:format("~ts~n~ts~n~ts~n", [Seperator, Header, Seperator]), + [io_lib:format("~ts~n~ts~n~ts~n", [Separator, Header, Separator]), lists:map(fun({Mod, Coverage}) -> Name = format(Mod, MaxLength), Cov = format(Coverage, 8), io_lib:format(" | ~ts | ~ts |~n", [Name, Cov]) end, Stats), - io_lib:format("~ts~n", [Seperator]), + io_lib:format("~ts~n", [Separator]), io_lib:format(" | ~ts | ~ts |~n", [TotalLabel, TotalCov]), - io_lib:format("~ts~n", [Seperator]), + io_lib:format("~ts~n", [Separator]), io_lib:format(" coverage calculated from:~n", []), lists:map(fun(File) -> io_lib:format(" ~ts~n", [File]) @@ -242,7 +242,7 @@ max_length({ModName, _}, Min) -> header(Width) -> [" | ", format("module", Width), " | ", format("coverage", 8), " |"]. -seperator(Width) -> +separator(Width) -> [" |--", io_lib:format("~*c", [Width, $-]), "--|------------|"]. format(String, Width) -> io_lib:format("~*.ts", [Width, String]). diff --git a/src/rebar_prv_path.erl b/src/rebar_prv_path.erl index d8e14a4..d32d2c2 100644 --- a/src/rebar_prv_path.erl +++ b/src/rebar_prv_path.erl @@ -118,7 +118,7 @@ path_opts(_State) -> {src, undefined, "src", boolean, help(src)}, {rel, undefined, "rel", boolean, help(rel)}]. -help(app) -> "Comma seperated list of applications to return paths for."; +help(app) -> "Comma separated list of applications to return paths for."; help(base) -> "Return the `base' path of the current profile."; help(bin) -> "Return the `bin' path of the current profile."; help(ebin) -> "Return all `ebin' paths of the current profile's applications."; -- cgit v1.1 From d15ea91b714c56c2d94996f3f5138edd062e5a5d Mon Sep 17 00:00:00 2001 From: Luis Rascao Date: Wed, 5 Apr 2017 14:13:22 +0100 Subject: Fix typo, occured -> occurred --- src/rebar_prv_common_test.erl | 2 +- src/rebar_prv_dialyzer.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index bbd2a11..62dd9f1 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -93,7 +93,7 @@ format_error({error, Reason}) -> format_error({error_running_tests, Reason}) -> format_error({error, Reason}); format_error({failures_running_tests, {Failed, AutoSkipped}}) -> - io_lib:format("Failures occured running tests: ~b", [Failed+AutoSkipped]); + io_lib:format("Failures occurred running tests: ~b", [Failed+AutoSkipped]); format_error({badconfig, {Msg, {Value, Key}}}) -> io_lib:format(Msg, [Value, Key]); format_error({badconfig, Msg}) -> diff --git a/src/rebar_prv_dialyzer.erl b/src/rebar_prv_dialyzer.erl index 21d7f5a..867b9cb 100644 --- a/src/rebar_prv_dialyzer.erl +++ b/src/rebar_prv_dialyzer.erl @@ -118,7 +118,7 @@ maybe_fix_env() -> format_error({error_processing_apps, Error}) -> io_lib:format("Error in dialyzing apps: ~s", [Error]); format_error({dialyzer_warnings, Warnings}) -> - io_lib:format("Warnings occured running dialyzer: ~b", [Warnings]); + io_lib:format("Warnings occurred running dialyzer: ~b", [Warnings]); format_error({unknown_application, App}) -> io_lib:format("Could not find application: ~s", [App]); format_error({unknown_module, Mod}) -> -- cgit v1.1 From 03425c788c0e1ac38a3172f6c13a42cd1ffa4b4a Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 24 Apr 2017 19:43:44 -0400 Subject: Abuse error_handler to get free metacalls in r3 This uses the very risky '$handle_undefined_function'/2 export from the r3 and rebar_agent modules to allow meta-calls that can support plugins and all other rebar3 extensions. This is nasty but very tempting. Currently we only support: - r3:do(Command) - r3:do(Namespace, Command) There is currently no way to pass arguments to the function such that we can, for example, run cover analysis or tests on a subset of suites. With the new abuse of '$handle_undefined_function'/2, we can detect the unused commands (since they are not exported) and re-route them: - r3:Command() - r3:Command("--args=as a string") - r3:Command(Namespace, "--args=as a string") Of course, in doing so, we make it impossible to use the 'do' provider (as in 'rebar3 do ct -c, cover') since the 'do' function is already required for things to work. Since the previous function had very strict guards, we can, without conflict, add manual overrides that simulate the meta-calls fine. Sample run: https://gist.github.com/ferd/2c06d59c7083c146d25e4ee301de0073 --- src/r3.erl | 5 +++++ src/rebar_agent.erl | 31 ++++++++++++++++++++++++------- 2 files changed, 29 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/r3.erl b/src/r3.erl index d0d6c47..bbf9eea 100644 --- a/src/r3.erl +++ b/src/r3.erl @@ -2,6 +2,7 @@ %%% calls from a shell. -module(r3). -export([do/1, do/2]). +-export(['$handle_undefined_function'/2]). %% @doc alias for `rebar_agent:do/1' -spec do(atom()) -> ok | {error, term()}. @@ -10,3 +11,7 @@ do(Command) -> rebar_agent:do(Command). %% @doc alias for `rebar_agent:do/2' -spec do(atom(), atom()) -> ok | {error, term()}. do(Namespace, Command) -> rebar_agent:do(Namespace, Command). + +%% @private defer to rebar_agent +'$handle_undefined_function'(Cmd, Args) -> + rebar_agent:'$handle_undefined_function'(Cmd, Args). diff --git a/src/rebar_agent.erl b/src/rebar_agent.erl index ed9e45d..627ed96 100644 --- a/src/rebar_agent.erl +++ b/src/rebar_agent.erl @@ -2,6 +2,7 @@ %%% to statefully maintain loaded project state into a running VM. -module(rebar_agent). -export([start_link/1, do/1, do/2]). +-export(['$handle_undefined_function'/2]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, code_change/3, terminate/2]). @@ -22,13 +23,24 @@ start_link(State) -> %% @doc runs a given command in the agent's context. -spec do(atom()) -> ok | {error, term()}. do(Command) when is_atom(Command) -> - gen_server:call(?MODULE, {cmd, Command}, infinity). + gen_server:call(?MODULE, {cmd, Command}, infinity); +do(Args) when is_list(Args) -> + gen_server:call(?MODULE, {cmd, default, do, Args}, infinity). %% @doc runs a given command in the agent's context, under a given %% namespace. -spec do(atom(), atom()) -> ok | {error, term()}. do(Namespace, Command) when is_atom(Namespace), is_atom(Command) -> - gen_server:call(?MODULE, {cmd, Namespace, Command}, infinity). + gen_server:call(?MODULE, {cmd, Namespace, Command}, infinity); +do(Namespace, Args) when is_atom(Namespace), is_list(Args) -> + gen_server:call(?MODULE, {cmd, Namespace, do, Args}, infinity). + +'$handle_undefined_function'(Cmd, [Namespace, Args]) -> + gen_server:call(?MODULE, {cmd, Namespace, Cmd, Args}, infinity); +'$handle_undefined_function'(Cmd, [Args]) -> + gen_server:call(?MODULE, {cmd, default, Cmd, Args}, infinity); +'$handle_undefined_function'(Cmd, []) -> + gen_server:call(?MODULE, {cmd, default, Cmd}, infinity). %%%%%%%%%%%%%%%%% %%% CALLBACKS %%% @@ -42,11 +54,15 @@ init(State) -> %% @private handle_call({cmd, Command}, _From, State=#state{state=RState, cwd=Cwd}) -> MidState = maybe_show_warning(State), - {Res, NewRState} = run(default, Command, RState, Cwd), + {Res, NewRState} = run(default, Command, "", RState, Cwd), {reply, Res, MidState#state{state=NewRState}, hibernate}; handle_call({cmd, Namespace, Command}, _From, State = #state{state=RState, cwd=Cwd}) -> MidState = maybe_show_warning(State), - {Res, NewRState} = run(Namespace, Command, RState, Cwd), + {Res, NewRState} = run(Namespace, Command, "", RState, Cwd), + {reply, Res, MidState#state{state=NewRState}, hibernate}; +handle_call({cmd, Namespace, Command, Args}, _From, State = #state{state=RState, cwd=Cwd}) -> + MidState = maybe_show_warning(State), + {Res, NewRState} = run(Namespace, Command, Args, RState, Cwd), {reply, Res, MidState#state{state=NewRState}, hibernate}; handle_call(_Call, _From, State) -> {noreply, State}. @@ -72,13 +88,14 @@ terminate(_Reason, _State) -> %%%%%%%%%%%%%%% %% @private runs the actual command and maintains the state changes --spec run(atom(), atom(), rebar_state:t(), file:filename()) -> +-spec run(atom(), atom(), string(), rebar_state:t(), file:filename()) -> {ok, rebar_state:t()} | {{error, term()}, rebar_state:t()}. -run(Namespace, Command, RState, Cwd) -> +run(Namespace, Command, StrArgs, RState, Cwd) -> try case rebar_dir:get_cwd() of Cwd -> - Args = [atom_to_list(Namespace), atom_to_list(Command)], + PArgs = getopt:tokenize(StrArgs), + Args = [atom_to_list(Namespace), atom_to_list(Command)] ++ PArgs, CmdState0 = refresh_state(RState, Cwd), CmdState1 = rebar_state:set(CmdState0, task, atom_to_list(Command)), CmdState = rebar_state:set(CmdState1, caller, api), -- cgit v1.1 From eb6cf4d721127695f18ec2d640be6c11dd5ace8d Mon Sep 17 00:00:00 2001 From: Anthony Molinaro Date: Wed, 26 Apr 2017 23:27:08 +0000 Subject: REBAR_COLOR supports all ec_cmd_log intensity options --- src/rebar_log.erl | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src') diff --git a/src/rebar_log.erl b/src/rebar_log.erl index b1a70c2..47c84c5 100644 --- a/src/rebar_log.erl +++ b/src/rebar_log.erl @@ -57,6 +57,8 @@ intensity() -> high; "low" -> low; + "none" -> + none; _ -> ?DFLT_INTENSITY end, -- cgit v1.1 From 1f9404e9cd5edc44627a17204c9ba01b32487dae Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 3 May 2017 08:36:11 -0400 Subject: Drop R15 support It can't even fetch packages from Hex anymore because of old SSL/TLS libraries, and so it can't bootstrap anymore either. Plus R20 comes out soon, and 5 major versions is quite enough. --- src/rebar_prv_common_test.erl | 3 --- 1 file changed, 3 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index 62dd9f1..e6788f8 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -567,9 +567,6 @@ get_tests_from_specs(Specs) -> case ct_testspec:collect_tests_from_file(Specs,true) of Tests when is_list(Tests) -> {ok,[{S,ct_testspec:prepare_tests(R)} || {S,R} <- Tests]}; - R when is_tuple(R), element(1,R)==testspec -> - %% R15 - {ok,[{Specs,ct_testspec:prepare_tests(R)}]}; Error -> Error end -- cgit v1.1 From c7f1554af6dbf4944d4b6a4fae2398aeb3b536d7 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 28 Apr 2017 23:35:01 -0400 Subject: Allow to upgrade profile-specific dependencies Only happens when calling `rebar3 as upgrade `, with the caveat that all profile deps get upgraded and lined up with the rebar config. --- src/rebar_prv_upgrade.erl | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl index 34631ff..a0ddbbc 100644 --- a/src/rebar_prv_upgrade.erl +++ b/src/rebar_prv_upgrade.erl @@ -70,7 +70,16 @@ do(State) -> is_atom(Dep) orelse is_atom(element(1, Dep))], Names = parse_names(ec_cnv:to_binary(proplists:get_value(package, Args, <<"">>)), Locks), DepsDict = deps_dict(rebar_state:all_deps(State)), - case prepare_locks(Names, Deps, Locks, [], DepsDict) of + %% Find alternative deps in non-default profiles since they may + %% need to be passed through (they are never locked) + AltProfiles = rebar_state:current_profiles(State) -- [default], + AltProfileDeps = lists:append([ + rebar_state:get(State, {deps, Profile}, []) || Profile <- AltProfiles] + ), + AltDeps = [Dep || Dep <- AltProfileDeps, + is_atom(Dep) orelse is_atom(element(1, Dep)) + andalso not lists:member(Dep, Deps)], + case prepare_locks(Names, Deps, Locks, [], DepsDict, AltDeps) of {error, Reason} -> {error, Reason}; {Locks0, _Unlocks0} -> @@ -115,20 +124,20 @@ parse_names(Bin, Locks) -> Other -> Other end. -prepare_locks([], _, Locks, Unlocks, _Dict) -> +prepare_locks([], _, Locks, Unlocks, _Dict, _AltDeps) -> {Locks, Unlocks}; -prepare_locks([Name|Names], Deps, Locks, Unlocks, Dict) -> +prepare_locks([Name|Names], Deps, Locks, Unlocks, Dict, AltDeps) -> AtomName = binary_to_atom(Name, utf8), case lists:keyfind(Name, 1, Locks) of {_, _, 0} = Lock -> case rebar_utils:tup_find(AtomName, Deps) of false -> ?WARN("Dependency ~s has been removed and will not be upgraded", [Name]), - prepare_locks(Names, Deps, Locks, Unlocks, Dict); + prepare_locks(Names, Deps, Locks, Unlocks, Dict, AltDeps); Dep -> {Source, NewLocks, NewUnlocks} = prepare_lock(Dep, Lock, Locks, Dict), prepare_locks(Names, Deps, NewLocks, - [{Name, Source, 0} | NewUnlocks ++ Unlocks], Dict) + [{Name, Source, 0} | NewUnlocks ++ Unlocks], Dict, AltDeps) end; {_, _, Level} = Lock when Level > 0 -> case rebar_utils:tup_find(AtomName, Deps) of @@ -137,10 +146,15 @@ prepare_locks([Name|Names], Deps, Locks, Unlocks, Dict) -> Dep -> % Dep has been promoted {Source, NewLocks, NewUnlocks} = prepare_lock(Dep, Lock, Locks, Dict), prepare_locks(Names, Deps, NewLocks, - [{Name, Source, 0} | NewUnlocks ++ Unlocks], Dict) + [{Name, Source, 0} | NewUnlocks ++ Unlocks], Dict, AltDeps) end; false -> - ?PRV_ERROR({unknown_dependency, Name}) + case rebar_utils:tup_find(AtomName, AltDeps) of + false -> + ?PRV_ERROR({unknown_dependency, Name}); + _ -> % non-default profile dependency found, pass through + prepare_locks(Names, Deps, Locks, Unlocks, Dict, AltDeps) + end end. prepare_lock(Dep, Lock, Locks, Dict) -> -- cgit v1.1 From a1f47427093daa060ab510e099de3947430bdb8d Mon Sep 17 00:00:00 2001 From: Thomas O'Dowd Date: Tue, 20 Dec 2016 17:03:37 +0900 Subject: Issue #1416: Merge erl_files_first separately and avoid sorting. The order of the files listed in "erl_files_first" option is important and should not be sorted. By handling the merge similarly to mib_files_first, the sort order is preserved. --- src/rebar_opts.erl | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src') diff --git a/src/rebar_opts.erl b/src/rebar_opts.erl index 444b760..589dbb8 100644 --- a/src/rebar_opts.erl +++ b/src/rebar_opts.erl @@ -118,6 +118,10 @@ merge_opt({plugins, _}, NewValue, _OldValue) -> NewValue; merge_opt(profiles, NewValue, OldValue) -> dict:to_list(merge_opts(dict:from_list(NewValue), dict:from_list(OldValue))); +merge_opt(erl_first_files, Value, Value) -> + Value; +merge_opt(erl_first_files, NewValue, OldValue) -> + OldValue ++ NewValue; merge_opt(mib_first_files, Value, Value) -> Value; merge_opt(mib_first_files, NewValue, OldValue) -> -- cgit v1.1 From f4ca27d58fa81521808f530cfa50b9b314e93aa2 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 11 May 2017 08:27:39 -0400 Subject: Handle internal CT failures This is based on issue #1517 where out of nowhere, CT has returned a user's error code directly. This in turn caused a crashdump in rebar3 itself. This patch handles the unexpected cases by: a) not trying to format them b) converting them to an error whenever they happen The execution flow is still interrupted, but we should fail with a clearer error than a crashdump. --- src/rebar_prv_common_test.erl | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index e6788f8..3df8b75 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -650,7 +650,11 @@ handle_results(_) -> sum_results({Passed, Failed, {UserSkipped, AutoSkipped}}, {Passed2, Failed2, {UserSkipped2, AutoSkipped2}}) -> {Passed+Passed2, Failed+Failed2, - {UserSkipped+UserSkipped2, AutoSkipped+AutoSkipped2}}. + {UserSkipped+UserSkipped2, AutoSkipped+AutoSkipped2}}; +sum_results(_, {error, Reason}) -> + {error, Reason}; +sum_results(Unknown, _) -> + {error, Unknown}. handle_quiet_results(_, {error, _} = Result) -> handle_results(Result); @@ -673,7 +677,10 @@ format_result({Passed, 0, {0, 0}}) -> format_result({Passed, Failed, Skipped}) -> Format = [format_failed(Failed), format_skipped(Skipped), format_passed(Passed)], - ?CONSOLE("~s", [Format]). + ?CONSOLE("~s", [Format]); +format_result(_Unknown) -> + %% Happens when CT itself encounters a bug + ok. format_failed(0) -> []; -- cgit v1.1 From 8d1d138a26131eaa23f50149fe09c472f7c8b0b3 Mon Sep 17 00:00:00 2001 From: Luis Rascao Date: Thu, 23 Mar 2017 00:40:58 +0000 Subject: Let relx know about application erl opts This is so that some of them can be enforced (such as warnings_as_errors) on relx's task like generating a release or a release upgrade. --- src/rebar_relx.erl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_relx.erl b/src/rebar_relx.erl index 5c653a3..abfb8fc 100644 --- a/src/rebar_relx.erl +++ b/src/rebar_relx.erl @@ -26,19 +26,21 @@ do(Module, Command, Provider, State) -> AllOptions = string:join([Command | Options], " "), Cwd = rebar_state:dir(State), Providers = rebar_state:providers(State), + RebarOpts = rebar_state:opts(State), + ErlOpts = rebar_opts:erl_opts(RebarOpts), rebar_hooks:run_project_and_app_hooks(Cwd, pre, Provider, Providers, State), try case rebar_state:get(State, relx, []) of [] -> relx:main([{lib_dirs, LibDirs} ,{caller, api} - ,{log_level, LogLevel} | output_dir(OutputDir, Options)], AllOptions); + ,{log_level, LogLevel} | output_dir(OutputDir, Options)] ++ ErlOpts, AllOptions); Config -> Config1 = merge_overlays(Config), relx:main([{lib_dirs, LibDirs} ,{config, Config1} ,{caller, api} - ,{log_level, LogLevel} | output_dir(OutputDir, Options)], AllOptions) + ,{log_level, LogLevel} | output_dir(OutputDir, Options)] ++ ErlOpts, AllOptions) end, rebar_hooks:run_project_and_app_hooks(Cwd, post, Provider, Providers, State), {ok, State} -- cgit v1.1 From 47db2e91fce39da12ec91742b41d80f142b30a4b Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 17 May 2017 10:47:46 +0200 Subject: Use rm_rf instead of ec_file:delete. In contrast to `ec_file:delete`, `rebar_file_utils:rm_rf` will also delete write-protected files on Windows which is needed for git object files. Fixes #1483. --- src/rebar_fetch.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index 47bfe1d..f5e5db5 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -46,7 +46,7 @@ download_source_(AppDir, Source, State) -> {ok, _} -> ec_file:mkdir_p(AppDir1), code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), - ec_file:remove(filename:absname(AppDir1), [recursive]), + ok = rebar_file_utils:rm_rf(filename:absname(AppDir1)), ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]), ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)), true; -- cgit v1.1 From 0ce1a4f5fea0a0a58b862e1e375043e7696f9bff Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 19 May 2017 22:34:26 -0400 Subject: Cull default dep names in non-default upgrade Prevents showing false upgrade messages when upgrading in a non-default profile. Also shows an info message explaining default deps don't get updated in a non-default upgrade. --- src/rebar_prv_upgrade.erl | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl index a0ddbbc..2de30e4 100644 --- a/src/rebar_prv_upgrade.erl +++ b/src/rebar_prv_upgrade.erl @@ -70,16 +70,9 @@ do(State) -> is_atom(Dep) orelse is_atom(element(1, Dep))], Names = parse_names(ec_cnv:to_binary(proplists:get_value(package, Args, <<"">>)), Locks), DepsDict = deps_dict(rebar_state:all_deps(State)), - %% Find alternative deps in non-default profiles since they may - %% need to be passed through (they are never locked) - AltProfiles = rebar_state:current_profiles(State) -- [default], - AltProfileDeps = lists:append([ - rebar_state:get(State, {deps, Profile}, []) || Profile <- AltProfiles] - ), - AltDeps = [Dep || Dep <- AltProfileDeps, - is_atom(Dep) orelse is_atom(element(1, Dep)) - andalso not lists:member(Dep, Deps)], - case prepare_locks(Names, Deps, Locks, [], DepsDict, AltDeps) of + AltDeps = find_non_default_deps(Deps, State), + FilteredNames = cull_default_names_if_profiles(Names, Deps, State), + case prepare_locks(FilteredNames, Deps, Locks, [], DepsDict, AltDeps) of {error, Reason} -> {error, Reason}; {Locks0, _Unlocks0} -> @@ -124,6 +117,31 @@ parse_names(Bin, Locks) -> Other -> Other end. +%% Find alternative deps in non-default profiles since they may +%% need to be passed through (they are never locked) +find_non_default_deps(Deps, State) -> + AltProfiles = rebar_state:current_profiles(State) -- [default], + AltProfileDeps = lists:append([ + rebar_state:get(State, {deps, Profile}, []) || Profile <- AltProfiles] + ), + [Dep || Dep <- AltProfileDeps, + is_atom(Dep) orelse is_atom(element(1, Dep)) + andalso not lists:member(Dep, Deps)]. + +%% If any alt profiles are used, remove the default profiles from +%% the upgrade list and warn about it. +cull_default_names_if_profiles(Names, Deps, State) -> + case rebar_state:current_profiles(State) of + [default] -> + Names; + _ -> + ?INFO("Dependencies in the default profile will not be upgraded", []), + lists:filter(fun(Name) -> + AtomName = binary_to_atom(Name, utf8), + rebar_utils:tup_find(AtomName, Deps) == false + end, Names) + end. + prepare_locks([], _, Locks, Unlocks, _Dict, _AltDeps) -> {Locks, Unlocks}; prepare_locks([Name|Names], Deps, Locks, Unlocks, Dict, AltDeps) -> -- cgit v1.1 From 3ac0e40181a3d747decbc6bca3017ed62dfd6368 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sun, 21 May 2017 11:17:46 -0400 Subject: Try to start epmd when distribution fails We do it by calling os:cmd on a named shell so that the automated daemon rules work for rebar3 as well. --- src/rebar_dist_utils.erl | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/rebar_dist_utils.erl b/src/rebar_dist_utils.erl index 93edf9d..5de858e 100644 --- a/src/rebar_dist_utils.erl +++ b/src/rebar_dist_utils.erl @@ -51,14 +51,27 @@ find_options(State) -> %%% PRIVATE %%% %%%%%%%%%%%%%%% start(Name, Type, Opts) -> - check_epmd(net_kernel:start([Name, Type])), + case dist_up(net_kernel:start([Name, Type])) of + false -> + start_epmd(), + dist_up(net_kernel:start([Name, Type])) orelse warn_dist(); + true -> + ok + end, setup_cookie(Opts). -check_epmd({error,{{shutdown, {_,net_kernel,{'EXIT',nodistribution}}},_}}) -> - ?ERROR("Erlang Distribution failed, falling back to nonode@nohost. " - "Verify that epmd is running and try again.",[]); -check_epmd(_) -> - ok. +dist_up({error,{{shutdown,{_,net_kernel,{'EXIT',nodistribution}}},_}}) -> false; +dist_up(_) -> true. + +start_epmd() -> + %% Indirectly boot EPMD through calling Erlang so that we don't risk + %% attaching it to the current proc + ?CONSOLE("Attempting to start epmd...", []), + os:cmd("erl -sname a -eval 'halt(0).'"). + +warn_dist() -> + ?ERROR("Erlang Distribution failed, falling back to nonode@nohost.", []). + setup_cookie(Opts) -> case {node(), proplists:get_value(setcookie, Opts, nocookie)} of -- cgit v1.1 From 46651f030c6a9560061ff5d4f5f5bf5b747a2738 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 22 May 2017 10:17:09 -0400 Subject: Bump to 3.4.0 Patches: - Try to start epmd when distribution fails - Allow non-default profile deps upgrade - Switch default license in templates to Apache 2.0, and hook it up in the .app file - Fix git SHA comparisons for dep update - Let relx know about rebar-specified erl_opts - Use rm_rf instead of ec_file:delete, fixing some deps-related issues on Windows - Handle internal CT failures - Fix profile merging of erl_first_files options - Fix CT output in skipped test cases and failing end of function hooks - Officially drop R15 support, since its SSL libs can no longer talk to hex.pm - REBAR_COLOR supports all ec_cmd_log intensity options, allowing the none value to disable all colors without dropping termcap support - Bump relx to 3.22.4 The one new feature supported here is for the interactive shell: - Support commands in the shell such as r3:Command(), r3:Command("--args=as a string"), or r3:Command(Namespace, "--args=as a string"), instead of just r3:do(Command). --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 74efe97..5a8f2b4 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.4.0"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 2d103f908ad9f7e3015b0dbd459644feb22160ac Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 22 May 2017 10:25:40 -0400 Subject: Back to git-based versioning --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 5a8f2b4..74efe97 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.4.0"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 2312aa0228b5deeacbcee4969aee22cdbf7daba3 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 24 May 2017 14:01:52 -0400 Subject: Bump to 3.4.1 Includes Relx fixes Signed-off-by: Fred Hebert --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 74efe97..8b18645 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.4.1"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 87e756d8dc30270a90d0fc12186bc4b14c1543a5 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 24 May 2017 14:17:01 -0400 Subject: Back to git-based versioning --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 8b18645..74efe97 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.4.1"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 249b5f13eb8f1859d7a70ab316bc7541543aaf57 Mon Sep 17 00:00:00 2001 From: Anthony Molinaro Date: Thu, 1 Jun 2017 18:32:30 +0000 Subject: Avoid reversing relx overlays. This mostly moves a lists:reverse/1 which ensures that profile overlays are run first but keeps the order of overlays otherwise. --- src/rebar_relx.erl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_relx.erl b/src/rebar_relx.erl index abfb8fc..17c0bd6 100644 --- a/src/rebar_relx.erl +++ b/src/rebar_relx.erl @@ -6,6 +6,10 @@ -export([do/4, format_error/1]). +-ifdef(TEST). +-export([merge_overlays/1]). +-endif. + -include("rebar.hrl"). %% =================================================================== @@ -64,5 +68,5 @@ merge_overlays(Config) -> (_) -> false end, Config), %% Have profile overlay entries come before others to match how profiles work elsewhere - NewOverlay = lists:reverse(lists:flatmap(fun({overlay, Overlay}) -> Overlay end, Overlays)), + NewOverlay = lists:flatmap(fun({overlay, Overlay}) -> Overlay end, lists:reverse(Overlays)), [{overlay, NewOverlay} | Others]. -- cgit v1.1 From 48dc9738bbab828381c15daab6035dda59cbd646 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 2 Jun 2017 12:46:17 -0700 Subject: reset hooks under profiles for application opts --- src/rebar_app_discover.erl | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index db74cd3..de72c09 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -97,11 +97,11 @@ merge_deps(AppInfo, State) -> %% the application they are defined at. If an umbrella structure is used and %% they are deifned at the top level they will instead run in the context of %% the State and at the top level, not as part of an application. - Default = reset_hooks(rebar_state:default(State)), + CurrentProfiles = rebar_state:current_profiles(State), + Default = reset_hooks(rebar_state:default(State), CurrentProfiles), {C, State1} = project_app_config(AppInfo, State), AppInfo0 = rebar_app_info:update_opts(AppInfo, Default, C), - CurrentProfiles = rebar_state:current_profiles(State1), Name = rebar_app_info:name(AppInfo0), %% We reset the opts here to default so no profiles are applied multiple times @@ -171,17 +171,33 @@ project_app_config(AppInfo, State) -> maybe_reset_hooks(Dir, Opts, State) -> case ec_file:real_dir_path(rebar_dir:root_dir(State)) of Dir -> - reset_hooks(Opts); + CurrentProfiles = rebar_state:current_profiles(State), + reset_hooks(Opts, CurrentProfiles); _ -> Opts end. %% @doc make the hooks empty for a given set of options --spec reset_hooks(Opts) -> Opts when Opts :: rebar_dict(). -reset_hooks(Opts) -> - lists:foldl(fun(Key, OptsAcc) -> - rebar_opts:set(OptsAcc, Key, []) - end, Opts, [post_hooks, pre_hooks, provider_hooks, artifacts]). +-spec reset_hooks(Opts, Profiles) -> + Opts when + Opts :: rebar_dict(), + Profiles :: [atom()]. +reset_hooks(Opts, CurrentProfiles) -> + AllHooks = [post_hooks, pre_hooks, provider_hooks, artifacts], + Opts1 = lists:foldl(fun(Key, OptsAcc) -> + rebar_opts:set(OptsAcc, Key, []) + end, Opts, AllHooks), + Profiles = rebar_opts:get(Opts1, profiles, []), + Profiles1 = lists:map(fun({P, ProfileOpts}) -> + case lists:member(P, CurrentProfiles) of + true -> + {P, [X || X={Key, _} <- ProfileOpts, + not lists:member(Key, AllHooks)]}; + false -> + {P, ProfileOpts} + end + end, Profiles), + rebar_opts:set(Opts1, profiles, Profiles1). %% @private find the directories for all apps, while detecting their source dirs %% Returns the app dir with the respective src_dirs for them, in that order, @@ -422,4 +438,3 @@ find_config_src(AppDir, Default) -> %% TODO: handle profiles I guess, but we don't have that info proplists:get_value(src_dirs, Terms, Default) end. - -- cgit v1.1 From 9a6f04a2d9d00167e0b853776b60b12b5c5dcc57 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 7 Jun 2017 20:12:49 -0400 Subject: Fix xref compile hook on deps If xref analysis is being run by a dependency during its compilation phase, the xref provider will try to add that deps' parents to the check job while the ebin/ directories for them do not exist. This causes a crash. This patch makes it so directories are only added if they are existing, preventing failure on any compile hook for dependencies and allowing successful compilation as a best effort. --- src/rebar_prv_xref.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_xref.erl b/src/rebar_prv_xref.erl index c4e72e7..51154b5 100644 --- a/src/rebar_prv_xref.erl +++ b/src/rebar_prv_xref.erl @@ -97,8 +97,11 @@ prepare(State) -> rebar_state:get(State, xref_warnings, false)}, {verbose, rebar_log:is_verbose(State)}]), - [{ok, _} = xref:add_directory(xref, rebar_app_info:ebin_dir(App)) - || App <- rebar_state:project_apps(State)], + [{ok, _} = xref:add_directory(xref, Dir) + || App <- rebar_state:project_apps(State), + %% the directory may not exist in rare cases of a compile + %% hook of a dep running xref prior to the full job being done + Dir <- [rebar_app_info:ebin_dir(App)], filelib:is_dir(Dir)], %% Get list of xref checks we want to run ConfXrefChecks = rebar_state:get(State, xref_checks, -- cgit v1.1 From 5ae96a50769f815047e1e098c7fb2a4ae0e9d3aa Mon Sep 17 00:00:00 2001 From: getong Date: Tue, 13 Jun 2017 15:01:47 +0800 Subject: the new added PATH value should be in the first position, in case there be other same name cmd --- src/rebar_prv_local_install.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_local_install.erl b/src/rebar_prv_local_install.erl index 1b58859..0b30fd4 100644 --- a/src/rebar_prv_local_install.erl +++ b/src/rebar_prv_local_install.erl @@ -90,6 +90,6 @@ extract_escript(State, ScriptPath) -> uid=Uid, gid=Gid}), - ?INFO("Add to $PATH for use: export PATH=$PATH:~s", [BinDir]), + ?INFO("Add to $PATH for use: export PATH=~s:$PATH", [BinDir]), {ok, State}. -- cgit v1.1 From 7f1301f635b8c07fd22ffdf1c9dcf6f5cbba8655 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Tue, 13 Jun 2017 07:17:07 -0400 Subject: Prevent xref crashes with undef behaviours When a given behaviour module does not exist, rebar3 brutally crashes. This patch makes it so instead, there is a warning output menitoning the missing behaviour, and this one is omitted from the unused function calls check. This means that unused calls will instead be shown for the module implementing a non-existing behaviour, as if no behaviour were declared in the first place. --- src/rebar_prv_xref.erl | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_xref.erl b/src/rebar_prv_xref.erl index 51154b5..e6a074f 100644 --- a/src/rebar_prv_xref.erl +++ b/src/rebar_prv_xref.erl @@ -168,8 +168,15 @@ keyall(Key, List) -> lists:flatmap(fun({K, L}) when Key =:= K -> L; (_) -> [] end, List). get_behaviour_callbacks(exports_not_used, Attributes) -> - [B:behaviour_info(callbacks) || B <- keyall(behaviour, Attributes) ++ - keyall(behavior, Attributes)]; + lists:map(fun(Mod) -> + try + Mod:behaviour_info(callbacks) + catch + error:undef -> + ?WARN("Behaviour ~p is used but cannot be found.", [Mod]), + [] + end + end, keyall(behaviour, Attributes) ++ keyall(behavior, Attributes)); get_behaviour_callbacks(_XrefCheck, _Attributes) -> []. -- cgit v1.1 From 083fc942b4becdef1400d1163e259a68bf8e6625 Mon Sep 17 00:00:00 2001 From: Anthony Molinaro Date: Thu, 15 Jun 2017 17:42:59 +0000 Subject: Application type of none was not working. I noticed this when trying to include entop in a release. Entop uses cecho which takes over the terminal, so you do not want it loaded or started. According to http://erlang.org/doc/man/rel.html, when you specify a Type of none it should not load or start, but the code for it's modules should be loaded. This patch ensures the code is not loaded or started, but doesn't do anything with the code paths. At the very least this allows me to start a shell in the case where I have an application of type none, and the application is neither loaded nor started. --- src/rebar_prv_shell.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_shell.erl b/src/rebar_prv_shell.erl index c1bf735..c958dde 100644 --- a/src/rebar_prv_shell.erl +++ b/src/rebar_prv_shell.erl @@ -360,8 +360,10 @@ boot_apps(Apps) -> ok. normalize_load_apps([]) -> []; +normalize_load_apps([{_App, none} | T]) -> normalize_load_apps(T); normalize_load_apps([{App, _} | T]) -> [App | normalize_load_apps(T)]; normalize_load_apps([{App, _Vsn, load} | T]) -> [App | normalize_load_apps(T)]; +normalize_load_apps([{_App, _Vsn, none} | T]) -> normalize_load_apps(T); normalize_load_apps([{App, _Vsn, Operator} | T]) when is_atom(Operator) -> [App | normalize_load_apps(T)]; normalize_load_apps([App | T]) when is_atom(App) -> [App | normalize_load_apps(T)]. @@ -369,12 +371,13 @@ normalize_load_apps([App | T]) when is_atom(App) -> [App | normalize_load_apps(T normalize_boot_apps([]) -> []; normalize_boot_apps([{_App, load} | T]) -> normalize_boot_apps(T); normalize_boot_apps([{_App, _Vsn, load} | T]) -> normalize_boot_apps(T); +normalize_boot_apps([{_App, none} | T]) -> normalize_boot_apps(T); +normalize_boot_apps([{_App, _Vsn, none} | T]) -> normalize_boot_apps(T); normalize_boot_apps([{App, _Vsn, Operator} | T]) when is_atom(Operator) -> [App | normalize_boot_apps(T)]; normalize_boot_apps([{App, _Vsn} | T]) -> [App | normalize_boot_apps(T)]; normalize_boot_apps([App | T]) when is_atom(App) -> [App | normalize_boot_apps(T)]. - remove_error_handler(0) -> ?WARN("Unable to remove simple error_logger handler", []); remove_error_handler(N) -> -- cgit v1.1 From c94d3d2a16ccfdb05f81ad235da4910f78c8d573 Mon Sep 17 00:00:00 2001 From: Gleb Peregud Date: Mon, 19 Jun 2017 22:44:11 +0200 Subject: Bare compile: support multiple paths wildcards. Similarly to PATH env variable, this allows to pass paths to bare compiler which do not fit nicely into a single wildcard structure. Colon (":") is used as separator. This provides more flexibility when rebar is run in offline/hermetic environment, e.g. #958 and #1281. --- src/rebar_prv_bare_compile.erl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_bare_compile.erl b/src/rebar_prv_bare_compile.erl index 201620a..6f1ac16 100644 --- a/src/rebar_prv_bare_compile.erl +++ b/src/rebar_prv_bare_compile.erl @@ -29,7 +29,8 @@ init(State) -> {example, ""}, {short_desc, ""}, {desc, ""}, - {opts, [{paths, $p, "paths", string, "Wildcard path of ebin directories to add to code path"}]}])), + {opts, [{paths, $p, "paths", string, "Wildcard paths of ebin directories to add to code path, separated by a colon"}, + {separator, $s, "separator", string, "In case of multiple return paths, the separator character to use to join them."}]}])), {ok, State1}. -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. @@ -39,8 +40,9 @@ do(State) -> %% Add code paths from --paths to the beginning of the code path {RawOpts, _} = rebar_state:command_parsed_args(State), Paths = proplists:get_value(paths, RawOpts), - CodePaths = filelib:wildcard(Paths), - code:add_pathsa(CodePaths), + Sep = proplists:get_value(separator, RawOpts, " "), + [ code:add_pathsa(filelib:wildcard(PathWildcard)) + || PathWildcard <- string:tokens(Paths, Sep) ], [AppInfo] = rebar_state:project_apps(State), AppInfo1 = rebar_app_info:out_dir(AppInfo, rebar_dir:get_cwd()), -- cgit v1.1 From 15ea087336ffe377dd1cbaf09806594dbeea3b67 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 13 Jul 2017 11:41:39 -0700 Subject: recompile only if new option effects code generation --- src/rebar_erlc_compiler.erl | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index f7244dc..9157aba 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -279,7 +279,7 @@ gather_src(Opts, BaseDirParts, [Dir|Rest], Srcs, CompileOpts) -> end, DirRecursive = dir_recursive(Opts, RelDir, CompileOpts), gather_src(Opts, BaseDirParts, Rest, Srcs ++ rebar_utils:find_files(Dir, ?RE_PREFIX".*\\.erl\$", DirRecursive), CompileOpts). - + %% Get files which need to be compiled first, i.e. those specified in erl_first_files %% and parse_transform options. Also produce specific erl_opts for these first %% files, so that yet to be compiled parse transformations are excluded from it. @@ -351,10 +351,25 @@ opts_changed(NewOpts, Target) -> false -> NewOpts end, case compile_info(Target) of - {ok, Opts} -> lists:sort(Opts) =/= lists:sort(TotalOpts); + {ok, Opts} -> lists:any(fun effects_code_generation/1, lists:usort(TotalOpts) -- lists:usort(Opts)); _ -> true end. +effects_code_generation(Option) -> + case Option of + beam -> false; + report_warnings -> false; + report_errors -> false; + return_errors-> false; + return_warnings-> false; + warnings_as_errors -> false; + binary -> false; + verbose -> false; + {cwd,_} -> false; + {outdir, _} -> false; + _ -> true + end. + compile_info(Target) -> case beam_lib:chunks(Target, [compile_info]) of {ok, {_mod, Chunks}} -> -- cgit v1.1 From 15461c9b1edf77ef91cc70b0db9c824bda876642 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 13 Jul 2017 18:13:16 -0700 Subject: add compile_only option to ct provider --- src/rebar_prv_common_test.erl | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index 3df8b75..dfade77 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -41,7 +41,14 @@ do(State) -> Tests = prepare_tests(State), case compile(State, Tests) of %% successfully compiled apps - {ok, S} -> do(S, Tests); + {ok, S} -> + {RawOpts, _} = rebar_state:command_parsed_args(S), + case proplists:get_value(compile_only, RawOpts, false) of + true -> + {ok, S}; + false -> + do(S, Tests) + end; %% this should look like a compiler error, not a ct error Error -> Error end. @@ -743,9 +750,12 @@ ct_opts(_State) -> {name, undefined, "name", atom, help(name)}, {sname, undefined, "sname", atom, help(sname)}, {setcookie, undefined, "setcookie", atom, help(setcookie)}, - {sys_config, undefined, "sys_config", string, help(sys_config)} %% comma-separated list + {sys_config, undefined, "sys_config", string, help(sys_config)}, %% comma-separated list + {compile_only, undefined, "compile_only", boolean, help(compile_only)} ]. +help(compile_only) -> + "Compile modules in the project with the test configuration but do not run the tests"; help(dir) -> "List of additional directories containing test suites"; help(suite) -> -- cgit v1.1 From b49987ca33ad8c1920635faa3128a85e2cdd80c1 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 14 Jul 2017 18:25:49 -0400 Subject: Bump to 3.4.2 --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 74efe97..3e12bc8 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.4.2"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From dee97b11845e00baa1acff92086627662ed8ff42 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 14 Jul 2017 18:32:03 -0400 Subject: Back to git-based versioning --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 3e12bc8..74efe97 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.4.2"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From d09aaed77d9bd82063fbeda2347e083b752f6e2c Mon Sep 17 00:00:00 2001 From: Sam Sawan Date: Mon, 17 Jul 2017 13:47:32 -0400 Subject: [#149002995] stop eating edoc path config be a little smarter when generating edocs; loof for some sort of doc path and then replace w/e was there with the updated doc paths --- src/rebar_prv_edoc.erl | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_edoc.erl b/src/rebar_prv_edoc.erl index 97f70a9..6e58ad5 100644 --- a/src/rebar_prv_edoc.erl +++ b/src/rebar_prv_edoc.erl @@ -84,9 +84,8 @@ format_error(Reason) -> has_configured_paths(EdocOpts) -> proplists:get_value(dir, EdocOpts) =/= undefined. -add_to_paths([], Path) -> - [{doc_path, [Path]}]; -add_to_paths([{doc_path, Paths}|T], Path) -> - [{doc_path, [Path | Paths]} | T]; -add_to_paths([H|T], Path) -> - [H | add_to_paths(Path, T)]. +add_to_paths(Opts, Path) -> + case proplists:get_value(doc_path, Opts) of + undefined -> [{doc_path, [Path]} | Opts]; + Paths -> lists:keyreplace(doc_path, 1, Opts, {doc_path, [Path | Paths]}) + end. -- cgit v1.1 From f62b7cb32e93fa1f2b30c649e25f555c1b216c75 Mon Sep 17 00:00:00 2001 From: Sam Sawan Date: Mon, 17 Jul 2017 16:29:18 -0400 Subject: [#149002995] fix flipped variables --- src/rebar_prv_edoc.erl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_edoc.erl b/src/rebar_prv_edoc.erl index 6e58ad5..d663b0c 100644 --- a/src/rebar_prv_edoc.erl +++ b/src/rebar_prv_edoc.erl @@ -84,8 +84,9 @@ format_error(Reason) -> has_configured_paths(EdocOpts) -> proplists:get_value(dir, EdocOpts) =/= undefined. -add_to_paths(Opts, Path) -> - case proplists:get_value(doc_path, Opts) of - undefined -> [{doc_path, [Path]} | Opts]; - Paths -> lists:keyreplace(doc_path, 1, Opts, {doc_path, [Path | Paths]}) - end. +add_to_paths([], Path) -> + [{doc_path, [Path]}]; +add_to_paths([{doc_path, Paths}|T], Path) -> + [{doc_path, [Path | Paths]} | T]; +add_to_paths([H|T], Path) -> + [H | add_to_paths(T, Path)]. -- cgit v1.1 From 639bd13aecd3ca9f4237790264728dabaad50db0 Mon Sep 17 00:00:00 2001 From: Mikhail Kalashnikov Date: Wed, 26 Jul 2017 12:08:17 +0300 Subject: Fix cleanup_code_path for xref compile hook --- src/rebar_prv_xref.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_xref.erl b/src/rebar_prv_xref.erl index e6a074f..db0f4e4 100644 --- a/src/rebar_prv_xref.erl +++ b/src/rebar_prv_xref.erl @@ -36,6 +36,7 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> + OldPath = code:get_path(), code:add_pathsa(rebar_state:code_paths(State, all_deps)), XrefChecks = prepare(State), XrefIgnores = rebar_state:get(State, xref_ignores, []), @@ -47,7 +48,7 @@ do(State) -> QueryChecks = rebar_state:get(State, xref_queries, []), QueryResults = lists:foldl(fun check_query/2, [], QueryChecks), stopped = xref:stop(xref), - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), + rebar_utils:cleanup_code_path(OldPath), case XrefResults =:= [] andalso QueryResults =:= [] of true -> {ok, State}; -- cgit v1.1 From 1c96de5e10bd51ba4da9a89c087c191a8c7cd8c5 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sat, 5 Aug 2017 10:35:51 -0400 Subject: Allow top-level apps to take precedence over deps The use case has been described in issue #1478 where a local application can exist while being declared as a dependency as well. This allows, for example, to work on a release where all applications may require to be published independently, or to provide some form of 'vendoring' with a local app. The fix is done by decoupling the dependency source resolution form the dependency parsing. The reason for this being that the discovery phase needs to parse apps for their top-level deps, and dep installation needs to resolve the packages with accuracy. In the current implementation, both code paths call to the same function. This patch splits up the precise discovery and makes it happen *only* when installing dependencies, and only if a top-level app does not already define the application needing resolving. One weakness of this fix is that it necessarily breaks cycle detection in dependencies that involve a root application depending on itself since its own version as a dep will not be expanded. There appears to be no possible way to prevent this, but should be rare enough to be worth the tradeoff for the common case. --- src/rebar_app_utils.erl | 10 +++++++++- src/rebar_prv_install_deps.erl | 26 ++++++++++++++++++++------ 2 files changed, 29 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index 7154999..4e87adc 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -34,6 +34,7 @@ validate_application_info/2, parse_deps/5, parse_deps/6, + expand_deps_sources/2, dep_to_app/7, format_error/1]). @@ -225,7 +226,7 @@ dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> not_found -> rebar_app_info:new(Parent, Name, Vsn, Dir, []) end, - update_source(AppInfo0, Source, State) + rebar_app_info:source(AppInfo0, Source) end, C = rebar_config:consult(rebar_app_info:dir(AppInfo)), AppInfo1 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), C), @@ -236,6 +237,13 @@ dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> AppInfo5 = rebar_app_info:profiles(AppInfo4, [default]), rebar_app_info:is_lock(AppInfo5, IsLock). +%% @doc Takes a given application app_info record along with the project. +%% If the app is a package, resolve and expand the package definition. +-spec expand_deps_sources(rebar_app_info:t(), rebar_state:t()) -> + rebar_app_info:t(). +expand_deps_sources(Dep, State) -> + update_source(Dep, rebar_app_info:source(Dep), State). + %% @doc sets the source for a given dependency or app along with metadata %% around version if required. -spec update_source(rebar_app_info:t(), Source, rebar_state:t()) -> diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index c9fe0ad..dc208a7 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -140,7 +140,9 @@ handle_deps_as_profile(Profile, State, Deps, Upgrade) -> DepsDir = profile_dep_dir(State, Profile), Deps1 = rebar_app_utils:parse_deps(DepsDir, Deps, State, Locks, Level), ProfileLevelDeps = [{Profile, Deps1, Level}], - handle_profile_level(ProfileLevelDeps, [], sets:new(), Upgrade, Locks, State). + RootSeen = sets:from_list([rebar_app_info:name(AppInfo) + || AppInfo <- rebar_state:project_apps(State)]), + handle_profile_level(ProfileLevelDeps, [], RootSeen, RootSeen, Upgrade, Locks, State). %% =================================================================== %% Internal functions @@ -153,7 +155,9 @@ deps_per_profile(Profiles, Upgrade, State) -> Deps = lists:foldl(fun(Profile, DepAcc) -> [parsed_profile_deps(State, Profile, Level) | DepAcc] end, [], Profiles), - handle_profile_level(Deps, [], sets:new(), Upgrade, Locks, State). + RootSeen = sets:from_list([rebar_app_info:name(AppInfo) + || AppInfo <- rebar_state:project_apps(State)]), + handle_profile_level(Deps, [], RootSeen, RootSeen, Upgrade, Locks, State). parsed_profile_deps(State, Profile, Level) -> ParsedDeps = rebar_state:get(State, {parsed_deps, Profile}, []), @@ -162,17 +166,27 @@ parsed_profile_deps(State, Profile, Level) -> %% Level-order traversal of all dependencies, across profiles. %% If profiles x,y,z are present, then the traversal will go: %% x0, y0, z0, x1, y1, z1, ..., xN, yN, zN. -handle_profile_level([], Apps, _Seen, _Upgrade, _Locks, State) -> +%% +%% There are two 'seen' sets: one for the top-level apps (`RootSeen') and +%% one for all dependencies (`Seen'). The former is used to know when +%% to skip the resolving of dependencies altogether (since they're already +%% top-level apps), while the latter is used to prevent reprocessing +%% deps more than one. +handle_profile_level([], Apps, _RootSeen, _Seen, _Upgrade, _Locks, State) -> {Apps, State}; -handle_profile_level([{Profile, Deps, Level} | Rest], Apps, Seen, Upgrade, Locks, State) -> +handle_profile_level([{Profile, Deps, Level} | Rest], Apps, RootSeen, Seen, Upgrade, Locks, State) -> + Deps0 = [rebar_app_utils:expand_deps_sources(Dep, State) + || Dep <- Deps, + %% skip top-level apps being double-declared + not sets:is_element(rebar_app_info:name(Dep), RootSeen)], {Deps1, Apps1, State1, Seen1} = - update_deps(Profile, Level, Deps, Apps + update_deps(Profile, Level, Deps0, Apps ,State, Upgrade, Seen, Locks), Deps2 = case Deps1 of [] -> Rest; _ -> Rest ++ [{Profile, Deps1, Level+1}] end, - handle_profile_level(Deps2, Apps1, sets:union(Seen, Seen1), Upgrade, Locks, State1). + handle_profile_level(Deps2, Apps1, RootSeen, sets:union(Seen, Seen1), Upgrade, Locks, State1). find_cycles(Apps) -> case rebar_digraph:compile_order(Apps) of -- cgit v1.1 From e8e20e32a8683daa6848c0bb0b715eae842bc4a9 Mon Sep 17 00:00:00 2001 From: Guilherme Andrade Date: Sat, 5 Aug 2017 22:14:40 +0100 Subject: Don't crash when determining the source of undefined functions in stripped modules This can be reproduced by running xref analysis against a rebar3 plugin project which doesn't list rebar3 as an explicit dependency -- calls to certain bundled modules ('rebar_state', 'rebar_api', 'ec_cnv', ...) will result in a failed pattern match as these modules appear to have had their abstract code stripped. --- src/rebar_prv_xref.erl | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_xref.erl b/src/rebar_prv_xref.erl index db0f4e4..f358787 100644 --- a/src/rebar_prv_xref.erl +++ b/src/rebar_prv_xref.erl @@ -281,12 +281,21 @@ find_mfa_source({M, F, A}) -> end. find_function_source(M, F, A, Bin) -> - AbstractCode = beam_lib:chunks(Bin, [abstract_code]), - {ok, {M, [{abstract_code, {raw_abstract_v1, Code}}]}} = AbstractCode, + ChunksLookup = beam_lib:chunks(Bin, [abstract_code]), + {ok, {M, [{abstract_code, AbstractCodeLookup}]}} = ChunksLookup, + case AbstractCodeLookup of + no_abstract_code -> + % There isn't much else we can do at this point + {module_not_found, function_not_found}; + {raw_abstract_v1, AbstractCode} -> + find_function_source_in_abstract_code(F, A, AbstractCode) + end. + +find_function_source_in_abstract_code(F, A, AbstractCode) -> %% Extract the original source filename from the abstract code - [{attribute, _, file, {Source, _}} | _] = Code, + [{attribute, _, file, {Source, _}} | _] = AbstractCode, %% Extract the line number for a given function def - Fn = [E || E <- Code, + Fn = [E || E <- AbstractCode, safe_element(1, E) == function, safe_element(3, E) == F, safe_element(4, E) == A], -- cgit v1.1 From 963c49f5eb9ab5b34e1843fb43305743720917ac Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sun, 6 Aug 2017 07:26:21 -0400 Subject: Unicode support in all the places This is done through 3 main change groups: - replacing `~s` by `~ts` in format strings, so that strings that contain unicode are properly printed rather than crashing - adding the `unicode` argument to all function of the `re` module to ensure transformations on strings containing unicode data are valid instead of crashing (see issue #1302) - replacing `ec_cnv:to_binary/1` and `ec_cnv:to_list/1` with matching functions in `rebar_utils`. The last point has been done, rather than modifying and updating erlware commons, because binary and list conversions can be a contentious subject. For example, if what is being handled is actually bytes from a given binary stream, then forcing a byte-oriented interpretation of the data can corrupt it. As such, it does not appear safe to modify erlware commons' conversion functions since it may not be safe for all its users. Instead, rebar3 reimplements a subset of them (only converting atoms and chardata, ignoring numbers) with the explicit purpose of handling unicode string data. Tests were left as unchanged as possible. This may impact the ability to run rebar3's own suites in a unicode path, but respects a principle of least change for such a large patch. --- src/rebar3.erl | 10 +-- src/rebar_agent.erl | 2 +- src/rebar_app_discover.erl | 6 +- src/rebar_app_info.erl | 54 ++++++------ src/rebar_app_utils.erl | 24 ++--- src/rebar_base_compiler.erl | 16 ++-- src/rebar_config.erl | 8 +- src/rebar_core.erl | 10 +-- src/rebar_dialyzer_format.erl | 178 +++++++++++++++++++------------------- src/rebar_digraph.erl | 4 +- src/rebar_dir.erl | 4 +- src/rebar_erlc_compiler.erl | 10 +-- src/rebar_fetch.erl | 20 ++--- src/rebar_file_utils.erl | 45 ++++++---- src/rebar_git_resource.erl | 47 +++++----- src/rebar_hg_resource.erl | 23 ++--- src/rebar_hooks.erl | 6 +- src/rebar_log.erl | 2 +- src/rebar_otp_app.erl | 8 +- src/rebar_packages.erl | 12 +-- src/rebar_pkg_resource.erl | 18 ++-- src/rebar_prv_app_discovery.erl | 12 +-- src/rebar_prv_as.erl | 6 +- src/rebar_prv_clean.erl | 2 +- src/rebar_prv_common_test.erl | 2 +- src/rebar_prv_compile.erl | 8 +- src/rebar_prv_cover.erl | 4 +- src/rebar_prv_deps.erl | 20 ++--- src/rebar_prv_deps_tree.erl | 4 +- src/rebar_prv_dialyzer.erl | 16 ++-- src/rebar_prv_edoc.erl | 6 +- src/rebar_prv_escriptize.erl | 14 +-- src/rebar_prv_install_deps.erl | 26 +++--- src/rebar_prv_local_install.erl | 8 +- src/rebar_prv_new.erl | 20 ++--- src/rebar_prv_packages.erl | 6 +- src/rebar_prv_path.erl | 16 ++-- src/rebar_prv_plugins.erl | 10 +-- src/rebar_prv_plugins_upgrade.erl | 2 +- src/rebar_prv_report.erl | 14 +-- src/rebar_prv_unlock.erl | 4 +- src/rebar_prv_update.erl | 10 +-- src/rebar_prv_upgrade.erl | 12 +-- src/rebar_prv_xref.erl | 24 ++--- src/rebar_state.erl | 6 +- src/rebar_templater.erl | 18 ++-- src/rebar_utils.erl | 69 ++++++++------- 47 files changed, 433 insertions(+), 413 deletions(-) (limited to 'src') diff --git a/src/rebar3.erl b/src/rebar3.erl index 1e1d0b0..eec8968 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -183,11 +183,11 @@ init_config() -> GlobalConfigFile = rebar_dir:global_config(), State = case filelib:is_regular(GlobalConfigFile) of true -> - ?DEBUG("Load global config file ~s", [GlobalConfigFile]), + ?DEBUG("Load global config file ~ts", [GlobalConfigFile]), try state_from_global_config(Config1, GlobalConfigFile) catch _:_ -> - ?WARN("Global config ~s exists but can not be read. Ignoring global config values.", [GlobalConfigFile]), + ?WARN("Global config ~ts exists but can not be read. Ignoring global config values.", [GlobalConfigFile]), rebar_state:new(Config1) end; false -> @@ -270,7 +270,7 @@ log_level() -> -spec version() -> ok. version() -> {ok, Vsn} = application:get_key(rebar, vsn), - ?CONSOLE("rebar ~s on Erlang/OTP ~s Erts ~s", + ?CONSOLE("rebar ~ts on Erlang/OTP ~ts Erts ~ts", [Vsn, erlang:system_info(otp_release), erlang:system_info(version)]). %% @private set global flag based on getopt option boolean value @@ -311,11 +311,11 @@ handle_error({error, {Module, Reason}}) -> ?DEBUG("Uncaught error: ~p ~p", [Module, Reason]), ?INFO("When submitting a bug report, please include the output of `rebar3 report \"your command\"`", []); _ -> - ?ERROR("~s", [Module:format_error(Reason)]) + ?ERROR("~ts", [Module:format_error(Reason)]) end, erlang:halt(1); handle_error({error, Error}) when is_list(Error) -> - ?ERROR("~s", [Error]), + ?ERROR("~ts", [Error]), erlang:halt(1); handle_error(Error) -> %% Nothing should percolate up from rebar_core; diff --git a/src/rebar_agent.erl b/src/rebar_agent.erl index 627ed96..b5dcfcf 100644 --- a/src/rebar_agent.erl +++ b/src/rebar_agent.erl @@ -157,7 +157,7 @@ refresh_paths(RState) -> {ok, Mods} -> case {length(Mods), length(Mods -- Blacklist)} of {X,X} -> - ?DEBUG("reloading ~p from ~s", [Modules, Path]), + ?DEBUG("reloading ~p from ~ts", [Modules, Path]), code:replace_path(App, Path), reload_modules(Modules); {_,_} -> diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index de72c09..cdd183c 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -60,7 +60,7 @@ do(State, LibDirs) -> rebar_state:project_apps(StateAcc1 ,rebar_app_info:deps(AppInfo2, ProjectDeps1)); false -> - ?INFO("Ignoring ~s", [Name]), + ?INFO("Ignoring ~ts", [Name]), StateAcc end end, State1, SortedApps). @@ -233,7 +233,7 @@ app_dirs(LibDir, SrcDirs) -> EbinPath = filename:join([LibDir, "ebin", "*.app"]), lists:usort(lists:foldl(fun(Path, Acc) -> - Files = filelib:wildcard(ec_cnv:to_list(Path)), + Files = filelib:wildcard(rebar_utils:to_list(Path)), [{app_dir(File), SrcDirs} || File <- Files] ++ Acc end, [], [EbinPath | Paths])). @@ -386,7 +386,7 @@ try_handle_app_file(AppInfo0, [File], AppDir, AppSrcFile, _, Validate) -> end catch throw:{error, {Module, Reason}} -> - ?DEBUG("Falling back to app.src file because .app failed: ~s", [Module:format_error(Reason)]), + ?DEBUG("Falling back to app.src file because .app failed: ~ts", [Module:format_error(Reason)]), try_handle_app_src_file(AppInfo0, File, AppDir, AppSrcFile, Validate) end; try_handle_app_file(_AppInfo, Other, _AppDir, _AppSrcFile, _, _Validate) -> diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index 62ec6dd..050ccc1 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -107,43 +107,43 @@ new() -> -spec new(atom() | binary() | string()) -> {ok, t()}. new(AppName) -> - {ok, #app_info_t{name=ec_cnv:to_binary(AppName)}}. + {ok, #app_info_t{name=rebar_utils:to_binary(AppName)}}. %% @doc Build a new app info value with only the name and version set. -spec new(atom() | binary() | string(), binary() | string()) -> {ok, t()}. new(AppName, Vsn) -> - {ok, #app_info_t{name=ec_cnv:to_binary(AppName), + {ok, #app_info_t{name=rebar_utils:to_binary(AppName), original_vsn=Vsn}}. %% @doc build a complete version of the app info with all fields set. -spec new(atom() | binary() | string(), binary() | string(), file:name()) -> {ok, t()}. new(AppName, Vsn, Dir) -> - {ok, #app_info_t{name=ec_cnv:to_binary(AppName), + {ok, #app_info_t{name=rebar_utils:to_binary(AppName), original_vsn=Vsn, - dir=ec_cnv:to_list(Dir), - out_dir=ec_cnv:to_list(Dir)}}. + dir=rebar_utils:to_list(Dir), + out_dir=rebar_utils:to_list(Dir)}}. %% @doc build a complete version of the app info with all fields set. -spec new(atom() | binary() | string(), binary() | string(), file:name(), list()) -> {ok, t()}. new(AppName, Vsn, Dir, Deps) -> - {ok, #app_info_t{name=ec_cnv:to_binary(AppName), + {ok, #app_info_t{name=rebar_utils:to_binary(AppName), original_vsn=Vsn, - dir=ec_cnv:to_list(Dir), - out_dir=ec_cnv:to_list(Dir), + dir=rebar_utils:to_list(Dir), + out_dir=rebar_utils:to_list(Dir), deps=Deps}}. %% @doc build a complete version of the app info with all fields set. -spec new(atom() | binary(), atom() | binary() | string(), binary() | string(), file:name(), list()) -> {ok, t()}. new(Parent, AppName, Vsn, Dir, Deps) -> - {ok, #app_info_t{name=ec_cnv:to_binary(AppName), + {ok, #app_info_t{name=rebar_utils:to_binary(AppName), parent=Parent, original_vsn=Vsn, - dir=ec_cnv:to_list(Dir), - out_dir=ec_cnv:to_list(Dir), + dir=rebar_utils:to_list(Dir), + out_dir=rebar_utils:to_list(Dir), deps=Deps}}. %% @doc update the opts based on the contents of a config @@ -199,7 +199,7 @@ name(#app_info_t{name=Name}) -> %% @doc set the name of the app. -spec name(t(), atom() | binary() | string()) -> t(). name(AppInfo=#app_info_t{}, AppName) -> - AppInfo#app_info_t{name=ec_cnv:to_binary(AppName)}. + AppInfo#app_info_t{name=rebar_utils:to_binary(AppName)}. %% @doc get the dictionary of options for the app. -spec opts(t()) -> rebar_dict(). @@ -249,14 +249,14 @@ set(AppInfo=#app_info_t{opts=Opts}, Key, Value) -> %% @doc finds the .app.src file for an app, if any. -spec app_file_src(t()) -> file:filename_all() | undefined. app_file_src(#app_info_t{app_file_src=undefined, dir=Dir, name=Name, opts=Opts}) -> - CandidatePaths = [filename:join([ec_cnv:to_list(Dir), Src, ec_cnv:to_list(Name)++".app.src"]) + CandidatePaths = [filename:join([rebar_utils:to_list(Dir), Src, rebar_utils:to_list(Name)++".app.src"]) || Src <- rebar_opts:get(Opts, src_dirs, ["src"])], case lists:dropwhile(fun(Path) -> not filelib:is_file(Path) end, CandidatePaths) of [] -> undefined; [AppFileSrc|_] -> AppFileSrc end; app_file_src(#app_info_t{app_file_src=AppFileSrc}) -> - ec_cnv:to_list(AppFileSrc). + rebar_utils:to_list(AppFileSrc). %% @doc sets the .app.src file for an app. An app without such a file %% can explicitly be set with `undefined'. @@ -264,12 +264,12 @@ app_file_src(#app_info_t{app_file_src=AppFileSrc}) -> app_file_src(AppInfo=#app_info_t{}, undefined) -> AppInfo#app_info_t{app_file_src=undefined}; app_file_src(AppInfo=#app_info_t{}, AppFileSrc) -> - AppInfo#app_info_t{app_file_src=ec_cnv:to_list(AppFileSrc)}. + AppInfo#app_info_t{app_file_src=rebar_utils:to_list(AppFileSrc)}. %% @doc finds the .app.src.script file for an app, if any. -spec app_file_src_script(t()) -> file:filename_all() | undefined. app_file_src_script(#app_info_t{app_file_src_script=undefined, dir=Dir, name=Name}) -> - AppFileSrcScript = filename:join([ec_cnv:to_list(Dir), "src", ec_cnv:to_list(Name)++".app.src.script"]), + AppFileSrcScript = filename:join([rebar_utils:to_list(Dir), "src", rebar_utils:to_list(Name)++".app.src.script"]), case filelib:is_file(AppFileSrcScript) of true -> AppFileSrcScript; @@ -277,7 +277,7 @@ app_file_src_script(#app_info_t{app_file_src_script=undefined, dir=Dir, name=Nam undefined end; app_file_src_script(#app_info_t{app_file_src_script=AppFileSrcScript}) -> - ec_cnv:to_list(AppFileSrcScript). + rebar_utils:to_list(AppFileSrcScript). %% @doc sets the .app.src.script file for an app. An app without such a file %% can explicitly be set with `undefined'. @@ -285,12 +285,12 @@ app_file_src_script(#app_info_t{app_file_src_script=AppFileSrcScript}) -> app_file_src_script(AppInfo=#app_info_t{}, undefined) -> AppInfo#app_info_t{app_file_src_script=undefined}; app_file_src_script(AppInfo=#app_info_t{}, AppFileSrcScript) -> - AppInfo#app_info_t{app_file_src_script=ec_cnv:to_list(AppFileSrcScript)}. + AppInfo#app_info_t{app_file_src_script=rebar_utils:to_list(AppFileSrcScript)}. %% @doc finds the .app file for an app, if any. -spec app_file(t()) -> file:filename_all() | undefined. app_file(#app_info_t{app_file=undefined, out_dir=Dir, name=Name}) -> - AppFile = filename:join([ec_cnv:to_list(Dir), "ebin", ec_cnv:to_list(Name)++".app"]), + AppFile = filename:join([rebar_utils:to_list(Dir), "ebin", rebar_utils:to_list(Name)++".app"]), case filelib:is_file(AppFile) of true -> AppFile; @@ -317,7 +317,7 @@ app_details(AppInfo=#app_info_t{app_details=[]}) -> rebar_file_utils:try_consult(AppFile) catch throw:{error, {Module, Reason}} -> - ?DEBUG("Warning, falling back to .app.src because of: ~s", + ?DEBUG("Warning, falling back to .app.src because of: ~ts", [Module:format_error(Reason)]), rebar_file_utils:try_consult(app_file_src(AppInfo)) end @@ -404,10 +404,10 @@ dir(#app_info_t{dir=Dir}) -> %% @doc sets the directory that contains the app. -spec dir(t(), file:name()) -> t(). dir(AppInfo=#app_info_t{out_dir=undefined}, Dir) -> - AppInfo#app_info_t{dir=ec_cnv:to_list(Dir), - out_dir=ec_cnv:to_list(Dir)}; + AppInfo#app_info_t{dir=rebar_utils:to_list(Dir), + out_dir=rebar_utils:to_list(Dir)}; dir(AppInfo=#app_info_t{}, Dir) -> - AppInfo#app_info_t{dir=ec_cnv:to_list(Dir)}. + AppInfo#app_info_t{dir=rebar_utils:to_list(Dir)}. %% @doc returns the directory where build artifacts for the app %% should go @@ -419,17 +419,17 @@ out_dir(#app_info_t{out_dir=OutDir}) -> %% should go -spec out_dir(t(), file:name()) -> t(). out_dir(AppInfo=#app_info_t{}, OutDir) -> - AppInfo#app_info_t{out_dir=ec_cnv:to_list(OutDir)}. + AppInfo#app_info_t{out_dir=rebar_utils:to_list(OutDir)}. %% @doc gets the directory where ebin files for the app should go -spec ebin_dir(t()) -> file:name(). ebin_dir(#app_info_t{out_dir=OutDir}) -> - ec_cnv:to_list(filename:join(OutDir, "ebin")). + rebar_utils:to_list(filename:join(OutDir, "ebin")). %% @doc gets the directory where private files for the app should go -spec priv_dir(t()) -> file:name(). priv_dir(#app_info_t{out_dir=OutDir}) -> - ec_cnv:to_list(filename:join(OutDir, "priv")). + rebar_utils:to_list(filename:join(OutDir, "priv")). %% @doc returns whether the app is source app or a package app. -spec resource_type(t()) -> pkg | src. @@ -519,7 +519,7 @@ all(Dir, Context, [File|Artifacts]) -> FilePath = filename:join(Dir, rebar_templater:render(File, Context)), case filelib:is_regular(FilePath) of false -> - ?DEBUG("Missing artifact ~s", [FilePath]), + ?DEBUG("Missing artifact ~ts", [FilePath]), {false, File}; true -> all(Dir, Context, Artifacts) diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index 7154999..9422240 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -145,7 +145,7 @@ parse_dep(Dep, Parent, DepsDir, State, Locks, Level) -> Dep -> Dep end, - case lists:keyfind(ec_cnv:to_binary(Name), 1, Locks) of + case lists:keyfind(rebar_utils:to_binary(Name), 1, Locks) of false -> parse_dep(Parent, Dep, DepsDir, false, State); LockedDep -> @@ -167,18 +167,20 @@ parse_dep(Dep, Parent, DepsDir, State, Locks, Level) -> IsLock :: boolean(), State :: rebar_state:t(). parse_dep(Parent, {Name, Vsn, {pkg, PkgName}}, DepsDir, IsLock, State) -> - {PkgName1, PkgVsn} = {ec_cnv:to_binary(PkgName), ec_cnv:to_binary(Vsn)}, + {PkgName1, PkgVsn} = {rebar_utils:to_binary(PkgName), + rebar_utils:to_binary(Vsn)}, dep_to_app(Parent, DepsDir, Name, PkgVsn, {pkg, PkgName1, PkgVsn, undefined}, IsLock, State); parse_dep(Parent, {Name, {pkg, PkgName}}, DepsDir, IsLock, State) -> %% Package dependency with different package name from app name - dep_to_app(Parent, DepsDir, Name, undefined, {pkg, ec_cnv:to_binary(PkgName), undefined, undefined}, IsLock, State); + dep_to_app(Parent, DepsDir, Name, undefined, {pkg, rebar_utils:to_binary(PkgName), undefined, undefined}, IsLock, State); parse_dep(Parent, {Name, Vsn}, DepsDir, IsLock, State) when is_list(Vsn); is_binary(Vsn) -> %% Versioned Package dependency - {PkgName, PkgVsn} = {ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}, + {PkgName, PkgVsn} = {rebar_utils:to_binary(Name), + rebar_utils:to_binary(Vsn)}, dep_to_app(Parent, DepsDir, PkgName, PkgVsn, {pkg, PkgName, PkgVsn, undefined}, IsLock, State); parse_dep(Parent, Name, DepsDir, IsLock, State) when is_atom(Name); is_binary(Name) -> %% Unversioned package dependency - dep_to_app(Parent, DepsDir, ec_cnv:to_binary(Name), undefined, {pkg, ec_cnv:to_binary(Name), undefined, undefined}, IsLock, State); + dep_to_app(Parent, DepsDir, rebar_utils:to_binary(Name), undefined, {pkg, rebar_utils:to_binary(Name), undefined, undefined}, IsLock, State); parse_dep(Parent, {Name, Source}, DepsDir, IsLock, State) when is_tuple(Source) -> dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); parse_dep(Parent, {Name, _Vsn, Source}, DepsDir, IsLock, State) when is_tuple(Source) -> @@ -212,12 +214,12 @@ parse_dep(_, Dep, _, _, _) -> IsLock :: boolean(), State :: rebar_state:t(). dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> - CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), + CheckoutsDir = rebar_utils:to_list(rebar_dir:checkouts_dir(State, Name)), AppInfo = case rebar_app_info:discover(CheckoutsDir) of {ok, App} -> rebar_app_info:source(rebar_app_info:is_checkout(App, true), checkout); not_found -> - Dir = ec_cnv:to_list(filename:join(DepsDir, Name)), + Dir = rebar_utils:to_list(filename:join(DepsDir, Name)), {ok, AppInfo0} = case rebar_app_info:discover(Dir) of {ok, App} -> @@ -275,7 +277,7 @@ fetch_checksum(PkgName, PkgVsn, Hash, State) -> rebar_packages:registry_checksum({pkg, PkgName, PkgVsn, Hash}, State) catch _:_ -> - ?INFO("Package ~s-~s not found. Fetching registry updates and trying again...", [PkgName, PkgVsn]), + ?INFO("Package ~ts-~ts not found. Fetching registry updates and trying again...", [PkgName, PkgVsn]), {ok, _} = rebar_prv_update:do(State), rebar_packages:registry_checksum({pkg, PkgName, PkgVsn, Hash}, State) end. @@ -283,7 +285,7 @@ fetch_checksum(PkgName, PkgVsn, Hash, State) -> %% @doc convert a given exception's payload into an io description. -spec format_error(any()) -> iolist(). format_error({missing_package, Package}) -> - io_lib:format("Package not found in registry: ~s", [Package]); + io_lib:format("Package not found in registry: ~ts", [Package]); format_error({parse_dep, Dep}) -> io_lib:format("Failed parsing dep ~p", [Dep]); format_error(Error) -> @@ -302,7 +304,7 @@ get_package(Dep, Vsn, State) -> {ok, HighestDepVsn} -> {Dep, HighestDepVsn}; none -> - throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Dep)})) + throw(?PRV_ERROR({missing_package, rebar_utils:to_binary(Dep)})) end. %% @private checks that all the beam files have been properly @@ -310,7 +312,7 @@ get_package(Dep, Vsn, State) -> -spec has_all_beams(file:filename_all(), [module()]) -> true | ?PRV_ERROR({missing_module, module()}). has_all_beams(EbinDir, [Module | ModuleList]) -> - BeamFile = filename:join([EbinDir, ec_cnv:to_list(Module) ++ ".beam"]), + BeamFile = filename:join([EbinDir, rebar_utils:to_list(Module) ++ ".beam"]), case filelib:is_file(BeamFile) of true -> has_all_beams(EbinDir, ModuleList); diff --git a/src/rebar_base_compiler.erl b/src/rebar_base_compiler.erl index 480e49c..3f273f1 100644 --- a/src/rebar_base_compiler.erl +++ b/src/rebar_base_compiler.erl @@ -199,15 +199,15 @@ compile_each([], _Config, _CompileFn) -> compile_each([Source | Rest], Config, CompileFn) -> case CompileFn(Source, Config) of ok -> - ?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), filename:basename(Source)]); + ?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]); {ok, Warnings} -> report(Warnings), - ?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), filename:basename(Source)]); + ?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]); skipped -> - ?DEBUG("~sSkipped ~s", [rebar_utils:indent(1), filename:basename(Source)]); + ?DEBUG("~tsSkipped ~ts", [rebar_utils:indent(1), filename:basename(Source)]); Error -> NewSource = format_error_source(Source, Config), - ?ERROR("Compiling ~s failed", [NewSource]), + ?ERROR("Compiling ~ts failed", [NewSource]), maybe_report(Error), ?DEBUG("Compilation failed: ~p", [Error]), ?FAIL @@ -258,7 +258,7 @@ maybe_report(_) -> %% @private Outputs a bunch of strings, including a newline -spec report([string()]) -> ok. report(Messages) -> - lists:foreach(fun(Msg) -> io:format("~s~n", [Msg]) end, Messages). + lists:foreach(fun(Msg) -> io:format("~ts~n", [Msg]) end, Messages). %% private format compiler errors into proper outputtable strings -spec format_errors(_, Extra, [err_or_warn()]) -> [string()] when @@ -274,10 +274,10 @@ format_errors(_MainSource, Extra, Errors) -> Extra :: string(). format_error(Source, Extra, {{Line, Column}, Mod, Desc}) -> ErrorDesc = Mod:format_error(Desc), - ?FMT("~s:~w:~w: ~s~s~n", [Source, Line, Column, Extra, ErrorDesc]); + ?FMT("~ts:~w:~w: ~ts~ts~n", [Source, Line, Column, Extra, ErrorDesc]); format_error(Source, Extra, {Line, Mod, Desc}) -> ErrorDesc = Mod:format_error(Desc), - ?FMT("~s:~w: ~s~s~n", [Source, Line, Extra, ErrorDesc]); + ?FMT("~ts:~w: ~ts~ts~n", [Source, Line, Extra, ErrorDesc]); format_error(Source, Extra, {Mod, Desc}) -> ErrorDesc = Mod:format_error(Desc), - ?FMT("~s: ~s~s~n", [Source, Extra, ErrorDesc]). + ?FMT("~ts: ~ts~ts~n", [Source, Extra, ErrorDesc]). diff --git a/src/rebar_config.erl b/src/rebar_config.erl index 82ff0d9..7316d83 100644 --- a/src/rebar_config.erl +++ b/src/rebar_config.erl @@ -120,7 +120,7 @@ write_lock_file(LockFile, Locks) -> file:write_file(LockFile, io_lib:format("~p.~n", [NewLocks])); _ -> file:write_file(LockFile, - io_lib:format("{~p,~n~p}.~n[~n~s~n].~n", + io_lib:format("{~p,~n~p}.~n[~n~ts~n].~n", [?CONFIG_VERSION, NewLocks, format_attrs(Attrs)])) end. @@ -359,7 +359,7 @@ check_newly_added_({Name, Source}, LockedDeps) -> false end; check_newly_added_(Dep, LockedDeps) when is_atom(Dep) -> - Name = ec_cnv:to_binary(Dep), + Name = rebar_utils:to_binary(Dep), case lists:keyfind(Name, 1, LockedDeps) of false -> {true, Name}; @@ -368,8 +368,8 @@ check_newly_added_(Dep, LockedDeps) when is_atom(Dep) -> 0 -> {true, Name}; _ -> - ?WARN("Newly added dep ~s is locked at a lower level. " - "If you really want to unlock it, use 'rebar3 upgrade ~s'", + ?WARN("Newly added dep ~ts is locked at a lower level. " + "If you really want to unlock it, use 'rebar3 upgrade ~ts'", [Name, Name]), false end diff --git a/src/rebar_core.erl b/src/rebar_core.erl index 14c4906..3ef7a0d 100644 --- a/src/rebar_core.erl +++ b/src/rebar_core.erl @@ -119,11 +119,11 @@ process_command(State, Command) -> State2 = rebar_state:command_parsed_args(State1, Args), do(TargetProviders, State2); {error, {invalid_option, Option}} -> - {error, io_lib:format("Invalid option ~s on task ~p", [Option, Command])}; + {error, io_lib:format("Invalid option ~ts on task ~p", [Option, Command])}; {error, {invalid_option_arg, {Option, Arg}}} -> - {error, io_lib:format("Invalid argument ~s to option ~s", [Arg, Option])}; + {error, io_lib:format("Invalid argument ~ts to option ~ts", [Arg, Option])}; {error, {missing_option_arg, Option}} -> - {error, io_lib:format("Missing argument to option ~s", [Option])} + {error, io_lib:format("Missing argument to option ~ts", [Option])} end end end. @@ -173,6 +173,6 @@ do([ProviderName | Rest], State) -> %% @doc convert a given exception's payload into an io description. -spec format_error(any()) -> iolist(). format_error({bad_provider_namespace, {Namespace, Name}}) -> - io_lib:format("Undefined command ~s in namespace ~s", [Name, Namespace]); + io_lib:format("Undefined command ~ts in namespace ~ts", [Name, Namespace]); format_error({bad_provider_namespace, Name}) -> - io_lib:format("Undefined command ~s", [Name]). + io_lib:format("Undefined command ~ts", [Name]). diff --git a/src/rebar_dialyzer_format.erl b/src/rebar_dialyzer_format.erl index be8cc48..7cf4e63 100644 --- a/src/rebar_dialyzer_format.erl +++ b/src/rebar_dialyzer_format.erl @@ -31,7 +31,7 @@ format_warnings(Opts, Warnings) -> format_warning_(_Opts, Warning = {_Tag, {File, Line}, Msg}, {File, Acc}) -> try String = message_to_string(Msg), - {File, [lists:flatten(fmt("~!c~4w~!!: ~s", [Line, String])) | Acc]} + {File, [lists:flatten(fmt("~!c~4w~!!: ~ts", [Line, String])) | Acc]} catch Error:Reason -> ?DEBUG("Failed to pretty format warning: ~p:~p", @@ -47,11 +47,11 @@ format_warning_(Opts, Warning = {_Tag, {SrcFile, Line}, Msg}, {_LastFile, Acc}) Dir = filename:dirname(File), Root = filename:rootname(Base), Ext = filename:extension(Base), - Path = re:replace(Dir, "^.*/_build/", "_build/", [{return, list}]), - Base1 = fmt("~!_c~s~!!~!__~s", [Root, Ext]), - F = fmt("~!__~s", [filename:join(Path, Base1)]), + Path = re:replace(Dir, "^.*/_build/", "_build/", [{return, list}, unicode]), + Base1 = fmt("~!_c~ts~!!~!__~ts", [Root, Ext]), + F = fmt("~!__~ts", [filename:join(Path, Base1)]), String = message_to_string(Msg), - {SrcFile, [lists:flatten(fmt("~n~s~n~!c~4w~!!: ~s", [F, Line, String])) | Acc]} + {SrcFile, [lists:flatten(fmt("~n~ts~n~!c~4w~!!: ~ts", [F, Line, String])) | Acc]} catch Error:Reason -> ?DEBUG("Failed to pretty format warning: ~p:~p~n~p", @@ -72,53 +72,53 @@ fmt(Fmt, Args) -> %%----- Warnings for general discrepancies ---------------- message_to_string({apply, [Args, ArgNs, FailReason, SigArgs, SigRet, Contract]}) -> - fmt("~!^Fun application with arguments ~!!~s ", + fmt("~!^Fun application with arguments ~!!~ts ", [bad_arg(ArgNs, Args)]) ++ call_or_apply_to_string(ArgNs, FailReason, SigArgs, SigRet, Contract); message_to_string({app_call, [M, F, Args, Culprit, ExpectedType, FoundType]}) -> - fmt("~!^The call~!! ~s:~s~s ~!^requires that" - "~!! ~s ~!^is of type ~!g~s~!^ not ~!r~s", + fmt("~!^The call~!! ~ts:~ts~ts ~!^requires that" + "~!! ~ts ~!^is of type ~!g~ts~!^ not ~!r~ts", [M, F, Args, Culprit, ExpectedType, FoundType]); message_to_string({bin_construction, [Culprit, Size, Seg, Type]}) -> - fmt("~!^Binary construction will fail since the ~!b~s~!^ field~!!" - " ~s~!^ in segment~!! ~s~!^ has type~!! ~s", + fmt("~!^Binary construction will fail since the ~!b~ts~!^ field~!!" + " ~ts~!^ in segment~!! ~ts~!^ has type~!! ~ts", [Culprit, Size, Seg, Type]); message_to_string({call, [M, F, Args, ArgNs, FailReason, SigArgs, SigRet, Contract]}) -> - fmt("~!^The call~!! ~w:~w~s ", [M, F, bad_arg(ArgNs, Args)]) ++ + fmt("~!^The call~!! ~w:~w~ts ", [M, F, bad_arg(ArgNs, Args)]) ++ call_or_apply_to_string(ArgNs, FailReason, SigArgs, SigRet, Contract); message_to_string({call_to_missing, [M, F, A]}) -> fmt("~!^Call to missing or unexported function ~!!~w:~w/~w", [M, F, A]); message_to_string({exact_eq, [Type1, Op, Type2]}) -> - fmt("~!^The test ~!!~s ~s ~s~!^ can never evaluate to 'true'", + fmt("~!^The test ~!!~ts ~ts ~ts~!^ can never evaluate to 'true'", [Type1, Op, Type2]); message_to_string({fun_app_args, [Args, Type]}) -> - fmt("~!^Fun application with arguments ~!!~s~!^ will fail" - " since the function has type ~!!~s", [Args, Type]); + fmt("~!^Fun application with arguments ~!!~ts~!^ will fail" + " since the function has type ~!!~ts", [Args, Type]); message_to_string({fun_app_no_fun, [Op, Type, Arity]}) -> - fmt("~!^Fun application will fail since ~!!~s ~!^::~!! ~s" + fmt("~!^Fun application will fail since ~!!~ts ~!^::~!! ~ts" " is not a function of arity ~!!~w", [Op, Type, Arity]); message_to_string({guard_fail, []}) -> "~!^Clause guard cannot succeed.~!!"; message_to_string({guard_fail, [Arg1, Infix, Arg2]}) -> - fmt("~!^Guard test ~!!~s ~s ~s~!^ can never succeed", + fmt("~!^Guard test ~!!~ts ~ts ~ts~!^ can never succeed", [Arg1, Infix, Arg2]); message_to_string({neg_guard_fail, [Arg1, Infix, Arg2]}) -> - fmt("~!^Guard test not(~!!~s ~s ~s~!^) can never succeed", + fmt("~!^Guard test not(~!!~ts ~ts ~ts~!^) can never succeed", [Arg1, Infix, Arg2]); message_to_string({guard_fail, [Guard, Args]}) -> - fmt("~!^Guard test ~!!~w~s~!^ can never succeed", + fmt("~!^Guard test ~!!~w~ts~!^ can never succeed", [Guard, Args]); message_to_string({neg_guard_fail, [Guard, Args]}) -> - fmt("~!^Guard test not(~!!~w~s~!^) can never succeed", + fmt("~!^Guard test not(~!!~w~ts~!^) can never succeed", [Guard, Args]); message_to_string({guard_fail_pat, [Pat, Type]}) -> - fmt("~!^Clause guard cannot succeed. The ~!!~s~!^ was matched" - " against the type ~!!~s", [Pat, Type]); + fmt("~!^Clause guard cannot succeed. The ~!!~ts~!^ was matched" + " against the type ~!!~ts", [Pat, Type]); message_to_string({improper_list_constr, [TlType]}) -> fmt("~!^Cons will produce an improper list" - " since its ~!b2~!!nd~!^ argument is~!! ~s", [TlType]); + " since its ~!b2~!!nd~!^ argument is~!! ~ts", [TlType]); message_to_string({no_return, [Type|Name]}) -> NameString = case Name of @@ -126,59 +126,59 @@ message_to_string({no_return, [Type|Name]}) -> [F, A] -> fmt("~!^Function ~!r~w/~w ", [F, A]) end, case Type of - no_match -> fmt("~s~!^has no clauses that will ever match",[NameString]); - only_explicit -> fmt("~s~!^only terminates with explicit exception", [NameString]); - only_normal -> fmt("~s~!^has no local return", [NameString]); - both -> fmt("~s~!^has no local return", [NameString]) + no_match -> fmt("~ts~!^has no clauses that will ever match",[NameString]); + only_explicit -> fmt("~ts~!^only terminates with explicit exception", [NameString]); + only_normal -> fmt("~ts~!^has no local return", [NameString]); + both -> fmt("~ts~!^has no local return", [NameString]) end; message_to_string({record_constr, [RecConstr, FieldDiffs]}) -> - fmt("~!^Record construction ~!!~s~!^ violates the" - " declared type of field ~!!~s", [RecConstr, FieldDiffs]); + fmt("~!^Record construction ~!!~ts~!^ violates the" + " declared type of field ~!!~ts", [RecConstr, FieldDiffs]); message_to_string({record_constr, [Name, Field, Type]}) -> fmt("~!^Record construction violates the declared type for ~!!#~w{}~!^" - " since ~!!~s~!^ cannot be of type ~!!~s", + " since ~!!~ts~!^ cannot be of type ~!!~ts", [Name, Field, Type]); message_to_string({record_matching, [String, Name]}) -> - fmt("~!^The ~!!~s~!^ violates the" + fmt("~!^The ~!!~ts~!^ violates the" " declared type for ~!!#~w{}", [String, Name]); message_to_string({record_match, [Pat, Type]}) -> - fmt("~!^Matching of ~!!~s~!^ tagged with a record name violates the" - " declared type of ~!!~s", [Pat, Type]); + fmt("~!^Matching of ~!!~ts~!^ tagged with a record name violates the" + " declared type of ~!!~ts", [Pat, Type]); message_to_string({pattern_match, [Pat, Type]}) -> - fmt("~!^The ~s~!^ can never match the type ~!g~s", + fmt("~!^The ~ts~!^ can never match the type ~!g~ts", [bad_pat(Pat), Type]); message_to_string({pattern_match_cov, [Pat, Type]}) -> - fmt("~!^The ~s~!^ can never match since previous" - " clauses completely covered the type ~!g~s", + fmt("~!^The ~ts~!^ can never match since previous" + " clauses completely covered the type ~!g~ts", [bad_pat(Pat), Type]); message_to_string({unmatched_return, [Type]}) -> - fmt("~!^Expression produces a value of type ~!!~s~!^," + fmt("~!^Expression produces a value of type ~!!~ts~!^," " but this value is unmatched", [Type]); message_to_string({unused_fun, [F, A]}) -> fmt("~!^Function ~!r~w/~w~!!~!^ will never be called", [F, A]); %%----- Warnings for specs and contracts ------------------- message_to_string({contract_diff, [M, F, _A, Contract, Sig]}) -> - fmt("~!^Type specification ~!!~w:~w~s~!^" - " is not equal to the success typing: ~!!~w:~w~s", + fmt("~!^Type specification ~!!~w:~w~ts~!^" + " is not equal to the success typing: ~!!~w:~w~ts", [M, F, Contract, M, F, Sig]); message_to_string({contract_subtype, [M, F, _A, Contract, Sig]}) -> - fmt("~!^Type specification ~!!~w:~w~s~!^" - " is a subtype of the success typing: ~!!~w:~w~s", + fmt("~!^Type specification ~!!~w:~w~ts~!^" + " is a subtype of the success typing: ~!!~w:~w~ts", [M, F, Contract, M, F, Sig]); message_to_string({contract_supertype, [M, F, _A, Contract, Sig]}) -> - fmt("~!^Type specification ~!!~w:~w~s~!^" - " is a supertype of the success typing: ~!!~w:~w~s", + fmt("~!^Type specification ~!!~w:~w~ts~!^" + " is a supertype of the success typing: ~!!~w:~w~ts", [M, F, Contract, M, F, Sig]); message_to_string({contract_range, [Contract, M, F, ArgStrings, Line, CRet]}) -> - fmt("~!^The contract ~!!~w:~w~s~!^ cannot be right because the" - " inferred return for ~!!~w~s~!^ on line ~!!~w~!^ is ~!!~s", + fmt("~!^The contract ~!!~w:~w~ts~!^ cannot be right because the" + " inferred return for ~!!~w~ts~!^ on line ~!!~w~!^ is ~!!~ts", [M, F, Contract, F, ArgStrings, Line, CRet]); message_to_string({invalid_contract, [M, F, A, Sig]}) -> fmt("~!^Invalid type specification for function~!! ~w:~w/~w." - "~!^ The success typing is~!! ~s", [M, F, A, Sig]); + "~!^ The success typing is~!! ~ts", [M, F, A, Sig]); message_to_string({extra_range, [M, F, A, ExtraRanges, SigRange]}) -> fmt("~!^The specification for ~!!~w:~w/~w~!^ states that the function" - " might also return ~!!~s~!^ but the inferred return is ~!!~s", + " might also return ~!!~ts~!^ but the inferred return is ~!!~ts", [M, F, A, ExtraRanges, SigRange]); message_to_string({overlapping_contract, [M, F, A]}) -> fmt("~!^Overloaded contract for ~!!~w:~w/~w~!^ has overlapping" @@ -189,62 +189,62 @@ message_to_string({spec_missing_fun, [M, F, A]}) -> [M, F, A]); %%----- Warnings for opaque type violations ------------------- message_to_string({call_with_opaque, [M, F, Args, ArgNs, ExpArgs]}) -> - fmt("~!^The call ~!!~w:~w~s~!^ contains ~!!~s~!^ when ~!!~s", + fmt("~!^The call ~!!~w:~w~ts~!^ contains ~!!~ts~!^ when ~!!~ts", [M, F, bad_arg(ArgNs, Args), form_positions(ArgNs), form_expected(ExpArgs)]); message_to_string({call_without_opaque, [M, F, Args, [{N,_,_}|_] = ExpectedTriples]}) -> - fmt("~!^The call ~!!~w:~w~s ~!^does not have~!! ~s", + fmt("~!^The call ~!!~w:~w~ts ~!^does not have~!! ~ts", [M, F, bad_arg(N, Args), form_expected_without_opaque(ExpectedTriples)]); message_to_string({opaque_eq, [Type, _Op, OpaqueType]}) -> - fmt("~!^Attempt to test for equality between a term of type ~!!~s~!^" - " and a term of opaque type ~!!~s", [Type, OpaqueType]); + fmt("~!^Attempt to test for equality between a term of type ~!!~ts~!^" + " and a term of opaque type ~!!~ts", [Type, OpaqueType]); message_to_string({opaque_guard, [Arg1, Infix, Arg2, ArgNs]}) -> - fmt("~!^Guard test ~!!~s ~s ~s~!^ contains ~!!~s", + fmt("~!^Guard test ~!!~ts ~ts ~ts~!^ contains ~!!~ts", [Arg1, Infix, Arg2, form_positions(ArgNs)]); message_to_string({opaque_guard, [Guard, Args]}) -> - fmt("~!^Guard test ~!!~w~s~!^ breaks the opaqueness of its" + fmt("~!^Guard test ~!!~w~ts~!^ breaks the opaqueness of its" " argument", [Guard, Args]); message_to_string({opaque_match, [Pat, OpaqueType, OpaqueTerm]}) -> Term = if OpaqueType =:= OpaqueTerm -> "the term"; true -> OpaqueTerm end, - fmt("~!^The attempt to match a term of type ~!!~s~!^ against the" - "~!! ~s~!^ breaks the opaqueness of ~!!~s", + fmt("~!^The attempt to match a term of type ~!!~ts~!^ against the" + "~!! ~ts~!^ breaks the opaqueness of ~!!~ts", [OpaqueType, Pat, Term]); message_to_string({opaque_neq, [Type, _Op, OpaqueType]}) -> - fmt("~!^Attempt to test for inequality between a term of type ~!!~s" - "~!^ and a term of opaque type ~!!~s", [Type, OpaqueType]); + fmt("~!^Attempt to test for inequality between a term of type ~!!~ts" + "~!^ and a term of opaque type ~!!~ts", [Type, OpaqueType]); message_to_string({opaque_type_test, [Fun, Args, Arg, ArgType]}) -> - fmt("~!^The type test ~!!~s~s~!^ breaks the opaqueness of the term " - "~!!~s~s", [Fun, Args, Arg, ArgType]); + fmt("~!^The type test ~!!~ts~ts~!^ breaks the opaqueness of the term " + "~!!~ts~ts", [Fun, Args, Arg, ArgType]); message_to_string({opaque_size, [SizeType, Size]}) -> - fmt("~!^The size ~!!~s~!^ breaks the opaqueness of ~!!~s", + fmt("~!^The size ~!!~ts~!^ breaks the opaqueness of ~!!~ts", [SizeType, Size]); message_to_string({opaque_call, [M, F, Args, Culprit, OpaqueType]}) -> - fmt("~!^The call ~!!~s:~s~s~!^ breaks the opaqueness of the term~!!" - " ~s :: ~s", [M, F, Args, Culprit, OpaqueType]); + fmt("~!^The call ~!!~ts:~ts~ts~!^ breaks the opaqueness of the term~!!" + " ~ts :: ~ts", [M, F, Args, Culprit, OpaqueType]); %%----- Warnings for concurrency errors -------------------- message_to_string({race_condition, [M, F, Args, Reason]}) -> - fmt("~!^The call ~!!~w:~w~s ~s", [M, F, Args, Reason]); + fmt("~!^The call ~!!~w:~w~ts ~ts", [M, F, Args, Reason]); %%----- Warnings for behaviour errors -------------------- message_to_string({callback_type_mismatch, [B, F, A, ST, CT]}) -> - fmt("~!^The inferred return type of~!! ~w/~w (~s) ~!^" - "has nothing in common with~!! ~s, ~!^which is the expected" + fmt("~!^The inferred return type of~!! ~w/~w (~ts) ~!^" + "has nothing in common with~!! ~ts, ~!^which is the expected" " return type for the callback of~!! ~w ~!^behaviour", [F, A, ST, CT, B]); message_to_string({callback_arg_type_mismatch, [B, F, A, N, ST, CT]}) -> - fmt("~!^The inferred type for the~!! ~s ~!^argument of~!!" - " ~w/~w (~s) ~!^is not a supertype of~!! ~s~!^, which is" + fmt("~!^The inferred type for the~!! ~ts ~!^argument of~!!" + " ~w/~w (~ts) ~!^is not a supertype of~!! ~ts~!^, which is" "expected type for this argument in the callback of the~!! ~w " "~!^behaviour", [ordinal(N), F, A, ST, CT, B]); message_to_string({callback_spec_type_mismatch, [B, F, A, ST, CT]}) -> - fmt("~!^The return type ~!!~s~!^ in the specification of ~!!" - "~w/~w~!^ is not a subtype of ~!!~s~!^, which is the expected" + fmt("~!^The return type ~!!~ts~!^ in the specification of ~!!" + "~w/~w~!^ is not a subtype of ~!!~ts~!^, which is the expected" " return type for the callback of ~!!~w~!^ behaviour", [ST, F, A, CT, B]); message_to_string({callback_spec_arg_type_mismatch, [B, F, A, N, ST, CT]}) -> - fmt("~!^The specified type for the ~!!~s~!^ argument of ~!!" - "~w/~w (~s)~!^ is not a supertype of ~!!~s~!^, which is" + fmt("~!^The specified type for the ~!!~ts~!^ argument of ~!!" + "~w/~w (~ts)~!^ is not a supertype of ~!!~ts~!^, which is" " expected type for this argument in the callback of the ~!!~w" "~!^ behaviour", [ordinal(N), F, A, ST, CT, B]); message_to_string({callback_missing, [B, F, A]}) -> @@ -274,26 +274,26 @@ call_or_apply_to_string(ArgNs, FailReason, SigArgs, SigRet, true -> %% We do not know which argument(s) caused the failure fmt("~!^will never return since the success typing arguments" - " are ~!!~s", [SigArgs]); + " are ~!!~ts", [SigArgs]); false -> fmt("~!^will never return since it differs in the~!!" - " ~s ~!^argument from the success typing" - " arguments:~!! ~s", + " ~ts ~!^argument from the success typing" + " arguments:~!! ~ts", [PositionString, good_arg(ArgNs, SigArgs)]) end; only_contract -> case (ArgNs =:= []) orelse IsOverloaded of true -> %% We do not know which arguments caused the failure - fmt("~!^breaks the contract~!! ~s", [good_arg(ArgNs, Contract)]); + fmt("~!^breaks the contract~!! ~ts", [good_arg(ArgNs, Contract)]); false -> - fmt("~!^breaks the contract~!! ~s ~!^in the~!!" - " ~s ~!^argument", + fmt("~!^breaks the contract~!! ~ts ~!^in the~!!" + " ~ts ~!^argument", [good_arg(ArgNs, Contract), PositionString]) end; both -> fmt("~!^will never return since the success typing is " - "~!!~s ~!^->~!! ~s ~!^and the contract is ~!!~s", + "~!!~ts ~!^->~!! ~ts ~!^and the contract is ~!!~ts", [good_arg(ArgNs, SigArgs), SigRet, good_arg(ArgNs, Contract)]) end. @@ -301,8 +301,8 @@ call_or_apply_to_string(ArgNs, FailReason, SigArgs, SigRet, form_positions(ArgNs) -> ArgS = form_position_string(ArgNs), case ArgNs of - [_] -> fmt("~!^an opaque term as ~!!~s~!^ argument", [ArgS]); - [_,_|_] -> fmt("~!^opaque terms as ~!!~s~!^ arguments", [ArgS]) + [_] -> fmt("~!^an opaque term as ~!!~ts~!^ argument", [ArgS]); + [_,_|_] -> fmt("~!^opaque terms as ~!!~ts~!^ arguments", [ArgS]) end. %% We know which positions N are to blame; @@ -310,9 +310,9 @@ form_positions(ArgNs) -> form_expected_without_opaque([{N, T, TStr}]) -> FStr = case erl_types:t_is_opaque(T) of true -> - "~!^an opaque term of type~!g ~s ~!^as "; + "~!^an opaque term of type~!g ~ts ~!^as "; false -> - "~!^a term of type ~!g~s ~!^(with opaque subterms) as " + "~!^a term of type ~!g~ts ~!^(with opaque subterms) as " end ++ form_position_string([N]) ++ "~!^ argument", fmt(FStr, [TStr]); @@ -325,9 +325,9 @@ form_expected(ExpectedArgs) -> [T] -> TS = erl_types:t_to_string(T), case erl_types:t_is_opaque(T) of - true -> fmt("~!^an opaque term of type ~!!~s~!^ is" + true -> fmt("~!^an opaque term of type ~!!~ts~!^ is" " expected", [TS]); - false -> fmt("~!^a structured term of type ~!!~s~!^ is" + false -> fmt("~!^a structured term of type ~!!~ts~!^ is" " expected", [TS]) end; [_,_|_] -> fmt("~!^terms of different types are expected in these" @@ -340,7 +340,7 @@ form_position_string(ArgNs) -> [N1] -> ordinal(N1); [_,_|_] -> [Last|Prevs] = lists:reverse(ArgNs), - ", " ++ Head = lists:flatten([fmt(", ~s",[ordinal(N)]) || + ", " ++ Head = lists:flatten([fmt(", ~ts",[ordinal(N)]) || N <- lists:reverse(Prevs)]), Head ++ " and " ++ ordinal(Last) end. @@ -352,11 +352,11 @@ ordinal(N) when is_integer(N) -> fmt("~!B~w~!!th", [N]). %% Format a pattern ad highlight errorous part in red. bad_pat("pattern " ++ P) -> - fmt("pattern ~!r~s",[P]); + fmt("pattern ~!r~ts",[P]); bad_pat("variable " ++ P) -> - fmt("variable ~!r~s",[P]); + fmt("variable ~!r~ts",[P]); bad_pat(P) -> - fmt("~!r~s",[P]). + fmt("~!r~ts",[P]). bad_arg(N, Args) -> @@ -378,10 +378,10 @@ highlight([], _N, _C, Rest) -> Rest; highlight([N | Nr], N, g, [Arg | Rest]) -> - [fmt("~!g~s", [Arg]) | highlight(Nr, N+1, g, Rest)]; + [fmt("~!g~ts", [Arg]) | highlight(Nr, N+1, g, Rest)]; highlight([N | Nr], N, r, [Arg | Rest]) -> - [fmt("~!r~s", [Arg]) | highlight(Nr, N+1, r, Rest)]; + [fmt("~!r~ts", [Arg]) | highlight(Nr, N+1, r, Rest)]; highlight(Ns, N, C, [Arg | Rest]) -> [Arg | highlight(Ns, N + 1, C, Rest)]. diff --git a/src/rebar_digraph.erl b/src/rebar_digraph.erl index a827735..776d7b8 100644 --- a/src/rebar_digraph.erl +++ b/src/rebar_digraph.erl @@ -53,9 +53,9 @@ add(Graph, {PkgName, Deps}) -> lists:foreach(fun(DepName) -> Name1 = case DepName of {Name, _Vsn} -> - ec_cnv:to_binary(Name); + rebar_utils:to_binary(Name); Name -> - ec_cnv:to_binary(Name) + rebar_utils:to_binary(Name) end, V3 = case digraph:vertex(Graph, Name1) of false -> diff --git a/src/rebar_dir.erl b/src/rebar_dir.erl index b61bfcc..d0c7805 100644 --- a/src/rebar_dir.erl +++ b/src/rebar_dir.erl @@ -41,7 +41,7 @@ base_dir(State) -> %% of profiles. -spec profile_dir(rebar_dict(), [atom()]) -> file:filename_all(). profile_dir(Opts, Profiles) -> - {BaseDir, ProfilesStrings} = case [ec_cnv:to_list(P) || P <- Profiles] of + {BaseDir, ProfilesStrings} = case [rebar_utils:to_list(P) || P <- Profiles] of ["global" | _] -> {?MODULE:global_cache_dir(Opts), [""]}; ["bootstrap", "default"] -> {rebar_opts:get(Opts, base_dir, ?DEFAULT_BASE_DIR), ["default"]}; ["default"] -> {rebar_opts:get(Opts, base_dir, ?DEFAULT_BASE_DIR), ["default"]}; @@ -363,7 +363,7 @@ warn_source_format_once(Format) -> ok; true -> ?WARN("Invalid argument ~p for compiler_source_format - " - "assuming ~s~n", [Format, ?DEFAULT_COMPILER_SOURCE_FORMAT]) + "assuming ~ts~n", [Format, ?DEFAULT_COMPILER_SOURCE_FORMAT]) end. %% @private takes a filename and canonicalizes its path if it is a link. diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index 9157aba..94cbe13 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -92,7 +92,7 @@ compile(AppInfo) when element(1, AppInfo) == app_info_t -> %% @doc compile an individual application. -spec compile(rebar_app_info:t(), compile_opts()) -> ok. compile(AppInfo, CompileOpts) when element(1, AppInfo) == app_info_t -> - Dir = ec_cnv:to_list(rebar_app_info:out_dir(AppInfo)), + Dir = rebar_utils:to_list(rebar_app_info:out_dir(AppInfo)), RebarOpts = rebar_app_info:opts(AppInfo), SrcOpts = [check_last_mod, @@ -237,8 +237,8 @@ clean(AppInfo) -> YrlFiles = rebar_utils:find_files(filename:join([AppDir, "src"]), ?RE_PREFIX".*\\.[x|y]rl\$"), rebar_file_utils:delete_each( - [ binary_to_list(iolist_to_binary(re:replace(F, "\\.[x|y]rl$", ".erl"))) - || F <- YrlFiles ]), + [rebar_utils:to_list(re:replace(F, "\\.[x|y]rl$", ".erl", [unicode])) + || F <- YrlFiles]), BinDirs = ["ebin"|rebar_dir:extra_src_dirs(rebar_app_info:opts(AppInfo))], ok = clean_dirs(AppDir, BinDirs), @@ -339,7 +339,7 @@ maybe_rm_beam_and_edge(G, OutDir, Source) -> false; false -> Target = target_base(OutDir, Source) ++ ".beam", - ?DEBUG("Source ~s is gone, deleting previous beam file if it exists ~s", [Source, Target]), + ?DEBUG("Source ~ts is gone, deleting previous beam file if it exists ~ts", [Source, Target]), file:delete(Target), digraph:del_vertex(G, Source), true @@ -400,7 +400,7 @@ init_erlcinfo(InclDirs, Erls, Dir, OutDir) -> try restore_erlcinfo(G, InclDirs, Dir) catch _:_ -> - ?WARN("Failed to restore ~s file. Discarding it.~n", [erlcinfo_file(Dir)]), + ?WARN("Failed to restore ~ts file. Discarding it.~n", [erlcinfo_file(Dir)]), file:delete(erlcinfo_file(Dir)) end, Dirs = source_and_include_dirs(InclDirs, Erls), diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index f5e5db5..f68a54d 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -41,7 +41,7 @@ download_source_(AppDir, Source, State) -> Resources = rebar_state:resources(State), Module = get_resource_type(Source, Resources), TmpDir = ec_file:insecure_mkdtemp(), - AppDir1 = ec_cnv:to_list(AppDir), + AppDir1 = rebar_utils:to_list(AppDir), case Module:download(TmpDir, Source, State) of {ok, _} -> ec_file:mkdir_p(AppDir1), @@ -66,25 +66,25 @@ needs_update(AppDir, Source, State) -> end. format_error({bad_download, CachePath}) -> - io_lib:format("Download of package does not match md5sum from server: ~s", [CachePath]); + io_lib:format("Download of package does not match md5sum from server: ~ts", [CachePath]); format_error({unexpected_hash, CachePath, Expected, Found}) -> - io_lib:format("The checksum for package at ~s (~s) does not match the " - "checksum previously locked (~s). Either unlock or " + io_lib:format("The checksum for package at ~ts (~ts) does not match the " + "checksum previously locked (~ts). Either unlock or " "upgrade the package, or make sure you fetched it from " "the same index from which it was initially fetched.", [CachePath, Found, Expected]); format_error({failed_extract, CachePath}) -> - io_lib:format("Failed to extract package: ~s", [CachePath]); + io_lib:format("Failed to extract package: ~ts", [CachePath]); format_error({bad_etag, Source}) -> - io_lib:format("MD5 Checksum comparison failed for: ~s", [Source]); + io_lib:format("MD5 Checksum comparison failed for: ~ts", [Source]); format_error({fetch_fail, Name, Vsn}) -> - io_lib:format("Failed to fetch and copy dep: ~s-~s", [Name, Vsn]); + io_lib:format("Failed to fetch and copy dep: ~ts-~ts", [Name, Vsn]); format_error({fetch_fail, Source}) -> io_lib:format("Failed to fetch and copy dep: ~p", [Source]); format_error({bad_checksum, File}) -> - io_lib:format("Checksum mismatch against tarball in ~s", [File]); + io_lib:format("Checksum mismatch against tarball in ~ts", [File]); format_error({bad_registry_checksum, File}) -> - io_lib:format("Checksum mismatch against registry in ~s", [File]). + io_lib:format("Checksum mismatch against registry in ~ts", [File]). get_resource_type({Type, Location}, Resources) -> find_resource_module(Type, Location, Resources); @@ -100,7 +100,7 @@ find_resource_module(Type, Location, Resources) -> false -> case code:which(Type) of non_existing -> - {error, io_lib:format("Cannot handle dependency ~s.~n" + {error, io_lib:format("Cannot handle dependency ~ts.~n" " No module for resource type ~p", [Location, Type])}; _ -> Type diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 8158312..d7716e5 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -84,7 +84,7 @@ consult_config(State, Filename) -> [JoinedConfig]. format_error({bad_term_file, AppFile, Reason}) -> - io_lib:format("Error reading file ~s: ~s", [AppFile, file:format_error(Reason)]). + io_lib:format("Error reading file ~ts: ~ts", [AppFile, file:format_error(Reason)]). symlink_or_copy(Source, Target) -> Link = case os:type() of @@ -121,7 +121,7 @@ symlink_or_copy(Source, Target) -> win32_symlink(Source, Target) -> Res = rebar_utils:sh( - ?FMT("cmd /c mklink /j \"~s\" \"~s\"", + ?FMT("cmd /c mklink /j \"~ts\" \"~ts\"", [rebar_utils:escape_double_quotes(filename:nativename(Target)), rebar_utils:escape_double_quotes(filename:nativename(Source))]), [{use_stdout, false}, return_on_error]), @@ -129,7 +129,7 @@ win32_symlink(Source, Target) -> true -> ok; false -> {error, lists:flatten( - io_lib:format("Failed to symlink ~s to ~s~n", + io_lib:format("Failed to symlink ~ts to ~ts~n", [Source, Target]))} end. @@ -141,7 +141,7 @@ rm_rf(Target) -> case os:type() of {unix, _} -> EscTarget = rebar_utils:escape_chars(Target), - {ok, []} = rebar_utils:sh(?FMT("rm -rf ~s", [EscTarget]), + {ok, []} = rebar_utils:sh(?FMT("rm -rf ~ts", [EscTarget]), [{use_stdout, false}, abort_on_error]), ok; {win32, _} -> @@ -161,7 +161,7 @@ cp_r(Sources, Dest) -> {unix, _} -> EscSources = [rebar_utils:escape_chars(Src) || Src <- Sources], SourceStr = string:join(EscSources, " "), - {ok, []} = rebar_utils:sh(?FMT("cp -Rp ~s \"~s\"", + {ok, []} = rebar_utils:sh(?FMT("cp -Rp ~ts \"~ts\"", [SourceStr, rebar_utils:escape_double_quotes(Dest)]), [{use_stdout, false}, abort_on_error]), ok; @@ -176,7 +176,7 @@ mv(Source, Dest) -> {unix, _} -> EscSource = rebar_utils:escape_chars(Source), EscDest = rebar_utils:escape_chars(Dest), - case rebar_utils:sh(?FMT("mv ~s ~s", [EscSource, EscDest]), + case rebar_utils:sh(?FMT("mv ~ts ~ts", [EscSource, EscDest]), [{use_stdout, false}, abort_on_error]) of {ok, []} -> ok; @@ -234,7 +234,7 @@ robocopy_mv_and_rename(Source, Dest, SrcDir, SrcName, DestDir, DestName) -> case ec_file:insecure_mkdtemp() of {error, _Reason} -> {error, lists:flatten( - io_lib:format("Failed to move ~s to ~s (tmpdir failed)~n", + io_lib:format("Failed to move ~ts to ~ts (tmpdir failed)~n", [Source, Dest]))}; TmpPath -> case robocopy_file(SrcDir, TmpPath, SrcName) of @@ -246,7 +246,7 @@ robocopy_mv_and_rename(Source, Dest, SrcDir, SrcName, DestDir, DestName) -> case file:rename(TmpSrc, TmpDst) of {error, _} -> {error, lists:flatten( - io_lib:format("Failed to move ~s to ~s (via rename)~n", + io_lib:format("Failed to move ~ts to ~ts (via rename)~n", [Source, Dest]))}; ok -> case robocopy_file(TmpPath, DestDir, DestName) of @@ -258,7 +258,7 @@ robocopy_mv_and_rename(Source, Dest, SrcDir, SrcName, DestDir, DestName) -> end. robocopy_file(SrcPath, DestPath, FileName) -> - Cmd = ?FMT("robocopy /move /e \"~s\" \"~s\" \"~s\"", + Cmd = ?FMT("robocopy /move /e \"~ts\" \"~ts\" \"~ts\"", [rebar_utils:escape_double_quotes(SrcPath), rebar_utils:escape_double_quotes(DestPath), rebar_utils:escape_double_quotes(FileName)]), @@ -266,7 +266,7 @@ robocopy_file(SrcPath, DestPath, FileName) -> case win32_ok(Res) of false -> {error, lists:flatten( - io_lib:format("Failed to move ~s to ~s~n", + io_lib:format("Failed to move ~ts to ~ts~n", [filename:join(SrcPath, FileName), filename:join(DestPath, FileName)]))}; true -> @@ -274,7 +274,7 @@ robocopy_file(SrcPath, DestPath, FileName) -> end. robocopy_dir(Source, Dest) -> - Cmd = ?FMT("robocopy /move /e \"~s\" \"~s\"", + Cmd = ?FMT("robocopy /move /e \"~ts\" \"~ts\"", [rebar_utils:escape_double_quotes(Source), rebar_utils:escape_double_quotes(Dest)]), Res = rebar_utils:sh(Cmd, @@ -283,7 +283,7 @@ robocopy_dir(Source, Dest) -> true -> ok; false -> {error, lists:flatten( - io_lib:format("Failed to move ~s to ~s~n", + io_lib:format("Failed to move ~ts to ~ts~n", [Source, Dest]))} end. @@ -301,12 +301,19 @@ delete_each([File | Rest]) -> {error, enoent} -> delete_each(Rest); {error, Reason} -> - ?ERROR("Failed to delete file ~s: ~p\n", [File, Reason]), + ?ERROR("Failed to delete file ~ts: ~p\n", [File, Reason]), ?FAIL end. write_file_if_contents_differ(Filename, Bytes) -> - ToWrite = iolist_to_binary(Bytes), + %% first try to convert directly to binaries, + %% but if it fails, we likely contain unicode and + %% need special treatment + ToWrite = try + iolist_to_binary(Bytes) + catch + error:badarg -> unicode:characters_to_binary(Bytes) + end, case file:read_file(Filename) of {ok, ToWrite} -> ok; @@ -401,13 +408,13 @@ split_dirname(Path) -> delete_each_dir_win32([]) -> ok; delete_each_dir_win32([Dir | Rest]) -> - {ok, []} = rebar_utils:sh(?FMT("rd /q /s \"~s\"", + {ok, []} = rebar_utils:sh(?FMT("rd /q /s \"~ts\"", [rebar_utils:escape_double_quotes(filename:nativename(Dir))]), [{use_stdout, false}, return_on_error]), delete_each_dir_win32(Rest). xcopy_win32(Source,Dest)-> - %% "xcopy \"~s\" \"~s\" /q /y /e 2> nul", Changed to robocopy to + %% "xcopy \"~ts\" \"~ts\" /q /y /e 2> nul", Changed to robocopy to %% handle long names. May have issues with older windows. Cmd = case filelib:is_dir(Source) of true -> @@ -417,11 +424,11 @@ xcopy_win32(Source,Dest)-> %% must manually add the last fragment of a directory to the `Dest` %% in order to properly replicate POSIX platforms NewDest = filename:join([Dest, filename:basename(Source)]), - ?FMT("robocopy \"~s\" \"~s\" /e /is 1> nul", + ?FMT("robocopy \"~ts\" \"~ts\" /e /is 1> nul", [rebar_utils:escape_double_quotes(filename:nativename(Source)), rebar_utils:escape_double_quotes(filename:nativename(NewDest))]); false -> - ?FMT("robocopy \"~s\" \"~s\" \"~s\" /e /is 1> nul", + ?FMT("robocopy \"~ts\" \"~ts\" \"~ts\" /e /is 1> nul", [rebar_utils:escape_double_quotes(filename:nativename(filename:dirname(Source))), rebar_utils:escape_double_quotes(filename:nativename(Dest)), rebar_utils:escape_double_quotes(filename:basename(Source))]) @@ -432,7 +439,7 @@ xcopy_win32(Source,Dest)-> true -> ok; false -> {error, lists:flatten( - io_lib:format("Failed to copy ~s to ~s~n", + io_lib:format("Failed to copy ~ts to ~ts~n", [Source, Dest]))} end. diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl index f666d17..a6b4d00 100644 --- a/src/rebar_git_resource.erl +++ b/src/rebar_git_resource.erl @@ -17,7 +17,7 @@ lock(AppDir, {git, Url, _}) -> lock(AppDir, {git, Url}); lock(AppDir, {git, Url}) -> - AbortMsg = lists:flatten(io_lib:format("Locking of git dependency failed in ~s", [AppDir])), + AbortMsg = lists:flatten(io_lib:format("Locking of git dependency failed in ~ts", [AppDir])), Dir = rebar_utils:escape_double_quotes(AppDir), {ok, VsnString} = case os:type() of @@ -38,17 +38,17 @@ needs_update(Dir, {git, Url, {tag, Tag}}) -> [{cd, Dir}]), Current1 = string:strip(string:strip(Current, both, $\n), both, $\r), - ?DEBUG("Comparing git tag ~s with ~s", [Tag, Current1]), + ?DEBUG("Comparing git tag ~ts with ~ts", [Tag, Current1]), not ((Current1 =:= Tag) andalso compare_url(Dir, Url)); needs_update(Dir, {git, Url, {branch, Branch}}) -> %% Fetch remote so we can check if the branch has changed SafeBranch = rebar_utils:escape_chars(Branch), - {ok, _} = rebar_utils:sh(?FMT("git fetch origin ~s", [SafeBranch]), + {ok, _} = rebar_utils:sh(?FMT("git fetch origin ~ts", [SafeBranch]), [{cd, Dir}]), %% Check for new commits to origin/Branch - {ok, Current} = rebar_utils:sh(?FMT("git log HEAD..origin/~s --oneline", [SafeBranch]), + {ok, Current} = rebar_utils:sh(?FMT("git log HEAD..origin/~ts --oneline", [SafeBranch]), [{cd, Dir}]), - ?DEBUG("Checking git branch ~s for updates", [Branch]), + ?DEBUG("Checking git branch ~ts for updates", [Branch]), not ((Current =:= []) andalso compare_url(Dir, Url)); needs_update(Dir, {git, Url, "master"}) -> needs_update(Dir, {git, Url, {branch, "master"}}); @@ -68,7 +68,7 @@ needs_update(Dir, {git, _, Ref}) -> Ref end, - ?DEBUG("Comparing git ref ~s with ~s", [Ref2, Current1]), + ?DEBUG("Comparing git ref ~ts with ~ts", [Ref2, Current1]), (Current1 =/= Ref2). compare_url(Dir, Url) -> @@ -82,7 +82,7 @@ compare_url(Dir, Url) -> parse_git_url(Url) -> %% Checks for standard scp style git remote - case re:run(Url, ?SCP_PATTERN, [{capture, [host, path], list}]) of + case re:run(Url, ?SCP_PATTERN, [{capture, [host, path], list}, unicode]) of {match, [Host, Path]} -> {ok, {Host, filename:rootname(Path, ".git")}}; nomatch -> @@ -119,41 +119,41 @@ download(Dir, {git, Url, Rev}, _State) -> %% Use different git clone commands depending on git --version git_clone(branch,Vsn,Url,Dir,Branch) when Vsn >= {1,7,10}; Vsn =:= undefined -> - rebar_utils:sh(?FMT("git clone ~s ~s -b ~s --single-branch", + rebar_utils:sh(?FMT("git clone ~ts ~ts -b ~ts --single-branch", [rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir)), rebar_utils:escape_chars(Branch)]), [{cd, filename:dirname(Dir)}]); git_clone(branch,_Vsn,Url,Dir,Branch) -> - rebar_utils:sh(?FMT("git clone ~s ~s -b ~s", + rebar_utils:sh(?FMT("git clone ~ts ~ts -b ~ts", [rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir)), rebar_utils:escape_chars(Branch)]), [{cd, filename:dirname(Dir)}]); git_clone(tag,Vsn,Url,Dir,Tag) when Vsn >= {1,7,10}; Vsn =:= undefined -> - rebar_utils:sh(?FMT("git clone ~s ~s -b ~s --single-branch", + rebar_utils:sh(?FMT("git clone ~ts ~ts -b ~ts --single-branch", [rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir)), rebar_utils:escape_chars(Tag)]), [{cd, filename:dirname(Dir)}]); git_clone(tag,_Vsn,Url,Dir,Tag) -> - rebar_utils:sh(?FMT("git clone ~s ~s -b ~s", + rebar_utils:sh(?FMT("git clone ~ts ~ts -b ~ts", [rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir)), rebar_utils:escape_chars(Tag)]), [{cd, filename:dirname(Dir)}]); git_clone(ref,_Vsn,Url,Dir,Ref) -> - rebar_utils:sh(?FMT("git clone -n ~s ~s", + rebar_utils:sh(?FMT("git clone -n ~ts ~ts", [rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]), - rebar_utils:sh(?FMT("git checkout -q ~s", [Ref]), [{cd, Dir}]); + rebar_utils:sh(?FMT("git checkout -q ~ts", [Ref]), [{cd, Dir}]); git_clone(rev,_Vsn,Url,Dir,Rev) -> - rebar_utils:sh(?FMT("git clone -n ~s ~s", + rebar_utils:sh(?FMT("git clone -n ~ts ~ts", [rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]), - rebar_utils:sh(?FMT("git checkout -q ~s", [rebar_utils:escape_chars(Rev)]), + rebar_utils:sh(?FMT("git checkout -q ~ts", [rebar_utils:escape_chars(Rev)]), [{cd, Dir}]). git_vsn() -> @@ -168,7 +168,7 @@ git_vsn() -> git_vsn_fetch() -> case rebar_utils:sh("git --version",[]) of {ok, VsnStr} -> - case re:run(VsnStr, "git version\\h+(\\d)\\.(\\d)\\.(\\d).*",[{capture,[1,2,3],list}]) of + case re:run(VsnStr, "git version\\h+(\\d)\\.(\\d)\\.(\\d).*", [{capture,[1,2,3],list}, unicode]) of {match,[Maj,Min,Patch]} -> {list_to_integer(Maj), list_to_integer(Min), @@ -198,7 +198,7 @@ collect_default_refcount(Dir) -> return_on_error, {cd, Dir}]) of {error, _} -> - ?WARN("Getting log of git dependency failed in ~s. Falling back to version 0.0.0", [rebar_dir:get_cwd()]), + ?WARN("Getting log of git dependency failed in ~ts. Falling back to version 0.0.0", [rebar_dir:get_cwd()]), {plain, "0.0.0"}; {ok, String} -> RawRef = string:strip(String, both, $\n), @@ -222,21 +222,20 @@ collect_default_refcount(Dir) -> build_vsn_string(Vsn, RawRef, Count) -> %% Cleanup the tag and the Ref information. Basically leading 'v's and %% whitespace needs to go away. - RefTag = [".ref", re:replace(RawRef, "\\s", "", [global])], + RefTag = [".ref", re:replace(RawRef, "\\s", "", [global, unicode])], %% Create the valid [semver](http://semver.org) version from the tag case Count of 0 -> - erlang:binary_to_list(erlang:iolist_to_binary(Vsn)); + rebar_utils:to_list(Vsn); _ -> - erlang:binary_to_list(erlang:iolist_to_binary([Vsn, "+build.", - integer_to_list(Count), RefTag])) + rebar_utils:to_list([Vsn, "+build.", integer_to_list(Count), RefTag]) end. get_patch_count(Dir, RawRef) -> AbortMsg = "Getting rev-list of git dep failed in " ++ Dir, - Ref = re:replace(RawRef, "\\s", "", [global]), - Cmd = io_lib:format("git rev-list ~s..HEAD", + Ref = re:replace(RawRef, "\\s", "", [global, unicode]), + Cmd = io_lib:format("git rev-list ~ts..HEAD", [rebar_utils:escape_chars(Ref)]), {ok, PatchLines} = rebar_utils:sh(Cmd, [{use_stdout, false}, @@ -252,7 +251,7 @@ parse_tags(Dir) -> {error, _} -> {undefined, "0.0.0"}; {ok, Line} -> - case re:run(Line, "(\\(|\\s)(HEAD[^,]*,\\s)tag:\\s(v?([^,\\)]+))", [{capture, [3, 4], list}]) of + case re:run(Line, "(\\(|\\s)(HEAD[^,]*,\\s)tag:\\s(v?([^,\\)]+))", [{capture, [3, 4], list}, unicode]) of {match,[Tag, Vsn]} -> {Tag, Vsn}; nomatch -> diff --git a/src/rebar_hg_resource.erl b/src/rebar_hg_resource.erl index 7d03eda..42e634c 100644 --- a/src/rebar_hg_resource.erl +++ b/src/rebar_hg_resource.erl @@ -22,7 +22,7 @@ lock(AppDir, {hg, Url}) -> needs_update(Dir, {hg, Url, {tag, Tag}}) -> Ref = get_ref(Dir), {ClosestTag, Distance} = get_tag_distance(Dir, Ref), - ?DEBUG("Comparing hg tag ~s with ref ~s (closest tag is ~s at distance ~s)", + ?DEBUG("Comparing hg tag ~ts with ref ~ts (closest tag is ~ts at distance ~ts)", [Tag, Ref, ClosestTag, Distance]), not ((Distance =:= "0") andalso (Tag =:= ClosestTag) andalso compare_url(Dir, Url)); @@ -45,7 +45,7 @@ needs_update(Dir, {hg, Url, Ref}) -> Ref1 -> Ref1 end, - ?DEBUG("Comparing hg ref ~s with ~s", [Ref1, LocalRef]), + ?DEBUG("Comparing hg ref ~ts with ~ts", [Ref1, LocalRef]), not ((LocalRef =:= TargetRef) andalso compare_url(Dir, Url)). download(Dir, {hg, Url}, State) -> @@ -56,28 +56,28 @@ download(Dir, {hg, Url, ""}, State) -> download(Dir, {hg, Url, {branch, "default"}}, State); download(Dir, {hg, Url, {branch, Branch}}, _State) -> ok = filelib:ensure_dir(Dir), - rebar_utils:sh(?FMT("hg clone -q -b ~s ~s ~s", + rebar_utils:sh(?FMT("hg clone -q -b ~ts ~ts ~ts", [rebar_utils:escape_chars(Branch), rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]); download(Dir, {hg, Url, {tag, Tag}}, _State) -> ok = filelib:ensure_dir(Dir), - rebar_utils:sh(?FMT("hg clone -q -u ~s ~s ~s", + rebar_utils:sh(?FMT("hg clone -q -u ~ts ~ts ~ts", [rebar_utils:escape_chars(Tag), rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]); download(Dir, {hg, Url, {ref, Ref}}, _State) -> ok = filelib:ensure_dir(Dir), - rebar_utils:sh(?FMT("hg clone -q -r ~s ~s ~s", + rebar_utils:sh(?FMT("hg clone -q -r ~ts ~ts ~ts", [rebar_utils:escape_chars(Ref), rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]); download(Dir, {hg, Url, Rev}, _State) -> ok = filelib:ensure_dir(Dir), - rebar_utils:sh(?FMT("hg clone -q -r ~s ~s ~s", + rebar_utils:sh(?FMT("hg clone -q -r ~ts ~ts ~ts", [rebar_utils:escape_chars(Rev), rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), @@ -88,7 +88,7 @@ make_vsn(Dir) -> Ref = get_ref(Dir), Cmd = BaseHg ++ "log --template \"{latesttag}+build.{latesttagdistance}.rev.{node|short}\"" " --rev " ++ Ref, - AbortMsg = io_lib:format("Version resolution of hg dependency failed in ~s", [Dir]), + AbortMsg = io_lib:format("Version resolution of hg dependency failed in ~ts", [Dir]), {ok, VsnString} = rebar_utils:sh(Cmd, [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]), @@ -108,14 +108,14 @@ compare_url(Dir, Url) -> parse_hg_url(CurrentUrl1) =:= parse_hg_url(Url). get_ref(Dir) -> - AbortMsg = io_lib:format("Get ref of hg dependency failed in ~s", [Dir]), + AbortMsg = io_lib:format("Get ref of hg dependency failed in ~ts", [Dir]), {ok, RefString} = rebar_utils:sh("hg -R \"" ++ rebar_utils:escape_double_quotes(Dir) ++ "\" --debug id -i", [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]), string:strip(RefString, both, $\n). get_tag_distance(Dir, Ref) -> - AbortMsg = io_lib:format("Get tag distance of hg dependency failed in ~s", [Dir]), + AbortMsg = io_lib:format("Get tag distance of hg dependency failed in ~ts", [Dir]), {ok, LogString} = rebar_utils:sh("hg -R \"" ++ rebar_utils:escape_double_quotes(Dir) ++ "\" " "log --template \"{latesttag}-{latesttagdistance}\n\" " @@ -123,11 +123,12 @@ get_tag_distance(Dir, Ref) -> [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]), Log = string:strip(LogString, both, $\n), - [Tag, Distance] = re:split(Log, "-([0-9]+)$", [{parts,0}, {return, list}]), + [Tag, Distance] = re:split(Log, "-([0-9]+)$", + [{parts,0}, {return,list}, unicode]), {Tag, Distance}. get_branch_ref(Dir, Branch) -> - AbortMsg = io_lib:format("Get branch ref of hg dependency failed in ~s", [Dir]), + AbortMsg = io_lib:format("Get branch ref of hg dependency failed in ~ts", [Dir]), {ok, BranchRefString} = rebar_utils:sh("hg -R \"" ++ rebar_utils:escape_double_quotes(Dir) ++ "\" log --template \"{node}\n\" --rev " ++ rebar_utils:escape_chars(Branch), diff --git a/src/rebar_hooks.erl b/src/rebar_hooks.erl index d6a0e2b..48aa928 100644 --- a/src/rebar_hooks.erl +++ b/src/rebar_hooks.erl @@ -57,9 +57,9 @@ run_provider_hooks_(Dir, Type, Command, Providers, TypeHooks, State) -> end. format_error({bad_provider, Type, Command, {Name, Namespace}}) -> - io_lib:format("Unable to run ~s hooks for '~p', command '~p' in namespace '~p' not found.", [Type, Command, Namespace, Name]); + io_lib:format("Unable to run ~ts hooks for '~p', command '~p' in namespace '~p' not found.", [Type, Command, Namespace, Name]); format_error({bad_provider, Type, Command, Name}) -> - io_lib:format("Unable to run ~s hooks for '~p', command '~p' not found.", [Type, Command, Name]). + io_lib:format("Unable to run ~ts hooks for '~p', command '~p' not found.", [Type, Command, Name]). %% @doc The following environment variables are exported when running %% a hook (absolute paths): @@ -143,7 +143,7 @@ join_dirs(BaseDir, Dirs) -> string:join([ filename:join(BaseDir, Dir) || Dir <- Dirs ], ":"). re_version(Path) -> - case re:run(Path, "^.*-(?[^/-]*)$", [{capture, [1], list}]) of + case re:run(Path, "^.*-(?[^/-]*)$", [{capture,[1],list}, unicode]) of nomatch -> ""; {match, [Ver]} -> Ver end. diff --git a/src/rebar_log.erl b/src/rebar_log.erl index 47c84c5..9150346 100644 --- a/src/rebar_log.erl +++ b/src/rebar_log.erl @@ -94,7 +94,7 @@ get_level() -> log(Level = error, Str, Args) -> {ok, LogState} = application:get_env(rebar, log), - ec_cmd_log:Level(LogState, lists:flatten(cf:format("~!^~s~n", [Str])), Args); + ec_cmd_log:Level(LogState, lists:flatten(cf:format("~!^~ts~n", [Str])), Args); log(Level, Str, Args) -> {ok, LogState} = application:get_env(rebar, log), ec_cmd_log:Level(LogState, Str++"~n", Args). diff --git a/src/rebar_otp_app.erl b/src/rebar_otp_app.erl index ddaa44b..ed573f2 100644 --- a/src/rebar_otp_app.erl +++ b/src/rebar_otp_app.erl @@ -58,11 +58,11 @@ compile(State, App) -> validate_app(State, App1). format_error({missing_app_file, Filename}) -> - io_lib:format("App file is missing: ~s", [Filename]); + io_lib:format("App file is missing: ~ts", [Filename]); format_error({file_read, File, Reason}) -> - io_lib:format("Failed to read required file ~s for processing: ~s", [File, file:format_error(Reason)]); + io_lib:format("Failed to read required file ~ts for processing: ~ts", [File, file:format_error(Reason)]); format_error({invalid_name, File, AppName}) -> - io_lib:format("Invalid ~s: name of application (~p) must match filename.", [File, AppName]). + io_lib:format("Invalid ~ts: name of application (~p) must match filename.", [File, AppName]). %% =================================================================== %% Internal functions @@ -222,7 +222,7 @@ app_vsn(AppData, AppFile, State) -> get_value(Key, AppInfo, AppFile) -> case proplists:get_value(Key, AppInfo) of undefined -> - ?ABORT("Failed to get app value '~p' from '~s'~n", [Key, AppFile]); + ?ABORT("Failed to get app value '~p' from '~ts'~n", [Key, AppFile]); Value -> Value end. diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 5b6ab5c..cba1d16 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -72,12 +72,12 @@ deps(Name, Vsn, State) -> deps_(Name, Vsn, State) -> ?MODULE:verify_table(State), - ets:lookup_element(?PACKAGE_TABLE, {ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}, 2). + ets:lookup_element(?PACKAGE_TABLE, {rebar_utils:to_binary(Name), rebar_utils:to_binary(Vsn)}, 2). handle_missing_package(Dep, State, Fun) -> case Dep of {Name, Vsn} -> - ?INFO("Package ~s-~s not found. Fetching registry updates and trying again...", [Name, Vsn]); + ?INFO("Package ~ts-~ts not found. Fetching registry updates and trying again...", [Name, Vsn]); _ -> ?INFO("Package ~p not found. Fetching registry updates and trying again...", [Dep]) end, @@ -128,7 +128,7 @@ registry_checksum({pkg, Name, Vsn, _Hash}, State) -> ets:lookup_element(?PACKAGE_TABLE, {Name, Vsn}, 3) catch _:_ -> - throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)})) + throw(?PRV_ERROR({missing_package, rebar_utils:to_binary(Name), rebar_utils:to_binary(Vsn)})) end. %% Hex supports use of ~> to specify the version required for a dependency. @@ -207,17 +207,17 @@ handle_single_vsn(Pkg, PkgVsn, Dep, Vsn, Constraint) -> false -> case {Pkg, PkgVsn} of {undefined, undefined} -> - ?DEBUG("Only existing version of ~s is ~s which does not match constraint ~~> ~s. " + ?DEBUG("Only existing version of ~ts is ~ts which does not match constraint ~~> ~ts. " "Using anyway, but it is not guaranteed to work.", [Dep, Vsn, Constraint]); _ -> - ?DEBUG("[~s:~s] Only existing version of ~s is ~s which does not match constraint ~~> ~s. " + ?DEBUG("[~ts:~ts] Only existing version of ~ts is ~ts which does not match constraint ~~> ~ts. " "Using anyway, but it is not guaranteed to work.", [Pkg, PkgVsn, Dep, Vsn, Constraint]) end, {ok, Vsn} end. format_error({missing_package, Name, Vsn}) -> - io_lib:format("Package not found in registry: ~s-~s.", [ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)]); + io_lib:format("Package not found in registry: ~ts-~ts.", [rebar_utils:to_binary(Name), rebar_utils:to_binary(Vsn)]); format_error({missing_package, Dep}) -> io_lib:format("Package not found in registry: ~p.", [Dep]). diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 88419bd..d588f24 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -21,7 +21,7 @@ lock(_AppDir, Source) -> needs_update(Dir, {pkg, _Name, Vsn, _Hash}) -> [AppInfo] = rebar_app_discover:find_apps([Dir], all), - case rebar_app_info:original_vsn(AppInfo) =:= ec_cnv:to_list(Vsn) of + case rebar_app_info:original_vsn(AppInfo) =:= rebar_utils:to_list(Vsn) of true -> false; false -> @@ -43,13 +43,13 @@ download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State) -> cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, Url, ETag, State) -> case request(Url, ETag) of {ok, cached} -> - ?INFO("Version cached at ~s is up to date, reusing it", [CachePath]), + ?INFO("Version cached at ~ts is up to date, reusing it", [CachePath]), serve_from_cache(TmpDir, CachePath, Pkg, State); {ok, Body, NewETag} -> - ?INFO("Downloaded package, caching at ~s", [CachePath]), + ?INFO("Downloaded package, caching at ~ts", [CachePath]), serve_from_download(TmpDir, CachePath, Pkg, NewETag, Body, State); error when ETag =/= false -> - ?INFO("Download error, using cached file at ~s", [CachePath]), + ?INFO("Download error, using cached file at ~ts", [CachePath]), serve_from_cache(TmpDir, CachePath, Pkg, State); error -> {fetch_fail, Name, Vsn} @@ -76,13 +76,13 @@ serve_from_cache(TmpDir, CachePath, Pkg, State) -> end. serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State) -> - ?DEBUG("Writing ~p to cache at ~s", [Package, CachePath]), + ?DEBUG("Writing ~p to cache at ~ts", [Package, CachePath]), file:write_file(CachePath, Binary), case etag(CachePath) of ETag -> serve_from_cache(TmpDir, CachePath, Package, State); FileETag -> - ?DEBUG("Downloaded file ~s ETag ~s doesn't match returned ETag ~s", [CachePath, ETag, FileETag]), + ?DEBUG("Downloaded file ~ts ETag ~ts doesn't match returned ETag ~ts", [CachePath, ETag, FileETag]), {bad_download, CachePath} end. @@ -114,11 +114,11 @@ request(Url, ETag) -> [{body_format, binary}], rebar) of {ok, {{_Version, 200, _Reason}, Headers, Body}} -> - ?DEBUG("Successfully downloaded ~s", [Url]), + ?DEBUG("Successfully downloaded ~ts", [Url]), {"etag", ETag1} = lists:keyfind("etag", 1, Headers), {ok, Body, string:strip(ETag1, both, $")}; {ok, {{_Version, 304, _Reason}, _Headers, _Body}} -> - ?DEBUG("Cached copy of ~s still valid", [Url]), + ?DEBUG("Cached copy of ~ts still valid", [Url]), {ok, cached}; {ok, {{_Version, Code, _Reason}, _Headers, _Body}} -> ?DEBUG("Request to ~p failed: status code ~p", [Url, Code]), @@ -154,7 +154,7 @@ ssl_opts(Url) -> ssl_opts(ssl_verify_enabled, Url) -> case check_ssl_version() of true -> - {ok, {_, _, Hostname, _, _, _}} = http_uri:parse(ec_cnv:to_list(Url)), + {ok, {_, _, Hostname, _, _, _}} = http_uri:parse(rebar_utils:to_list(Url)), VerifyFun = {fun ssl_verify_hostname:verify_fun/3, [{check_hostname, Hostname}]}, CACerts = certifi:cacerts(), [{verify, verify_peer}, {depth, 2}, {cacerts, CACerts} diff --git a/src/rebar_prv_app_discovery.erl b/src/rebar_prv_app_discovery.erl index 1954214..3f10a3f 100644 --- a/src/rebar_prv_app_discovery.erl +++ b/src/rebar_prv_app_discovery.erl @@ -49,19 +49,19 @@ do(State) -> -spec format_error(any()) -> iolist(). format_error({multiple_app_files, Files}) -> - io_lib:format("Multiple app files found in one app dir: ~s", [string:join(Files, " and ")]); + io_lib:format("Multiple app files found in one app dir: ~ts", [string:join(Files, " and ")]); format_error({invalid_app_file, File, Reason}) -> case Reason of {Line, erl_parse, Description} -> - io_lib:format("Invalid app file ~s at line ~b: ~p", + io_lib:format("Invalid app file ~ts at line ~b: ~p", [File, Line, lists:flatten(Description)]); _ -> - io_lib:format("Invalid app file ~s: ~p", [File, Reason]) + io_lib:format("Invalid app file ~ts: ~p", [File, Reason]) end; %% Provide a slightly more informative error message for consult of app file failure format_error({rebar_file_utils, {bad_term_file, AppFile, Reason}}) -> - io_lib:format("Error in app file ~s: ~s", [rebar_dir:make_relative_path(AppFile, - rebar_dir:get_cwd()), - file:format_error(Reason)]); + io_lib:format("Error in app file ~ts: ~ts", [rebar_dir:make_relative_path(AppFile, + rebar_dir:get_cwd()), + file:format_error(Reason)]); format_error(Reason) -> io_lib:format("~p", [Reason]). diff --git a/src/rebar_prv_as.erl b/src/rebar_prv_as.erl index e7c6d68..562ce99 100644 --- a/src/rebar_prv_as.erl +++ b/src/rebar_prv_as.erl @@ -64,7 +64,7 @@ args_to_profiles_and_tasks(Args) -> first_profile([]) -> {[], []}; first_profile([ProfileList|Rest]) -> - case re:split(ProfileList, ",", [{return, list}, {parts, 2}]) of + case re:split(ProfileList, ",", [{return, list}, {parts, 2}, unicode]) of %% `foo, bar` [P, ""] -> profiles(Rest, [P]); %% `foo,bar` @@ -75,7 +75,7 @@ first_profile([ProfileList|Rest]) -> profiles([], Acc) -> {lists:reverse(Acc), rebar_utils:args_to_tasks([])}; profiles([ProfileList|Rest], Acc) -> - case re:split(ProfileList, ",", [{return, list}, {parts, 2}]) of + case re:split(ProfileList, ",", [{return, list}, {parts, 2}, unicode]) of %% `foo, bar` [P, ""] -> profiles(Rest, [P|Acc]); %% `foo,bar` @@ -101,5 +101,5 @@ warn_on_empty_profile(Profiles, State) -> ProjectApps = rebar_state:project_apps(State), DefinedProfiles = rebar_state:get(State, profiles, []) ++ lists:flatten([rebar_app_info:get(AppInfo, profiles, []) || AppInfo <- ProjectApps]), - [?WARN("No entry for profile ~s in config.", [Profile]) || + [?WARN("No entry for profile ~ts in config.", [Profile]) || Profile <- Profiles, not(lists:keymember(list_to_atom(Profile), 1, DefinedProfiles))]. diff --git a/src/rebar_prv_clean.erl b/src/rebar_prv_clean.erl index d185f75..aa0b5af 100644 --- a/src/rebar_prv_clean.erl +++ b/src/rebar_prv_clean.erl @@ -68,7 +68,7 @@ format_error(Reason) -> clean_apps(State, Providers, Apps) -> [begin - ?INFO("Cleaning out ~s...", [rebar_app_info:name(AppInfo)]), + ?INFO("Cleaning out ~ts...", [rebar_app_info:name(AppInfo)]), AppDir = rebar_app_info:dir(AppInfo), AppInfo1 = rebar_hooks:run_all_hooks(AppDir, pre, ?PROVIDER, Providers, AppInfo, State), rebar_erlc_compiler:clean(AppInfo1), diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index dfade77..bf788d2 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -684,7 +684,7 @@ format_result({Passed, 0, {0, 0}}) -> format_result({Passed, Failed, Skipped}) -> Format = [format_failed(Failed), format_skipped(Skipped), format_passed(Passed)], - ?CONSOLE("~s", [Format]); + ?CONSOLE("~ts", [Format]); format_result(_Unknown) -> %% Happens when CT itself encounters a bug ok. diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index effc763..959ecb0 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -73,7 +73,7 @@ do(State) -> -spec format_error(any()) -> iolist(). format_error({missing_artifact, File}) -> - io_lib:format("Missing artifact ~s", [File]); + io_lib:format("Missing artifact ~ts", [File]); format_error(Reason) -> io_lib:format("~p", [Reason]). @@ -114,7 +114,7 @@ compile(State, AppInfo) -> compile(State, rebar_state:providers(State), AppInfo). compile(State, Providers, AppInfo) -> - ?INFO("Compiling ~s", [rebar_app_info:name(AppInfo)]), + ?INFO("Compiling ~ts", [rebar_app_info:name(AppInfo)]), AppDir = rebar_app_info:dir(AppInfo), AppInfo1 = rebar_hooks:run_all_hooks(AppDir, pre, ?PROVIDER, Providers, AppInfo, State), @@ -173,8 +173,8 @@ has_all_artifacts(AppInfo1) -> end. copy_app_dirs(AppInfo, OldAppDir, AppDir) -> - case ec_cnv:to_binary(filename:absname(OldAppDir)) =/= - ec_cnv:to_binary(filename:absname(AppDir)) of + case rebar_utils:to_binary(filename:absname(OldAppDir)) =/= + rebar_utils:to_binary(filename:absname(AppDir)) of true -> EbinDir = filename:join([OldAppDir, "ebin"]), %% copy all files from ebin if it exists diff --git a/src/rebar_prv_cover.erl b/src/rebar_prv_cover.erl index bf142c4..fca7c40 100644 --- a/src/rebar_prv_cover.erl +++ b/src/rebar_prv_cover.erl @@ -278,7 +278,7 @@ write_index(State, Coverage) -> write_index_section(_F, []) -> ok; write_index_section(F, [{Section, DataFile, Mods}|Rest]) -> %% Write the report - ok = file:write(F, ?FMT("

~s summary

\n", [Section])), + ok = file:write(F, ?FMT("

~ts summary

\n", [Section])), ok = file:write(F, "coverage calculated from:\n
    "), ok = lists:foreach(fun(D) -> ok = file:write(F, io_lib:format("
  • ~ts
    • ", [D])) end, DataFile), @@ -303,7 +303,7 @@ strip_coverdir(File) -> 2))). cover_compile(State, apps) -> - ExclApps = [list_to_binary(A) || A <- rebar_state:get(State, cover_excl_apps, [])], + ExclApps = [rebar_utils:to_binary(A) || A <- rebar_state:get(State, cover_excl_apps, [])], Apps = filter_checkouts_and_excluded(rebar_state:project_apps(State), ExclApps), AppDirs = app_dirs(Apps), cover_compile(State, lists:filter(fun(D) -> ec_file:is_dir(D) end, AppDirs)); diff --git a/src/rebar_prv_deps.erl b/src/rebar_prv_deps.erl index 9ff2bfa..a88b014 100644 --- a/src/rebar_prv_deps.erl +++ b/src/rebar_prv_deps.erl @@ -55,7 +55,7 @@ merge(Deps, SourceDeps) -> normalize(Name) when is_binary(Name) -> Name; normalize(Name) when is_atom(Name) -> - ec_cnv:to_binary(Name); + atom_to_binary(Name, unicode); normalize(Dep) when is_tuple(Dep) -> Name = element(1, Dep), setelement(1, Dep, normalize(Name)). @@ -87,31 +87,31 @@ display_deps(State, Deps) -> %% packages display_dep(_State, {Name, Vsn}) when is_list(Vsn) -> - ?CONSOLE("~s* (package ~s)", [ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)]); + ?CONSOLE("~ts* (package ~ts)", [rebar_utils:to_binary(Name), rebar_utils:to_binary(Vsn)]); display_dep(_State, Name) when is_binary(Name) -> - ?CONSOLE("~s* (package)", [Name]); + ?CONSOLE("~ts* (package)", [Name]); display_dep(_State, {Name, Source}) when is_tuple(Source) -> - ?CONSOLE("~s* (~s source)", [ec_cnv:to_binary(Name), type(Source)]); + ?CONSOLE("~ts* (~ts source)", [rebar_utils:to_binary(Name), type(Source)]); display_dep(_State, {Name, _Vsn, Source}) when is_tuple(Source) -> - ?CONSOLE("~s* (~s source)", [ec_cnv:to_binary(Name), type(Source)]); + ?CONSOLE("~ts* (~ts source)", [rebar_utils:to_binary(Name), type(Source)]); display_dep(_State, {Name, _Vsn, Source, _Opts}) when is_tuple(Source) -> - ?CONSOLE("~s* (~s source)", [ec_cnv:to_binary(Name), type(Source)]); + ?CONSOLE("~ts* (~ts source)", [rebar_utils:to_binary(Name), type(Source)]); %% Locked display_dep(State, {Name, Source={pkg, _, Vsn}, Level}) when is_integer(Level) -> DepsDir = rebar_dir:deps_dir(State), - AppDir = filename:join([DepsDir, ec_cnv:to_binary(Name)]), + AppDir = filename:join([DepsDir, rebar_utils:to_binary(Name)]), NeedsUpdate = case rebar_fetch:needs_update(AppDir, Source, State) of true -> "*"; false -> "" end, - ?CONSOLE("~s~s (locked package ~s)", [Name, NeedsUpdate, Vsn]); + ?CONSOLE("~ts~ts (locked package ~ts)", [Name, NeedsUpdate, Vsn]); display_dep(State, {Name, Source, Level}) when is_tuple(Source), is_integer(Level) -> DepsDir = rebar_dir:deps_dir(State), - AppDir = filename:join([DepsDir, ec_cnv:to_binary(Name)]), + AppDir = filename:join([DepsDir, rebar_utils:to_binary(Name)]), NeedsUpdate = case rebar_fetch:needs_update(AppDir, Source, State) of true -> "*"; false -> "" end, - ?CONSOLE("~s~s (locked ~s source)", [Name, NeedsUpdate, type(Source)]). + ?CONSOLE("~ts~ts (locked ~ts source)", [Name, NeedsUpdate, type(Source)]). type(Source) when is_tuple(Source) -> element(1, Source). diff --git a/src/rebar_prv_deps_tree.erl b/src/rebar_prv_deps_tree.erl index c0c8bab..7c6978b 100644 --- a/src/rebar_prv_deps_tree.erl +++ b/src/rebar_prv_deps_tree.erl @@ -90,7 +90,7 @@ type(Source, Verbose) when is_tuple(Source) -> {pkg, _} -> "hex package"; {Other, false} -> - io_lib:format("~s repo", [Other]); + io_lib:format("~ts repo", [Other]); {_, true} -> - io_lib:format("~s", [element(2, Source)]) + io_lib:format("~ts", [element(2, Source)]) end. diff --git a/src/rebar_prv_dialyzer.erl b/src/rebar_prv_dialyzer.erl index 867b9cb..a74eefb 100644 --- a/src/rebar_prv_dialyzer.erl +++ b/src/rebar_prv_dialyzer.erl @@ -116,18 +116,18 @@ maybe_fix_env() -> -spec format_error(any()) -> iolist(). format_error({error_processing_apps, Error}) -> - io_lib:format("Error in dialyzing apps: ~s", [Error]); + io_lib:format("Error in dialyzing apps: ~ts", [Error]); format_error({dialyzer_warnings, Warnings}) -> io_lib:format("Warnings occurred running dialyzer: ~b", [Warnings]); format_error({unknown_application, App}) -> - io_lib:format("Could not find application: ~s", [App]); + io_lib:format("Could not find application: ~ts", [App]); format_error({unknown_module, Mod}) -> - io_lib:format("Could not find module: ~s", [Mod]); + io_lib:format("Could not find module: ~ts", [Mod]); format_error({duplicate_module, Mod, File1, File2}) -> - io_lib:format("Duplicates of module ~s: ~s ~s", [Mod, File1, File2]); + io_lib:format("Duplicates of module ~ts: ~ts ~ts", [Mod, File1, File2]); format_error({output_file_error, File, Error}) -> Error1 = file:format_error(Error), - io_lib:format("Failed to write to ~s: ~s", [File, Error1]); + io_lib:format("Failed to write to ~ts: ~ts", [File, Error1]); format_error(Reason) -> io_lib:format("~p", [Reason]). @@ -155,7 +155,7 @@ do(State, Plt) -> 0 -> {ok, State2}; TotalWarnings -> - ?INFO("Warnings written to ~s", [Output]), + ?INFO("Warnings written to ~ts", [Output]), throw({dialyzer_warnings, TotalWarnings}) end. @@ -229,7 +229,7 @@ apps_files([AppName | DepApps], SkipApps, Files) -> apps_files(DepApps, SkipApps, Files); false -> AppFiles = app_files(AppName), - ?DEBUG("~s modules: ~p", [AppName, dict:fetch_keys(AppFiles)]), + ?DEBUG("~ts modules: ~p", [AppName, dict:fetch_keys(AppFiles)]), Files2 = merge_files(Files, AppFiles), apps_files(DepApps, [AppName | SkipApps], Files2) end. @@ -505,7 +505,7 @@ format_warnings(Opts, Output, Warnings) -> length(Warnings). console_warnings(Warnings) -> - _ = [?CONSOLE("~s", [Warning]) || Warning <- Warnings], + _ = [?CONSOLE("~ts", [Warning]) || Warning <- Warnings], ok. file_warnings(_, []) -> diff --git a/src/rebar_prv_edoc.erl b/src/rebar_prv_edoc.erl index d663b0c..9517335 100644 --- a/src/rebar_prv_edoc.erl +++ b/src/rebar_prv_edoc.erl @@ -42,8 +42,8 @@ do(State) -> Res = try lists:foldl(fun(AppInfo, EdocOptsAcc) -> rebar_hooks:run_all_hooks(Cwd, pre, ?PROVIDER, Providers, AppInfo, State), - AppName = ec_cnv:to_list(rebar_app_info:name(AppInfo)), - ?INFO("Running edoc for ~s", [AppName]), + AppName = rebar_utils:to_list(rebar_app_info:name(AppInfo)), + ?INFO("Running edoc for ~ts", [AppName]), AppDir = rebar_app_info:dir(AppInfo), AppRes = (catch edoc:application(list_to_atom(AppName), AppDir, EdocOptsAcc)), rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, AppInfo, State), @@ -74,7 +74,7 @@ do(State) -> -spec format_error(any()) -> iolist(). format_error({app_failed, AppName}) -> - io_lib:format("Failed to generate documentation for app '~s'", [AppName]); + io_lib:format("Failed to generate documentation for app '~ts'", [AppName]); format_error(Reason) -> io_lib:format("~p", [Reason]). diff --git a/src/rebar_prv_escriptize.erl b/src/rebar_prv_escriptize.erl index 56b0d8a..cffbbdc 100644 --- a/src/rebar_prv_escriptize.erl +++ b/src/rebar_prv_escriptize.erl @@ -75,7 +75,7 @@ do(State) -> end; Name -> AllApps = rebar_state:all_deps(State)++rebar_state:project_apps(State), - case rebar_app_utils:find(ec_cnv:to_binary(Name), AllApps) of + case rebar_app_utils:find(rebar_utils:to_binary(Name), AllApps) of {ok, AppInfo} -> escriptize(State, AppInfo); _ -> @@ -87,12 +87,12 @@ do(State) -> escriptize(State0, App) -> AppName = rebar_app_info:name(App), - AppNameStr = ec_cnv:to_list(AppName), + AppNameStr = rebar_utils:to_list(AppName), %% Get the output filename for the escript -- this may include dirs Filename = filename:join([rebar_dir:base_dir(State0), "bin", rebar_state:get(State0, escript_name, AppName)]), - ?DEBUG("Creating escript file ~s", [Filename]), + ?DEBUG("Creating escript file ~ts", [Filename]), ok = filelib:ensure_dir(Filename), State = rebar_state:escript_path(State0, Filename), @@ -116,7 +116,7 @@ escriptize(State0, App) -> ExtraFiles = usort(InclBeams ++ InclExtra), Files = get_nonempty(EbinFiles ++ ExtraFiles), - DefaultEmuArgs = ?FMT("%%! -escript main ~s -pz ~s/~s/ebin\n", + DefaultEmuArgs = ?FMT("%%! -escript main ~ts -pz ~ts/~ts/ebin\n", [AppNameStr, AppNameStr, AppNameStr]), EscriptSections = [ {shebang, @@ -163,7 +163,7 @@ get_apps_beams(Apps, AllApps) -> get_apps_beams([], _, Acc) -> Acc; get_apps_beams([App | Rest], AllApps, Acc) -> - case rebar_app_utils:find(ec_cnv:to_binary(App), AllApps) of + case rebar_app_utils:find(rebar_utils:to_binary(App), AllApps) of {ok, App1} -> OutDir = filename:absname(rebar_app_info:ebin_dir(App1)), Beams = get_app_beams(App, OutDir), @@ -235,7 +235,7 @@ get_nonempty(Files) -> [{FName,FBin} || {FName,FBin} <- Files, FBin =/= <<>>]. find_deps(AppNames, AllApps) -> - BinAppNames = [ec_cnv:to_binary(Name) || Name <- AppNames], + BinAppNames = [rebar_utils:to_binary(Name) || Name <- AppNames], [ec_cnv:to_atom(Name) || Name <- find_deps_of_deps(BinAppNames, AllApps, BinAppNames)]. @@ -245,7 +245,7 @@ find_deps_of_deps([Name|Names], Apps, Acc) -> ?DEBUG("processing ~p", [Name]), {ok, App} = rebar_app_utils:find(Name, Apps), DepNames = proplists:get_value(applications, rebar_app_info:app_details(App), []), - BinDepNames = [ec_cnv:to_binary(Dep) || Dep <- DepNames, + BinDepNames = [rebar_utils:to_binary(Dep) || Dep <- DepNames, %% ignore system libs; shouldn't include them. DepDir <- [code:lib_dir(Dep)], DepDir =:= {error, bad_name} orelse % those are all local diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index c9fe0ad..55dca3d 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -104,22 +104,22 @@ do_(State) -> %% @doc convert a given exception's payload into an io description. -spec format_error(any()) -> iolist(). format_error({dep_app_not_found, AppDir, AppName}) -> - io_lib:format("Dependency failure: Application ~s not found at the top level of directory ~s", [AppName, AppDir]); + io_lib:format("Dependency failure: Application ~ts not found at the top level of directory ~ts", [AppName, AppDir]); format_error({load_registry_fail, Dep}) -> - io_lib:format("Error loading registry to resolve version of ~s. Try fixing by running 'rebar3 update'", [Dep]); + io_lib:format("Error loading registry to resolve version of ~ts. Try fixing by running 'rebar3 update'", [Dep]); format_error({bad_constraint, Name, Constraint}) -> - io_lib:format("Unable to parse version for package ~s: ~s", [Name, Constraint]); + io_lib:format("Unable to parse version for package ~ts: ~ts", [Name, Constraint]); format_error({parse_dep, Dep}) -> io_lib:format("Failed parsing dep ~p", [Dep]); format_error({not_rebar_package, Package, Version}) -> - io_lib:format("Package not buildable with rebar3: ~s-~s", [Package, Version]); + io_lib:format("Package not buildable with rebar3: ~ts-~ts", [Package, Version]); format_error({missing_package, Package, Version}) -> - io_lib:format("Package not found in registry: ~s-~s", [Package, Version]); + io_lib:format("Package not found in registry: ~ts-~ts", [Package, Version]); format_error({missing_package, Package}) -> - io_lib:format("Package not found in registry: ~s", [Package]); + io_lib:format("Package not found in registry: ~ts", [Package]); format_error({cycles, Cycles}) -> Prints = [["applications: ", - [io_lib:format("~s ", [Dep]) || Dep <- Cycle], + [io_lib:format("~ts ", [Dep]) || Dep <- Cycle], "depend on each other\n"] || Cycle <- Cycles], ["Dependency cycle(s) detected:\n", Prints]; @@ -291,7 +291,7 @@ handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) -> -spec maybe_fetch(rebar_app_info:t(), atom(), boolean(), sets:set(binary()), rebar_state:t()) -> {boolean(), rebar_app_info:t()}. maybe_fetch(AppInfo, Profile, Upgrade, Seen, State) -> - AppDir = ec_cnv:to_list(rebar_app_info:dir(AppInfo)), + AppDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)), %% Don't fetch dep if it exists in the _checkouts dir case rebar_app_info:is_checkout(AppInfo) of true -> @@ -346,7 +346,7 @@ symlink_dep(State, From, To) -> ok -> RelativeFrom = make_relative_to_root(State, From), RelativeTo = make_relative_to_root(State, To), - ?INFO("Linking ~s to ~s", [RelativeFrom, RelativeTo]), + ?INFO("Linking ~ts to ~ts", [RelativeFrom, RelativeTo]), ok; exists -> ok @@ -359,7 +359,7 @@ make_relative_to_root(State, Path) when is_list(Path) -> rebar_dir:make_relative_path(Path, Root). fetch_app(AppInfo, AppDir, State) -> - ?INFO("Fetching ~s (~p)", [rebar_app_info:name(AppInfo), + ?INFO("Fetching ~ts (~p)", [rebar_app_info:name(AppInfo), format_source(rebar_app_info:source(AppInfo))]), Source = rebar_app_info:source(AppInfo), true = rebar_fetch:download_source(AppDir, Source, State). @@ -384,12 +384,12 @@ maybe_upgrade(AppInfo, AppDir, Upgrade, State) -> true -> case rebar_fetch:needs_update(AppDir, Source, State) of true -> - ?INFO("Upgrading ~s (~p)", [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)]), + ?INFO("Upgrading ~ts (~p)", [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)]), true = rebar_fetch:download_source(AppDir, Source, State); false -> case Upgrade of true -> - ?INFO("No upgrade needed for ~s", [rebar_app_info:name(AppInfo)]), + ?INFO("No upgrade needed for ~ts", [rebar_app_info:name(AppInfo)]), false; false -> false @@ -400,7 +400,7 @@ maybe_upgrade(AppInfo, AppDir, Upgrade, State) -> end. warn_skip_deps(AppInfo, State) -> - Msg = "Skipping ~s (from ~p) as an app of the same name " + Msg = "Skipping ~ts (from ~p) as an app of the same name " "has already been fetched", Args = [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)], diff --git a/src/rebar_prv_local_install.erl b/src/rebar_prv_local_install.erl index 0b30fd4..282c548 100644 --- a/src/rebar_prv_local_install.erl +++ b/src/rebar_prv_local_install.erl @@ -60,7 +60,7 @@ format_error(Reason) -> bin_contents(OutputDir) -> <<"#!/usr/bin/env sh -erl -pz ", (ec_cnv:to_binary(OutputDir))/binary,"/*/ebin +sbtu +A0 -noshell -boot start_clean -s rebar3 main $REBAR3_ERL_ARGS -extra \"$@\" +erl -pz ", (rebar_utils:to_binary(OutputDir))/binary,"/*/ebin +sbtu +A0 -noshell -boot start_clean -s rebar3 main $REBAR3_ERL_ARGS -extra \"$@\" ">>. extract_escript(State, ScriptPath) -> @@ -73,7 +73,7 @@ extract_escript(State, ScriptPath) -> OutputDir = filename:join(rebar_dir:global_cache_dir(Opts), "lib"), filelib:ensure_dir(filename:join(OutputDir, "empty")), - ?INFO("Extracting rebar3 libs to ~s...", [OutputDir]), + ?INFO("Extracting rebar3 libs to ~ts...", [OutputDir]), zip:extract(Archive, [{cwd, OutputDir}]), BinDir = filename:join(rebar_dir:global_cache_dir(Opts), "bin"), @@ -84,12 +84,12 @@ extract_escript(State, ScriptPath) -> uid = Uid, gid = Gid}} = file:read_file_info(ScriptPath), - ?INFO("Writing rebar3 run script ~s...", [BinFile]), + ?INFO("Writing rebar3 run script ~ts...", [BinFile]), file:write_file(BinFile, bin_contents(OutputDir)), ok = file:write_file_info(BinFile, #file_info{mode=33277, uid=Uid, gid=Gid}), - ?INFO("Add to $PATH for use: export PATH=~s:$PATH", [BinDir]), + ?INFO("Add to $PATH for use: export PATH=~ts:$PATH", [BinDir]), {ok, State}. diff --git a/src/rebar_prv_new.erl b/src/rebar_prv_new.erl index 152a56e..f632362 100644 --- a/src/rebar_prv_new.erl +++ b/src/rebar_prv_new.erl @@ -60,7 +60,7 @@ do(State) -> -spec format_error(any()) -> iolist(). format_error({consult, File, Reason}) -> - io_lib:format("Error consulting file at ~s for reason ~p", [File, Reason]); + io_lib:format("Error consulting file at ~ts for reason ~p", [File, Reason]); format_error(Reason) -> io_lib:format("~p", [Reason]). @@ -70,7 +70,7 @@ format_error(Reason) -> list_templates(State) -> lists:foldl(fun({error, {consult, File, Reason}}, Acc) -> - ?WARN("Error consulting template file ~s for reason ~p", + ?WARN("Error consulting template file ~ts for reason ~p", [File, Reason]), Acc ; (Tpl, Acc) -> @@ -116,16 +116,16 @@ show_short_templates(List) -> lists:map(fun show_short_template/1, lists:sort(List)). show_short_template({Name, Type, _Location, Description, _Vars}) -> - io:format("~s (~s): ~s~n", + io:format("~ts (~ts): ~ts~n", [Name, format_type(Type), format_description(Description)]). show_template({Name, Type, Location, Description, Vars}) -> - io:format("~s:~n" - "\t~s~n" - "\tDescription: ~s~n" - "\tVariables:~n~s~n", + io:format("~ts:~n" + "\t~ts~n" + "\tDescription: ~ts~n" + "\tVariables:~n~ts~n", [Name, format_type(Type, Location), format_description(Description), @@ -141,9 +141,9 @@ format_type(escript, _) -> format_type(builtin, _) -> "built-in template"; format_type(plugin, Loc) -> - io_lib:format("plugin template (~s)", [Loc]); + io_lib:format("plugin template (~ts)", [Loc]); format_type(file, Loc) -> - io_lib:format("custom template (~s)", [Loc]). + io_lib:format("custom template (~ts)", [Loc]). format_description(Description) -> case Description of @@ -156,4 +156,4 @@ format_vars(Vars) -> [format_var(Var) || Var <- Vars]. format_var({Var, Default}) -> io_lib:format("\t\t~p=~p~n",[Var, Default]); format_var({Var, Default, Doc}) -> - io_lib:format("\t\t~p=~p (~s)~n", [Var, Default, Doc]). + io_lib:format("\t\t~p=~p (~ts)~n", [Var, Default, Doc]). diff --git a/src/rebar_prv_packages.erl b/src/rebar_prv_packages.erl index 7217ab8..6e8e683 100644 --- a/src/rebar_prv_packages.erl +++ b/src/rebar_prv_packages.erl @@ -30,7 +30,7 @@ do(State) -> rebar_packages:packages(State), case rebar_state:command_args(State) of [Name] -> - print_packages(get_packages(iolist_to_binary(Name))); + print_packages(get_packages(rebar_utils:to_binary(Name))); _ -> print_packages(sort_packages()) end, @@ -47,7 +47,7 @@ print_packages(Pkgs) -> ,ec_semver:parse(B)) end, Vsns), VsnStr = join(SortedVsns, <<", ">>), - ?CONSOLE("~s:~n Versions: ~s~n", [Name, VsnStr]) + ?CONSOLE("~ts:~n Versions: ~ts~n", [Name, VsnStr]) end, Pkgs). sort_packages() -> @@ -71,4 +71,4 @@ join([Bin | T], Sep) -> info(Description) -> - io_lib:format("~s.~n", [Description]). + io_lib:format("~ts.~n", [Description]). diff --git a/src/rebar_prv_path.erl b/src/rebar_prv_path.erl index d32d2c2..75d38eb 100644 --- a/src/rebar_prv_path.erl +++ b/src/rebar_prv_path.erl @@ -75,23 +75,23 @@ paths([{src, true}|Rest], Apps, State, Acc) -> paths([{rel, true}|Rest], Apps, State, Acc) -> paths(Rest, Apps, State, [rel_dir(State)|Acc]). -base_dir(State) -> io_lib:format("~s", [rebar_dir:base_dir(State)]). -bin_dir(State) -> io_lib:format("~s/bin", [rebar_dir:base_dir(State)]). -lib_dir(State) -> io_lib:format("~s", [rebar_dir:deps_dir(State)]). -rel_dir(State) -> io_lib:format("~s/rel", [rebar_dir:base_dir(State)]). +base_dir(State) -> io_lib:format("~ts", [rebar_dir:base_dir(State)]). +bin_dir(State) -> io_lib:format("~ts/bin", [rebar_dir:base_dir(State)]). +lib_dir(State) -> io_lib:format("~ts", [rebar_dir:deps_dir(State)]). +rel_dir(State) -> io_lib:format("~ts/rel", [rebar_dir:base_dir(State)]). ebin_dirs(Apps, State) -> - lists:map(fun(App) -> io_lib:format("~s/~s/ebin", [rebar_dir:deps_dir(State), App]) end, Apps). + lists:map(fun(App) -> io_lib:format("~ts/~ts/ebin", [rebar_dir:deps_dir(State), App]) end, Apps). priv_dirs(Apps, State) -> - lists:map(fun(App) -> io_lib:format("~s/~s/priv", [rebar_dir:deps_dir(State), App]) end, Apps). + lists:map(fun(App) -> io_lib:format("~ts/~ts/priv", [rebar_dir:deps_dir(State), App]) end, Apps). src_dirs(Apps, State) -> - lists:map(fun(App) -> io_lib:format("~s/~s/src", [rebar_dir:deps_dir(State), App]) end, Apps). + lists:map(fun(App) -> io_lib:format("~ts/~ts/src", [rebar_dir:deps_dir(State), App]) end, Apps). print_paths_if_exist(Paths, State) -> {RawOpts, _} = rebar_state:command_parsed_args(State), Sep = proplists:get_value(separator, RawOpts, " "), RealPaths = lists:filter(fun(P) -> ec_file:is_dir(P) end, Paths), - io:format("~s", [string:join(RealPaths, Sep)]). + io:format("~ts", [string:join(RealPaths, Sep)]). project_deps(State) -> Profiles = rebar_state:current_profiles(State), diff --git a/src/rebar_prv_plugins.erl b/src/rebar_prv_plugins.erl index c76dae1..4bea3b3 100644 --- a/src/rebar_prv_plugins.erl +++ b/src/rebar_prv_plugins.erl @@ -55,19 +55,19 @@ format_error(Reason) -> display_plugins(_Header, _Apps, []) -> ok; display_plugins(Header, Apps, Plugins) -> - ?CONSOLE("--- ~s ---", [Header]), + ?CONSOLE("--- ~ts ---", [Header]), display_plugins(Apps, Plugins), ?CONSOLE("", []). display_plugins(Apps, Plugins) -> lists:foreach(fun(Plugin) -> - Name = if is_atom(Plugin) -> ec_cnv:to_binary(Plugin); - is_tuple(Plugin) -> ec_cnv:to_binary(element(1, Plugin)) + Name = if is_atom(Plugin) -> atom_to_binary(Plugin, unicode); + is_tuple(Plugin) -> rebar_utils:to_binary(element(1, Plugin)) end, case rebar_app_utils:find(Name, Apps) of {ok, _App} -> - ?CONSOLE("~s", [Name]); + ?CONSOLE("~ts", [Name]); error -> - ?DEBUG("Unable to find plugin ~s", [Name]) + ?DEBUG("Unable to find plugin ~ts", [Name]) end end, Plugins). diff --git a/src/rebar_prv_plugins_upgrade.erl b/src/rebar_prv_plugins_upgrade.erl index 03521c7..7420c83 100644 --- a/src/rebar_prv_plugins_upgrade.erl +++ b/src/rebar_prv_plugins_upgrade.erl @@ -44,7 +44,7 @@ do(State) -> format_error(no_plugin_arg) -> io_lib:format("Must give an installed plugin to upgrade as an argument", []); format_error({not_found, Plugin}) -> - io_lib:format("Plugin to upgrade not found: ~s", [Plugin]); + io_lib:format("Plugin to upgrade not found: ~ts", [Plugin]); format_error(Reason) -> io_lib:format("~p", [Reason]). diff --git a/src/rebar_prv_report.erl b/src/rebar_prv_report.erl index d6c8b60..73e9624 100644 --- a/src/rebar_prv_report.erl +++ b/src/rebar_prv_report.erl @@ -44,7 +44,7 @@ do(State) -> {ok, Vsn} = application:get_key(rebar, vsn), {ok, Apps} = application:get_key(rebar, applications), [application:load(App) || App <- Apps], - Vsns = [io_lib:format("~p: ~s~n", [App, AVsn]) + Vsns = [io_lib:format("~p: ~ts~n", [App, AVsn]) || App <- lists:sort(Apps), {ok, AVsn} <- [application:get_key(App, vsn)]], %% Show OS and versions @@ -59,10 +59,10 @@ do(State) -> %% ?CONSOLE( "Rebar3 report~n" - " version ~s~n" - " generated at ~s~n" + " version ~ts~n" + " generated at ~ts~n" "=================~n" - "Please submit this along with your issue at ~s " + "Please submit this along with your issue at ~ts " "(and feel free to edit out private information, if any)~n" "-----------------~n" "Task: ~ts~n" @@ -75,11 +75,11 @@ do(State) -> "Library directory: ~ts~n" "-----------------~n" "Loaded Applications:~n" - "~s~n" + "~ts~n" "-----------------~n" "Escript path: ~ts~n" "Providers:~n" - " ~s", + " ~ts", [Vsn, time_to_string(UTC), ?ISSUES_URL, Command, Task, OS, ERTS, Root, Lib, @@ -100,4 +100,4 @@ time_to_string({{Y,M,D},{H,Min,S}}) -> [Y,M,D,H,Min,S])). parse_task(Str) -> - hd(re:split(Str, " ")). + hd(re:split(Str, " ", [unicode])). diff --git a/src/rebar_prv_unlock.erl b/src/rebar_prv_unlock.erl index 7ff0d89..51c57ab 100644 --- a/src/rebar_prv_unlock.erl +++ b/src/rebar_prv_unlock.erl @@ -66,7 +66,7 @@ format_error(Reason) -> handle_unlocks(State, Locks, LockFile) -> {Args, _} = rebar_state:command_parsed_args(State), - Names = parse_names(ec_cnv:to_binary(proplists:get_value(package, Args, <<"">>))), + Names = parse_names(rebar_utils:to_binary(proplists:get_value(package, Args, <<"">>))), case [Lock || Lock = {Name, _, _} <- Locks, not lists:member(Name, Names)] of [] -> file:delete(LockFile); @@ -77,7 +77,7 @@ handle_unlocks(State, Locks, LockFile) -> end. parse_names(Bin) -> - case lists:usort(re:split(Bin, <<" *, *">>, [trim])) of + case lists:usort(re:split(Bin, <<" *, *">>, [trim, unicode])) of [<<"">>] -> []; % nothing submitted Other -> Other end. diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index 046c864..a019c5a 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -150,7 +150,7 @@ update_deps_list(Pkg, PkgVsn, Deps, HexRegistry, State) -> %% and doubled since spaces seem not to be %% enforced {false, Vsn} -> - ?WARN("[~s:~s], Bad dependency version for ~s: ~s.", + ?WARN("[~ts:~ts], Bad dependency version for ~ts: ~ts.", [Pkg, PkgVsn, Dep, Vsn]), DepsListAcc; {_, <<"~>", Vsn/binary>>} -> @@ -202,14 +202,14 @@ valid_vsn(Vsn) -> SemVerRegExp = "v?(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)(\\.(0|[1-9][0-9]*))?" "(-[0-9a-z-]+(\\.[0-9a-z-]+)*)?(\\+[0-9a-z-]+(\\.[0-9a-z-]+)*)?", SupportedVersions = "^(>=?|<=?|~>|==)?\\s*" ++ SemVerRegExp ++ "$", - re:run(Vsn, SupportedVersions) =/= nomatch. + re:run(Vsn, SupportedVersions, [unicode]) =/= nomatch. highest_matching({Pkg, PkgVsn, Dep, App}, Vsn, HexRegistry, State, DepsListAcc) -> case rebar_packages:find_highest_matching(Pkg, PkgVsn, Dep, Vsn, HexRegistry, State) of {ok, HighestDepVsn} -> [{App, {pkg, Dep, HighestDepVsn, undefined}} | DepsListAcc]; none -> - ?WARN("[~s:~s] Missing registry entry for package ~s. Try to fix with `rebar3 update`", + ?WARN("[~ts:~ts] Missing registry entry for package ~ts. Try to fix with `rebar3 update`", [Pkg, PkgVsn, Dep]), DepsListAcc end. @@ -220,7 +220,7 @@ cmp({_Pkg, _PkgVsn, Dep, _App} = Dep1, Vsn, HexRegistry, State, DepsListAcc, Cmp cmp_(undefined, _MinVsn, [], DepsListAcc, {Pkg, PkgVsn, Dep, _App}, _CmpFun) -> - ?WARN("[~s:~s] Missing registry entry for package ~s. Try to fix with `rebar3 update`", + ?WARN("[~ts:~ts] Missing registry entry for package ~ts. Try to fix with `rebar3 update`", [Pkg, PkgVsn, Dep]), DepsListAcc; cmp_(HighestDepVsn, _MinVsn, [], DepsListAcc, {_Pkg, _PkgVsn, Dep, App}, _CmpFun) -> @@ -241,7 +241,7 @@ cmpl({_Pkg, _PkgVsn, Dep, _App} = Dep1, Vsn, HexRegistry, State, DepsListAcc, Cm cmpl_(undefined, Vsn, Vsns, DepsListAcc, Dep1, CmpFun). cmpl_(undefined, _MaxVsn, [], DepsListAcc, {Pkg, PkgVsn, Dep, _App}, _CmpFun) -> - ?WARN("[~s:~s] Missing registry entry for package ~s. Try to fix with `rebar3 update`", + ?WARN("[~ts:~ts] Missing registry entry for package ~ts. Try to fix with `rebar3 update`", [Pkg, PkgVsn, Dep]), DepsListAcc; diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl index 2de30e4..5a7dff8 100644 --- a/src/rebar_prv_upgrade.erl +++ b/src/rebar_prv_upgrade.erl @@ -68,7 +68,7 @@ do(State) -> ProfileDeps = rebar_state:get(State, {deps, default}, []), Deps = [Dep || Dep <- TopDeps ++ ProfileDeps, % TopDeps > ProfileDeps is_atom(Dep) orelse is_atom(element(1, Dep))], - Names = parse_names(ec_cnv:to_binary(proplists:get_value(package, Args, <<"">>)), Locks), + Names = parse_names(rebar_utils:to_binary(proplists:get_value(package, Args, <<"">>)), Locks), DepsDict = deps_dict(rebar_state:all_deps(State)), AltDeps = find_non_default_deps(Deps, State), FilteredNames = cull_default_names_if_profiles(Names, Deps, State), @@ -109,7 +109,7 @@ format_error(Reason) -> io_lib:format("~p", [Reason]). parse_names(Bin, Locks) -> - case lists:usort(re:split(Bin, <<" *, *">>, [trim])) of + case lists:usort(re:split(Bin, <<" *, *">>, [trim, unicode])) of %% Nothing submitted, use *all* apps [<<"">>] -> [Name || {Name, _, 0} <- Locks]; [] -> [Name || {Name, _, 0} <- Locks]; @@ -150,7 +150,7 @@ prepare_locks([Name|Names], Deps, Locks, Unlocks, Dict, AltDeps) -> {_, _, 0} = Lock -> case rebar_utils:tup_find(AtomName, Deps) of false -> - ?WARN("Dependency ~s has been removed and will not be upgraded", [Name]), + ?WARN("Dependency ~ts has been removed and will not be upgraded", [Name]), prepare_locks(Names, Deps, Locks, Unlocks, Dict, AltDeps); Dep -> {Source, NewLocks, NewUnlocks} = prepare_lock(Dep, Lock, Locks, Dict), @@ -181,7 +181,7 @@ prepare_lock(Dep, Lock, Locks, Dict) -> {Name, _, Src} -> {Name, Src}; _ when is_atom(Dep) -> %% version-free package. Must unlock whatever matches in locks - {_, Vsn, _} = lists:keyfind(ec_cnv:to_binary(Dep), 1, Locks), + {_, Vsn, _} = lists:keyfind(rebar_utils:to_binary(Dep), 1, Locks), {Dep, Vsn} end, Children = all_children(Name1, Dict), @@ -197,7 +197,7 @@ unlock_children(Children, Locks) -> unlock_children(_, [], Locks, Unlocks) -> {Locks, Unlocks}; unlock_children(Children, [App = {Name,_,_} | Apps], Locks, Unlocks) -> - case lists:member(ec_cnv:to_binary(Name), Children) of + case lists:member(rebar_utils:to_binary(Name), Children) of true -> unlock_children(Children, Apps, Locks, [App | Unlocks]); false -> @@ -215,7 +215,7 @@ all_children(Name, Dict) -> lists:flatten(all_children_(Name, Dict)). all_children_(Name, Dict) -> - case dict:find(ec_cnv:to_binary(Name), Dict) of + case dict:find(rebar_utils:to_binary(Name), Dict) of {ok, Children} -> [Children | [all_children_(Child, Dict) || Child <- Children]]; error -> diff --git a/src/rebar_prv_xref.erl b/src/rebar_prv_xref.erl index f358787..2405ebb 100644 --- a/src/rebar_prv_xref.erl +++ b/src/rebar_prv_xref.erl @@ -71,7 +71,7 @@ short_desc() -> desc() -> io_lib:format( - "~s~n" + "~ts~n" "~n" "Valid rebar.config options:~n" " ~p~n" @@ -204,7 +204,7 @@ display_results(XrefResults, QueryResults) -> lists:map(fun display_query_result/1, QueryResults)]. display_query_result({Query, Answer, Value}) -> - io_lib:format("Query ~s~n answer ~p~n did not match ~p~n", + io_lib:format("Query ~ts~n answer ~p~n did not match ~p~n", [Query, Answer, Value]). display_xref_results_for_type({Type, XrefResults}) -> @@ -225,37 +225,37 @@ display_xref_result_fun(Type) -> end, case Type of undefined_function_calls -> - io_lib:format("~sWarning: ~s calls undefined function ~s (Xref)\n", + io_lib:format("~tsWarning: ~ts calls undefined function ~ts (Xref)\n", [Source, SMFA, TMFA]); undefined_functions -> - io_lib:format("~sWarning: ~s is undefined function (Xref)\n", + io_lib:format("~tsWarning: ~ts is undefined function (Xref)\n", [Source, SMFA]); locals_not_used -> - io_lib:format("~sWarning: ~s is unused local function (Xref)\n", + io_lib:format("~tsWarning: ~ts is unused local function (Xref)\n", [Source, SMFA]); exports_not_used -> - io_lib:format("~sWarning: ~s is unused export (Xref)\n", + io_lib:format("~tsWarning: ~ts is unused export (Xref)\n", [Source, SMFA]); deprecated_function_calls -> - io_lib:format("~sWarning: ~s calls deprecated function ~s (Xref)\n", + io_lib:format("~tsWarning: ~ts calls deprecated function ~ts (Xref)\n", [Source, SMFA, TMFA]); deprecated_functions -> - io_lib:format("~sWarning: ~s is deprecated function (Xref)\n", + io_lib:format("~tsWarning: ~ts is deprecated function (Xref)\n", [Source, SMFA]); Other -> - io_lib:format("~sWarning: ~s - ~s xref check: ~s (Xref)\n", + io_lib:format("~tsWarning: ~ts - ~ts xref check: ~ts (Xref)\n", [Source, SMFA, TMFA, Other]) end end. format_mfa({M, F, A}) -> - ?FMT("~s:~s/~w", [M, F, A]). + ?FMT("~ts:~ts/~w", [M, F, A]). format_mfa_source(MFA) -> case find_mfa_source(MFA) of {module_not_found, function_not_found} -> ""; - {Source, function_not_found} -> ?FMT("~s: ", [Source]); - {Source, Line} -> ?FMT("~s:~w: ", [Source, Line]) + {Source, function_not_found} -> ?FMT("~ts: ", [Source]); + {Source, Line} -> ?FMT("~ts:~w: ", [Source, Line]) end. %% diff --git a/src/rebar_state.erl b/src/rebar_state.erl index 9683709..3314a11 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -182,7 +182,7 @@ all(_, []) -> all(Dir, [File|Artifacts]) -> case filelib:is_regular(filename:join(Dir, File)) of false -> - ?DEBUG("Missing artifact ~s", [filename:join(Dir, File)]), + ?DEBUG("Missing artifact ~ts", [filename:join(Dir, File)]), {false, File}; true -> all(Dir, Artifacts) @@ -302,9 +302,9 @@ dir(State=#state_t{}, Dir) -> deps_names(Deps) when is_list(Deps) -> lists:map(fun(Dep) when is_tuple(Dep) -> - ec_cnv:to_binary(element(1, Dep)); + rebar_utils:to_binary(element(1, Dep)); (Dep) when is_atom(Dep) -> - ec_cnv:to_binary(Dep) + rebar_utils:to_binary(Dep) end, Deps); deps_names(State) -> Deps = rebar_state:get(State, deps, []), diff --git a/src/rebar_templater.erl b/src/rebar_templater.erl index 1c28788..75190ec 100644 --- a/src/rebar_templater.erl +++ b/src/rebar_templater.erl @@ -129,7 +129,7 @@ default_author_and_email() -> %% Ok, try mecurial case rebar_utils:sh("hg showconfig ui.username", [return_on_error]) of {ok, NameEmail} -> - case re:run(NameEmail, "^(.*) <(.*)>$", [{capture, [1,2], list}]) of + case re:run(NameEmail, "^(.*) <(.*)>$", [{capture, [1,2], list}, unicode]) of {match, [Name, Email]} -> {Name, Email}; _ -> @@ -169,7 +169,7 @@ maybe_warn_about_name(Vars) -> invalid -> ?WARN("The 'name' variable is often associated with Erlang " "module names and/or file names. The value submitted " - "(~s) isn't an unquoted Erlang atom. Templates " + "(~ts) isn't an unquoted Erlang atom. Templates " "generated may contain errors.", [Name]); valid -> @@ -189,7 +189,7 @@ validate_atom(Str) -> %% Run template instructions one at a time. execute_template([], _, {Template,_,_}, _, _) -> - ?DEBUG("Template ~s applied", [Template]), + ?DEBUG("Template ~ts applied", [Template]), ok; %% We can't execute the description execute_template([{description, _} | Terms], Files, Template, Vars, Force) -> @@ -242,7 +242,7 @@ execute_template([{template, From, To} | Terms], Files, {Template, Type, Cwd}, V execute_template(Terms, Files, {Template, Type, Cwd}, Vars, Force); %% Unknown execute_template([Instruction|Terms], Files, Tpl={Template,_,_}, Vars, Force) -> - ?WARN("Unknown template instruction ~p in template ~s", + ?WARN("Unknown template instruction ~p in template ~ts", [Instruction, Template]), execute_template(Terms, Files, Tpl, Vars, Force). @@ -305,7 +305,7 @@ cache_escript_files(State) -> find_escript_templates(Files) -> [{escript, Name} || {Name, _Bin} <- Files, - re:run(Name, ?TEMPLATE_RE, [{capture, none}]) == match]. + re:run(Name, ?TEMPLATE_RE, [{capture, none}, unicode]) == match]. find_localinstall_templates(_State) -> Templates = rebar_utils:find_files(code:priv_dir(rebar), ?TEMPLATE_RE), @@ -374,7 +374,7 @@ prioritize_templates([{Name, Type, File} | Rest], Valid) -> prioritize_templates(Rest, Valid); {_, file, _} -> ?DEBUG("Skipping template ~p, due to presence of a custom " - "template at ~s", [Name, File]), + "template at ~ts", [Name, File]), prioritize_templates(Rest, Valid) end. @@ -429,10 +429,10 @@ write_file(Output, Data, Force) -> ok = filelib:ensure_dir(Output), case {Force, FileExists} of {true, true} -> - ?INFO("Writing ~s (forcibly overwriting)", + ?INFO("Writing ~ts (forcibly overwriting)", [Output]); _ -> - ?INFO("Writing ~s", [Output]) + ?INFO("Writing ~ts", [Output]) end, case file:write_file(Output, Data) of ok -> @@ -449,4 +449,4 @@ write_file(Output, Data, Force) -> %% Render a binary to a string, using mustache and the specified context %% render(Bin, Context) -> - bbmustache:render(ec_cnv:to_binary(Bin), Context, [{key_type, atom}]). + bbmustache:render(rebar_utils:to_binary(Bin), Context, [{key_type, atom}]). diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index c684e2d..016847e 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -55,6 +55,8 @@ get_arch/0, wordsize/0, deps_to_binary/1, + to_binary/1, + to_list/1, tup_dedup/1, tup_umerge/2, tup_sort/1, @@ -115,7 +117,7 @@ filtermap(F, [Hd|Tail]) -> filtermap(F, []) when is_function(F, 1) -> []. is_arch(ArchRegex) -> - case re:run(get_arch(), ArchRegex, [{capture, none}]) of + case re:run(get_arch(), ArchRegex, [{capture, none}, unicode]) of match -> true; nomatch -> @@ -142,7 +144,7 @@ wordsize() -> end. sh_send(Command0, String, Options0) -> - ?INFO("sh_send info:\n\tcwd: ~p\n\tcmd: ~s < ~s\n", + ?INFO("sh_send info:\n\tcwd: ~p\n\tcmd: ~ts < ~ts\n", [rebar_dir:get_cwd(), Command0, String]), ?DEBUG("\topts: ~p\n", [Options0]), @@ -171,7 +173,7 @@ sh_send(Command0, String, Options0) -> %% Val = string() | false %% sh(Command0, Options0) -> - ?DEBUG("sh info:\n\tcwd: ~p\n\tcmd: ~s\n", [rebar_dir:get_cwd(), Command0]), + ?DEBUG("sh info:\n\tcwd: ~p\n\tcmd: ~ts\n", [rebar_dir:get_cwd(), Command0]), ?DEBUG("\topts: ~p\n", [Options0]), DefaultOptions = [{use_stdout, false}, debug_and_abort_on_error], @@ -184,7 +186,7 @@ sh(Command0, Options0) -> Command = lists:flatten(patch_on_windows(Command0, proplists:get_value(env, Options, []))), PortSettings = proplists:get_all_values(port_settings, Options) ++ [exit_status, {line, 16384}, use_stdio, stderr_to_stdout, hide, eof], - ?DEBUG("Port Cmd: ~s\nPort Opts: ~p\n", [Command, PortSettings]), + ?DEBUG("Port Cmd: ~ts\nPort Opts: ~p\n", [Command, PortSettings]), Port = open_port({spawn, Command}, PortSettings), try @@ -232,7 +234,7 @@ deprecated(Old, New, When) -> <<"WARNING: deprecated ~p option used~n" "Option '~p' has been deprecated~n" "in favor of '~p'.~n" - "'~p' will be removed ~s.~n">>, + "'~p' will be removed ~ts.~n">>, [Old, Old, New, Old, When]). %% for use by `do` task @@ -244,11 +246,17 @@ args_to_tasks(Args) -> new_task(Args, []). deps_to_binary([]) -> []; deps_to_binary([{Name, _, Source} | T]) -> - [{ec_cnv:to_binary(Name), Source} | deps_to_binary(T)]; + [{to_binary(Name), Source} | deps_to_binary(T)]; deps_to_binary([{Name, Source} | T]) -> - [{ec_cnv:to_binary(Name), Source} | deps_to_binary(T)]; + [{to_binary(Name), Source} | deps_to_binary(T)]; deps_to_binary([Name | T]) -> - [ec_cnv:to_binary(Name) | deps_to_binary(T)]. + [to_binary(Name) | deps_to_binary(T)]. + +to_binary(A) when is_atom(A) -> atom_to_binary(A, unicode); +to_binary(Str) -> unicode:characters_to_binary(Str). + +to_list(A) when is_atom(A) -> atom_to_list(A); +to_list(Str) -> unicode:characters_to_list(Str). tup_dedup(List) -> tup_dedup_(tup_sort(List)). @@ -396,10 +404,10 @@ check_min_otp_version(MinOtpVersion) -> case ParsedVsn >= ParsedMin of true -> - ?DEBUG("~s satisfies the requirement for minimum OTP version ~s", + ?DEBUG("~ts satisfies the requirement for minimum OTP version ~ts", [OtpRelease, MinOtpVersion]); false -> - ?ABORT("OTP release ~s or later is required. Version in use: ~s", + ?ABORT("OTP release ~ts or later is required. Version in use: ~ts", [MinOtpVersion, OtpRelease]) end. @@ -415,16 +423,16 @@ check_blacklisted_otp_versions(BlacklistedRegexes) -> abort_if_blacklisted(BlacklistedRegex, OtpRelease) -> case re:run(OtpRelease, BlacklistedRegex, [{capture, none}]) of match -> - ?ABORT("OTP release ~s matches blacklisted version ~s", + ?ABORT("OTP release ~ts matches blacklisted version ~ts", [OtpRelease, BlacklistedRegex]); nomatch -> - ?DEBUG("~s does not match blacklisted OTP version ~s", + ?DEBUG("~ts does not match blacklisted OTP version ~ts", [OtpRelease, BlacklistedRegex]) end. user_agent() -> {ok, Vsn} = application:get_key(rebar, vsn), - ?FMT("Rebar/~s (OTP/~s)", [Vsn, otp_release()]). + ?FMT("Rebar/~ts (OTP/~ts)", [Vsn, otp_release()]). reread_config(ConfigList) -> %% NB: we attempt to mimic -config here, which survives app reload, @@ -457,7 +465,7 @@ version_tuple(OtpRelease) -> {match, [_Full, Maj]} -> {list_to_integer(Maj), 0, 0}; nomatch -> - ?ABORT("Minimum OTP release unable to be parsed: ~s", [OtpRelease]) + ?ABORT("Minimum OTP release unable to be parsed: ~ts", [OtpRelease]) end. otp_release() -> @@ -510,7 +518,7 @@ patch_on_windows(Cmd, Env) -> end, Cmd, Env), %% Remove left-over vars re:replace(Cmd1, "\\\$\\w+|\\\${\\w+}", "", - [global, {return, list}]); + [global, {return, list}, unicode]); _ -> Cmd end. @@ -529,7 +537,7 @@ expand_env_variable(InStr, VarName, RawVarValue) -> VarValue = re:replace(RawVarValue, "\\\\", "\\\\\\\\", ReOpts), %% Use a regex to match/replace: %% Given variable "FOO": match $FOO\s | $FOOeol | ${FOO} - RegEx = io_lib:format("\\\$(~s(\\W|$)|{~s})", [VarName, VarName]), + RegEx = io_lib:format("\\\$(~ts(\\W|$)|{~ts})", [VarName, VarName]), re:replace(InStr, RegEx, [VarValue, "\\2"], ReOpts) end. @@ -554,7 +562,7 @@ expand_sh_flag(use_stdout) -> {output_handler, fun(Line, Acc) -> %% Line already has a newline so don't use ?CONSOLE which adds one - io:format("~s", [Line]), + io:format("~ts", [Line]), [Line | Acc] end}; expand_sh_flag({use_stdout, false}) -> @@ -577,23 +585,23 @@ log_msg_and_abort(Message) -> -spec debug_log_msg_and_abort(string()) -> err_handler(). debug_log_msg_and_abort(Message) -> fun(Command, {Rc, Output}) -> - ?DEBUG("sh(~s)~n" + ?DEBUG("sh(~ts)~n" "failed with return code ~w and the following output:~n" - "~s", [Command, Rc, Output]), + "~ts", [Command, Rc, Output]), ?ABORT(Message, []) end. -spec log_and_abort(string(), {integer(), string()}) -> no_return(). log_and_abort(Command, {Rc, Output}) -> - ?ABORT("sh(~s)~n" + ?ABORT("sh(~ts)~n" "failed with return code ~w and the following output:~n" - "~s", [Command, Rc, Output]). + "~ts", [Command, Rc, Output]). -spec debug_and_abort(string(), {integer(), string()}) -> no_return(). debug_and_abort(Command, {Rc, Output}) -> - ?DEBUG("sh(~s)~n" + ?DEBUG("sh(~ts)~n" "failed with return code ~w and the following output:~n" - "~s", [Command, Rc, Output]), + "~ts", [Command, Rc, Output]), throw(rebar_abort). sh_loop(Port, Fun, Acc) -> @@ -662,7 +670,7 @@ vcs_vsn(Vcs, Dir, Resources) -> unknown -> ?ABORT("vcs_vsn: Unknown vsn format: ~p", [Vcs]); {error, Reason} -> - ?ABORT("vcs_vsn: ~s", [Reason]) + ?ABORT("vcs_vsn: ~ts", [Reason]) end. %% Temp work around for repos like relx that use "semver" @@ -779,7 +787,7 @@ cleanup_code_path(OrigPath) -> new_task([], Acc) -> lists:reverse(Acc); new_task([TaskList|Rest], Acc) -> - case re:split(TaskList, ",", [{return, list}, {parts, 2}]) of + case re:split(TaskList, ",", [{return, list}, {parts, 2}, unicode]) of %% `do` consumes all remaining args ["do" = Task] -> lists:reverse([{Task, Rest}|Acc]); @@ -806,7 +814,7 @@ arg_or_flag(["-" ++ _ = Flag|Rest], [{Task, Args}|Acc]) -> end; %% an argument or a sequence of arguments arg_or_flag([ArgList|Rest], [{Task, Args}|Acc]) -> - case re:split(ArgList, ",", [{return, list}, {parts, 2}]) of + case re:split(ArgList, ",", [{return, list}, {parts, 2}, unicode]) of %% single arg terminated by a comma [Arg, ""] -> new_task(Rest, [{Task, lists:reverse([Arg|Args])}|Acc]); @@ -857,15 +865,18 @@ url_append_path(Url, ExtraPath) -> escape_chars(Str) when is_atom(Str) -> escape_chars(atom_to_list(Str)); escape_chars(Str) -> - re:replace(Str, "([ ()?`!$&;\"\'])", "\\\\&", [global, {return, list}]). + re:replace(Str, "([ ()?`!$&;\"\'])", "\\\\&", + [global, {return, list}, unicode]). %% "escape inside these" escape_double_quotes(Str) -> - re:replace(Str, "([\"\\\\`!$&*;])", "\\\\&", [global, {return, list}]). + re:replace(Str, "([\"\\\\`!$&*;])", "\\\\&", + [global, {return, list}, unicode]). %% "escape inside these" but allow * escape_double_quotes_weak(Str) -> - re:replace(Str, "([\"\\\\`!$&;])", "\\\\&", [global, {return, list}]). + re:replace(Str, "([\"\\\\`!$&;])", "\\\\&", + [global, {return, list}, unicode]). info_useless(Old, New) -> [?INFO("App ~ts is no longer needed and can be deleted.", [Name]) -- cgit v1.1 From 0588936de12fe5b1607c6a23e300904e8f0fc6c0 Mon Sep 17 00:00:00 2001 From: William H Date: Thu, 10 Aug 2017 10:05:01 -0400 Subject: Support Windows with non-NTFS filesystems The code for symlink handling failed whenever a win32 platform with no symlink capability would be detected. This patch is provided by William H from the support ticket at http://www.rebar3.org/v3/discuss/598225c90365bb00144bc07f, which adds detection of failures in non-NTFS scenarios on Win32, and then copies files instead of bailing out. The end result should be appropriate support for such a platform. --- src/rebar_file_utils.erl | 42 +++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index d7716e5..c860513 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -105,7 +105,7 @@ symlink_or_copy(Source, Target) -> T = unicode:characters_to_list(Target), case filelib:is_dir(S) of true -> - win32_symlink(S, T); + win32_symlink_or_copy(S, T); false -> cp_r([S], T) end; @@ -119,20 +119,48 @@ symlink_or_copy(Source, Target) -> end end. -win32_symlink(Source, Target) -> +%% @private Compatibility function for windows +win32_symlink_or_copy(Source, Target) -> Res = rebar_utils:sh( ?FMT("cmd /c mklink /j \"~ts\" \"~ts\"", [rebar_utils:escape_double_quotes(filename:nativename(Target)), rebar_utils:escape_double_quotes(filename:nativename(Source))]), [{use_stdout, false}, return_on_error]), - case win32_ok(Res) of + case win32_mklink_ok(Res, Target) of true -> ok; - false -> - {error, lists:flatten( - io_lib:format("Failed to symlink ~ts to ~ts~n", - [Source, Target]))} + false -> cp_r_win32(Source, drop_last_dir_from_path(Target)) end. +%% @private specifically pattern match against the output +%% of the windows 'mklink' shell call; different values from +%% what win32_ok/1 handles +win32_mklink_ok({ok, _}, _) -> + true; +win32_mklink_ok({error,{1,"Local NTFS volumes are required to complete the operation.\n"}}, _) -> + false; +win32_mklink_ok({error,{1,"Cannot create a file when that file already exists.\n"}}, Target) -> + % File or dir is already in place; find if it is already a symlink (true) or + % if it is a directory (copy-required; false) + is_symlink(Target); +win32_mklink_ok(_, _) -> + false. + +%% @private +is_symlink(Filename) -> + {ok, Info} = file:read_link_info(Filename), + Info#file_info.type == symlink. + +%% @private +%% drops the last 'node' of the filename, presumably the last dir such as 'src' +%% this is because cp_r_win32/2 automatically adds the dir name, to appease +%% robocopy and be more uniform with POSIX +drop_last_dir_from_path([]) -> + []; +drop_last_dir_from_path(Path) -> + case lists:droplast(filename:split(Path)) of + [] -> []; + Dirs -> filename:join(Dirs) + end. %% @doc Remove files and directories. %% Target is a single filename, directoryname or wildcard expression. -- cgit v1.1 From 21246698fb08c1e7da8aec7ce5eb4676135d8416 Mon Sep 17 00:00:00 2001 From: Vitor Enes Duarte Date: Thu, 10 Aug 2017 16:07:48 +0200 Subject: Fix total coverage --- src/rebar_prv_cover.erl | 44 ++++++++++++++++++-------------------------- 1 file changed, 18 insertions(+), 26 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_cover.erl b/src/rebar_prv_cover.erl index fca7c40..2e2c882 100644 --- a/src/rebar_prv_cover.erl +++ b/src/rebar_prv_cover.erl @@ -196,10 +196,7 @@ mod_to_filename(TaskDir, M) -> process(Coverage) -> process(Coverage, {0, 0}). -process([], {0, 0}) -> - "0%"; -process([], {Cov, Not}) -> - integer_to_list(trunc((Cov / (Cov + Not)) * 100)) ++ "%"; +process([], Acc) -> Acc; %% line 0 is a line added by eunit and never executed so ignore it process([{{_, 0}, _}|Rest], Acc) -> process(Rest, Acc); process([{_, {Cov, Not}}|Rest], {Covered, NotCovered}) -> @@ -208,20 +205,19 @@ process([{_, {Cov, Not}}|Rest], {Covered, NotCovered}) -> print_analysis(_, false) -> ok; print_analysis(Analysis, true) -> {_, CoverFiles, Stats} = lists:keyfind("aggregate", 1, Analysis), - ConsoleStats = [ {atom_to_list(M), C} || {M, C, _} <- Stats ], - Table = format_table(ConsoleStats, CoverFiles), + Table = format_table(Stats, CoverFiles), io:format("~ts", [Table]). format_table(Stats, CoverFiles) -> - MaxLength = max(lists:foldl(fun max_length/2, 0, Stats), 20), + MaxLength = lists:max([20 | lists:map(fun({M, _, _}) -> mod_length(M) end, Stats)]), Header = header(MaxLength), Separator = separator(MaxLength), TotalLabel = format("total", MaxLength), TotalCov = format(calculate_total(Stats), 8), [io_lib:format("~ts~n~ts~n~ts~n", [Separator, Header, Separator]), - lists:map(fun({Mod, Coverage}) -> + lists:map(fun({Mod, Coverage, _}) -> Name = format(Mod, MaxLength), - Cov = format(Coverage, 8), + Cov = format(percentage(Coverage), 8), io_lib:format(" | ~ts | ~ts |~n", [Name, Cov]) end, Stats), io_lib:format("~ts~n", [Separator]), @@ -232,12 +228,8 @@ format_table(Stats, CoverFiles) -> io_lib:format(" ~ts~n", [File]) end, CoverFiles)]. -max_length({ModName, _}, Min) -> - Length = length(lists:flatten(ModName)), - case Length > Min of - true -> Length; - false -> Min - end. +mod_length(Mod) when is_atom(Mod) -> mod_length(atom_to_list(Mod)); +mod_length(Mod) -> length(Mod). header(Width) -> [" | ", format("module", Width), " | ", format("coverage", 8), " |"]. @@ -247,17 +239,17 @@ separator(Width) -> format(String, Width) -> io_lib:format("~*.ts", [Width, String]). -calculate_total(Stats) when length(Stats) =:= 0 -> - "0%"; calculate_total(Stats) -> - TotalStats = length(Stats), - TotalCovInt = round(lists:foldl( - fun({_Mod, Coverage, _File}, Acc) -> - Acc + (list_to_integer(string:strip(Coverage, right, $%)) / TotalStats); - ({_Mod, Coverage}, Acc) -> - Acc + (list_to_integer(string:strip(Coverage, right, $%)) / TotalStats) - end, 0, Stats)), - integer_to_list(TotalCovInt) ++ "%". + percentage(lists:foldl( + fun({_Mod, {Cov, Not}, _File}, {CovAcc, NotAcc}) -> + {CovAcc + Cov, NotAcc + Not} + end, + {0, 0}, + Stats + )). + +percentage({0, 0}) -> "0%"; +percentage({Cov, Not}) -> integer_to_list(trunc((Cov / (Cov + Not)) * 100)) ++ "%". write_index(State, Coverage) -> CoverDir = cover_dir(State), @@ -287,7 +279,7 @@ write_index_section(F, [{Section, DataFile, Mods}|Rest]) -> FmtLink = fun({Mod, Cov, Report}) -> ?FMT("~ts~ts\n", - [strip_coverdir(Report), Mod, Cov]) + [strip_coverdir(Report), Mod, percentage(Cov)]) end, lists:foreach(fun(M) -> ok = file:write(F, FmtLink(M)) end, Mods), ok = file:write(F, ?FMT("Total~ts\n", -- cgit v1.1 From fa82f5bee491002fd4c3718697becbdddb7d1161 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 10 Aug 2017 11:12:05 -0400 Subject: Remove duplicate ebins from escripts During the building of escripts, multiple passes are done. Two of them may end up duplicating content: one that gathers all of the beam files that will be needed for the app to work, and a second one that goes over the ebin/ directory of the root application to grab all the stuff in there, prior to the run. This allows to grab whatever could be required for runtime without breaking the rest (or so I think), and sticks them at the front of the archive, where it needs to sit for things to work fine. Whenever the ebin/ directory contains a pre-compile .beam file, it gets fetched both from the first pass described and the latter one. This results in duplicate entries in the resulting zip files used for the escript and makes the executable larger than it needs to be. The patch is a simple 1:1 removal of duplicate values. Since large pre-populated ebin/ directories are pretty rare, this should not be too costly for the vast majority of users. Fixes #1577 --- src/rebar_prv_escriptize.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_escriptize.erl b/src/rebar_prv_escriptize.erl index cffbbdc..1da70f8 100644 --- a/src/rebar_prv_escriptize.erl +++ b/src/rebar_prv_escriptize.erl @@ -114,7 +114,7 @@ escriptize(State0, App) -> EbinFiles = usort(load_files(EbinPrefix, "*", "ebin")), ExtraFiles = usort(InclBeams ++ InclExtra), - Files = get_nonempty(EbinFiles ++ ExtraFiles), + Files = get_nonempty(EbinFiles ++ (ExtraFiles -- EbinFiles)), % drop dupes DefaultEmuArgs = ?FMT("%%! -escript main ~ts -pz ~ts/~ts/ebin\n", [AppNameStr, AppNameStr, AppNameStr]), -- cgit v1.1 From 77c85cb99471a725f8bd4dadd958eeeeaeb443e8 Mon Sep 17 00:00:00 2001 From: Vitor Enes Duarte Date: Thu, 10 Aug 2017 19:04:33 +0200 Subject: Fix coverage percentage on modules with 0 lines to be covered --- src/rebar_prv_cover.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_cover.erl b/src/rebar_prv_cover.erl index 2e2c882..c890452 100644 --- a/src/rebar_prv_cover.erl +++ b/src/rebar_prv_cover.erl @@ -248,7 +248,7 @@ calculate_total(Stats) -> Stats )). -percentage({0, 0}) -> "0%"; +percentage({_, 0}) -> "100%"; percentage({Cov, Not}) -> integer_to_list(trunc((Cov / (Cov + Not)) * 100)) ++ "%". write_index(State, Coverage) -> -- cgit v1.1 From 13bdb75b2961e1ae43e43e7c8d06bf463571d541 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 11 Aug 2017 13:58:13 -0400 Subject: Fix recursive profile merging in umbrella apps When a config file exists at the root of a project, defines a given configuration value for a given profile, and that a sub-application (umbrella app) also has the same profile defined with the same key (but different values), the configuration values of the sub-application's profile would get silently dropped. The problem being that when the function to merge profiles is applied recursively, it is applied to each profile (so it will merge on the keys test, prod, etc.) rather than to each of the values of each profile. This patch reworks the profile merging so that the current behaviour is respected overall (a profile cannot be cancelled by a subdep's non-existant profile since its value should have been ignored), but ensures that sub-deps' profiles are otherwise applied recursively with the proper rules: - dependencies favor prior values - plugins favor new values - erl_first_files combine the lists - relx uses the tuple merge algorithm - erl_opts has its own custom merge as well - otherwise the new value takes precedence A test has also been added. There is a risk of breakage in some applications that may have relied on the buggy behaviour to work, though at this time we are aware of none of them. --- src/rebar3.erl | 4 ++-- src/rebar_opts.erl | 18 +++++++++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rebar3.erl b/src/rebar3.erl index eec8968..2b24bca 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -396,8 +396,8 @@ safe_define_test_macro(Opts) -> %% defining a compile macro twice results in an exception so %% make sure 'TEST' is only defined once case test_defined(Opts) of - true -> []; - false -> [{d, 'TEST'}] + true -> Opts; + false -> [{d, 'TEST'}|Opts] end. test_defined([{d, 'TEST'}|_]) -> true; diff --git a/src/rebar_opts.erl b/src/rebar_opts.erl index 589dbb8..e327e31 100644 --- a/src/rebar_opts.erl +++ b/src/rebar_opts.erl @@ -117,7 +117,10 @@ merge_opt(plugins, NewValue, _OldValue) -> merge_opt({plugins, _}, NewValue, _OldValue) -> NewValue; merge_opt(profiles, NewValue, OldValue) -> - dict:to_list(merge_opts(dict:from_list(NewValue), dict:from_list(OldValue))); + ToMerge = fill_profile_gaps(lists:sort(NewValue), + lists:sort(OldValue)), + [{K,dict:to_list(merge_opts(dict:from_list(New), dict:from_list(Old)))} + || {K,New,Old} <- ToMerge]; merge_opt(erl_first_files, Value, Value) -> Value; merge_opt(erl_first_files, NewValue, OldValue) -> @@ -190,3 +193,16 @@ filter_defines([{platform_define, ArchRegex, Key, Value} | Rest], Acc) -> end; filter_defines([Opt | Rest], Acc) -> filter_defines(Rest, [Opt | Acc]). + +fill_profile_gaps([], []) -> + []; +fill_profile_gaps([{P,V}|Ps], []) -> + [{P,V,[]} | fill_profile_gaps(Ps, [])]; +fill_profile_gaps([], [{P,V}|Ps]) -> + [{P,[],V} | fill_profile_gaps([], Ps)]; +fill_profile_gaps([{P,VA}|PAs], [{P,VB}|PBs]) -> + [{P,VA,VB} | fill_profile_gaps(PAs, PBs)]; +fill_profile_gaps([{PA,VA}|PAs], [{PB,VB}|PBs]) when PA < PB -> + [{PA,VA,[]} | fill_profile_gaps(PAs, [{PB, VB}|PBs])]; +fill_profile_gaps([{PA,VA}|PAs], [{PB,VB}|PBs]) when PA > PB -> + [{PB,[],VB} | fill_profile_gaps([{PA,VA}|PAs], PBs)]. \ No newline at end of file -- cgit v1.1 From bed661aef80d113ae04e8e9da035f904b7c4aec4 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sun, 13 Aug 2017 15:30:03 -0400 Subject: Clarify function to normalise profile pairs --- src/rebar_opts.erl | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) (limited to 'src') diff --git a/src/rebar_opts.erl b/src/rebar_opts.erl index e327e31..8863b4c 100644 --- a/src/rebar_opts.erl +++ b/src/rebar_opts.erl @@ -117,8 +117,10 @@ merge_opt(plugins, NewValue, _OldValue) -> merge_opt({plugins, _}, NewValue, _OldValue) -> NewValue; merge_opt(profiles, NewValue, OldValue) -> - ToMerge = fill_profile_gaps(lists:sort(NewValue), - lists:sort(OldValue)), + %% Merge up sparse pairs of {Profile, Opts} into a joined up + %% {Profile, OptsNew, OptsOld} list. + ToMerge = normalise_profile_pairs(lists:sort(NewValue), + lists:sort(OldValue)), [{K,dict:to_list(merge_opts(dict:from_list(New), dict:from_list(Old)))} || {K,New,Old} <- ToMerge]; merge_opt(erl_first_files, Value, Value) -> @@ -194,15 +196,25 @@ filter_defines([{platform_define, ArchRegex, Key, Value} | Rest], Acc) -> filter_defines([Opt | Rest], Acc) -> filter_defines(Rest, [Opt | Acc]). -fill_profile_gaps([], []) -> +%% @private takes two lists of profile tuples and merges them +%% into one list of 3-tuples containing the values of either +%% profiles. +%% Any missing profile in one of the keys is replaced by an +%% empty one. +-spec normalise_profile_pairs([Profile], [Profile]) -> [Pair] when + Profile :: {Name, Opts}, + Pair :: {Name, Opts, Opts}, + Name :: atom(), + Opts :: [term()]. +normalise_profile_pairs([], []) -> []; -fill_profile_gaps([{P,V}|Ps], []) -> - [{P,V,[]} | fill_profile_gaps(Ps, [])]; -fill_profile_gaps([], [{P,V}|Ps]) -> - [{P,[],V} | fill_profile_gaps([], Ps)]; -fill_profile_gaps([{P,VA}|PAs], [{P,VB}|PBs]) -> - [{P,VA,VB} | fill_profile_gaps(PAs, PBs)]; -fill_profile_gaps([{PA,VA}|PAs], [{PB,VB}|PBs]) when PA < PB -> - [{PA,VA,[]} | fill_profile_gaps(PAs, [{PB, VB}|PBs])]; -fill_profile_gaps([{PA,VA}|PAs], [{PB,VB}|PBs]) when PA > PB -> - [{PB,[],VB} | fill_profile_gaps([{PA,VA}|PAs], PBs)]. \ No newline at end of file +normalise_profile_pairs([{P,V}|Ps], []) -> + [{P,V,[]} | normalise_profile_pairs(Ps, [])]; +normalise_profile_pairs([], [{P,V}|Ps]) -> + [{P,[],V} | normalise_profile_pairs([], Ps)]; +normalise_profile_pairs([{P,VA}|PAs], [{P,VB}|PBs]) -> + [{P,VA,VB} | normalise_profile_pairs(PAs, PBs)]; +normalise_profile_pairs([{PA,VA}|PAs], [{PB,VB}|PBs]) when PA < PB -> + [{PA,VA,[]} | normalise_profile_pairs(PAs, [{PB, VB}|PBs])]; +normalise_profile_pairs([{PA,VA}|PAs], [{PB,VB}|PBs]) when PA > PB -> + [{PB,[],VB} | normalise_profile_pairs([{PA,VA}|PAs], PBs)]. -- cgit v1.1 From bc2f06220003499ac998f519c9eb595a0e957e15 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Tue, 15 Aug 2017 11:20:28 -0400 Subject: Fix ordering of overlays and overlay vars in Relx Specifically, this impacts profiles. It appears that relx as a whole requires its configuration to be merged in one tuple order (New takes precedence over Old), whereas the overlays require the opposite (Old takes precedence over New) since the operation order on disk is important to work well. This patch reorders overlay values such that the overlay of a profile takes place *after* the basic overlay, ensuring that the profile actions take place after the basic ones; this allows profiles to properly overwrite files as expected (see #1609) This is done while adequately maintaining the order of operations that were required as part of #1563 Overlay vars of profiles are also checked to be working fine, along with a test. This fixes #1247 and #1609 --- src/rebar_opts.erl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_opts.erl b/src/rebar_opts.erl index 8863b4c..b7156b2 100644 --- a/src/rebar_opts.erl +++ b/src/rebar_opts.erl @@ -132,7 +132,11 @@ merge_opt(mib_first_files, Value, Value) -> merge_opt(mib_first_files, NewValue, OldValue) -> OldValue ++ NewValue; merge_opt(relx, NewValue, OldValue) -> - rebar_utils:tup_umerge(OldValue, NewValue); + Partition = fun(C) -> is_tuple(C) andalso element(1, C) =:= overlay end, + {NewOverlays, NewOther} = lists:partition(Partition, NewValue), + {OldOverlays, OldOther} = lists:partition(Partition, OldValue), + rebar_utils:tup_umerge(NewOverlays, OldOverlays) + ++ rebar_utils:tup_umerge(OldOther, NewOther); merge_opt(Key, NewValue, OldValue) when Key == erl_opts; Key == eunit_compile_opts; Key == ct_compile_opts -> merge_erl_opts(lists:reverse(OldValue), NewValue); -- cgit v1.1 From 9a76737d91177961cc3df917bea7d69df263d7cb Mon Sep 17 00:00:00 2001 From: patrick cieplak Date: Thu, 17 Aug 2017 21:39:53 -0700 Subject: fix `rebar3 shell` when relx section of rebar.config contains releases with independent configurations --- src/rebar_prv_shell.erl | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src') diff --git a/src/rebar_prv_shell.erl b/src/rebar_prv_shell.erl index c958dde..0244833 100644 --- a/src/rebar_prv_shell.erl +++ b/src/rebar_prv_shell.erl @@ -322,6 +322,9 @@ find_apps_relx(State) -> {_, _, Apps} -> ?DEBUG("Found shell apps from relx.", []), Apps; + {_, _, Apps, _} -> + ?DEBUG("Found shell apps from relx.", []), + Apps; false -> no_value end. -- cgit v1.1 From 533a27731bd39243fca8775686f498494f2ae182 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 21 Aug 2017 17:24:37 -0400 Subject: Bump to 3.4.3 --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 74efe97..c33d9c9 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.4.3"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 92d3fa9272582eea6ce77faab19fb74be0d4c6ce Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 21 Aug 2017 17:27:34 -0400 Subject: Back to git-based versionning --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index c33d9c9..74efe97 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.4.3"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 7095382e2c769f43214c32c4cb7f357b15ee095c Mon Sep 17 00:00:00 2001 From: suexcxine Date: Fri, 25 Aug 2017 22:50:22 +0800 Subject: Recompile when include files change --- src/rebar_erlc_compiler.erl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index 94cbe13..c588a25 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -787,8 +787,9 @@ outdir(RebarOpts) -> proplists:get_value(outdir, ErlOpts, ?DEFAULT_OUTDIR). include_abs_dirs(ErlOpts, BaseDir) -> - InclDirs = ["include"|proplists:get_all_values(i, ErlOpts)], - lists:map(fun(Incl) -> filename:join([BaseDir, Incl]) end, InclDirs). + ErlOptIncludes = proplists:get_all_values(i, ErlOpts), + InclDirs = lists:map(fun(Incl) -> filename:absname(Incl) end, ErlOptIncludes), + [filename:join([BaseDir, "include"])|InclDirs]. dir_recursive(Opts, Dir, CompileOpts) when is_list(CompileOpts) -> case proplists:get_value(recursive,CompileOpts) of -- cgit v1.1 From 7c959ccb4f91cdfdb956aa090b3789cf8fd396ed Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 30 Aug 2017 08:08:52 -0400 Subject: Fix relative src dir specifications When fetching src_dir values, some relative paths can be inserted. When deduplicating the paths on the fetch, this fact means that logically duplicate (but literally different) directories can be returned at once. By normalizing the names, duplication bugs can be resolved. --- src/rebar_dir.erl | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_dir.erl b/src/rebar_dir.erl index d0c7805..7a779d1 100644 --- a/src/rebar_dir.erl +++ b/src/rebar_dir.erl @@ -258,8 +258,11 @@ extra_src_dirs(Opts, Default) -> %% @private agnostic version of src_dirs and extra_src_dirs. src_dirs(Type, Opts, Default) -> - lists:usort([case D0 of {D,_} -> D; _ -> D0 end || - D0 <- raw_src_dirs(Type,Opts,Default)]). + lists:usort([ + case D0 of + {D,_} -> normalize_relative_path(D); + _ -> normalize_relative_path(D0) + end || D0 <- raw_src_dirs(Type,Opts,Default)]). %% @private extracts the un-formatted src_dirs or extra_src_dirs %% options as configured. @@ -271,6 +274,10 @@ raw_src_dirs(Type, Opts, Default) -> Dirs -> Dirs end. +%% @private normalizes relative paths so that ./a/b/c/ => a/b/c +normalize_relative_path(Path) -> + make_normalized_path(filename:split(Path), []). + %% @doc returns all the source directories (`src_dirs' and %% `extra_src_dirs'). -spec all_src_dirs(rebar_dict()) -> list(file:filename_all()). -- cgit v1.1 From 8e4f90fa08bb44e041b4225c49d5a75124adcb3c Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 1 Sep 2017 12:21:25 -0700 Subject: fix sys config merging --- src/rebar_file_utils.erl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index c860513..4a783f2 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -75,8 +75,9 @@ consult_config(State, Filename) -> JoinedConfig = lists:flatmap( fun (SubConfig) when is_list(SubConfig) -> case lists:suffix(".config", SubConfig) of - false -> consult_config(State, SubConfig ++ ".config"); - true -> consult_config(State, SubConfig) + %% since consult_config returns a list in a list we take the head here + false -> hd(consult_config(State, SubConfig ++ ".config")); + true -> hd(consult_config(State, SubConfig)) end; (Entry) -> [Entry] end, Config), -- cgit v1.1 From 6ec3b6e63a770df7d9178f5e0883a918a2f96526 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sun, 10 Sep 2017 21:43:42 -0400 Subject: Bump to 3.4.4 - fix sys config merging - Fix relative src_dir specifications to avoid double .app.src file detection - Recompile when include files change in non-default directories --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 74efe97..4be8b76 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.4.4"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 71fe6c9b5f31fbd16c5f71f7b3ea51b42e11722a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jimmy=20Z=C3=B6ger?= Date: Thu, 14 Sep 2017 16:51:09 +0200 Subject: Changes word 'transient' to 'transitive' which is what it is supposed to say --- src/rebar_prv_upgrade.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl index 5a7dff8..cd75672 100644 --- a/src/rebar_prv_upgrade.erl +++ b/src/rebar_prv_upgrade.erl @@ -102,7 +102,7 @@ do(State) -> format_error({unknown_dependency, Name}) -> io_lib:format("Dependency ~ts not found", [Name]); format_error({transitive_dependency, Name}) -> - io_lib:format("Dependency ~ts is transient and cannot be safely upgraded. " + io_lib:format("Dependency ~ts is transitive and cannot be safely upgraded. " "Promote it to your top-level rebar.config file to upgrade it.", [Name]); format_error(Reason) -> -- cgit v1.1 From cf83eb7fc45fac677a83b5f38e87c581cf516cf2 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 27 Sep 2017 19:01:44 -0400 Subject: Corrects a fix to src_dir values The previous patch at #7c959cc fixed the usage of duplicate values for directories through relative paths, but mistakenly went overboard and dropped the `./` path, which is still fairly common. Similarly for `../". The code is modified to special-case such values and keep the code working. --- src/rebar_dir.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_dir.erl b/src/rebar_dir.erl index 7a779d1..7182c10 100644 --- a/src/rebar_dir.erl +++ b/src/rebar_dir.erl @@ -197,8 +197,10 @@ make_normalized_path([], NormalizedPath) -> filename:join(lists:reverse(NormalizedPath)); make_normalized_path([H|T], NormalizedPath) -> case H of + "." when NormalizedPath == [], T == [] -> make_normalized_path(T, ["."]); "." -> make_normalized_path(T, NormalizedPath); - ".." -> make_normalized_path(T, tl(NormalizedPath)); + ".." when NormalizedPath == [] -> make_normalized_path(T, [".."]); + ".." when hd(NormalizedPath) =/= ".." -> make_normalized_path(T, tl(NormalizedPath)); _ -> make_normalized_path(T, [H|NormalizedPath]) end. -- cgit v1.1 From 0d339db49ad1a3b8410f2619e0796ee8845c30fb Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 4 Oct 2017 16:16:23 -0400 Subject: Fix messed up rollback to git versioning was still hardcoded to 3.4.4 --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 4be8b76..74efe97 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.4.4"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 1ef0ed5b2d1fa289575152bb001e6dd3beb963ee Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 4 Oct 2017 16:17:02 -0400 Subject: Normalize return values of app_info data The parsing functions were used inconsistently, and the returned values were bad in some clauses; things only worked because they are never used within rebar3. This ensures the return types are consistent on all clauses. --- src/rebar_app_info.erl | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index 050ccc1..de497d5 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -311,15 +311,22 @@ app_file(AppInfo=#app_info_t{}, AppFile) -> app_details(AppInfo=#app_info_t{app_details=[]}) -> case app_file(AppInfo) of undefined -> - rebar_file_utils:try_consult(app_file_src(AppInfo)); + case rebar_config:consult_app_file(app_file_src(AppInfo)) of + [] -> []; + [{application, _Name, AppDetails}] -> AppDetails + end; AppFile -> - try - rebar_file_utils:try_consult(AppFile) + try rebar_file_utils:try_consult(AppFile) of + [] -> []; + [{application, _Name, AppDetails}] -> AppDetails catch throw:{error, {Module, Reason}} -> ?DEBUG("Warning, falling back to .app.src because of: ~ts", [Module:format_error(Reason)]), - rebar_file_utils:try_consult(app_file_src(AppInfo)) + case rebar_config:consult_app_file(app_file_src(AppInfo)) of + [] -> []; + [{application, _Name, AppDetails}] -> AppDetails + end end end; app_details(#app_info_t{app_details=AppDetails}) -> -- cgit v1.1 From 13260ed62126970421d42a2874b100dbf8763ec3 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 4 Oct 2017 16:21:04 -0400 Subject: Add a description in compiled app file if undef Same default value as used in relx and other environments, but as reported in #979 some tools don't like having no description available. --- src/rebar_otp_app.erl | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_otp_app.erl b/src/rebar_otp_app.erl index ed573f2..1bc33b9 100644 --- a/src/rebar_otp_app.erl +++ b/src/rebar_otp_app.erl @@ -117,8 +117,11 @@ preprocess(State, AppInfo, AppSrcFile) -> %% without a 'registered' value. A3 = ensure_registered(A2), + %% some tools complain if a description is not present. + A4 = ensure_description(A3), + %% Build the final spec as a string - Spec = io_lib:format("~p.\n", [{application, AppName, A3}]), + Spec = io_lib:format("~p.\n", [{application, AppName, A4}]), %% Setup file .app filename and write new contents EbinDir = rebar_app_info:ebin_dir(AppInfo), @@ -195,6 +198,15 @@ ensure_registered(AppData) -> AppData end. +ensure_description(AppData) -> + case lists:keyfind(description, 1, AppData) of + false -> + %% Required for releases to work. + [{description, ""} | AppData]; + {description, _} -> + AppData + end. + %% In the case of *.app.src we want to give the user the ability to %% dynamically script the application resource file (think dynamic version %% string, etc.), in a way similar to what can be done with the rebar -- cgit v1.1 From 228df89f3bdf92b9cd4aa36fca69f7d2738a1951 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 5 Oct 2017 13:14:50 -0400 Subject: Warn user when a local git or hg resource is used Those aren't supported and so a warning should be output. Fixes #1003 --- src/rebar_git_resource.erl | 14 ++++++++++++++ src/rebar_hg_resource.erl | 14 ++++++++++++++ 2 files changed, 28 insertions(+) (limited to 'src') diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl index a6b4d00..c63d10d 100644 --- a/src/rebar_git_resource.erl +++ b/src/rebar_git_resource.erl @@ -105,18 +105,32 @@ download(Dir, {git, Url, ""}, State) -> download(Dir, {git, Url, {branch, "master"}}, State); download(Dir, {git, Url, {branch, Branch}}, _State) -> ok = filelib:ensure_dir(Dir), + maybe_warn_local_url(Url), git_clone(branch, git_vsn(), Url, Dir, Branch); download(Dir, {git, Url, {tag, Tag}}, _State) -> ok = filelib:ensure_dir(Dir), + maybe_warn_local_url(Url), git_clone(tag, git_vsn(), Url, Dir, Tag); download(Dir, {git, Url, {ref, Ref}}, _State) -> ok = filelib:ensure_dir(Dir), + maybe_warn_local_url(Url), git_clone(ref, git_vsn(), Url, Dir, Ref); download(Dir, {git, Url, Rev}, _State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), ok = filelib:ensure_dir(Dir), + maybe_warn_local_url(Url), git_clone(rev, git_vsn(), Url, Dir, Rev). +maybe_warn_local_url(Url) -> + WarnStr = "Local git resources (~ts) are unsupported and may have odd behaviour. " + "Use remote git resources, or a plugin for local dependencies.", + case parse_git_url(Url) of + {error, no_scheme} -> ?WARN(WarnStr, [Url]); + {error, {no_default_port, _, _}} -> ?WARN(WarnStr, [Url]); + {error, {malformed_url, _, _}} -> ?WARN(WarnStr, [Url]); + _ -> ok + end. + %% Use different git clone commands depending on git --version git_clone(branch,Vsn,Url,Dir,Branch) when Vsn >= {1,7,10}; Vsn =:= undefined -> rebar_utils:sh(?FMT("git clone ~ts ~ts -b ~ts --single-branch", diff --git a/src/rebar_hg_resource.erl b/src/rebar_hg_resource.erl index 42e634c..0a77c1f 100644 --- a/src/rebar_hg_resource.erl +++ b/src/rebar_hg_resource.erl @@ -56,6 +56,7 @@ download(Dir, {hg, Url, ""}, State) -> download(Dir, {hg, Url, {branch, "default"}}, State); download(Dir, {hg, Url, {branch, Branch}}, _State) -> ok = filelib:ensure_dir(Dir), + maybe_warn_local_url(Url), rebar_utils:sh(?FMT("hg clone -q -b ~ts ~ts ~ts", [rebar_utils:escape_chars(Branch), rebar_utils:escape_chars(Url), @@ -63,6 +64,7 @@ download(Dir, {hg, Url, {branch, Branch}}, _State) -> [{cd, filename:dirname(Dir)}]); download(Dir, {hg, Url, {tag, Tag}}, _State) -> ok = filelib:ensure_dir(Dir), + maybe_warn_local_url(Url), rebar_utils:sh(?FMT("hg clone -q -u ~ts ~ts ~ts", [rebar_utils:escape_chars(Tag), rebar_utils:escape_chars(Url), @@ -70,6 +72,7 @@ download(Dir, {hg, Url, {tag, Tag}}, _State) -> [{cd, filename:dirname(Dir)}]); download(Dir, {hg, Url, {ref, Ref}}, _State) -> ok = filelib:ensure_dir(Dir), + maybe_warn_local_url(Url), rebar_utils:sh(?FMT("hg clone -q -r ~ts ~ts ~ts", [rebar_utils:escape_chars(Ref), rebar_utils:escape_chars(Url), @@ -77,6 +80,7 @@ download(Dir, {hg, Url, {ref, Ref}}, _State) -> [{cd, filename:dirname(Dir)}]); download(Dir, {hg, Url, Rev}, _State) -> ok = filelib:ensure_dir(Dir), + maybe_warn_local_url(Url), rebar_utils:sh(?FMT("hg clone -q -r ~ts ~ts ~ts", [rebar_utils:escape_chars(Rev), rebar_utils:escape_chars(Url), @@ -135,6 +139,16 @@ get_branch_ref(Dir, Branch) -> [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]), string:strip(BranchRefString, both, $\n). + +maybe_warn_local_url(Url) -> + try + _ = parse_hg_url(Url), + ok + catch + _:_ -> + ?WARN("URL format (~ts) unsupported.", []) + end. + parse_hg_url("ssh://" ++ HostPath) -> [Host | Path] = string:tokens(HostPath, "/"), {Host, filename:rootname(filename:join(Path), ".hg")}; -- cgit v1.1 From 09284a8af93e742798ca61d64fb8699e18e53207 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 13 Oct 2017 10:09:46 -0400 Subject: Avoid guessing on utf8 decoding of app files Rather than trying one method and then the other, allow the caller to specify the encoding of the expected file. All other schemes are risky and won't work well. Rollback the function's default interface to the binary format in case any plugin used it for non-unicode content, preserving backwards compat. --- src/rebar_file_utils.erl | 21 +++++++++++++-------- src/rebar_otp_app.erl | 2 +- 2 files changed, 14 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 4a783f2..b0755ed 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -35,6 +35,7 @@ mv/2, delete_each/1, write_file_if_contents_differ/2, + write_file_if_contents_differ/3, system_tmpdir/0, system_tmpdir/1, reset_dir/1, @@ -334,15 +335,19 @@ delete_each([File | Rest]) -> ?FAIL end. +%% @doc backwards compat layer to pre-utf8 support write_file_if_contents_differ(Filename, Bytes) -> - %% first try to convert directly to binaries, - %% but if it fails, we likely contain unicode and - %% need special treatment - ToWrite = try - iolist_to_binary(Bytes) - catch - error:badarg -> unicode:characters_to_binary(Bytes) - end, + write_file_if_contents_differ(Filename, Bytes, raw). + +%% @doc let the user pick the encoding required; there are no good +%% heuristics for data encoding +write_file_if_contents_differ(Filename, Bytes, raw) -> + write_file_if_contents_differ_(Filename, iolist_to_binary(Bytes)); +write_file_if_contents_differ(Filename, Bytes, utf8) -> + write_file_if_contents_differ_(Filename, unicode:characters_to_binary(Bytes, utf8)). + +%% @private compare raw strings and check contents +write_file_if_contents_differ_(Filename, ToWrite) -> case file:read_file(Filename) of {ok, ToWrite} -> ok; diff --git a/src/rebar_otp_app.erl b/src/rebar_otp_app.erl index 1bc33b9..f5bb9cf 100644 --- a/src/rebar_otp_app.erl +++ b/src/rebar_otp_app.erl @@ -127,7 +127,7 @@ preprocess(State, AppInfo, AppSrcFile) -> EbinDir = rebar_app_info:ebin_dir(AppInfo), filelib:ensure_dir(filename:join(EbinDir, "dummy.beam")), AppFile = rebar_app_utils:app_src_to_app(OutDir, AppSrcFile), - ok = rebar_file_utils:write_file_if_contents_differ(AppFile, Spec), + ok = rebar_file_utils:write_file_if_contents_differ(AppFile, Spec, utf8), AppFile; {error, Reason} -> -- cgit v1.1 From 73fc54235f6b3cec5201e23c7baee4c5086ac2b1 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 13 Oct 2017 10:19:14 -0400 Subject: Fixing the carry of unlocks When composed with 'do', not carrying the unlocks in state may create problems. --- src/rebar_prv_unlock.erl | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_unlock.erl b/src/rebar_prv_unlock.erl index 51c57ab..8480991 100644 --- a/src/rebar_prv_unlock.erl +++ b/src/rebar_prv_unlock.erl @@ -49,8 +49,8 @@ do(State) -> {ok, _} -> Locks = rebar_config:consult_lock_file(LockFile), case handle_unlocks(State, Locks, LockFile) of - ok -> - {ok, State}; + {ok, NewLocks} -> + {ok, rebar_state:set(State, {locks, default}, NewLocks)}; {error, Reason} -> ?PRV_ERROR({file,Reason}) end @@ -69,11 +69,14 @@ handle_unlocks(State, Locks, LockFile) -> Names = parse_names(rebar_utils:to_binary(proplists:get_value(package, Args, <<"">>))), case [Lock || Lock = {Name, _, _} <- Locks, not lists:member(Name, Names)] of [] -> - file:delete(LockFile); + file:delete(LockFile), + {ok, []}; _ when Names =:= [] -> % implicitly all locks - file:delete(LockFile); + file:delete(LockFile), + {ok, []}; NewLocks -> - rebar_config:write_lock_file(LockFile, NewLocks) + rebar_config:write_lock_file(LockFile, NewLocks), + {ok, NewLocks} end. parse_names(Bin) -> -- cgit v1.1 From e57c8b74274cc43cc23ca0f4f76dd18f70a9241d Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 20 Oct 2017 09:42:51 -0400 Subject: Prevent hard crash on duplicate plugin paths When a global plugin is used both locally and within the project, there are cases when the rebar3 program will hard crash (killed in do_boot). This has been traced to plugin-handling in compilation, where the same code path may be purged twice in a row without further reloading for the compile operation. This of course yields the result where the code handling on the VM kills all processes holding references to the module in memory, in this case the rebar3 process itself. By deduplicating the paths first, we ensure at most one purge before reloading plugins and paths, and this prevents a hard crash. --- src/rebar_core.erl | 1 + src/rebar_utils.erl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_core.erl b/src/rebar_core.erl index 3ef7a0d..6132a5e 100644 --- a/src/rebar_core.erl +++ b/src/rebar_core.erl @@ -136,6 +136,7 @@ process_command(State, Command) -> do([], State) -> {ok, State}; do([ProviderName | Rest], State) -> + ?DEBUG("Provider: ~p", [ProviderName]), %% Special providers like 'as', 'do' or some hooks may be passed %% as a tuple {Namespace, Name}, otherwise not. Handle them %% on a per-need basis. diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 016847e..ee8f179 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -766,7 +766,7 @@ remove_from_code_path(Paths) -> [begin code:purge(M), code:delete(M) end || M <- Modules] end, code:del_path(Path) - end, Paths). + end, lists:usort(Paths)). %% @doc Revert to only having the beams necessary for running rebar3 and %% plugins in the path -- cgit v1.1 From 65e07082018dec2a47f309258761db760100f5f2 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 23 Oct 2017 10:11:06 -0400 Subject: Fix include paths in profile multiapp edge case The compiling of OTP applications is done by first topographically sorting them according to their dependencies, deps-first. This allows all compilation to take place in order. In the current code, the same logic extends to top-level applications in an umbrella project. Unfortunately, there are cases where this is not going to be true: when an application has extra_src_dirs entries (or additional directories or files) to conditionally compile under some profiles, it may start depending on another top-level application dedicated to that profile for include files. However, such an app will never make it to production and neither will the compilation artifacts that create the dependency. Under that scenario, current rebar3 is unusable. This patch makes it so that the compilation provider instead changes the logic for top-level apps: rather than copying their directories one by one and compiling them in order, it: 1. copies all top-level apps to the build directory so the files are in their proper locations 2. adds the top-level apps to the path (after the global hooks have run, so the existing scope and env has not changed) 3. runs the compilation as usual. Fixes #1651 --- src/rebar_prv_compile.erl | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index 959ecb0..c9a77a5 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -45,13 +45,13 @@ do(State) -> Deps = rebar_state:deps_to_build(State), Cwd = rebar_state:dir(State), - build_apps(State, Providers, Deps), + copy_and_build_apps(State, Providers, Deps), {ok, ProjectApps1} = rebar_digraph:compile_order(ProjectApps), %% Run top level hooks *before* project apps compiled but *after* deps are rebar_hooks:run_all_hooks(Cwd, pre, ?PROVIDER, Providers, State), - ProjectApps2 = build_apps(State, Providers, ProjectApps1), + ProjectApps2 = copy_and_build_project_apps(State, Providers, ProjectApps1), State2 = rebar_state:project_apps(State, ProjectApps2), %% projects with structures like /apps/foo,/apps/bar,/test @@ -77,7 +77,7 @@ format_error({missing_artifact, File}) -> format_error(Reason) -> io_lib:format("~p", [Reason]). -build_apps(State, Providers, Apps) -> +copy_and_build_apps(State, Providers, Apps) -> [build_app(State, Providers, AppInfo) || AppInfo <- Apps]. build_app(State, Providers, AppInfo) -> @@ -86,6 +86,19 @@ build_app(State, Providers, AppInfo) -> copy_app_dirs(AppInfo, AppDir, OutDir), compile(State, Providers, AppInfo). +copy_and_build_project_apps(State, Providers, Apps) -> + %% Top-level apps, because of profile usage and specific orderings (i.e. + %% may require an include file from a profile-specific app for an extra_dirs + %% entry that only exists in a test context), need to be + %% copied and added to the path at once, and not just in compile order. + [copy_app_dirs(AppInfo, + rebar_app_info:dir(AppInfo), + rebar_app_info:out_dir(AppInfo)) + || AppInfo <- Apps], + code:add_pathsa([rebar_app_info:out_dir(AppInfo) || AppInfo <- Apps]), + [compile(State, Providers, AppInfo) || AppInfo <- Apps]. + + build_extra_dirs(State, Apps) -> BaseDir = rebar_state:dir(State), F = fun(App) -> rebar_app_info:dir(App) == BaseDir end, -- cgit v1.1 From b383dcc1a965a568c8301d3afd73d28a4089823a Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 2 Nov 2017 11:40:19 -0700 Subject: git vsn from tag both strip 'v' prefix --- src/rebar_git_resource.erl | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src') diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl index c63d10d..2855dd0 100644 --- a/src/rebar_git_resource.erl +++ b/src/rebar_git_resource.erl @@ -273,6 +273,9 @@ parse_tags(Dir) -> [{use_stdout, false}, return_on_error, {cd, Dir}]) of {error, _} -> {undefined, "0.0.0"}; + %% strip the v prefix if it exists like is done in the above match + {ok, [$v | LatestVsn]} -> + {undefined, string:strip(LatestVsn, both, $\n)}; {ok, LatestVsn} -> {undefined, string:strip(LatestVsn, both, $\n)} end -- cgit v1.1 From bf1597ba280155f1004b5a2bfd7c553d8d102860 Mon Sep 17 00:00:00 2001 From: Andrey Kanyuka Date: Fri, 3 Nov 2017 14:02:16 +0200 Subject: fixed handling of proxy username and password when fetching registry --- src/rebar_prv_update.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index a019c5a..9d632d4 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -51,9 +51,10 @@ do(State) -> CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), case rebar_utils:url_append_path(CDN, ?REMOTE_REGISTRY_FILE) of {ok, Url} -> + HttpOptions = rebar_utils:get_proxy_auth(), ?DEBUG("Fetching registry from ~p", [Url]), case httpc:request(get, {Url, [{"User-Agent", rebar_utils:user_agent()}]}, - [], [{stream, TmpFile}, {sync, true}], + HttpOptions, [{stream, TmpFile}, {sync, true}], rebar) of {ok, saved_to_file} -> {ok, Data} = file:read_file(TmpFile), -- cgit v1.1 From cc660d717a7c4fd5d290eddf9c1b1d9772a8924e Mon Sep 17 00:00:00 2001 From: Andrey Kanyuka Date: Sun, 5 Nov 2017 18:27:10 +0200 Subject: added http option {relaxed, true} when fetching registry --- src/rebar_prv_update.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index 9d632d4..79804d3 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -51,7 +51,7 @@ do(State) -> CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), case rebar_utils:url_append_path(CDN, ?REMOTE_REGISTRY_FILE) of {ok, Url} -> - HttpOptions = rebar_utils:get_proxy_auth(), + HttpOptions = [{relaxed, true} | rebar_utils:get_proxy_auth()], ?DEBUG("Fetching registry from ~p", [Url]), case httpc:request(get, {Url, [{"User-Agent", rebar_utils:user_agent()}]}, HttpOptions, [{stream, TmpFile}, {sync, true}], -- cgit v1.1 From 2d5cd9c00cfa4e58066b48beee4057fdd52cc7be Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 1 Nov 2017 19:38:03 -0400 Subject: OTP-21 readiness, Full Unicode support This replaces all deprecated function usage by alternative ones based on a version switch enacted at compile time, preventing all warnings. This will likely introduce some possible runtime errors in using a Rebar3 compiled on OTP-20 or OTP-21 back in versions 19 and earlier, but we can't really work around that. A bunch of dependencies have been updated to support OTP-21 without warnings as well. --- src/rebar_dialyzer_format.erl | 2 +- src/rebar_dir.erl | 2 +- src/rebar_file_utils.erl | 2 +- src/rebar_git_resource.erl | 19 ++++++++++--------- src/rebar_hg_resource.erl | 20 ++++++++++---------- src/rebar_hooks.erl | 2 +- src/rebar_packages.erl | 2 +- src/rebar_pkg_resource.erl | 8 ++++---- src/rebar_prv_app_discovery.erl | 2 +- src/rebar_prv_bare_compile.erl | 2 +- src/rebar_prv_common_test.erl | 2 +- src/rebar_prv_dialyzer.erl | 2 +- src/rebar_prv_eunit.erl | 3 ++- src/rebar_prv_local_upgrade.erl | 2 +- src/rebar_prv_path.erl | 4 ++-- src/rebar_prv_shell.erl | 2 +- src/rebar_relx.erl | 2 +- src/rebar_string.erl | 41 +++++++++++++++++++++++++++++++++++++++++ src/rebar_templater.erl | 3 ++- src/rebar_utils.erl | 13 ++++++------- 20 files changed, 89 insertions(+), 46 deletions(-) create mode 100644 src/rebar_string.erl (limited to 'src') diff --git a/src/rebar_dialyzer_format.erl b/src/rebar_dialyzer_format.erl index 7cf4e63..5583633 100644 --- a/src/rebar_dialyzer_format.erl +++ b/src/rebar_dialyzer_format.erl @@ -427,4 +427,4 @@ separate_args(D, [C | R], Arg, Args) -> separate_args(D, R, [C | Arg], Args). join_args(Args) -> - [$(, string:join(Args, ", "), $)]. + [$(, rebar_string:join(Args, ", "), $)]. diff --git a/src/rebar_dir.erl b/src/rebar_dir.erl index 7182c10..d7be423 100644 --- a/src/rebar_dir.erl +++ b/src/rebar_dir.erl @@ -49,7 +49,7 @@ profile_dir(Opts, Profiles) -> %% of profiles to match order passed to `as` ["default"|Rest] -> {rebar_opts:get(Opts, base_dir, ?DEFAULT_BASE_DIR), Rest} end, - ProfilesDir = string:join(ProfilesStrings, "+"), + ProfilesDir = rebar_string:join(ProfilesStrings, "+"), filename:join(BaseDir, ProfilesDir). %% @doc returns the directory where dependencies should be placed diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index b0755ed..7a48a6a 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -190,7 +190,7 @@ cp_r(Sources, Dest) -> case os:type() of {unix, _} -> EscSources = [rebar_utils:escape_chars(Src) || Src <- Sources], - SourceStr = string:join(EscSources, " "), + SourceStr = rebar_string:join(EscSources, " "), {ok, []} = rebar_utils:sh(?FMT("cp -Rp ~ts \"~ts\"", [SourceStr, rebar_utils:escape_double_quotes(Dest)]), [{use_stdout, false}, abort_on_error]), diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl index 2855dd0..ea77b89 100644 --- a/src/rebar_git_resource.erl +++ b/src/rebar_git_resource.erl @@ -28,7 +28,7 @@ lock(AppDir, {git, Url}) -> rebar_utils:sh("git --git-dir=\"" ++ Dir ++ "/.git\" rev-parse --verify HEAD", [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]) end, - Ref = string:strip(VsnString, both, $\n), + Ref = rebar_string:trim(VsnString, both, "\n"), {git, Url, {ref, Ref}}. %% Return true if either the git url or tag/branch/ref is not the same as the currently @@ -36,8 +36,8 @@ lock(AppDir, {git, Url}) -> needs_update(Dir, {git, Url, {tag, Tag}}) -> {ok, Current} = rebar_utils:sh(?FMT("git describe --tags --exact-match", []), [{cd, Dir}]), - Current1 = string:strip(string:strip(Current, both, $\n), both, $\r), - + Current1 = rebar_string:trim(rebar_string:trim(Current, both, "\n"), + both, "\r"), ?DEBUG("Comparing git tag ~ts with ~ts", [Tag, Current1]), not ((Current1 =:= Tag) andalso compare_url(Dir, Url)); needs_update(Dir, {git, Url, {branch, Branch}}) -> @@ -55,8 +55,8 @@ needs_update(Dir, {git, Url, "master"}) -> needs_update(Dir, {git, _, Ref}) -> {ok, Current} = rebar_utils:sh(?FMT("git rev-parse --short=7 -q HEAD", []), [{cd, Dir}]), - Current1 = string:strip(string:strip(Current, both, $\n), both, $\r), - + Current1 = rebar_string:trim(rebar_string:trim(Current, both, "\n"), + both, "\r"), Ref2 = case Ref of {ref, Ref1} -> Length = length(Current1), @@ -74,7 +74,8 @@ needs_update(Dir, {git, _, Ref}) -> compare_url(Dir, Url) -> {ok, CurrentUrl} = rebar_utils:sh(?FMT("git config --get remote.origin.url", []), [{cd, Dir}]), - CurrentUrl1 = string:strip(string:strip(CurrentUrl, both, $\n), both, $\r), + CurrentUrl1 = rebar_string:trim(rebar_string:trim(CurrentUrl, both, "\n"), + both, "\r"), {ok, ParsedUrl} = parse_git_url(Url), {ok, ParsedCurrentUrl} = parse_git_url(CurrentUrl1), ?DEBUG("Comparing git url ~p with ~p", [ParsedUrl, ParsedCurrentUrl]), @@ -215,7 +216,7 @@ collect_default_refcount(Dir) -> ?WARN("Getting log of git dependency failed in ~ts. Falling back to version 0.0.0", [rebar_dir:get_cwd()]), {plain, "0.0.0"}; {ok, String} -> - RawRef = string:strip(String, both, $\n), + RawRef = rebar_string:trim(String, both, "\n"), {Tag, TagVsn} = parse_tags(Dir), {ok, RawCount} = @@ -275,9 +276,9 @@ parse_tags(Dir) -> {undefined, "0.0.0"}; %% strip the v prefix if it exists like is done in the above match {ok, [$v | LatestVsn]} -> - {undefined, string:strip(LatestVsn, both, $\n)}; + {undefined, rebar_string:trim(LatestVsn, both, "\n")}; {ok, LatestVsn} -> - {undefined, string:strip(LatestVsn, both, $\n)} + {undefined, rebar_string:trim(LatestVsn,both, "\n")} end end end. diff --git a/src/rebar_hg_resource.erl b/src/rebar_hg_resource.erl index 0a77c1f..6d25783 100644 --- a/src/rebar_hg_resource.erl +++ b/src/rebar_hg_resource.erl @@ -96,7 +96,7 @@ make_vsn(Dir) -> {ok, VsnString} = rebar_utils:sh(Cmd, [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]), - RawVsn = string:strip(VsnString, both, $\n), + RawVsn = rebar_string:trim(VsnString, both, "\n"), Vsn = case RawVsn of "null+" ++ Rest -> "0.0.0+" ++ Rest; @@ -107,8 +107,8 @@ make_vsn(Dir) -> %%% Internal functions compare_url(Dir, Url) -> - CurrentUrl = string:strip(os:cmd("hg -R \"" ++ rebar_utils:escape_double_quotes(Dir) ++"\" paths default"), both, $\n), - CurrentUrl1 = string:strip(CurrentUrl, both, $\r), + CurrentUrl = rebar_string:trim(os:cmd("hg -R \"" ++ rebar_utils:escape_double_quotes(Dir) ++"\" paths default"), both, "\n"), + CurrentUrl1 = rebar_string:trim(CurrentUrl, both, "\r"), parse_hg_url(CurrentUrl1) =:= parse_hg_url(Url). get_ref(Dir) -> @@ -116,7 +116,7 @@ get_ref(Dir) -> {ok, RefString} = rebar_utils:sh("hg -R \"" ++ rebar_utils:escape_double_quotes(Dir) ++ "\" --debug id -i", [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]), - string:strip(RefString, both, $\n). + rebar_string:trim(RefString, both, "\n"). get_tag_distance(Dir, Ref) -> AbortMsg = io_lib:format("Get tag distance of hg dependency failed in ~ts", [Dir]), @@ -125,8 +125,8 @@ get_tag_distance(Dir, Ref) -> "log --template \"{latesttag}-{latesttagdistance}\n\" " "--rev " ++ rebar_utils:escape_chars(Ref), [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]), - Log = string:strip(LogString, - both, $\n), + Log = rebar_string:trim(LogString, + both, "\n"), [Tag, Distance] = re:split(Log, "-([0-9]+)$", [{parts,0}, {return,list}, unicode]), {Tag, Distance}. @@ -137,7 +137,7 @@ get_branch_ref(Dir, Branch) -> rebar_utils:sh("hg -R \"" ++ rebar_utils:escape_double_quotes(Dir) ++ "\" log --template \"{node}\n\" --rev " ++ rebar_utils:escape_chars(Branch), [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]), - string:strip(BranchRefString, both, $\n). + rebar_string:strip(BranchRefString, both, "\n"). maybe_warn_local_url(Url) -> @@ -150,11 +150,11 @@ maybe_warn_local_url(Url) -> end. parse_hg_url("ssh://" ++ HostPath) -> - [Host | Path] = string:tokens(HostPath, "/"), + [Host | Path] = rebar_string:lexemes(HostPath, "/"), {Host, filename:rootname(filename:join(Path), ".hg")}; parse_hg_url("http://" ++ HostPath) -> - [Host | Path] = string:tokens(HostPath, "/"), + [Host | Path] = rebar_string:lexemes(HostPath, "/"), {Host, filename:rootname(filename:join(Path), ".hg")}; parse_hg_url("https://" ++ HostPath) -> - [Host | Path] = string:tokens(HostPath, "/"), + [Host | Path] = rebar_string:lexemes(HostPath, "/"), {Host, filename:rootname(filename:join(Path), ".hg")}. diff --git a/src/rebar_hooks.erl b/src/rebar_hooks.erl index 48aa928..8893f2a 100644 --- a/src/rebar_hooks.erl +++ b/src/rebar_hooks.erl @@ -140,7 +140,7 @@ create_env(State, Opts) -> ]. join_dirs(BaseDir, Dirs) -> - string:join([ filename:join(BaseDir, Dir) || Dir <- Dirs ], ":"). + rebar_string:join([filename:join(BaseDir, Dir) || Dir <- Dirs], ":"). re_version(Path) -> case re:run(Path, "^.*-(?[^/-]*)$", [{capture,[1],list}, unicode]) of diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index cba1d16..d17b54f 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -102,7 +102,7 @@ registry_dir(State) -> case rebar_utils:url_append_path(CDN, ?REMOTE_PACKAGE_DIR) of {ok, Parsed} -> {ok, {_, _, Host, _, Path, _}} = http_uri:parse(Parsed), - CDNHostPath = lists:reverse(string:tokens(Host, ".")), + CDNHostPath = lists:reverse(rebar_string:lexemes(Host, ".")), CDNPath = tl(filename:split(Path)), RegistryDir = filename:join([CacheDir, "hex"] ++ CDNHostPath ++ CDNPath), ok = filelib:ensure_dir(filename:join(RegistryDir, "placeholder")), diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index d588f24..60ad8f9 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -98,7 +98,7 @@ extract(TmpDir, CachePath) -> checksums(Pkg={pkg, _Name, _Vsn, Hash}, Files, Contents, Version, Meta, State) -> Blob = <>, <> = crypto:hash(sha256, Blob), - BinChecksum = list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))), + BinChecksum = list_to_binary(rebar_string:uppercase(lists:flatten(io_lib:format("~64.16.0b", [X])))), RegistryChecksum = rebar_packages:registry_checksum(Pkg, State), {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files), {Hash, BinChecksum, RegistryChecksum, TarChecksum}. @@ -116,7 +116,7 @@ request(Url, ETag) -> {ok, {{_Version, 200, _Reason}, Headers, Body}} -> ?DEBUG("Successfully downloaded ~ts", [Url]), {"etag", ETag1} = lists:keyfind("etag", 1, Headers), - {ok, Body, string:strip(ETag1, both, $")}; + {ok, Body, rebar_string:trim(ETag1, both, [$"])}; {ok, {{_Version, 304, _Reason}, _Headers, _Body}} -> ?DEBUG("Cached copy of ~ts still valid", [Url]), {ok, cached}; @@ -132,7 +132,7 @@ etag(Path) -> case file:read_file(Path) of {ok, Binary} -> <> = crypto:hash(md5, Binary), - string:to_lower(lists:flatten(io_lib:format("~32.16.0b", [X]))); + rebar_string:lowercase(lists:flatten(io_lib:format("~32.16.0b", [X]))); {error, _} -> false end. @@ -205,7 +205,7 @@ get_ssl_config() -> end. parse_vsn(Vsn) -> - version_pad(string:tokens(Vsn, ".-")). + version_pad(rebar_string:lexemes(Vsn, ".-")). version_pad([Major]) -> {list_to_integer(Major), 0, 0}; diff --git a/src/rebar_prv_app_discovery.erl b/src/rebar_prv_app_discovery.erl index 3f10a3f..f5bab49 100644 --- a/src/rebar_prv_app_discovery.erl +++ b/src/rebar_prv_app_discovery.erl @@ -49,7 +49,7 @@ do(State) -> -spec format_error(any()) -> iolist(). format_error({multiple_app_files, Files}) -> - io_lib:format("Multiple app files found in one app dir: ~ts", [string:join(Files, " and ")]); + io_lib:format("Multiple app files found in one app dir: ~ts", [rebar_string:join(Files, " and ")]); format_error({invalid_app_file, File, Reason}) -> case Reason of {Line, erl_parse, Description} -> diff --git a/src/rebar_prv_bare_compile.erl b/src/rebar_prv_bare_compile.erl index 6f1ac16..c29a711 100644 --- a/src/rebar_prv_bare_compile.erl +++ b/src/rebar_prv_bare_compile.erl @@ -42,7 +42,7 @@ do(State) -> Paths = proplists:get_value(paths, RawOpts), Sep = proplists:get_value(separator, RawOpts, " "), [ code:add_pathsa(filelib:wildcard(PathWildcard)) - || PathWildcard <- string:tokens(Paths, Sep) ], + || PathWildcard <- rebar_string:lexemes(Paths, Sep) ], [AppInfo] = rebar_state:project_apps(State), AppInfo1 = rebar_app_info:out_dir(AppInfo, rebar_dir:get_cwd()), diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index bf788d2..2443710 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -173,7 +173,7 @@ transform_opts([Opt|Rest], Acc) -> transform_opts(Rest, [Opt|Acc]). split_string(String) -> - string:tokens(String, [$,]). + rebar_string:lexemes(String, [$,]). cfgopts(State) -> case rebar_state:get(State, ct_opts, []) of diff --git a/src/rebar_prv_dialyzer.erl b/src/rebar_prv_dialyzer.erl index a74eefb..99a7698 100644 --- a/src/rebar_prv_dialyzer.erl +++ b/src/rebar_prv_dialyzer.erl @@ -565,7 +565,7 @@ collect_nested_dependent_apps(App, Seen) -> dialyzer_version() -> _ = application:load(dialyzer), {ok, Vsn} = application:get_key(dialyzer, vsn), - case string:tokens(Vsn, ".") of + case rebar_string:lexemes(Vsn, ".") of [Major, Minor] -> version_tuple(Major, Minor, "0"); [Major, Minor, Patch | _] -> diff --git a/src/rebar_prv_eunit.erl b/src/rebar_prv_eunit.erl index 65addc3..2361432 100644 --- a/src/rebar_prv_eunit.erl +++ b/src/rebar_prv_eunit.erl @@ -141,7 +141,8 @@ resolve(Flag, RawOpts) -> resolve(Flag, Flag, RawOpts). resolve(Flag, EUnitKey, RawOpts) -> case proplists:get_value(Flag, RawOpts) of undefined -> []; - Args -> lists:map(fun(Arg) -> normalize(EUnitKey, Arg) end, string:tokens(Args, [$,])) + Args -> lists:map(fun(Arg) -> normalize(EUnitKey, Arg) end, + rebar_string:lexemes(Args, [$,])) end. normalize(Key, Value) when Key == dir; Key == file -> {Key, Value}; diff --git a/src/rebar_prv_local_upgrade.erl b/src/rebar_prv_local_upgrade.erl index aa9ee44..1ac3adb 100644 --- a/src/rebar_prv_local_upgrade.erl +++ b/src/rebar_prv_local_upgrade.erl @@ -72,7 +72,7 @@ get_md5(Rebar3Path) -> {ok, Rebar3File} = file:read_file(Rebar3Path), Digest = crypto:hash(md5, Rebar3File), DigestHex = lists:flatten([io_lib:format("~2.16.0B", [X]) || X <- binary_to_list(Digest)]), - string:to_lower(DigestHex). + rebar_string:lowercase(DigestHex). maybe_fetch_rebar3(Rebar3Md5) -> TmpDir = ec_file:insecure_mkdtemp(), diff --git a/src/rebar_prv_path.erl b/src/rebar_prv_path.erl index 75d38eb..5374b0c 100644 --- a/src/rebar_prv_path.erl +++ b/src/rebar_prv_path.erl @@ -49,7 +49,7 @@ format_error(Reason) -> filter_apps(RawOpts, State) -> RawApps = proplists:get_all_values(app, RawOpts), - Apps = lists:foldl(fun(String, Acc) -> string:tokens(String, ",") ++ Acc end, [], RawApps), + Apps = lists:foldl(fun(String, Acc) -> rebar_string:lexemes(String, ",") ++ Acc end, [], RawApps), case Apps of [] -> ProjectDeps = project_deps(State), @@ -91,7 +91,7 @@ print_paths_if_exist(Paths, State) -> {RawOpts, _} = rebar_state:command_parsed_args(State), Sep = proplists:get_value(separator, RawOpts, " "), RealPaths = lists:filter(fun(P) -> ec_file:is_dir(P) end, Paths), - io:format("~ts", [string:join(RealPaths, Sep)]). + io:format("~ts", [rebar_string:join(RealPaths, Sep)]). project_deps(State) -> Profiles = rebar_state:current_profiles(State), diff --git a/src/rebar_prv_shell.erl b/src/rebar_prv_shell.erl index 0244833..47e0366 100644 --- a/src/rebar_prv_shell.erl +++ b/src/rebar_prv_shell.erl @@ -307,7 +307,7 @@ find_apps_option(State) -> no_value -> no_value; AppsStr -> [ list_to_atom(AppStr) - || AppStr <- string:tokens(AppsStr, " ,:") ] + || AppStr <- rebar_string:lexemes(AppsStr, " ,:") ] end. -spec find_apps_rebar(rebar_state:t()) -> no_value | list(). diff --git a/src/rebar_relx.erl b/src/rebar_relx.erl index 17c0bd6..4548761 100644 --- a/src/rebar_relx.erl +++ b/src/rebar_relx.erl @@ -27,7 +27,7 @@ do(Module, Command, Provider, State) -> LibDirs = rebar_utils:filtermap(fun ec_file:exists/1, [rebar_dir:checkouts_dir(State), DepsDir | ProjectAppDirs]), OutputDir = filename:join(rebar_dir:base_dir(State), ?DEFAULT_RELEASE_DIR), - AllOptions = string:join([Command | Options], " "), + AllOptions = rebar_string:join([Command | Options], " "), Cwd = rebar_state:dir(State), Providers = rebar_state:providers(State), RebarOpts = rebar_state:opts(State), diff --git a/src/rebar_string.erl b/src/rebar_string.erl new file mode 100644 index 0000000..c1858f9 --- /dev/null +++ b/src/rebar_string.erl @@ -0,0 +1,41 @@ +%%% @doc Compatibility module for string functionality +%%% for pre- and post-unicode support. +-module(rebar_string). +-export([join/2, lexemes/2, trim/3, uppercase/1, lowercase/1, chr/2]). + +-ifdef(unicode_str). + +%% string:join/2 copy; string:join/2 is getting obsoleted +%% and replaced by lists:join/2, but lists:join/2 is too new +%% for version support (only appeared in 19.0) so it cannot be +%% used. Instead we just adopt join/2 locally and hope it works +%% for most unicode use cases anyway. +join([], Sep) when is_list(Sep) -> + []; +join([H|T], Sep) -> + H ++ lists:append([Sep ++ X || X <- T]). + +lexemes(Str, SepList) -> string:lexemes(Str, SepList). +trim(Str, Direction, Cluster=[_]) -> string:trim(Str, Direction, Cluster). +uppercase(Str) -> string:uppercase(Str). +lowercase(Str) -> string:lowercase(Str). + +chr(S, C) when is_integer(C) -> chr(S, C, 1). +chr([C|_Cs], C, I) -> I; +chr([_|Cs], C, I) -> chr(Cs, C, I+1); +chr([], _C, _I) -> 0. +-else. + +join(Strings, Separator) -> string:join(Strings, Separator). +lexemes(Str, SepList) -> string:tokens(Str, SepList). +trim(Str, Direction, [Char]) -> + Dir = case Direction of + both -> both; + leading -> left; + trailing -> right + end, + string:strip(Str, Dir, Char). +uppercase(Str) -> string:to_upper(Str). +lowercase(Str) -> string:to_lower(Str). +chr(Str, Char) -> string:chr(Str, Char). +-endif. diff --git a/src/rebar_templater.erl b/src/rebar_templater.erl index 75190ec..929ca47 100644 --- a/src/rebar_templater.erl +++ b/src/rebar_templater.erl @@ -120,7 +120,8 @@ default_author_and_email() -> {ok, Name} -> case rebar_utils:sh("git config --global user.email", [return_on_error]) of {ok, Email} -> - {string:strip(Name, both, $\n), string:strip(Email, both, $\n)}; + {rebar_string:trim(Name, both, "\n"), + rebar_string:trim(Email, both, "\n")}; {error, _} -> %% Use neither if one doesn't exist {"Anonymous", "anonymous@example.org"} diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index ee8f179..5ea0452 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -391,7 +391,7 @@ compare({Priority, A}, {Secondary, B}) when not is_tuple(A), is_tuple(B) -> %% Implements wc -l functionality used to determine patchcount from git output line_count(PatchLines) -> - Tokenized = string:tokens(PatchLines, "\n"), + Tokenized = rebar_string:lexemes(PatchLines, "\n"), {ok, length(Tokenized)}. check_min_otp_version(undefined) -> @@ -528,7 +528,7 @@ patch_on_windows(Cmd, Env) -> %% The end of form `$FOO' is delimited with whitespace or EOL -spec expand_env_variable(string(), string(), term()) -> string(). expand_env_variable(InStr, VarName, RawVarValue) -> - case string:chr(InStr, $$) of + case rebar_string:chr(InStr, $$) of 0 -> %% No variables to expand InStr; @@ -622,7 +622,7 @@ sh_loop(Port, Fun, Acc) -> beam_to_mod(Dir, Filename) -> [Dir | Rest] = filename:split(Filename), - list_to_atom(filename:basename(string:join(Rest, "."), ".beam")). + list_to_atom(filename:basename(rebar_string:join(Rest, "."), ".beam")). beam_to_mod(Filename) -> list_to_atom(filename:basename(Filename, ".beam")). @@ -703,7 +703,7 @@ vcs_vsn_cmd(_, _, _) -> vcs_vsn_invoke(Cmd, Dir) -> {ok, VsnString} = rebar_utils:sh(Cmd, [{cd, Dir}, {use_stdout, false}]), - string:strip(VsnString, right, $\n). + rebar_string:trim(VsnString, trailing, "\n"). find_resource_module(Type, Resources) -> case lists:keyfind(Type, 1, Resources) of @@ -896,9 +896,8 @@ list_dir(Dir) -> set_proxy_auth([]) -> ok; set_proxy_auth(UserInfo) -> - Idx = string:chr(UserInfo, $:), - Username = string:sub_string(UserInfo, 1, Idx-1), - Password = string:sub_string(UserInfo, Idx+1), + [Username, Password] = re:split(UserInfo, ":", + [{return, list}, {parts,2}, unicode]), %% password may contain url encoded characters, need to decode them first application:set_env(rebar, proxy_auth, [{proxy_auth, {Username, http_uri:decode(Password)}}]). -- cgit v1.1 From 6c9b3e361d33cb2f6f54108bdb71ea3cfa40fa12 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 17 Nov 2017 12:02:18 -0500 Subject: Bump to 3.4.5 - OTP-21 readiness, Full Unicode support, massive dep upgrade - fixed handling of proxy username and password when fetching registry - git versions from tag made consistent and all strip 'v' prefix - Prevent hard crash on duplicate plugin paths - Fix include paths in profile multiapp edge case - Fix unlock state carry, which broke do sequences with unlock in them. - Avoid guessing on utf8 decoding of app files - Various fixes related to .app files - Warn user when an unsupported local git or hg resource is used - Corrects a fix to src_dir values - Update eunit_formatters to latest version - Changes in wording of warnings for more accuracy --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 74efe97..0b6f76d 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.4.5"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 8bdee78b9a617515e255adb60c9599dc54fad930 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 17 Nov 2017 12:05:11 -0500 Subject: Return to git-based versioning --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 0b6f76d..74efe97 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.4.5"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 9e4cd942d0ae70d3541ea542df0cad0595874eb1 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 17 Nov 2017 17:21:41 -0500 Subject: Bump to 3.4.6 Updates relx (windows fixes) and erlware commons (strings) --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 74efe97..43e65b6 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.4.6"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From eff06dd9ed9f32befde46578853ec6e0f29a1205 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 17 Nov 2017 17:25:31 -0500 Subject: Back to git-based versioning --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 43e65b6..74efe97 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.4.6"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 64056bf9f9fee47227c60609a7684b04d8df577a Mon Sep 17 00:00:00 2001 From: Liam McNamara Date: Mon, 20 Nov 2017 10:09:32 +0100 Subject: Allow silencing skip warnings when fetching deps When fetching deps, if this is a clean repo there will be extensive messages warning that dependencies which have already been fetched are being skipped. For large projects being built and tested in a clean environment this significantly increases the noise level of the build. This modification adds an additional rebar option (deps_warning_on_conflict) that will allow disabling these warning messages. If deps_error_on_conflict is set, an error will still be thrown. This will not change default behaviour of rebar. There is a similar outstanding issue: https://github.com/erlang/rebar3/issues/1105 However this seems to be a push for not outputting warnings when the dep version is the same, rather than disabling warnings altogether. --- src/rebar_prv_install_deps.erl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index 65aabff..b735ed0 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -419,8 +419,13 @@ warn_skip_deps(AppInfo, State) -> Args = [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)], case rebar_state:get(State, deps_error_on_conflict, false) of - false -> ?WARN(Msg, Args); - true -> ?ERROR(Msg, Args), ?FAIL + false -> + case rebar_state:get(State, deps_warning_on_conflict, true) of + true -> ?WARN(Msg, Args); + false -> ok + end; + true -> + ?ERROR(Msg, Args), ?FAIL end. not_needs_compile(App) -> -- cgit v1.1 From bf7ec3866f1819619286cc875fa88e1dc8bb9cfc Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sun, 5 Nov 2017 22:24:58 -0500 Subject: Make debug_info rules clear Current rebar3 uses debug_info rules where debug_info is added by default, and no_debug_info prevents the default from being added, and removes any explicit debug_info, if any. The problem is that if 'no_debug_info' is anywhere in the config for a run, it cannot be removed even with other profiles. additionally, no_debug_info ignores special tuples like {debug_info, {Mod, Data}} and {debug_info_key, Key}, which can be used to add debug info and encrypt it (in lieu of plain debug_info) respectively. This patch makes it so that the following rules are in place: - the last option seen takes priority, allowing profile overrides by the ordering rules the compiler expects - the overriden options shall be explicitly deleted to avoid confusing the compiler - any option related to debug info seen last cancels any no_debug_info that preceded it - any no_debug_info option seen last cancels all of the other debug_info options - if debug_info is seen last, it cancels out {debug_info_key, Key} - if {debug_info_key, Key} is seen last, it cancels out debug_info - All other options are left untouched in that context (defines can still be expanded and so on) This should allow proper profile rules to be followed. Note that erl_opt profile merging puts precedence on the *last* element of a list, to match the compiler. --- src/rebar_opts.erl | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/rebar_opts.erl b/src/rebar_opts.erl index b7156b2..1a927ba 100644 --- a/src/rebar_opts.erl +++ b/src/rebar_opts.erl @@ -35,12 +35,40 @@ erl_opts(Opts) -> Defines = [{d, list_to_atom(D)} || D <- ?MODULE:get(Opts, defines, [])], AllOpts = Defines ++ RawErlOpts, - case proplists:is_defined(no_debug_info, AllOpts) of - true -> - [O || O <- AllOpts, O =/= no_debug_info]; - false -> - [debug_info|AllOpts] - end. + lists:reverse(filter_debug_info(lists:reverse(AllOpts))). + +filter_debug_info([]) -> + %% Default == ON + [debug_info]; +filter_debug_info([debug_info|_] = L) -> + %% drop no_debug_info and {debug_info_key, _} since those would + %% conflict with a plain debug_info + [debug_info | + lists:filter(fun(K) -> + K =/= no_debug_info andalso K =/= debug_info andalso + not (is_tuple(K) andalso element(1,K) =:= debug_info_key) + end, L)]; +filter_debug_info([{debug_info, _} = H | T]) -> + %% custom debug_info field; keep and filter the rest except + %% without no_debug_info. Still have to filter for regular or crypto + %% debug_info. + [H | filter_debug_info(lists:filter(fun(K) -> K =/= no_debug_info end, T))]; +filter_debug_info([{debug_info_key, _}=H | T]) -> + %% Drop no_debug_info and regular debug_info + [H | lists:filter(fun(K) -> + K =/= no_debug_info andalso K =/= debug_info andalso + not (is_tuple(K) andalso element(1,K) =:= debug_info_key) + end, T)]; +filter_debug_info([no_debug_info|T]) -> + %% Drop all debug info + lists:filter(fun(debug_info) -> false + ; ({debug_info, _}) -> false + ; ({debug_info_key, _}) -> false + ; (no_debug_info) -> false + ; (_Other) -> true + end, T); +filter_debug_info([H|T]) -> + [H|filter_debug_info(T)]. apply_overrides(Opts, Name, Overrides) -> %% Inefficient. We want the order we get here though. -- cgit v1.1 From c550cf0c4c9df09ff1fe7eca0fbe8e26fa800502 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 22 Nov 2017 09:35:17 -0500 Subject: Fix compilation for custom resources dynamic vsn The Erlang compiler runs based on a global state built from currently loaded libraries and the configured code path that is available. For this reason, the rebar3 compiler job unloads all plugin paths before calling the Erlang compiler. However, this causes a problem when an application uses a custom resource handler with a dynamic version in their .app.src file since the plugin that can be used to find the version has been unloaded. Fortunately, the compile phase that runs the version handling is distinct from the phase that uses the Erlang compiler. This patch fixes the problem by re-loading the plugins' paths in memory before generating the .app file, and before unloading them afterwards. It appears that unloading them is unnecessary because the hooks after that will re-load them, but it is likely better to play it safe with that global state and clean up after ourselves. It offers better protection for future changes. Fixes #1657 --- src/rebar_prv_compile.erl | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index c9a77a5..72320fb 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -136,7 +136,18 @@ compile(State, Providers, AppInfo) -> AppInfo3 = rebar_hooks:run_all_hooks(AppDir, post, ?ERLC_HOOK, Providers, AppInfo2, State), AppInfo4 = rebar_hooks:run_all_hooks(AppDir, pre, ?APP_HOOK, Providers, AppInfo3, State), - case rebar_otp_app:compile(State, AppInfo4) of + + %% Load plugins back for make_vsn calls in custom resources. + %% The rebar_otp_app compilation step is safe regarding the + %% overall path management, so we can just load all plugins back + %% in memory. + PluginDepsPaths = rebar_state:code_paths(State, all_plugin_deps), + code:add_pathsa(PluginDepsPaths), + AppFileCompileResult = rebar_otp_app:compile(State, AppInfo4), + %% Clean up after ourselves, leave things as they were. + rebar_utils:remove_from_code_path(PluginDepsPaths), + + case AppFileCompileResult of {ok, AppInfo5} -> AppInfo6 = rebar_hooks:run_all_hooks(AppDir, post, ?APP_HOOK, Providers, AppInfo5, State), AppInfo7 = rebar_hooks:run_all_hooks(AppDir, post, ?PROVIDER, Providers, AppInfo6, State), -- cgit v1.1 From 81a5711b6db4dedb1d151d39464f326be80e2285 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 22 Nov 2017 14:55:21 -0500 Subject: Bump to 3.4.7 --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 74efe97..ae5afca 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.4.7"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 462f078e309f6faca5975700668173af516f2b1a Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 22 Nov 2017 14:58:17 -0500 Subject: Back to git-based versioning --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index ae5afca..74efe97 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.4.7"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 38bbbafb7e2546e301d027ab67b587882cf333c2 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 23 Nov 2017 13:52:38 -0500 Subject: Add experimental support for ct --retry option This commit adds a common test hook along with the cth_readable stuff whose role is to track failing test cases, and create a test specification out of them. The test specification is dumped on disk at _build//logs/retry.spec and can be accessed by calling 'rebar3 ct --retry'. This will auto-load the spec file if it can be found and re-run the failing cases. If any other argument is found on the list specifying tests, the '--retry' argument is ignored. All code for this is marked as experimental in case we end up (keeping and then) dropping the feature. --- src/cth_retry.erl | 161 ++++++++++++++++++++++++++++++++++++++++++ src/rebar_prv_common_test.erl | 28 ++++++-- 2 files changed, 184 insertions(+), 5 deletions(-) create mode 100644 src/cth_retry.erl (limited to 'src') diff --git a/src/cth_retry.erl b/src/cth_retry.erl new file mode 100644 index 0000000..e7b1663 --- /dev/null +++ b/src/cth_retry.erl @@ -0,0 +1,161 @@ +-module(cth_retry). + +%% Callbacks +-export([id/1]). +-export([init/2]). + +-export([pre_init_per_suite/3]). +-export([post_init_per_suite/4]). +-export([pre_end_per_suite/3]). +-export([post_end_per_suite/4]). + +-export([pre_init_per_group/3]). +-export([post_init_per_group/4]). +-export([pre_end_per_group/3]). +-export([post_end_per_group/4]). + +-export([pre_init_per_testcase/3]). +-export([post_end_per_testcase/4]). + +-export([on_tc_fail/3]). +-export([on_tc_skip/3, on_tc_skip/4]). + +-export([terminate/1]). + +-record(state, {id, suite, groups, acc=[]}). + +%% @doc Return a unique id for this CTH. +id(_Opts) -> + {?MODULE, make_ref()}. + +%% @doc Always called before any other callback function. Use this to initiate +%% any common state. +init(Id, _Opts) -> + {ok, #state{id=Id}}. + +%% @doc Called before init_per_suite is called. +pre_init_per_suite(Suite,Config,State) -> + {Config, State#state{suite=Suite, groups=[]}}. + +%% @doc Called after init_per_suite. +post_init_per_suite(_Suite,_Config,Return,State) -> + {Return, State}. + +%% @doc Called before end_per_suite. +pre_end_per_suite(_Suite,Config,State) -> + {Config, State}. + +%% @doc Called after end_per_suite. +post_end_per_suite(_Suite,_Config,Return,State) -> + {Return, State#state{suite=undefined, groups=[]}}. + +%% @doc Called before each init_per_group. +pre_init_per_group(_Group,Config,State) -> + {Config, State}. + +%% @doc Called after each init_per_group. +post_init_per_group(Group,_Config,Return, State=#state{groups=Groups}) -> + {Return, State#state{groups=[Group|Groups]}}. + +%% @doc Called after each end_per_group. +pre_end_per_group(_Group,Config,State) -> + {Config, State}. + +%% @doc Called after each end_per_group. +post_end_per_group(_Group,_Config,Return, State=#state{groups=Groups}) -> + {Return, State#state{groups=tl(Groups)}}. + +%% @doc Called before each test case. +pre_init_per_testcase(_TC,Config,State) -> + {Config, State}. + +%% @doc Called after each test case. +post_end_per_testcase(_TC,_Config,ok,State) -> + {ok, State}; +post_end_per_testcase(_TC,_Config,{skip,_},State) -> + {ok, State}; % manual skip +post_end_per_testcase(TC,_Config,Error,State=#state{suite=Suite, groups=Groups, acc=Acc}) -> + Test = case TC of + {_Group, Case} -> Case; + TC -> TC + end, + {Error, State#state{acc=[{Suite, Groups, Test}|Acc]}}. + +%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group, +%% post_end_per_group and post_end_per_testcase if the suite, group or test case failed. +on_tc_fail(_TC, _Reason, State) -> + State. + +%% @doc Called when a test case is skipped by either user action +%% or due to an init function failing. (>= 19.3) +on_tc_skip(Suite, TC, {tc_auto_skip, _R}, State=#state{suite=Suite, groups=Groups, acc=Acc}) -> + NewAcc = case TC of + init_per_testcase -> Acc; + end_per_testcase -> Acc; + {init_per_group,_} -> Acc; + {end_per_group, _} -> Acc; + init_per_suite -> Acc; + end_per_suite -> Acc; + {Case,_Group} -> [{Suite, Groups, Case}|Acc]; + TC -> [{Suite, Groups, TC}|Acc] + end, + State#state{suite=Suite, acc=NewAcc}; +on_tc_skip(Suite, _TC, _Reason, State) -> + State#state{suite=Suite}. + +%% @doc Called when a test case is skipped by either user action +%% or due to an init function failing. (Pre-19.3) +on_tc_skip(TC, {tc_auto_skip, _R}, State=#state{suite=Suite, groups=Groups, acc=Acc}) -> + NewAcc = case TC of + init_per_testcase -> Acc; + end_per_testcase -> Acc; + {init_per_group,_} -> Acc; + {end_per_group, _} -> Acc; + init_per_suite -> Acc; + end_per_suite -> Acc; + {Case, _Group} -> [{Suite, Groups, Case}|Acc]; + TC -> [{Suite, Groups, TC}|Acc] + end, + State#state{acc=NewAcc}; +on_tc_skip(_TC, _Reason, State) -> + State. + +%% @doc Called when the scope of the CTH is done +terminate(#state{acc=[]}) -> + ok; +terminate(#state{acc=Acc}) -> + Spec = to_spec(Acc), + {ok, Cwd} = file:get_cwd(), + Path = filename:join(lists:droplast(filename:split(Cwd))++["retry.spec"]), + io:format(user, + "EXPERIMENTAL: Writing retry specification at ~s~n" + " call rebar3 ct with '--retry' to re-run failing cases.~n", + [Path]), + file:write_file(Path, Spec), + ok. + +%%% Helpers +to_spec(List) -> + [to_spec_entry(X) || X <- merge(List)]. + +merge([]) -> []; +merge([{Suite, Groups, Case}|T]) when is_atom(Case) -> + merge([{Suite, Groups, [Case]}|T]); +merge([{Suite, Groups, Cases}, {Suite, Groups, Case} | T]) -> + merge([{Suite, Groups, [Case|Cases]}|T]); +merge([{Suite, Groups, Cases} | T]) -> + [{Suite, Groups, Cases} | merge(T)]. + +to_spec_entry({Suite, [], Cases}) -> + Dir = filename:dirname(proplists:get_value(source, Suite:module_info(compile))), + io_lib:format("~p.~n", [{cases, Dir, Suite, Cases}]); +to_spec_entry({Suite, Groups, Cases}) -> + Dir = filename:dirname(proplists:get_value(source, Suite:module_info(compile))), + ExpandedGroups = expand_groups(lists:reverse(Groups)), + io_lib:format("~p.~n", [{groups, Dir, Suite, ExpandedGroups, {cases,Cases}}]). + +expand_groups([Group]) -> + {Group, []}; +expand_groups([H|T]) -> + {H,[],[expand_groups(T)]}. + diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index 2443710..a88753c 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -213,10 +213,10 @@ add_hooks(Opts, State) -> {false, _} -> Opts; {true, false} -> - [{ct_hooks, [cth_readable_failonly, cth_readable_shell]} | Opts]; + [{ct_hooks, [cth_readable_failonly, cth_readable_shell, cth_retry]} | Opts]; {true, {ct_hooks, Hooks}} -> %% Make sure hooks are there once only. - ReadableHooks = [cth_readable_failonly, cth_readable_shell], + ReadableHooks = [cth_readable_failonly, cth_readable_shell, cth_retry], NewHooks = (Hooks -- ReadableHooks) ++ ReadableHooks, lists:keyreplace(ct_hooks, 1, Opts, {ct_hooks, NewHooks}) end. @@ -272,6 +272,15 @@ is_any_defined([Key|Keys],Opts) -> is_any_defined([],_Opts) -> false. +should_retry(State, Opts) -> + case proplists:get_value(retry, Opts, false) of + false -> + false; + true -> + Path = filename:join([rebar_dir:base_dir(State), "logs", "retry.spec"]), + filelib:is_file(Path) andalso {true, Path} + end. + sys_config_list(CmdOpts, CfgOpts) -> CmdSysConfigs = split_string(proplists:get_value(sys_config, CmdOpts, "")), case proplists:get_value(sys_config, CfgOpts, []) of @@ -285,9 +294,15 @@ sys_config_list(CmdOpts, CfgOpts) -> discover_tests(State, ProjectApps, Opts) -> case is_any_defined([spec,dir,suite],Opts) of + true -> {ok, Opts}; %% no tests defined, try using `$APP/test` and `$ROOT/test` as dirs - false -> {ok, [default_tests(State, ProjectApps)|Opts]}; - true -> {ok, Opts} + false -> + case should_retry(State, Opts) of + false -> + {ok, [default_tests(State, ProjectApps)|Opts]}; + {true, Path} -> + {ok, [{spec, Path} | Opts]} + end end. default_tests(State, ProjectApps) -> @@ -751,7 +766,8 @@ ct_opts(_State) -> {sname, undefined, "sname", atom, help(sname)}, {setcookie, undefined, "setcookie", atom, help(setcookie)}, {sys_config, undefined, "sys_config", string, help(sys_config)}, %% comma-separated list - {compile_only, undefined, "compile_only", boolean, help(compile_only)} + {compile_only, undefined, "compile_only", boolean, help(compile_only)}, + {retry, undefined, "retry", boolean, help(retry)} ]. help(compile_only) -> @@ -820,5 +836,7 @@ help(sname) -> "Gives a short name to the node"; help(setcookie) -> "Sets the cookie if the node is distributed"; +help(retry) -> + "Experimental feature. If any specification for previously failing test is found, runs them."; help(_) -> "". -- cgit v1.1 From 26136b6b54e95d3832c1ffe2f8bbab642c01ba1d Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 24 Nov 2017 14:27:02 -0500 Subject: Revert "Add experimental support for ct --retry option" --- src/cth_retry.erl | 161 ------------------------------------------ src/rebar_prv_common_test.erl | 28 ++------ 2 files changed, 5 insertions(+), 184 deletions(-) delete mode 100644 src/cth_retry.erl (limited to 'src') diff --git a/src/cth_retry.erl b/src/cth_retry.erl deleted file mode 100644 index e7b1663..0000000 --- a/src/cth_retry.erl +++ /dev/null @@ -1,161 +0,0 @@ --module(cth_retry). - -%% Callbacks --export([id/1]). --export([init/2]). - --export([pre_init_per_suite/3]). --export([post_init_per_suite/4]). --export([pre_end_per_suite/3]). --export([post_end_per_suite/4]). - --export([pre_init_per_group/3]). --export([post_init_per_group/4]). --export([pre_end_per_group/3]). --export([post_end_per_group/4]). - --export([pre_init_per_testcase/3]). --export([post_end_per_testcase/4]). - --export([on_tc_fail/3]). --export([on_tc_skip/3, on_tc_skip/4]). - --export([terminate/1]). - --record(state, {id, suite, groups, acc=[]}). - -%% @doc Return a unique id for this CTH. -id(_Opts) -> - {?MODULE, make_ref()}. - -%% @doc Always called before any other callback function. Use this to initiate -%% any common state. -init(Id, _Opts) -> - {ok, #state{id=Id}}. - -%% @doc Called before init_per_suite is called. -pre_init_per_suite(Suite,Config,State) -> - {Config, State#state{suite=Suite, groups=[]}}. - -%% @doc Called after init_per_suite. -post_init_per_suite(_Suite,_Config,Return,State) -> - {Return, State}. - -%% @doc Called before end_per_suite. -pre_end_per_suite(_Suite,Config,State) -> - {Config, State}. - -%% @doc Called after end_per_suite. -post_end_per_suite(_Suite,_Config,Return,State) -> - {Return, State#state{suite=undefined, groups=[]}}. - -%% @doc Called before each init_per_group. -pre_init_per_group(_Group,Config,State) -> - {Config, State}. - -%% @doc Called after each init_per_group. -post_init_per_group(Group,_Config,Return, State=#state{groups=Groups}) -> - {Return, State#state{groups=[Group|Groups]}}. - -%% @doc Called after each end_per_group. -pre_end_per_group(_Group,Config,State) -> - {Config, State}. - -%% @doc Called after each end_per_group. -post_end_per_group(_Group,_Config,Return, State=#state{groups=Groups}) -> - {Return, State#state{groups=tl(Groups)}}. - -%% @doc Called before each test case. -pre_init_per_testcase(_TC,Config,State) -> - {Config, State}. - -%% @doc Called after each test case. -post_end_per_testcase(_TC,_Config,ok,State) -> - {ok, State}; -post_end_per_testcase(_TC,_Config,{skip,_},State) -> - {ok, State}; % manual skip -post_end_per_testcase(TC,_Config,Error,State=#state{suite=Suite, groups=Groups, acc=Acc}) -> - Test = case TC of - {_Group, Case} -> Case; - TC -> TC - end, - {Error, State#state{acc=[{Suite, Groups, Test}|Acc]}}. - -%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group, -%% post_end_per_group and post_end_per_testcase if the suite, group or test case failed. -on_tc_fail(_TC, _Reason, State) -> - State. - -%% @doc Called when a test case is skipped by either user action -%% or due to an init function failing. (>= 19.3) -on_tc_skip(Suite, TC, {tc_auto_skip, _R}, State=#state{suite=Suite, groups=Groups, acc=Acc}) -> - NewAcc = case TC of - init_per_testcase -> Acc; - end_per_testcase -> Acc; - {init_per_group,_} -> Acc; - {end_per_group, _} -> Acc; - init_per_suite -> Acc; - end_per_suite -> Acc; - {Case,_Group} -> [{Suite, Groups, Case}|Acc]; - TC -> [{Suite, Groups, TC}|Acc] - end, - State#state{suite=Suite, acc=NewAcc}; -on_tc_skip(Suite, _TC, _Reason, State) -> - State#state{suite=Suite}. - -%% @doc Called when a test case is skipped by either user action -%% or due to an init function failing. (Pre-19.3) -on_tc_skip(TC, {tc_auto_skip, _R}, State=#state{suite=Suite, groups=Groups, acc=Acc}) -> - NewAcc = case TC of - init_per_testcase -> Acc; - end_per_testcase -> Acc; - {init_per_group,_} -> Acc; - {end_per_group, _} -> Acc; - init_per_suite -> Acc; - end_per_suite -> Acc; - {Case, _Group} -> [{Suite, Groups, Case}|Acc]; - TC -> [{Suite, Groups, TC}|Acc] - end, - State#state{acc=NewAcc}; -on_tc_skip(_TC, _Reason, State) -> - State. - -%% @doc Called when the scope of the CTH is done -terminate(#state{acc=[]}) -> - ok; -terminate(#state{acc=Acc}) -> - Spec = to_spec(Acc), - {ok, Cwd} = file:get_cwd(), - Path = filename:join(lists:droplast(filename:split(Cwd))++["retry.spec"]), - io:format(user, - "EXPERIMENTAL: Writing retry specification at ~s~n" - " call rebar3 ct with '--retry' to re-run failing cases.~n", - [Path]), - file:write_file(Path, Spec), - ok. - -%%% Helpers -to_spec(List) -> - [to_spec_entry(X) || X <- merge(List)]. - -merge([]) -> []; -merge([{Suite, Groups, Case}|T]) when is_atom(Case) -> - merge([{Suite, Groups, [Case]}|T]); -merge([{Suite, Groups, Cases}, {Suite, Groups, Case} | T]) -> - merge([{Suite, Groups, [Case|Cases]}|T]); -merge([{Suite, Groups, Cases} | T]) -> - [{Suite, Groups, Cases} | merge(T)]. - -to_spec_entry({Suite, [], Cases}) -> - Dir = filename:dirname(proplists:get_value(source, Suite:module_info(compile))), - io_lib:format("~p.~n", [{cases, Dir, Suite, Cases}]); -to_spec_entry({Suite, Groups, Cases}) -> - Dir = filename:dirname(proplists:get_value(source, Suite:module_info(compile))), - ExpandedGroups = expand_groups(lists:reverse(Groups)), - io_lib:format("~p.~n", [{groups, Dir, Suite, ExpandedGroups, {cases,Cases}}]). - -expand_groups([Group]) -> - {Group, []}; -expand_groups([H|T]) -> - {H,[],[expand_groups(T)]}. - diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index a88753c..2443710 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -213,10 +213,10 @@ add_hooks(Opts, State) -> {false, _} -> Opts; {true, false} -> - [{ct_hooks, [cth_readable_failonly, cth_readable_shell, cth_retry]} | Opts]; + [{ct_hooks, [cth_readable_failonly, cth_readable_shell]} | Opts]; {true, {ct_hooks, Hooks}} -> %% Make sure hooks are there once only. - ReadableHooks = [cth_readable_failonly, cth_readable_shell, cth_retry], + ReadableHooks = [cth_readable_failonly, cth_readable_shell], NewHooks = (Hooks -- ReadableHooks) ++ ReadableHooks, lists:keyreplace(ct_hooks, 1, Opts, {ct_hooks, NewHooks}) end. @@ -272,15 +272,6 @@ is_any_defined([Key|Keys],Opts) -> is_any_defined([],_Opts) -> false. -should_retry(State, Opts) -> - case proplists:get_value(retry, Opts, false) of - false -> - false; - true -> - Path = filename:join([rebar_dir:base_dir(State), "logs", "retry.spec"]), - filelib:is_file(Path) andalso {true, Path} - end. - sys_config_list(CmdOpts, CfgOpts) -> CmdSysConfigs = split_string(proplists:get_value(sys_config, CmdOpts, "")), case proplists:get_value(sys_config, CfgOpts, []) of @@ -294,15 +285,9 @@ sys_config_list(CmdOpts, CfgOpts) -> discover_tests(State, ProjectApps, Opts) -> case is_any_defined([spec,dir,suite],Opts) of - true -> {ok, Opts}; %% no tests defined, try using `$APP/test` and `$ROOT/test` as dirs - false -> - case should_retry(State, Opts) of - false -> - {ok, [default_tests(State, ProjectApps)|Opts]}; - {true, Path} -> - {ok, [{spec, Path} | Opts]} - end + false -> {ok, [default_tests(State, ProjectApps)|Opts]}; + true -> {ok, Opts} end. default_tests(State, ProjectApps) -> @@ -766,8 +751,7 @@ ct_opts(_State) -> {sname, undefined, "sname", atom, help(sname)}, {setcookie, undefined, "setcookie", atom, help(setcookie)}, {sys_config, undefined, "sys_config", string, help(sys_config)}, %% comma-separated list - {compile_only, undefined, "compile_only", boolean, help(compile_only)}, - {retry, undefined, "retry", boolean, help(retry)} + {compile_only, undefined, "compile_only", boolean, help(compile_only)} ]. help(compile_only) -> @@ -836,7 +820,5 @@ help(sname) -> "Gives a short name to the node"; help(setcookie) -> "Sets the cookie if the node is distributed"; -help(retry) -> - "Experimental feature. If any specification for previously failing test is found, runs them."; help(_) -> "". -- cgit v1.1 From 32ae99650110637d479bb0db64dc1e6583d7b094 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 23 Nov 2017 13:52:38 -0500 Subject: Add experimental support for ct --retry option This commit adds a common test hook along with the cth_readable stuff whose role is to track failing test cases, and create a test specification out of them. The test specification is dumped on disk at _build//logs/retry.spec and can be accessed by calling 'rebar3 ct --retry'. This will auto-load the spec file if it can be found and re-run the failing cases. If any other argument is found on the list specifying tests, the '--retry' argument is ignored. All code for this is marked as experimental in case we end up (keeping and then) dropping the feature. --- src/cth_retry.erl | 159 ++++++++++++++++++++++++++++++++++++++++++ src/rebar_prv_common_test.erl | 23 ++++-- 2 files changed, 178 insertions(+), 4 deletions(-) create mode 100644 src/cth_retry.erl (limited to 'src') diff --git a/src/cth_retry.erl b/src/cth_retry.erl new file mode 100644 index 0000000..7056c71 --- /dev/null +++ b/src/cth_retry.erl @@ -0,0 +1,159 @@ +-module(cth_retry). + +%% Callbacks +-export([id/1]). +-export([init/2]). + +-export([pre_init_per_suite/3]). +-export([post_init_per_suite/4]). +-export([pre_end_per_suite/3]). +-export([post_end_per_suite/4]). + +-export([pre_init_per_group/3]). +-export([post_init_per_group/4]). +-export([pre_end_per_group/3]). +-export([post_end_per_group/4]). + +-export([pre_init_per_testcase/3]). +-export([post_end_per_testcase/4]). + +-export([on_tc_fail/3]). +-export([on_tc_skip/3, on_tc_skip/4]). + +-export([terminate/1]). + +-record(state, {id, suite, groups, acc=[]}). + +%% @doc Return a unique id for this CTH. +id(_Opts) -> + {?MODULE, make_ref()}. + +%% @doc Always called before any other callback function. Use this to initiate +%% any common state. +init(Id, _Opts) -> + {ok, #state{id=Id}}. + +%% @doc Called before init_per_suite is called. +pre_init_per_suite(Suite,Config,State) -> + {Config, State#state{suite=Suite, groups=[]}}. + +%% @doc Called after init_per_suite. +post_init_per_suite(_Suite,_Config,Return,State) -> + {Return, State}. + +%% @doc Called before end_per_suite. +pre_end_per_suite(_Suite,Config,State) -> + {Config, State}. + +%% @doc Called after end_per_suite. +post_end_per_suite(_Suite,_Config,Return,State) -> + {Return, State#state{suite=undefined, groups=[]}}. + +%% @doc Called before each init_per_group. +pre_init_per_group(_Group,Config,State) -> + {Config, State}. + +%% @doc Called after each init_per_group. +post_init_per_group(Group,_Config,Return, State=#state{groups=Groups}) -> + {Return, State#state{groups=[Group|Groups]}}. + +%% @doc Called after each end_per_group. +pre_end_per_group(_Group,Config,State) -> + {Config, State}. + +%% @doc Called after each end_per_group. +post_end_per_group(_Group,_Config,Return, State=#state{groups=Groups}) -> + {Return, State#state{groups=tl(Groups)}}. + +%% @doc Called before each test case. +pre_init_per_testcase(_TC,Config,State) -> + {Config, State}. + +%% @doc Called after each test case. +post_end_per_testcase(_TC,_Config,ok,State) -> + {ok, State}; +post_end_per_testcase(TC,_Config,Error,State=#state{suite=Suite, groups=Groups, acc=Acc}) -> + Test = case TC of + {_Group, Case} -> Case; + TC -> TC + end, + {Error, State#state{acc=[{Suite, Groups, Test}|Acc]}}. + +%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group, +%% post_end_per_group and post_end_per_testcase if the suite, group or test case failed. +on_tc_fail(_TC, _Reason, State) -> + State. + +%% @doc Called when a test case is skipped by either user action +%% or due to an init function failing. (>= 19.3) +on_tc_skip(Suite, TC, {tc_auto_skip, _}, State=#state{suite=Suite, groups=Groups, acc=Acc}) -> + NewAcc = case TC of + init_per_testcase -> Acc; + end_per_testcase -> Acc; + {init_per_group,_} -> Acc; + {end_per_group, _} -> Acc; + init_per_suite -> Acc; + end_per_suite -> Acc; + {_Group, Case} -> [{Suite, Groups, Case}|Acc]; + TC -> [{Suite, Groups, TC}|Acc] + end, + State#state{suite=Suite, acc=NewAcc}; +on_tc_skip(Suite, _TC, _Reason, State) -> + State#state{suite=Suite}. + +%% @doc Called when a test case is skipped by either user action +%% or due to an init function failing. (Pre-19.3) +on_tc_skip(TC, {tc_auto_skip, _}, State=#state{suite=Suite, groups=Groups, acc=Acc}) -> + NewAcc = case TC of + init_per_testcase -> Acc; + end_per_testcase -> Acc; + {init_per_group,_} -> Acc; + {end_per_group, _} -> Acc; + init_per_suite -> Acc; + end_per_suite -> Acc; + {_Group, Case} -> [{Suite, Groups, Case}|Acc]; + TC -> [{Suite, Groups, TC}|Acc] + end, + State#state{acc=NewAcc}; +on_tc_skip(_TC, _Reason, State) -> + State. + +%% @doc Called when the scope of the CTH is done +terminate(#state{acc=[]}) -> + ok; +terminate(#state{acc=Acc}) -> + Spec = to_spec(Acc), + {ok, Cwd} = file:get_cwd(), + Path = filename:join(lists:droplast(filename:split(Cwd))++["retry.spec"]), + io:format(user, + "EXPERIMENTAL: Writing retry specification at ~s~n" + " call rebar3 ct with '--retry' to re-run failing cases.~n", + [Path]), + file:write_file(Path, Spec), + ok. + +%%% Helpers +to_spec(List) -> + [to_spec_entry(X) || X <- merge(List)]. + +merge([]) -> []; +merge([{Suite, Groups, Case}|T]) when is_atom(Case) -> + merge([{Suite, Groups, [Case]}|T]); +merge([{Suite, Groups, Cases}, {Suite, Groups, Case} | T]) -> + merge([{Suite, Groups, [Case|Cases]}|T]); +merge([{Suite, Groups, Cases} | T]) -> + [{Suite, Groups, Cases} | merge(T)]. + +to_spec_entry({Suite, [], Cases}) -> + Dir = filename:dirname(proplists:get_value(source, Suite:module_info(compile))), + io_lib:format("~p.~n", [{cases, Dir, Suite, Cases}]); +to_spec_entry({Suite, Groups, Cases}) -> + Dir = filename:dirname(proplists:get_value(source, Suite:module_info(compile))), + ExpandedGroups = expand_groups(lists:reverse(Groups)), + io_lib:format("~p.~n", [{groups, Dir, Suite, ExpandedGroups, {cases,Cases}}]). + +expand_groups([Group]) -> + {Group, []}; +expand_groups([H|T]) -> + {H,[],[expand_groups(T)]}. + diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index 2443710..9631a0d 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -135,7 +135,7 @@ cmdopts(State) -> {RawOpts, _} = rebar_state:command_parsed_args(State), %% filter out opts common_test doesn't know about and convert %% to ct acceptable forms - transform_opts(RawOpts, []). + transform_retry(transform_opts(RawOpts, []), State). transform_opts([], Acc) -> lists:reverse(Acc); transform_opts([{dir, Dirs}|Rest], Acc) -> @@ -172,6 +172,18 @@ transform_opts([{verbose, _}|Rest], Acc) -> transform_opts([Opt|Rest], Acc) -> transform_opts(Rest, [Opt|Acc]). +%% @private only retry if specified and if no other spec +%% is given. +transform_retry(Opts, State) -> + case proplists:get_value(retry, Opts, false) andalso + not is_any_defined([spec,dir,suite], Opts) of + false -> + Opts; + true -> + Path = filename:join([rebar_dir:base_dir(State), "logs", "retry.spec"]), + filelib:is_file(Path) andalso [{spec, Path}|Opts] + end. + split_string(String) -> rebar_string:lexemes(String, [$,]). @@ -213,10 +225,10 @@ add_hooks(Opts, State) -> {false, _} -> Opts; {true, false} -> - [{ct_hooks, [cth_readable_failonly, cth_readable_shell]} | Opts]; + [{ct_hooks, [cth_readable_failonly, cth_readable_shell, cth_retry]} | Opts]; {true, {ct_hooks, Hooks}} -> %% Make sure hooks are there once only. - ReadableHooks = [cth_readable_failonly, cth_readable_shell], + ReadableHooks = [cth_readable_failonly, cth_readable_shell, cth_retry], NewHooks = (Hooks -- ReadableHooks) ++ ReadableHooks, lists:keyreplace(ct_hooks, 1, Opts, {ct_hooks, NewHooks}) end. @@ -751,7 +763,8 @@ ct_opts(_State) -> {sname, undefined, "sname", atom, help(sname)}, {setcookie, undefined, "setcookie", atom, help(setcookie)}, {sys_config, undefined, "sys_config", string, help(sys_config)}, %% comma-separated list - {compile_only, undefined, "compile_only", boolean, help(compile_only)} + {compile_only, undefined, "compile_only", boolean, help(compile_only)}, + {retry, undefined, "retry", boolean, help(retry)} ]. help(compile_only) -> @@ -820,5 +833,7 @@ help(sname) -> "Gives a short name to the node"; help(setcookie) -> "Sets the cookie if the node is distributed"; +help(retry) -> + "Experimental feature. If any specification for previously failing test is found, runs them."; help(_) -> "". -- cgit v1.1 From 5fe44046f74d6a78af72b88a077d4d99cd05a9bd Mon Sep 17 00:00:00 2001 From: Juan Facorro Date: Wed, 29 Nov 2017 17:42:50 +0100 Subject: [#1675] Disable color in git log command --- src/rebar_git_resource.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl index ea77b89..3aa875f 100644 --- a/src/rebar_git_resource.erl +++ b/src/rebar_git_resource.erl @@ -261,7 +261,7 @@ get_patch_count(Dir, RawRef) -> parse_tags(Dir) -> %% Don't abort on error, we want the bad return to be turned into 0.0.0 - case rebar_utils:sh("git log --oneline --no-walk --tags --decorate", + case rebar_utils:sh("git -c color.ui=false log --oneline --no-walk --tags --decorate", [{use_stdout, false}, return_on_error, {cd, Dir}]) of {error, _} -> {undefined, "0.0.0"}; -- cgit v1.1 From 805dc0fa2aad53a5ac551033fd0da3b433f6b6e1 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sat, 2 Dec 2017 17:32:15 -0500 Subject: Support minimal coverage validation in tests Adds an option (-m, --min_coverage, or {cover_opts, {min_coverage,X}}) to the 'cover' command, where the value is an integer between 0 and 100. If the total coverage during the analysis is below the value received, the command will fail with output like: ===> Requiring 64% coverage to pass. Only 62% obtained If the rate is correct, the command silently passes as it currently does. This feature allows to enforce code coverage standards in a project if desired. --- src/rebar_prv_cover.erl | 57 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 13 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_cover.erl b/src/rebar_prv_cover.erl index c890452..376ba72 100644 --- a/src/rebar_prv_cover.erl +++ b/src/rebar_prv_cover.erl @@ -12,6 +12,7 @@ maybe_write_coverdata/2, format_error/1]). +-include_lib("providers/include/providers.hrl"). -include("rebar.hrl"). -define(PROVIDER, cover). @@ -62,6 +63,9 @@ maybe_write_coverdata(State, Task) -> end. -spec format_error(any()) -> iolist(). +format_error({min_coverage_failed, {PassRate, Total}}) -> + io_lib:format("Requiring ~p% coverage to pass. Only ~p% obtained", + [PassRate, Total]); format_error(Reason) -> io_lib:format("~p", [Reason]). @@ -102,13 +106,13 @@ do_analyze(State) -> %% redirect cover output true = redirect_cover_output(State, CoverPid), %% analyze! - ok = case analyze(State, CoverFiles) of - [] -> ok; + case analyze(State, CoverFiles) of + [] -> {ok, State}; Analysis -> print_analysis(Analysis, verbose(State)), - write_index(State, Analysis) - end, - {ok, State}. + write_index(State, Analysis), + maybe_fail_coverage(Analysis, State) + end. get_all_coverdata(CoverDir) -> ok = filelib:ensure_dir(filename:join([CoverDir, "dummy.log"])), @@ -213,11 +217,11 @@ format_table(Stats, CoverFiles) -> Header = header(MaxLength), Separator = separator(MaxLength), TotalLabel = format("total", MaxLength), - TotalCov = format(calculate_total(Stats), 8), + TotalCov = format(calculate_total_string(Stats), 8), [io_lib:format("~ts~n~ts~n~ts~n", [Separator, Header, Separator]), lists:map(fun({Mod, Coverage, _}) -> Name = format(Mod, MaxLength), - Cov = format(percentage(Coverage), 8), + Cov = format(percentage_string(Coverage), 8), io_lib:format(" | ~ts | ~ts |~n", [Name, Cov]) end, Stats), io_lib:format("~ts~n", [Separator]), @@ -239,6 +243,9 @@ separator(Width) -> format(String, Width) -> io_lib:format("~*.ts", [Width, String]). +calculate_total_string(Stats) -> + integer_to_list(calculate_total(Stats))++"%". + calculate_total(Stats) -> percentage(lists:foldl( fun({_Mod, {Cov, Not}, _File}, {CovAcc, NotAcc}) -> @@ -248,8 +255,10 @@ calculate_total(Stats) -> Stats )). -percentage({_, 0}) -> "100%"; -percentage({Cov, Not}) -> integer_to_list(trunc((Cov / (Cov + Not)) * 100)) ++ "%". +percentage_string(Data) -> integer_to_list(percentage(Data))++"%". + +percentage({_, 0}) -> 100; +percentage({Cov, Not}) -> trunc((Cov / (Cov + Not)) * 100). write_index(State, Coverage) -> CoverDir = cover_dir(State), @@ -279,14 +288,25 @@ write_index_section(F, [{Section, DataFile, Mods}|Rest]) -> FmtLink = fun({Mod, Cov, Report}) -> ?FMT("~ts~ts\n", - [strip_coverdir(Report), Mod, percentage(Cov)]) + [strip_coverdir(Report), Mod, percentage_string(Cov)]) end, lists:foreach(fun(M) -> ok = file:write(F, FmtLink(M)) end, Mods), ok = file:write(F, ?FMT("Total~ts\n", - [calculate_total(Mods)])), + [calculate_total_string(Mods)])), ok = file:write(F, "\n"), write_index_section(F, Rest). +maybe_fail_coverage(Analysis, State) -> + {_, _CoverFiles, Stats} = lists:keyfind("aggregate", 1, Analysis), + Total = calculate_total(Stats), + PassRate = min_coverage(State), + ?DEBUG("Comparing ~p to pass rate ~p", [Total, PassRate]), + if Total >= PassRate -> + {ok, State} + ; Total < PassRate -> + ?PRV_ERROR({min_coverage_failed, {PassRate, Total}}) + end. + %% fix for r15b which doesn't put the correct path in the `source` section %% of `module_info(compile)` strip_coverdir([]) -> ""; @@ -401,12 +421,23 @@ verbose(State) -> {Verbose, _} -> Verbose end. +min_coverage(State) -> + Command = proplists:get_value(min_coverage, command_line_opts(State), undefined), + Config = proplists:get_value(min_coverage, config_opts(State), undefined), + case {Command, Config} of + {undefined, undefined} -> 0; + {undefined, Rate} -> Rate; + {Rate, _} -> Rate + end. + cover_dir(State) -> filename:join([rebar_dir:base_dir(State), "cover"]). cover_opts(_State) -> [{reset, $r, "reset", boolean, help(reset)}, - {verbose, $v, "verbose", boolean, help(verbose)}]. + {verbose, $v, "verbose", boolean, help(verbose)}, + {min_coverage, $m, "min_coverage", integer, help(min_coverage)}]. help(reset) -> "Reset all coverdata."; -help(verbose) -> "Print coverage analysis.". +help(verbose) -> "Print coverage analysis."; +help(min_coverage) -> "Mandate a coverage percentage required to succeed (0..100)". -- cgit v1.1 From 6e672852a72f51d088e16f3662b0299744435127 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 1 Dec 2017 16:44:00 -0800 Subject: fix code path when validating plugins --- src/rebar_plugins.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_plugins.erl b/src/rebar_plugins.erl index 68ba6da..1fc01ff 100644 --- a/src/rebar_plugins.erl +++ b/src/rebar_plugins.erl @@ -105,7 +105,7 @@ handle_plugin(Profile, Plugin, State, Upgrade) -> %% Add newly built deps and plugin to code path State3 = rebar_state:update_all_plugin_deps(State2, Apps), NewCodePaths = [rebar_app_info:ebin_dir(A) || A <- ToBuild], - code:add_pathsa(CodePaths), + code:add_pathsa(NewCodePaths++CodePaths), %% Store plugin code paths so we can remove them when compiling project apps State4 = rebar_state:update_code_paths(State3, all_plugin_deps, CodePaths++NewCodePaths), -- cgit v1.1 From fb67eb0ee95b2f03dc6c51535ab4b18ea70d315f Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 4 Dec 2017 09:34:52 -0500 Subject: Fix Plugin path handling (again!) The path reloading of plugins had been fixed properly, but the problem is that the paths it was using to re-load only considered the current compile step, rather than the overall state of plugins. As such, the reloaded paths after plugin compilation only reloaded the *latest* plugin and not the other ones. This fix forces the addition of all built plugin paths to the code paths after a plugin compile job is run. This ensures that the path is clean for initial plugin deps (only add those that are required), and is re-made total after the fact (add all the plugins possible). This commit also includes a system tests suite that can be run optionally; the problem with this plugin mechanism was impossible to find through mocked dependencies, and a working counterexample was provided to us. The systest suite can be run against real projects without conflict to make sure no regressions are hit. --- src/rebar_plugins.erl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rebar_plugins.erl b/src/rebar_plugins.erl index 1fc01ff..57c34bc 100644 --- a/src/rebar_plugins.erl +++ b/src/rebar_plugins.erl @@ -96,8 +96,8 @@ handle_plugin(Profile, Plugin, State, Upgrade) -> ToBuild = rebar_prv_install_deps:cull_compile(Sorted, []), %% Add already built plugin deps to the code path - CodePaths = [rebar_app_info:ebin_dir(A) || A <- Apps -- ToBuild], - code:add_pathsa(CodePaths), + PreBuiltPaths = [rebar_app_info:ebin_dir(A) || A <- Apps] -- ToBuild, + code:add_pathsa(PreBuiltPaths), %% Build plugin and its deps [build_plugin(AppInfo, Apps, State2) || AppInfo <- ToBuild], @@ -105,10 +105,12 @@ handle_plugin(Profile, Plugin, State, Upgrade) -> %% Add newly built deps and plugin to code path State3 = rebar_state:update_all_plugin_deps(State2, Apps), NewCodePaths = [rebar_app_info:ebin_dir(A) || A <- ToBuild], + AllPluginEbins = filelib:wildcard(filename:join([rebar_dir:plugins_dir(State), "*", "ebin"])), + CodePaths = PreBuiltPaths++(AllPluginEbins--ToBuild), code:add_pathsa(NewCodePaths++CodePaths), %% Store plugin code paths so we can remove them when compiling project apps - State4 = rebar_state:update_code_paths(State3, all_plugin_deps, CodePaths++NewCodePaths), + State4 = rebar_state:update_code_paths(State3, all_plugin_deps, PreBuiltPaths++NewCodePaths), {plugin_providers(Plugin), State4} catch -- cgit v1.1 From d477715b1a869bdb112895ad28c31e5c94943f2c Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 4 Dec 2017 11:01:47 -0500 Subject: Drop the /is switch to robocopy for win7 Has no ill effect as tested on Win10 --- src/rebar_file_utils.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 7a48a6a..bb3ca71 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -458,11 +458,11 @@ xcopy_win32(Source,Dest)-> %% must manually add the last fragment of a directory to the `Dest` %% in order to properly replicate POSIX platforms NewDest = filename:join([Dest, filename:basename(Source)]), - ?FMT("robocopy \"~ts\" \"~ts\" /e /is 1> nul", + ?FMT("robocopy \"~ts\" \"~ts\" /e 1> nul", [rebar_utils:escape_double_quotes(filename:nativename(Source)), rebar_utils:escape_double_quotes(filename:nativename(NewDest))]); false -> - ?FMT("robocopy \"~ts\" \"~ts\" \"~ts\" /e /is 1> nul", + ?FMT("robocopy \"~ts\" \"~ts\" \"~ts\" /e 1> nul", [rebar_utils:escape_double_quotes(filename:nativename(filename:dirname(Source))), rebar_utils:escape_double_quotes(filename:nativename(Dest)), rebar_utils:escape_double_quotes(filename:basename(Source))]) -- cgit v1.1 From 72b649cc664747d7682eec7150f6ad022b3838ae Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Mon, 4 Dec 2017 11:35:22 -0800 Subject: add eunit and ct option to set coverdata file name --- src/rebar_prv_common_test.erl | 13 ++++++++++--- src/rebar_prv_cover.erl | 4 ++-- src/rebar_prv_eunit.erl | 5 ++++- 3 files changed, 16 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index 9631a0d..f800610 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -8,8 +8,11 @@ -export([init/1, do/1, format_error/1]). -%% exported for test purposes, consider private --export([compile/2, prepare_tests/1, translate_paths/2]). + +-ifdef(TEST). +%% exported for test purposes +-export([compile/2, prepare_tests/1, translate_paths/2, maybe_write_coverdata/1]). +-endif. -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). @@ -728,7 +731,8 @@ maybe_write_coverdata(State) -> true -> rebar_state:set(State, cover_enabled, true); false -> State end, - rebar_prv_cover:maybe_write_coverdata(State1, ?PROVIDER). + Name = proplists:get_value(cover_export_name, RawOpts, ?PROVIDER), + rebar_prv_cover:maybe_write_coverdata(State1, Name). ct_opts(_State) -> [{dir, undefined, "dir", string, help(dir)}, %% comma-separated list @@ -744,6 +748,7 @@ ct_opts(_State) -> {logopts, undefined, "logopts", string, help(logopts)}, %% comma-separated list {verbosity, undefined, "verbosity", integer, help(verbosity)}, %% Integer {cover, $c, "cover", {boolean, false}, help(cover)}, + {cover_export_name, undefined, "cover_export_name", string, help(cover_export_name)}, {repeat, undefined, "repeat", integer, help(repeat)}, %% integer {duration, undefined, "duration", string, help(duration)}, % format: HHMMSS {until, undefined, "until", string, help(until)}, %% format: YYMoMoDD[HHMMSS] @@ -797,6 +802,8 @@ help(verbosity) -> "Verbosity"; help(cover) -> "Generate cover data"; +help(cover_export_name) -> + "Base name of the coverdata file to write"; help(repeat) -> "How often to repeat tests"; help(duration) -> diff --git a/src/rebar_prv_cover.erl b/src/rebar_prv_cover.erl index c890452..2345b18 100644 --- a/src/rebar_prv_cover.erl +++ b/src/rebar_prv_cover.erl @@ -372,10 +372,10 @@ redirect_cover_output(State, CoverPid) -> [append]), group_leader(F, CoverPid). -write_coverdata(State, Task) -> +write_coverdata(State, Name) -> DataDir = cover_dir(State), ok = filelib:ensure_dir(filename:join([DataDir, "dummy.log"])), - ExportFile = filename:join([DataDir, atom_to_list(Task) ++ ".coverdata"]), + ExportFile = filename:join([DataDir, rebar_utils:to_list(Name) ++ ".coverdata"]), case cover:export(ExportFile) of ok -> %% dump accumulated coverdata after writing diff --git a/src/rebar_prv_eunit.erl b/src/rebar_prv_eunit.erl index 2361432..4b71416 100644 --- a/src/rebar_prv_eunit.erl +++ b/src/rebar_prv_eunit.erl @@ -472,7 +472,8 @@ maybe_write_coverdata(State) -> true -> rebar_state:set(State, cover_enabled, true); false -> State end, - rebar_prv_cover:maybe_write_coverdata(State1, ?PROVIDER). + Name = proplists:get_value(cover_export_name, RawOpts, ?PROVIDER), + rebar_prv_cover:maybe_write_coverdata(State1, Name). handle_results(ok) -> ok; handle_results(error) -> @@ -484,6 +485,7 @@ eunit_opts(_State) -> [{app, undefined, "app", string, help(app)}, {application, undefined, "application", string, help(app)}, {cover, $c, "cover", boolean, help(cover)}, + {cover_export_name, undefined, "cover_export_name", string, help(cover_export_name)}, {dir, $d, "dir", string, help(dir)}, {file, $f, "file", string, help(file)}, {module, $m, "module", string, help(module)}, @@ -495,6 +497,7 @@ eunit_opts(_State) -> help(app) -> "Comma separated list of application test suites to run. Equivalent to `[{application, App}]`."; help(cover) -> "Generate cover data. Defaults to false."; +help(cover_export_name) -> "Base name of the coverdata file to write"; help(dir) -> "Comma separated list of dirs to load tests from. Equivalent to `[{dir, Dir}]`."; help(file) -> "Comma separated list of files to load tests from. Equivalent to `[{file, File}]`."; help(module) -> "Comma separated list of modules to load tests from. Equivalent to `[{module, Module}]`."; -- cgit v1.1 From 553a579b36fe0fb4a8bf464cd282d43c07d4e192 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Tue, 5 Dec 2017 07:47:07 -0500 Subject: Alias plugin promoted to built-in command - Uses the code at https://github.com/tsloughter/rebar_alias and brings it within rebar3 - adds safety checks to prevent redefining built-in commands or obvious circular dependencies between commands (indirect circular deps are still possible) - adds tests - adds a systest to ensure no clash with the existing plugin --- src/rebar.app.src | 3 +- src/rebar_prv_alias.erl | 102 ++++++++++++++++++++++++++++++++++++++++++++++++ src/rebar_state.erl | 2 +- 3 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 src/rebar_prv_alias.erl (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 74efe97..43ad620 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -73,6 +73,7 @@ rebar_prv_update, rebar_prv_upgrade, rebar_prv_version, - rebar_prv_xref]} + rebar_prv_xref, + rebar_prv_alias]} % must run last to prevent overloads ]} ]}. diff --git a/src/rebar_prv_alias.erl b/src/rebar_prv_alias.erl new file mode 100644 index 0000000..2a03668 --- /dev/null +++ b/src/rebar_prv_alias.erl @@ -0,0 +1,102 @@ +%%% @doc Meta-provider that dynamically compiles providers +%%% to run aliased commands. +%%% +%%% This is hackish and out-there, but this module has graduated +%%% from a plugin at https://github.com/tsloughter/rebar_alias after +%%% years of stability. Only some error checks were added +-module(rebar_prv_alias). + +-export([init/1]). +-include("rebar.hrl"). + +%% =================================================================== +%% Public API +%% =================================================================== +-spec init(rebar_state:t()) -> {ok, rebar_state:t()}. +init(State) -> + Aliases = rebar_state:get(State, alias, []), + lists:foldl(fun({Alias, Cmds}, {ok, StateAcc}) -> + case validate_provider(Alias, Cmds, State) of + true -> init_alias(Alias, Cmds, StateAcc); + false -> {ok, State} + end + end, {ok, State}, Aliases). + +init_alias(Alias, Cmds, State) -> + Module = list_to_atom("rebar_prv_alias_" ++ atom_to_list(Alias)), + + MF = module(Module), + EF = exports(), + FF = do_func(Cmds), + + {ok, _, Bin} = compile:forms([MF, EF, FF]), + code:load_binary(Module, "none", Bin), + + Provider = providers:create([ + {name, Alias}, + {module, Module}, + {bare, true}, + {deps, []}, + {example, example(Alias)}, + {opts, []}, + {short_desc, desc(Cmds)}, + {desc, desc(Cmds)} + ]), + {ok, rebar_state:add_provider(State, Provider)}. + +validate_provider(Alias, Cmds, State) -> + %% This would be caught and prevented anyway, but the warning + %% is friendlier + case providers:get_provider(Alias, rebar_state:providers(State)) of + not_found -> + %% check for circular deps in the alias. + case not proplists:is_defined(Alias, Cmds) of + true -> true; + false -> + ?WARN("Alias ~p contains itself and would never " + "terminate. It will be ignored.", + [Alias]), + false + end; + _ -> + ?WARN("Alias ~p is already the name of a command in " + "the default namespace and will be ignored.", + [Alias]), + false + end. + + +example(Alias) -> + "rebar3 " ++ atom_to_list(Alias). + +desc(Cmds) -> + "Equivalent to running: rebar3 do " ++ + rebar_string:join(lists:map(fun({Cmd, Args}) -> + atom_to_list(Cmd) ++ " " ++ Args; + (Cmd) -> + atom_to_list(Cmd) + end, Cmds), ","). + +module(Name) -> + {attribute,1,module,Name}. + +exports() -> + {attribute,1,export,[{do,1}]}. + +do_func(Cmds) -> + {function,1,do,1, + [{clause,1, + [{var,1,'State'}], + [], + [{call,1, + {remote,1,{atom,1,rebar_prv_do},{atom,1,do_tasks}}, + [to_args(Cmds),{var,1,'State'}]}]}]}. + + +to_args([]) -> + {nil,1}; +to_args([{Cmd, Args} | Rest]) -> + {cons,1,{tuple,1,[{string,1,atom_to_list(Cmd)},{string,1,Args}]}, to_args(Rest)}; +to_args([Cmd | Rest]) -> + {cons,1,{tuple,1,[{string,1,atom_to_list(Cmd)},{nil,1}]}, to_args(Rest)}. + diff --git a/src/rebar_state.erl b/src/rebar_state.erl index 3314a11..577ed23 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -391,7 +391,7 @@ add_provider(State=#state_t{providers=Providers, allow_provider_overrides=false} case {providers:impl(P), providers:namespace(P)} of {Name, Namespace} -> ?DEBUG("Not adding provider ~p ~p from module ~p because it already exists from module ~p", - [Namespace, Name, providers:module(P), Module]), + [Namespace, Name, Module, providers:module(P)]), true; _ -> false -- cgit v1.1 From d45bacb73bd1a255a5042929a49c81ab298df946 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Tue, 5 Dec 2017 13:05:14 -0500 Subject: Run a soft purge while within the compiler step Prevents the killing of a plugin with itself --- src/rebar_prv_compile.erl | 2 +- src/rebar_utils.erl | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index 72320fb..75e6eee 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -145,7 +145,7 @@ compile(State, Providers, AppInfo) -> code:add_pathsa(PluginDepsPaths), AppFileCompileResult = rebar_otp_app:compile(State, AppInfo4), %% Clean up after ourselves, leave things as they were. - rebar_utils:remove_from_code_path(PluginDepsPaths), + rebar_utils:remove_from_code_path(PluginDepsPaths, soft_purge), case AppFileCompileResult of {ok, AppInfo5} -> diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 5ea0452..64d4952 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -49,6 +49,7 @@ update_code/1, update_code/2, remove_from_code_path/1, + remove_from_code_path/2, cleanup_code_path/1, args_to_tasks/1, expand_env_variable/3, @@ -753,6 +754,9 @@ update_code(Paths, Opts) -> end, Paths). remove_from_code_path(Paths) -> + remove_from_code_path(Paths, purge). + +remove_from_code_path(Paths, Type) when Type == purge; Type == soft_purge -> lists:foreach(fun(Path) -> Name = filename:basename(Path, "/ebin"), App = list_to_atom(Name), @@ -763,7 +767,7 @@ remove_from_code_path(Paths) -> ok; {ok, Modules} -> application:unload(App), - [begin code:purge(M), code:delete(M) end || M <- Modules] + [begin code:Type(M), code:delete(M) end || M <- Modules] end, code:del_path(Path) end, lists:usort(Paths)). -- cgit v1.1 From f12871451f2c60bc35da053198ccc48d1b1db687 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 6 Dec 2017 08:53:12 -0500 Subject: Safer purge switch Rather than the caller having to think of what to purge or not, use erlang:check_process_code/2 to detect if the caller (rebar3) may die because of the operation. If so, do a soft purge with a conditional delete instead of a hard purge with a mandatory delete. --- src/rebar_prv_compile.erl | 2 +- src/rebar_utils.erl | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index 75e6eee..72320fb 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -145,7 +145,7 @@ compile(State, Providers, AppInfo) -> code:add_pathsa(PluginDepsPaths), AppFileCompileResult = rebar_otp_app:compile(State, AppInfo4), %% Clean up after ourselves, leave things as they were. - rebar_utils:remove_from_code_path(PluginDepsPaths, soft_purge), + rebar_utils:remove_from_code_path(PluginDepsPaths), case AppFileCompileResult of {ok, AppInfo5} -> diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 64d4952..b633760 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -49,7 +49,6 @@ update_code/1, update_code/2, remove_from_code_path/1, - remove_from_code_path/2, cleanup_code_path/1, args_to_tasks/1, expand_env_variable/3, @@ -754,9 +753,6 @@ update_code(Paths, Opts) -> end, Paths). remove_from_code_path(Paths) -> - remove_from_code_path(Paths, purge). - -remove_from_code_path(Paths, Type) when Type == purge; Type == soft_purge -> lists:foreach(fun(Path) -> Name = filename:basename(Path, "/ebin"), App = list_to_atom(Name), @@ -767,7 +763,13 @@ remove_from_code_path(Paths, Type) when Type == purge; Type == soft_purge -> ok; {ok, Modules} -> application:unload(App), - [begin code:Type(M), code:delete(M) end || M <- Modules] + [case erlang:check_process_code(self(), M) of + false -> + code:purge(M), code:delete(M); + _ -> + ?DEBUG("~p can't purge ~p safely, doing a soft purge", [self(), M]), + code:soft_purge(M) andalso code:delete(M) + end || M <- Modules] end, code:del_path(Path) end, lists:usort(Paths)). -- cgit v1.1 From d0f958d59d1f87bda8d1d6c724a1af18af288324 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 14 Dec 2017 10:14:53 -0500 Subject: Fix escriptize failure when ebin/ has subdirs Although fairly irregular, the presence of a subdirectory in ebin/ while running rebar3 escriptize would crash the run and generate a dump. Since ebin subdirectories are generally not supported (adding a path for an application only adds ebin/ and no subdirectories to the VM's path set), we just silently ignore the directory altogether. Fixes #1693 --- src/rebar_prv_escriptize.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_escriptize.erl b/src/rebar_prv_escriptize.erl index 1da70f8..3d6e708 100644 --- a/src/rebar_prv_escriptize.erl +++ b/src/rebar_prv_escriptize.erl @@ -194,7 +194,8 @@ load_files(Wildcard, Dir) -> load_files(Prefix, Wildcard, Dir) -> [read_file(Prefix, Filename, Dir) - || Filename <- filelib:wildcard(Wildcard, Dir)]. + || Filename <- filelib:wildcard(Wildcard, Dir), + not filelib:is_dir(filename:join(Dir, Filename))]. read_file(Prefix, Filename, Dir) -> Filename1 = case Prefix of -- cgit v1.1 From e569a2c1b7af599f0d0c0bab873e9a128bebce45 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 22 Dec 2017 15:37:13 -0500 Subject: Bump to 3.5.0 --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 43ad620..9382238 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.5.0"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From b5d7d358ffe68c0cd87715033427b55cd6b2dc4b Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 22 Dec 2017 15:41:54 -0500 Subject: Back to git-based versioning --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 9382238..43ad620 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.5.0"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From fee322edb91bd04fc24287f28d94c08fe0632812 Mon Sep 17 00:00:00 2001 From: Carl-Johan Kjellander Date: Tue, 19 Dec 2017 13:29:53 +0100 Subject: run hooks and plugins during clean for deps find_apps didn't read config files so no hooks were in the app_infos, and now that hooks are being done rebar needs plugins to be able to run clean plugin hooks in deps. --- src/rebar_app_discover.erl | 11 ++++++++--- src/rebar_prv_clean.erl | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index cdd183c..1c02a48 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -331,14 +331,19 @@ create_app_info(AppInfo, AppDir, AppFile) -> AppInfo2 = rebar_app_info:applications( rebar_app_info:app_details(AppInfo1, AppDetails), IncludedApplications++Applications), - Valid = case rebar_app_utils:validate_application_info(AppInfo2) =:= true - andalso rebar_app_info:has_all_artifacts(AppInfo2) =:= true of + C = rebar_config:consult(AppDir), + AppInfo3 = rebar_app_info:update_opts(AppInfo2, + rebar_app_info:opts(AppInfo2), C), + ?DEBUG("create_app_info(~p, ~p, ~p) -> ~n~p~n", + [AppInfo, AppDir, AppFile, AppInfo3]), + Valid = case rebar_app_utils:validate_application_info(AppInfo3) =:= true + andalso rebar_app_info:has_all_artifacts(AppInfo3) =:= true of true -> true; _ -> false end, - rebar_app_info:dir(rebar_app_info:valid(AppInfo2, Valid), AppDir). + rebar_app_info:dir(rebar_app_info:valid(AppInfo3, Valid), AppDir). %% @doc Read in and parse the .app file if it is availabe. Do the same for %% the .app.src file if it exists. diff --git a/src/rebar_prv_clean.erl b/src/rebar_prv_clean.erl index aa0b5af..4da0a64 100644 --- a/src/rebar_prv_clean.erl +++ b/src/rebar_prv_clean.erl @@ -12,7 +12,7 @@ -include("rebar.hrl"). -define(PROVIDER, clean). --define(DEPS, [app_discovery]). +-define(DEPS, [app_discovery, install_deps]). %% =================================================================== %% Public API -- cgit v1.1 From cc70e4fe2ff7b3e318d353972021e548fc513440 Mon Sep 17 00:00:00 2001 From: Tobias Schlager Date: Wed, 24 Jan 2018 12:12:40 +0100 Subject: Issue #1704: Fix alias provider argument passing The way arguments are passed by the alias provider is not compatible with all rebar providers/commands. Especially the release (relx) provider does not like getting its arguments as a plain string. It expects its arguments in a pre-parsed format as returned by getopt:parse/2. Other commands, e.g. eunit, seem to be fine with both ways of argument passing. Therefore, this fix changes the alias provider argument passing to the getopt format in general. --- src/rebar_prv_alias.erl | 69 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 23 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_alias.erl b/src/rebar_prv_alias.erl index 2a03668..736417b 100644 --- a/src/rebar_prv_alias.erl +++ b/src/rebar_prv_alias.erl @@ -70,33 +70,56 @@ example(Alias) -> "rebar3 " ++ atom_to_list(Alias). desc(Cmds) -> - "Equivalent to running: rebar3 do " ++ - rebar_string:join(lists:map(fun({Cmd, Args}) -> - atom_to_list(Cmd) ++ " " ++ Args; - (Cmd) -> - atom_to_list(Cmd) - end, Cmds), ","). + "Equivalent to running: rebar3 do " + ++ rebar_string:join(lists:map(fun to_desc/1, Cmds), ","). + +to_desc({Cmd, Args}) -> + atom_to_list(Cmd) ++ " " ++ Args; +to_desc(Cmd) -> + atom_to_list(Cmd). module(Name) -> - {attribute,1,module,Name}. + {attribute, 1, module, Name}. exports() -> - {attribute,1,export,[{do,1}]}. + {attribute, 1, export, [{do, 1}]}. do_func(Cmds) -> - {function,1,do,1, - [{clause,1, - [{var,1,'State'}], + {function, 1, do, 1, + [{clause, 1, + [{var, 1, 'State'}], [], - [{call,1, - {remote,1,{atom,1,rebar_prv_do},{atom,1,do_tasks}}, - [to_args(Cmds),{var,1,'State'}]}]}]}. - - -to_args([]) -> - {nil,1}; -to_args([{Cmd, Args} | Rest]) -> - {cons,1,{tuple,1,[{string,1,atom_to_list(Cmd)},{string,1,Args}]}, to_args(Rest)}; -to_args([Cmd | Rest]) -> - {cons,1,{tuple,1,[{string,1,atom_to_list(Cmd)},{nil,1}]}, to_args(Rest)}. - + [{call, 1, + {remote, 1, {atom, 1, rebar_prv_do}, {atom, 1, do_tasks}}, + [make_args(Cmds), {var, 1, 'State'}]}]}]}. + +make_args(Cmds) -> + make_list( + lists:map(fun make_tuple/1, + lists:map(fun make_arg/1, Cmds))). + +make_arg({Cmd, Args}) -> + {make_string(Cmd), make_list([make_string(A) || A <- split_args(Args)])}; +make_arg(Cmd) -> + {make_string(Cmd), make_list([])}. + +make_tuple(Tuple) -> + {tuple, 1, tuple_to_list(Tuple)}. + +make_list(List) -> + lists:foldr( + fun(Elem, Acc) -> {cons, 1, Elem, Acc} end, + {nil, 1}, + List). + +make_string(Atom) when is_atom(Atom) -> + make_string(atom_to_list(Atom)); +make_string(String) when is_list(String) -> + {string, 1, String}. + +%% In case someone used the long option format, the option needs to get +%% separated from its value. +split_args(Args) -> + rebar_string:lexemes( + lists:map(fun($=) -> 32; (C) -> C end, Args), + " "). -- cgit v1.1 From 1effb745a38ed0f3d001b24393bf8dc177033af2 Mon Sep 17 00:00:00 2001 From: Shunichi Shinohara Date: Thu, 22 Feb 2018 09:22:07 +0900 Subject: Set unicode option for starndard_io at entry point --- src/rebar3.erl | 1 + src/rebar_prv_deps_tree.erl | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rebar3.erl b/src/rebar3.erl index 2b24bca..80b4e4d 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -116,6 +116,7 @@ run(RawArgs) -> -spec run_aux(rebar_state:t(), [string()]) -> {ok, rebar_state:t()} | {error, term()}. run_aux(State, RawArgs) -> + io:setopts([{encoding, unicode}]), %% Profile override; can only support one profile State1 = case os:getenv("REBAR_PROFILE") of false -> diff --git a/src/rebar_prv_deps_tree.erl b/src/rebar_prv_deps_tree.erl index 7c6978b..07c7972 100644 --- a/src/rebar_prv_deps_tree.erl +++ b/src/rebar_prv_deps_tree.erl @@ -52,14 +52,12 @@ print_deps_tree(SrcDeps, Verbose, State) -> ProjectAppNames = [{rebar_app_info:name(App) ,rebar_utils:vcs_vsn(rebar_app_info:original_vsn(App), rebar_app_info:dir(App), Resources) ,project} || App <- rebar_state:project_apps(State)], - io:setopts([{encoding, unicode}]), case dict:find(root, D) of {ok, Children} -> print_children("", lists:keysort(1, Children++ProjectAppNames), D, Verbose); error -> print_children("", lists:keysort(1, ProjectAppNames), D, Verbose) - end, - io:setopts([{encoding, latin1}]). + end. print_children(_, [], _, _) -> ok; -- cgit v1.1 From 950161357fe96816d1cd3515d232de70b968b15d Mon Sep 17 00:00:00 2001 From: Carl-Johan Kjellander Date: Fri, 19 Jan 2018 11:43:11 +0100 Subject: export env expansion --- src/rebar_utils.erl | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) (limited to 'src') diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index b633760..9bbe54e 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -453,6 +453,24 @@ reread_config(ConfigList) -> "and will be ignored.", []) end. +%% @doc Given env. variable `FOO' we want to expand all references to +%% it in `InStr'. References can have two forms: `$FOO' and `${FOO}' +%% The end of form `$FOO' is delimited with whitespace or EOL +-spec expand_env_variable(string(), string(), term()) -> string(). +expand_env_variable(InStr, VarName, RawVarValue) -> + case rebar_string:chr(InStr, $$) of + 0 -> + %% No variables to expand + InStr; + _ -> + ReOpts = [global, unicode, {return, list}], + VarValue = re:replace(RawVarValue, "\\\\", "\\\\\\\\", ReOpts), + %% Use a regex to match/replace: + %% Given variable "FOO": match $FOO\s | $FOOeol | ${FOO} + RegEx = io_lib:format("\\\$(~ts(\\W|$)|{~ts})", [VarName, VarName]), + re:replace(InStr, RegEx, [VarValue, "\\2"], ReOpts) + end. + %% ==================================================================== %% Internal functions %% ==================================================================== @@ -523,24 +541,6 @@ patch_on_windows(Cmd, Env) -> Cmd end. -%% @doc Given env. variable `FOO' we want to expand all references to -%% it in `InStr'. References can have two forms: `$FOO' and `${FOO}' -%% The end of form `$FOO' is delimited with whitespace or EOL --spec expand_env_variable(string(), string(), term()) -> string(). -expand_env_variable(InStr, VarName, RawVarValue) -> - case rebar_string:chr(InStr, $$) of - 0 -> - %% No variables to expand - InStr; - _ -> - ReOpts = [global, unicode, {return, list}], - VarValue = re:replace(RawVarValue, "\\\\", "\\\\\\\\", ReOpts), - %% Use a regex to match/replace: - %% Given variable "FOO": match $FOO\s | $FOOeol | ${FOO} - RegEx = io_lib:format("\\\$(~ts(\\W|$)|{~ts})", [VarName, VarName]), - re:replace(InStr, RegEx, [VarValue, "\\2"], ReOpts) - end. - expand_sh_flag(return_on_error) -> {error_handler, fun(_Command, Err) -> -- cgit v1.1 From dd9680fe42b42774190c2e95e56d052080f2e939 Mon Sep 17 00:00:00 2001 From: Carl-Johan Kjellander Date: Fri, 23 Feb 2018 15:24:55 +0100 Subject: move and export create_env for use by providers --- src/rebar_env.erl | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++ src/rebar_hooks.erl | 58 +--------------------------------------------- 2 files changed, 67 insertions(+), 57 deletions(-) create mode 100644 src/rebar_env.erl (limited to 'src') diff --git a/src/rebar_env.erl b/src/rebar_env.erl new file mode 100644 index 0000000..dc12419 --- /dev/null +++ b/src/rebar_env.erl @@ -0,0 +1,66 @@ +-module(rebar_env). + +-export([create_env/2]). + +-include("rebar.hrl"). + +%% @doc The following environment variables are exported when running +%% a hook (absolute paths): +%% +%% REBAR_DEPS_DIR = rebar_dir:deps_dir/1 +%% REBAR_BUILD_DIR = rebar_dir:base_dir/1 +%% REBAR_ROOT_DIR = rebar_dir:root_dir/1 +%% REBAR_CHECKOUTS_DIR = rebar_dir:checkouts_dir/1 +%% REBAR_PLUGINS_DIR = rebar_dir:plugins_dir/1 +%% REBAR_GLOBAL_CONFIG_DIR = rebar_dir:global_config_dir/1 +%% REBAR_GLOBAL_CACHE_DIR = rebar_dir:global_cache_dir/1 +%% REBAR_TEMPLATE_DIR = rebar_dir:template_dir/1 +%% REBAR_APP_DIRS = rebar_dir:lib_dirs/1 +%% REBAR_SRC_DIRS = rebar_dir:src_dirs/1 +%% +%% autoconf compatible variables +%% (see: http://www.gnu.org/software/autoconf/manual/autoconf.html#Erlang-Libraries): +%% ERLANG_ERTS_VER = erlang:system_info(version) +%% ERLANG_ROOT_DIR = code:root_dir/0 +%% ERLANG_LIB_DIR_erl_interface = code:lib_dir(erl_interface) +%% ERLANG_LIB_VER_erl_interface = version part of path returned by code:lib_dir(erl_interface) +%% ERL = ERLANG_ROOT_DIR/bin/erl +%% ERLC = ERLANG_ROOT_DIR/bin/erl +%% +-spec create_env(rebar_state:t(), rebar_dict()) -> proplists:proplist(). +create_env(State, Opts) -> + BaseDir = rebar_dir:base_dir(State), + [ + {"REBAR_DEPS_DIR", filename:absname(rebar_dir:deps_dir(State))}, + {"REBAR_BUILD_DIR", filename:absname(rebar_dir:base_dir(State))}, + {"REBAR_ROOT_DIR", filename:absname(rebar_dir:root_dir(State))}, + {"REBAR_CHECKOUTS_DIR", filename:absname(rebar_dir:checkouts_dir(State))}, + {"REBAR_PLUGINS_DIR", filename:absname(rebar_dir:plugins_dir(State))}, + {"REBAR_GLOBAL_CONFIG_DIR", filename:absname(rebar_dir:global_config_dir(State))}, + {"REBAR_GLOBAL_CACHE_DIR", filename:absname(rebar_dir:global_cache_dir(Opts))}, + {"REBAR_TEMPLATE_DIR", filename:absname(rebar_dir:template_dir(State))}, + {"REBAR_APP_DIRS", join_dirs(BaseDir, rebar_dir:lib_dirs(State))}, + {"REBAR_SRC_DIRS", join_dirs(BaseDir, rebar_dir:all_src_dirs(Opts))}, + {"ERLANG_ERTS_VER", erlang:system_info(version)}, + {"ERLANG_ROOT_DIR", code:root_dir()}, + {"ERLANG_LIB_DIR_erl_interface", code:lib_dir(erl_interface)}, + {"ERLANG_LIB_VER_erl_interface", re_version(code:lib_dir(erl_interface))}, + {"ERL", filename:join([code:root_dir(), "bin", "erl"])}, + {"ERLC", filename:join([code:root_dir(), "bin", "erlc"])}, + {"ERLANG_ARCH" , rebar_api:wordsize()}, + {"ERLANG_TARGET", rebar_api:get_arch()} + + ]. + +%% ==================================================================== +%% Internal functions +%% ==================================================================== + +join_dirs(BaseDir, Dirs) -> + rebar_string:join([filename:join(BaseDir, Dir) || Dir <- Dirs], ":"). + +re_version(Path) -> + case re:run(Path, "^.*-(?[^/-]*)$", [{capture,[1],list}, unicode]) of + nomatch -> ""; + {match, [Ver]} -> Ver + end. diff --git a/src/rebar_hooks.erl b/src/rebar_hooks.erl index 8893f2a..ec6fe31 100644 --- a/src/rebar_hooks.erl +++ b/src/rebar_hooks.erl @@ -61,29 +61,6 @@ format_error({bad_provider, Type, Command, {Name, Namespace}}) -> format_error({bad_provider, Type, Command, Name}) -> io_lib:format("Unable to run ~ts hooks for '~p', command '~p' not found.", [Type, Command, Name]). -%% @doc The following environment variables are exported when running -%% a hook (absolute paths): -%% -%% REBAR_DEPS_DIR = rebar_dir:deps_dir/1 -%% REBAR_BUILD_DIR = rebar_dir:base_dir/1 -%% REBAR_ROOT_DIR = rebar_dir:root_dir/1 -%% REBAR_CHECKOUTS_DIR = rebar_dir:checkouts_dir/1 -%% REBAR_PLUGINS_DIR = rebar_dir:plugins_dir/1 -%% REBAR_GLOBAL_CONFIG_DIR = rebar_dir:global_config_dir/1 -%% REBAR_GLOBAL_CACHE_DIR = rebar_dir:global_cache_dir/1 -%% REBAR_TEMPLATE_DIR = rebar_dir:template_dir/1 -%% REBAR_APP_DIRS = rebar_dir:lib_dirs/1 -%% REBAR_SRC_DIRS = rebar_dir:src_dirs/1 -%% -%% autoconf compatible variables -%% (see: http://www.gnu.org/software/autoconf/manual/autoconf.html#Erlang-Libraries): -%% ERLANG_ERTS_VER = erlang:system_info(version) -%% ERLANG_ROOT_DIR = code:root_dir/0 -%% ERLANG_LIB_DIR_erl_interface = code:lib_dir(erl_interface) -%% ERLANG_LIB_VER_erl_interface = version part of path returned by code:lib_dir(erl_interface) -%% ERL = ERLANG_ROOT_DIR/bin/erl -%% ERLC = ERLANG_ROOT_DIR/bin/erl -%% run_hooks(Dir, pre, Command, Opts, State) -> run_hooks(Dir, pre_hooks, Command, Opts, State); run_hooks(Dir, post, Command, Opts, State) -> @@ -94,7 +71,7 @@ run_hooks(Dir, Type, Command, Opts, State) -> ?DEBUG("run_hooks(~p, ~p, ~p) -> no hooks defined\n", [Dir, Type, Command]), ok; Hooks -> - Env = create_env(State, Opts), + Env = rebar_env:create_env(State, Opts), lists:foreach(fun({_, C, _}=Hook) when C =:= Command -> apply_hook(Dir, Env, Hook); ({C, _}=Hook) when C =:= Command -> @@ -114,36 +91,3 @@ apply_hook(Dir, Env, {Arch, Command, Hook}) -> apply_hook(Dir, Env, {Command, Hook}) -> Msg = lists:flatten(io_lib:format("Hook for ~p failed!~n", [Command])), rebar_utils:sh(Hook, [use_stdout, {cd, Dir}, {env, Env}, {abort_on_error, Msg}]). - -create_env(State, Opts) -> - BaseDir = rebar_dir:base_dir(State), - [ - {"REBAR_DEPS_DIR", filename:absname(rebar_dir:deps_dir(State))}, - {"REBAR_BUILD_DIR", filename:absname(rebar_dir:base_dir(State))}, - {"REBAR_ROOT_DIR", filename:absname(rebar_dir:root_dir(State))}, - {"REBAR_CHECKOUTS_DIR", filename:absname(rebar_dir:checkouts_dir(State))}, - {"REBAR_PLUGINS_DIR", filename:absname(rebar_dir:plugins_dir(State))}, - {"REBAR_GLOBAL_CONFIG_DIR", filename:absname(rebar_dir:global_config_dir(State))}, - {"REBAR_GLOBAL_CACHE_DIR", filename:absname(rebar_dir:global_cache_dir(Opts))}, - {"REBAR_TEMPLATE_DIR", filename:absname(rebar_dir:template_dir(State))}, - {"REBAR_APP_DIRS", join_dirs(BaseDir, rebar_dir:lib_dirs(State))}, - {"REBAR_SRC_DIRS", join_dirs(BaseDir, rebar_dir:all_src_dirs(Opts))}, - {"ERLANG_ERTS_VER", erlang:system_info(version)}, - {"ERLANG_ROOT_DIR", code:root_dir()}, - {"ERLANG_LIB_DIR_erl_interface", code:lib_dir(erl_interface)}, - {"ERLANG_LIB_VER_erl_interface", re_version(code:lib_dir(erl_interface))}, - {"ERL", filename:join([code:root_dir(), "bin", "erl"])}, - {"ERLC", filename:join([code:root_dir(), "bin", "erlc"])}, - {"ERLANG_ARCH" , rebar_api:wordsize()}, - {"ERLANG_TARGET", rebar_api:get_arch()} - - ]. - -join_dirs(BaseDir, Dirs) -> - rebar_string:join([filename:join(BaseDir, Dir) || Dir <- Dirs], ":"). - -re_version(Path) -> - case re:run(Path, "^.*-(?[^/-]*)$", [{capture,[1],list}, unicode]) of - nomatch -> ""; - {match, [Ver]} -> Ver - end. -- cgit v1.1 From 6d29233a16ae94cf4618ddadb381f91e4e5eb882 Mon Sep 17 00:00:00 2001 From: Carl-Johan Kjellander Date: Fri, 23 Feb 2018 15:29:05 +0100 Subject: remove whitespace --- src/rebar_env.erl | 1 - 1 file changed, 1 deletion(-) (limited to 'src') diff --git a/src/rebar_env.erl b/src/rebar_env.erl index dc12419..10bc5b8 100644 --- a/src/rebar_env.erl +++ b/src/rebar_env.erl @@ -49,7 +49,6 @@ create_env(State, Opts) -> {"ERLC", filename:join([code:root_dir(), "bin", "erlc"])}, {"ERLANG_ARCH" , rebar_api:wordsize()}, {"ERLANG_TARGET", rebar_api:get_arch()} - ]. %% ==================================================================== -- cgit v1.1 From 57768ba22ed39d2b099ac09c834ed90f422f2336 Mon Sep 17 00:00:00 2001 From: Carl-Johan Kjellander Date: Mon, 26 Feb 2018 16:06:51 +0100 Subject: make it easier to create env --- src/rebar_env.erl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_env.erl b/src/rebar_env.erl index 10bc5b8..eea47de 100644 --- a/src/rebar_env.erl +++ b/src/rebar_env.erl @@ -1,6 +1,7 @@ -module(rebar_env). --export([create_env/2]). +-export([create_env/1, + create_env/2]). -include("rebar.hrl"). @@ -27,6 +28,12 @@ %% ERL = ERLANG_ROOT_DIR/bin/erl %% ERLC = ERLANG_ROOT_DIR/bin/erl %% + +-spec create_env(rebar_state:t()) -> proplists:proplist(). +create_env(State) -> + Opts = rebar_state:opts(State), + create_env(State, Opts). + -spec create_env(rebar_state:t(), rebar_dict()) -> proplists:proplist(). create_env(State, Opts) -> BaseDir = rebar_dir:base_dir(State), -- cgit v1.1 From 8229d15b6c8384e8d974b32d587b05d64fce185b Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Mon, 26 Feb 2018 15:31:42 -0800 Subject: rebar_package: do not return first package version for constraint with no match --- src/rebar_packages.erl | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index d17b54f..7910fe2 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -170,8 +170,13 @@ find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State) -> try find_all(Dep, Table, State) of {ok, [Vsn]} -> handle_single_vsn(Pkg, PkgVsn, Dep, Vsn, Constraint); - {ok, [HeadVsn | VsnTail]} -> - {ok, handle_vsns(Constraint, HeadVsn, VsnTail)} + {ok, Vsns} -> + case handle_vsns(Constraint, Vsns) of + none -> + none; + FoundVsn -> + {ok, FoundVsn} + end catch error:badarg -> none @@ -189,16 +194,16 @@ find_all(Dep, Table, State) -> none end. -handle_vsns(Constraint, HeadVsn, VsnTail) -> +handle_vsns(Constraint, Vsns) -> lists:foldl(fun(Version, Highest) -> case ec_semver:pes(Version, Constraint) andalso - ec_semver:gt(Version, Highest) of + (Highest =:= none orelse ec_semver:gt(Version, Highest)) of true -> Version; false -> Highest end - end, HeadVsn, VsnTail). + end, none, Vsns). handle_single_vsn(Pkg, PkgVsn, Dep, Vsn, Constraint) -> case ec_semver:pes(Vsn, Constraint) of -- cgit v1.1 From d839a0be8dbd6355dc7d71b1e0e2f425b3f47fe9 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 27 Feb 2018 13:26:21 -0800 Subject: don't attempt retry of missing packages when updating index --- src/rebar_packages.erl | 1 + src/rebar_prv_update.erl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 7910fe2..50d4025 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -9,6 +9,7 @@ ,registry_checksum/2 ,find_highest_matching/6 ,find_highest_matching/4 + ,find_highest_matching_/6 ,find_all/3 ,verify_table/1 ,format_error/1]). diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index 79804d3..6dee7fd 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -206,7 +206,7 @@ valid_vsn(Vsn) -> re:run(Vsn, SupportedVersions, [unicode]) =/= nomatch. highest_matching({Pkg, PkgVsn, Dep, App}, Vsn, HexRegistry, State, DepsListAcc) -> - case rebar_packages:find_highest_matching(Pkg, PkgVsn, Dep, Vsn, HexRegistry, State) of + case rebar_packages:find_highest_matching_(Pkg, PkgVsn, Dep, Vsn, HexRegistry, State) of {ok, HighestDepVsn} -> [{App, {pkg, Dep, HighestDepVsn, undefined}} | DepsListAcc]; none -> -- cgit v1.1 From c59c5dd3a85e8b531db06d17b15aa58a5d1d2652 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 27 Feb 2018 14:12:26 -0800 Subject: use debug level for logs about bad package versioning --- src/rebar_prv_update.erl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index 6dee7fd..6dde024 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -151,8 +151,8 @@ update_deps_list(Pkg, PkgVsn, Deps, HexRegistry, State) -> %% and doubled since spaces seem not to be %% enforced {false, Vsn} -> - ?WARN("[~ts:~ts], Bad dependency version for ~ts: ~ts.", - [Pkg, PkgVsn, Dep, Vsn]), + ?DEBUG("[~ts:~ts], Bad dependency version for ~ts: ~ts.", + [Pkg, PkgVsn, Dep, Vsn]), DepsListAcc; {_, <<"~>", Vsn/binary>>} -> highest_matching(Dep1, rm_ws(Vsn), HexRegistry, @@ -210,7 +210,7 @@ highest_matching({Pkg, PkgVsn, Dep, App}, Vsn, HexRegistry, State, DepsListAcc) {ok, HighestDepVsn} -> [{App, {pkg, Dep, HighestDepVsn, undefined}} | DepsListAcc]; none -> - ?WARN("[~ts:~ts] Missing registry entry for package ~ts. Try to fix with `rebar3 update`", + ?DEBUG("[~ts:~ts] Missing registry entry for package ~ts. Try to fix with `rebar3 update`", [Pkg, PkgVsn, Dep]), DepsListAcc end. @@ -221,8 +221,8 @@ cmp({_Pkg, _PkgVsn, Dep, _App} = Dep1, Vsn, HexRegistry, State, DepsListAcc, Cmp cmp_(undefined, _MinVsn, [], DepsListAcc, {Pkg, PkgVsn, Dep, _App}, _CmpFun) -> - ?WARN("[~ts:~ts] Missing registry entry for package ~ts. Try to fix with `rebar3 update`", - [Pkg, PkgVsn, Dep]), + ?DEBUG("[~ts:~ts] Missing registry entry for package ~ts. Try to fix with `rebar3 update`", + [Pkg, PkgVsn, Dep]), DepsListAcc; cmp_(HighestDepVsn, _MinVsn, [], DepsListAcc, {_Pkg, _PkgVsn, Dep, App}, _CmpFun) -> [{App, {pkg, Dep, HighestDepVsn, undefined}} | DepsListAcc]; @@ -242,8 +242,8 @@ cmpl({_Pkg, _PkgVsn, Dep, _App} = Dep1, Vsn, HexRegistry, State, DepsListAcc, Cm cmpl_(undefined, Vsn, Vsns, DepsListAcc, Dep1, CmpFun). cmpl_(undefined, _MaxVsn, [], DepsListAcc, {Pkg, PkgVsn, Dep, _App}, _CmpFun) -> - ?WARN("[~ts:~ts] Missing registry entry for package ~ts. Try to fix with `rebar3 update`", - [Pkg, PkgVsn, Dep]), + ?DEBUG("[~ts:~ts] Missing registry entry for package ~ts. Try to fix with `rebar3 update`", + [Pkg, PkgVsn, Dep]), DepsListAcc; cmpl_(HighestDepVsn, _MaxVsn, [], DepsListAcc, {_Pkg, _PkgVsn, Dep, App}, _CmpFun) -> -- cgit v1.1 From 3146a285fb1c9e0028a9c09977c6da11df76d6e6 Mon Sep 17 00:00:00 2001 From: Pierre Fenoll Date: Sat, 10 Feb 2018 23:02:32 +0100 Subject: sort-as: force an order on multiple profiles --- src/rebar_dir.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_dir.erl b/src/rebar_dir.erl index d7be423..ee4d3d4 100644 --- a/src/rebar_dir.erl +++ b/src/rebar_dir.erl @@ -49,7 +49,7 @@ profile_dir(Opts, Profiles) -> %% of profiles to match order passed to `as` ["default"|Rest] -> {rebar_opts:get(Opts, base_dir, ?DEFAULT_BASE_DIR), Rest} end, - ProfilesDir = rebar_string:join(ProfilesStrings, "+"), + ProfilesDir = rebar_string:join(lists:sort(ProfilesStrings), "+"), filename:join(BaseDir, ProfilesDir). %% @doc returns the directory where dependencies should be placed -- cgit v1.1 From 9f0e3a2e5b6ea3eccda14660bc64570738ee7796 Mon Sep 17 00:00:00 2001 From: Pierre Fenoll Date: Fri, 16 Feb 2018 17:48:17 +0100 Subject: Revert "sort-as: force an order on multiple profiles" This reverts commit 3f8dd5eacebb913144f3615fdf44658b6223a791. --- src/rebar_dir.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_dir.erl b/src/rebar_dir.erl index ee4d3d4..d7be423 100644 --- a/src/rebar_dir.erl +++ b/src/rebar_dir.erl @@ -49,7 +49,7 @@ profile_dir(Opts, Profiles) -> %% of profiles to match order passed to `as` ["default"|Rest] -> {rebar_opts:get(Opts, base_dir, ?DEFAULT_BASE_DIR), Rest} end, - ProfilesDir = rebar_string:join(lists:sort(ProfilesStrings), "+"), + ProfilesDir = rebar_string:join(ProfilesStrings, "+"), filename:join(BaseDir, ProfilesDir). %% @doc returns the directory where dependencies should be placed -- cgit v1.1 From e504ba71e152a1e47719269f0108ad47c8f3f8b9 Mon Sep 17 00:00:00 2001 From: Pierre Fenoll Date: Sat, 17 Feb 2018 01:20:29 +0100 Subject: sort-as: found the issue. Will look into tests now --- src/rebar_state.erl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/rebar_state.erl b/src/rebar_state.erl index 577ed23..7ebe6da 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -87,8 +87,8 @@ new(Config) when is_list(Config) -> opts = Opts }. -spec new(t() | atom(), list()) -> t(). -new(Profile, Config) when is_atom(Profile) - , is_list(Config) -> +new(Profile, Config) when is_atom(Profile), + is_list(Config) -> BaseState = base_state(), Opts = base_opts(Config), BaseState#state_t { dir = rebar_dir:get_cwd(), @@ -283,11 +283,12 @@ apply_profiles(State=#state_t{default = Defaults, current_profiles=CurrentProfil end, Defaults, AppliedProfiles), State#state_t{current_profiles = AppliedProfiles, opts=NewOpts}. +%% @doc A stable deduplicator. deduplicate(Profiles) -> - do_deduplicate(lists:reverse(Profiles), []). + do_deduplicate(Profiles, []). do_deduplicate([], Acc) -> - Acc; + lists:reverse(Acc); do_deduplicate([Head | Rest], Acc) -> case lists:member(Head, Acc) of true -> do_deduplicate(Rest, Acc); -- cgit v1.1 From abad0e14bb5a6ea5df21f2732bd72bf6efb0ca1e Mon Sep 17 00:00:00 2001 From: Pierre Fenoll Date: Fri, 23 Feb 2018 04:07:24 +0100 Subject: sort-as: show issue more clearly --- src/rebar_state.erl | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src') diff --git a/src/rebar_state.erl b/src/rebar_state.erl index 7ebe6da..8c100aa 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -43,6 +43,10 @@ allow_provider_overrides/1, allow_provider_overrides/2 ]). +-ifdef(TEST). +-export([deduplicate/1]). +-endif. + -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). -- cgit v1.1 From 6aeff6300bf8986079ac6e8f83fa37150e239810 Mon Sep 17 00:00:00 2001 From: Pierre Fenoll Date: Fri, 23 Feb 2018 12:50:29 +0100 Subject: Revert "sort-as: show issue more clearly" This reverts commit 4ad1db97336a3ac070880876ada07d4c7b769327. --- src/rebar_state.erl | 4 ---- 1 file changed, 4 deletions(-) (limited to 'src') diff --git a/src/rebar_state.erl b/src/rebar_state.erl index 8c100aa..7ebe6da 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -43,10 +43,6 @@ allow_provider_overrides/1, allow_provider_overrides/2 ]). --ifdef(TEST). --export([deduplicate/1]). --endif. - -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). -- cgit v1.1 From 1d2aa24ef0d0bc6d1177cce361d7718f887eb03e Mon Sep 17 00:00:00 2001 From: Pierre Fenoll Date: Fri, 23 Feb 2018 12:50:38 +0100 Subject: Revert "sort-as: found the issue. Will look into tests now" This reverts commit 0f7e6c31e97c238649e7ae0a1b7087e342174ecc. --- src/rebar_state.erl | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/rebar_state.erl b/src/rebar_state.erl index 7ebe6da..577ed23 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -87,8 +87,8 @@ new(Config) when is_list(Config) -> opts = Opts }. -spec new(t() | atom(), list()) -> t(). -new(Profile, Config) when is_atom(Profile), - is_list(Config) -> +new(Profile, Config) when is_atom(Profile) + , is_list(Config) -> BaseState = base_state(), Opts = base_opts(Config), BaseState#state_t { dir = rebar_dir:get_cwd(), @@ -283,12 +283,11 @@ apply_profiles(State=#state_t{default = Defaults, current_profiles=CurrentProfil end, Defaults, AppliedProfiles), State#state_t{current_profiles = AppliedProfiles, opts=NewOpts}. -%% @doc A stable deduplicator. deduplicate(Profiles) -> - do_deduplicate(Profiles, []). + do_deduplicate(lists:reverse(Profiles), []). do_deduplicate([], Acc) -> - lists:reverse(Acc); + Acc; do_deduplicate([Head | Rest], Acc) -> case lists:member(Head, Acc) of true -> do_deduplicate(Rest, Acc); -- cgit v1.1 From 0d88ff8891d1545721b7b61bc866713a13d0905a Mon Sep 17 00:00:00 2001 From: Pierre Fenoll Date: Fri, 23 Feb 2018 13:06:39 +0100 Subject: do not append test profile if already there. Note that it comes from the prv list passed to providers:create/1 --- src/rebar_state.erl | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src') diff --git a/src/rebar_state.erl b/src/rebar_state.erl index 577ed23..ac77325 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -257,12 +257,15 @@ apply_profiles(State, Profile) when not is_list(Profile) -> apply_profiles(State, [default]) -> State; apply_profiles(State=#state_t{default = Defaults, current_profiles=CurrentProfiles}, Profiles) -> + IsTesting = lists:member(test, CurrentProfiles), AppliedProfiles = case Profiles of %% Head of list global profile is special, only for use by rebar3 %% It does not clash if a user does `rebar3 as global...` but when %% it is the head we must make sure not to prepend `default` [global | _] -> Profiles; + [test] when IsTesting -> + deduplicate(CurrentProfiles); _ -> deduplicate(CurrentProfiles ++ Profiles) end, -- cgit v1.1 From 1271458c511100a5402a7aec94e48904ad792ac7 Mon Sep 17 00:00:00 2001 From: Pierre Fenoll Date: Sun, 25 Feb 2018 13:39:45 +0100 Subject: sort-as: a more general pattern --- src/rebar_state.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_state.erl b/src/rebar_state.erl index ac77325..dd1f43f 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -257,14 +257,14 @@ apply_profiles(State, Profile) when not is_list(Profile) -> apply_profiles(State, [default]) -> State; apply_profiles(State=#state_t{default = Defaults, current_profiles=CurrentProfiles}, Profiles) -> - IsTesting = lists:member(test, CurrentProfiles), + ProvidedProfiles = lists:prefix([default|Profiles], CurrentProfiles), AppliedProfiles = case Profiles of %% Head of list global profile is special, only for use by rebar3 %% It does not clash if a user does `rebar3 as global...` but when %% it is the head we must make sure not to prepend `default` [global | _] -> Profiles; - [test] when IsTesting -> + _ when ProvidedProfiles -> deduplicate(CurrentProfiles); _ -> deduplicate(CurrentProfiles ++ Profiles) -- cgit v1.1 From 4a35276bdf93869f95344eaafb2571d3c731a615 Mon Sep 17 00:00:00 2001 From: Jean Bouchard Date: Fri, 16 Mar 2018 14:23:43 -0400 Subject: remove the .app from app_info --- src/rebar_app_discover.erl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index 1c02a48..441521b 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -411,12 +411,13 @@ try_handle_app_src_file(_AppInfo, _, _AppDir, _AppSrcFile, valid) -> false; try_handle_app_src_file(AppInfo, _, AppDir, [File], Validate) when Validate =:= invalid ; Validate =:= all -> - AppInfo1 = create_app_info(AppInfo, AppDir, File), + AppInfo1 = rebar_app_info:app_file(AppInfo, undefined), + AppInfo2 = create_app_info(AppInfo1, AppDir, File), case filename:extension(File) of ".script" -> - {true, rebar_app_info:app_file_src_script(AppInfo1, File)}; + {true, rebar_app_info:app_file_src_script(AppInfo2, File)}; _ -> - {true, rebar_app_info:app_file_src(AppInfo1, File)} + {true, rebar_app_info:app_file_src(AppInfo2, File)} end; try_handle_app_src_file(_AppInfo, _, _AppDir, Other, _Validate) -> throw({error, {multiple_app_files, Other}}). -- cgit v1.1 From 15ec1ce305065ae0cb8b659b29111c7f275eeff7 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 30 Mar 2018 10:59:59 -0400 Subject: Remove noisy debug message It makes things worse than if it weren't there. Fixes issue #1726 --- src/rebar_app_discover.erl | 2 -- 1 file changed, 2 deletions(-) (limited to 'src') diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index 441521b..382b36b 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -334,8 +334,6 @@ create_app_info(AppInfo, AppDir, AppFile) -> C = rebar_config:consult(AppDir), AppInfo3 = rebar_app_info:update_opts(AppInfo2, rebar_app_info:opts(AppInfo2), C), - ?DEBUG("create_app_info(~p, ~p, ~p) -> ~n~p~n", - [AppInfo, AppDir, AppFile, AppInfo3]), Valid = case rebar_app_utils:validate_application_info(AppInfo3) =:= true andalso rebar_app_info:has_all_artifacts(AppInfo3) =:= true of true -> -- cgit v1.1 From e754621e73db9533f9670124206b91869b2b35b6 Mon Sep 17 00:00:00 2001 From: tothlac Date: Mon, 9 Apr 2018 17:45:51 +0200 Subject: (#1741): Fix quotes in etag values --- src/rebar_pkg_resource.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 60ad8f9..7d42ccd 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -109,7 +109,8 @@ make_vsn(_) -> request(Url, ETag) -> HttpOptions = [{ssl, ssl_opts(Url)}, {relaxed, true} | rebar_utils:get_proxy_auth()], - case httpc:request(get, {Url, [{"if-none-match", ETag} || ETag =/= false]++[{"User-Agent", rebar_utils:user_agent()}]}, + case httpc:request(get, {Url, [{"if-none-match", "\"" ++ ETag ++ "\""} || ETag =/= false]++ + [{"User-Agent", rebar_utils:user_agent()}]}, HttpOptions, [{body_format, binary}], rebar) of -- cgit v1.1 From a03da56fb39bddcf6b5ef75cb5b5e40b5b5ab1d9 Mon Sep 17 00:00:00 2001 From: Laszlo Toth Date: Wed, 11 Apr 2018 14:03:57 +0200 Subject: (#1743): store the etags in cache files --- src/rebar_pkg_resource.erl | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 7d42ccd..f77a81a 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -6,6 +6,7 @@ -export([lock/2 ,download/3 + ,download/4 ,needs_update/2 ,make_vsn/1]). @@ -13,6 +14,8 @@ ,etag/1 ,ssl_opts/1]). +-export([store_etag_in_cache/2]). + -include("rebar.hrl"). -include_lib("public_key/include/OTP-PUB-KEY.hrl"). @@ -28,27 +31,34 @@ needs_update(Dir, {pkg, _Name, Vsn, _Hash}) -> true end. -download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State) -> +download(TmpDir, Pkg, State) -> + download(TmpDir, Pkg, State, true). + +download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State, UpdateETag) -> CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), {ok, PackageDir} = rebar_packages:package_dir(State), Package = binary_to_list(<>), + ETagFile = binary_to_list(<>), CachePath = filename:join(PackageDir, Package), + ETagPath = filename:join(PackageDir, ETagFile), case rebar_utils:url_append_path(CDN, filename:join(?REMOTE_PACKAGE_DIR, Package)) of {ok, Url} -> - cached_download(TmpDir, CachePath, Pkg, Url, etag(CachePath), State); + cached_download(TmpDir, CachePath, Pkg, Url, etag(ETagPath), State, ETagPath, UpdateETag); _ -> {fetch_fail, Name, Vsn} end. -cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, Url, ETag, State) -> +cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, Url, ETag, State, ETagPath, UpdateETag) -> case request(Url, ETag) of {ok, cached} -> ?INFO("Version cached at ~ts is up to date, reusing it", [CachePath]), serve_from_cache(TmpDir, CachePath, Pkg, State); {ok, Body, NewETag} -> ?INFO("Downloaded package, caching at ~ts", [CachePath]), - serve_from_download(TmpDir, CachePath, Pkg, NewETag, Body, State); + maybe_store_etag_in_cache(UpdateETag, ETagPath, NewETag), + serve_from_download(TmpDir, CachePath, Pkg, NewETag, Body, State, ETagPath); error when ETag =/= false -> + store_etag_in_cache(ETagPath, ETag), ?INFO("Download error, using cached file at ~ts", [CachePath]), serve_from_cache(TmpDir, CachePath, Pkg, State); error -> @@ -75,10 +85,10 @@ serve_from_cache(TmpDir, CachePath, Pkg, State) -> {bad_checksum, CachePath} end. -serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State) -> +serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State, ETagPath) -> ?DEBUG("Writing ~p to cache at ~ts", [Package, CachePath]), file:write_file(CachePath, Binary), - case etag(CachePath) of + case etag(ETagPath) of ETag -> serve_from_cache(TmpDir, CachePath, Package, State); FileETag -> @@ -86,7 +96,6 @@ serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State) -> {bad_download, CachePath} end. - extract(TmpDir, CachePath) -> ec_file:mkdir_p(TmpDir), {ok, Files} = erl_tar:extract(CachePath, [memory]), @@ -131,9 +140,8 @@ request(Url, ETag) -> etag(Path) -> case file:read_file(Path) of - {ok, Binary} -> - <> = crypto:hash(md5, Binary), - rebar_string:lowercase(lists:flatten(io_lib:format("~32.16.0b", [X]))); + {ok, Bin} -> + binary_to_list(Bin); {error, _} -> false end. @@ -216,3 +224,11 @@ version_pad([Major, Minor, Patch]) -> {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}; version_pad([Major, Minor, Patch | _]) -> {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}. + +maybe_store_etag_in_cache(false = _UpdateETag, _Path, _ETag) -> + ok; +maybe_store_etag_in_cache(true = _UpdateETag, Path, ETag) -> + store_etag_in_cache(Path, ETag). + +store_etag_in_cache(Path, ETag) -> + ok = file:write_file(Path, ETag). -- cgit v1.1 From 97c48e86bb7ca8969fd19c999f55259e88d4a6db Mon Sep 17 00:00:00 2001 From: Laszlo Toth Date: Wed, 11 Apr 2018 17:03:34 +0200 Subject: (#1743): Add specs for dialyzer --- src/rebar_pkg_resource.erl | 121 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 118 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index f77a81a..8806c8b 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -19,9 +19,26 @@ -include("rebar.hrl"). -include_lib("public_key/include/OTP-PUB-KEY.hrl"). +-type cached_result() :: {'bad_checksum',string()} | + {'bad_registry_checksum',string()} | + {'failed_extract',string()} | + {'ok','true'} | + {'unexpected_hash',string(),_,binary()}. + +-type download_result() :: {bad_download, binary() | string()} | + {fetch_fail, _, _} | cached_result(). + +-spec lock(AppDir, Source) -> Res when + AppDir :: file:name(), + Source :: tuple(), + Res :: {atom(), string(), any()}. lock(_AppDir, Source) -> Source. +-spec needs_update(Dir, Pkg) -> Res when + Dir :: file:name(), + Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + Res :: boolean(). needs_update(Dir, {pkg, _Name, Vsn, _Hash}) -> [AppInfo] = rebar_app_discover:find_apps([Dir], all), case rebar_app_info:original_vsn(AppInfo) =:= rebar_utils:to_list(Vsn) of @@ -31,9 +48,20 @@ needs_update(Dir, {pkg, _Name, Vsn, _Hash}) -> true end. +-spec download(TmpDir, Pkg, State) -> Res when + TmpDir :: file:name(), + Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + State :: rebar_state:t(), + Res :: {'error',_} | {'ok',_} | {'tarball',binary() | string()}. download(TmpDir, Pkg, State) -> download(TmpDir, Pkg, State, true). +-spec download(TmpDir, Pkg, State, UpdateETag) -> Res when + TmpDir :: file:name(), + Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + State :: rebar_state:t(), + UpdateETag :: boolean(), + Res :: download_result(). download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State, UpdateETag) -> CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), {ok, PackageDir} = rebar_packages:package_dir(State), @@ -48,6 +76,16 @@ download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State, UpdateETag) -> {fetch_fail, Name, Vsn} end. +-spec cached_download(TmpDir, CachePath, Pkg, Url, ETag, State, ETagPath, UpdateETag) -> Res when + TmpDir :: file:name(), + CachePath :: file:name(), + Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + Url :: string(), + ETag :: false | string(), + State :: rebar_state:t(), + ETagPath :: file:name(), + UpdateETag :: boolean(), + Res :: download_result(). cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, Url, ETag, State, ETagPath, UpdateETag) -> case request(Url, ETag) of {ok, cached} -> @@ -65,6 +103,12 @@ cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, Url, ETag, State {fetch_fail, Name, Vsn} end. +-spec serve_from_cache(TmpDir, CachePath, Pkg, State) -> Res when + TmpDir :: file:name(), + CachePath :: file:name(), + Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + State :: rebar_state:t(), + Res :: cached_result(). serve_from_cache(TmpDir, CachePath, Pkg, State) -> {Files, Contents, Version, Meta} = extract(TmpDir, CachePath), case checksums(Pkg, Files, Contents, Version, Meta, State) of @@ -85,6 +129,15 @@ serve_from_cache(TmpDir, CachePath, Pkg, State) -> {bad_checksum, CachePath} end. +-spec serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State, ETagPath) -> Res when + TmpDir :: file:name(), + CachePath :: file:name(), + Package :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + ETag :: string(), + Binary :: binary(), + State :: rebar_state:t(), + ETagPath :: file:name(), + Res :: download_result(). serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State, ETagPath) -> ?DEBUG("Writing ~p to cache at ~ts", [Package, CachePath]), file:write_file(CachePath, Binary), @@ -96,6 +149,14 @@ serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State, ETagPath) - {bad_download, CachePath} end. +-spec extract(TmpDir, CachePath) -> Res when + TmpDir :: file:name(), + CachePath :: file:name(), + Res :: {Files, Contents, Version, Meta}, + Files :: list({file:name(), binary()}), + Contents :: binary(), + Version :: binary(), + Meta :: binary(). extract(TmpDir, CachePath) -> ec_file:mkdir_p(TmpDir), {ok, Files} = erl_tar:extract(CachePath, [memory]), @@ -104,6 +165,18 @@ extract(TmpDir, CachePath) -> {"metadata.config", Meta} = lists:keyfind("metadata.config", 1, Files), {Files, Contents, Version, Meta}. +-spec checksums(Pkg, Files, Contents, Version, Meta, State) -> Res when + Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + Files :: list({file:name(), binary()}), + Contents :: binary(), + Version :: binary(), + Meta :: binary(), + State :: rebar_state:t(), + Res :: {Hash, BinChecksum, RegistryChecksum, TarChecksum}, + Hash :: binary(), + BinChecksum :: binary(), + RegistryChecksum :: any(), + TarChecksum :: binary(). checksums(Pkg={pkg, _Name, _Vsn, Hash}, Files, Contents, Version, Meta, State) -> Blob = <>, <> = crypto:hash(sha256, Blob), @@ -112,12 +185,18 @@ checksums(Pkg={pkg, _Name, _Vsn, Hash}, Files, Contents, Version, Meta, State) - {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files), {Hash, BinChecksum, RegistryChecksum, TarChecksum}. +-spec make_vsn(Vsn) -> Res when + Vsn :: any(), + Res :: {'error',[1..255,...]}. make_vsn(_) -> {error, "Replacing version of type pkg not supported."}. +-spec request(Url, ETag) -> Res when + Url :: string(), + ETag :: false | string(), + Res :: 'error' | {ok, cached} | {ok, any(), string()}. request(Url, ETag) -> HttpOptions = [{ssl, ssl_opts(Url)}, {relaxed, true} | rebar_utils:get_proxy_auth()], - case httpc:request(get, {Url, [{"if-none-match", "\"" ++ ETag ++ "\""} || ETag =/= false]++ [{"User-Agent", rebar_utils:user_agent()}]}, HttpOptions, @@ -138,6 +217,9 @@ request(Url, ETag) -> error end. +-spec etag(Path) -> Res when + Path :: file:name(), + Res :: false | string(). etag(Path) -> case file:read_file(Path) of {ok, Bin} -> @@ -148,7 +230,9 @@ etag(Path) -> %% @doc returns the SSL options adequate for the project based on %% its configuration, including for validation of certs. --spec ssl_opts(string()) -> [term()]. +-spec ssl_opts(Url) -> Res when + Url :: string(), + Res :: proplists:proplist(). ssl_opts(Url) -> case get_ssl_config() of ssl_verify_enabled -> @@ -159,7 +243,10 @@ ssl_opts(Url) -> %% @doc returns the SSL options adequate for the project based on %% its configuration, including for validation of certs. --spec ssl_opts(atom(), string()) -> [term()]. +-spec ssl_opts(Enabled, Url) -> Res when + Enabled :: atom(), + Url :: string(), + Res :: proplists:proplist(). ssl_opts(ssl_verify_enabled, Url) -> case check_ssl_version() of true -> @@ -173,6 +260,9 @@ ssl_opts(ssl_verify_enabled, Url) -> [{verify, verify_none}] end. +-spec partial_chain(Certs) -> Res when + Certs :: list(any()), + Res :: unknown_ca | {trusted_ca, any()}. partial_chain(Certs) -> Certs1 = [{Cert, public_key:pkix_decode_cert(Cert, otp)} || Cert <- Certs], CACerts = certifi:cacerts(), @@ -187,14 +277,23 @@ partial_chain(Certs) -> unknown_ca end. +-spec extract_public_key_info(Cert) -> Res when + Cert :: #'OTPCertificate'{tbsCertificate::#'OTPTBSCertificate'{}}, + Res :: any(). extract_public_key_info(Cert) -> ((Cert#'OTPCertificate'.tbsCertificate)#'OTPTBSCertificate'.subjectPublicKeyInfo). +-spec check_cert(CACerts, Cert) -> Res when + CACerts :: list(any()), + Cert :: any(), + Res :: boolean(). check_cert(CACerts, Cert) -> lists:any(fun(CACert) -> extract_public_key_info(CACert) == extract_public_key_info(Cert) end, CACerts). +-spec check_ssl_version() -> + boolean(). check_ssl_version() -> case application:get_key(ssl, vsn) of {ok, Vsn} -> @@ -203,6 +302,8 @@ check_ssl_version() -> false end. +-spec get_ssl_config() -> + ssl_verify_disabled | ssl_verify_enabled. get_ssl_config() -> GlobalConfigFile = rebar_dir:global_config(), Config = rebar_config:consult_file(GlobalConfigFile), @@ -213,9 +314,14 @@ get_ssl_config() -> ssl_verify_enabled end. +-spec parse_vsn(Vsn) -> Res when + Vsn :: string(), + Res :: {integer(), integer(), integer()}. parse_vsn(Vsn) -> version_pad(rebar_string:lexemes(Vsn, ".-")). +-spec version_pad(list(nonempty_string())) -> Res when + Res :: {integer(), integer(), integer()}. version_pad([Major]) -> {list_to_integer(Major), 0, 0}; version_pad([Major, Minor]) -> @@ -225,10 +331,19 @@ version_pad([Major, Minor, Patch]) -> version_pad([Major, Minor, Patch | _]) -> {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}. +-spec maybe_store_etag_in_cache(UpdateETag, Path, ETag) -> Res when + UpdateETag :: boolean(), + Path :: file:name(), + ETag :: string(), + Res :: ok. maybe_store_etag_in_cache(false = _UpdateETag, _Path, _ETag) -> ok; maybe_store_etag_in_cache(true = _UpdateETag, Path, ETag) -> store_etag_in_cache(Path, ETag). +-spec store_etag_in_cache(File, ETag) -> Res when + File :: file:name(), + ETag :: string(), + Res :: ok. store_etag_in_cache(Path, ETag) -> ok = file:write_file(Path, ETag). -- cgit v1.1 From 66143a65e0adb3ef9b4c46a4a5389940aa182494 Mon Sep 17 00:00:00 2001 From: Laszlo Toth Date: Thu, 12 Apr 2018 09:22:51 +0200 Subject: (#1743): Refactor rebar_pkg_resource, add documentation --- src/rebar_pkg_resource.erl | 233 +++++++++++++++++++++++++++++---------------- 1 file changed, 152 insertions(+), 81 deletions(-) (limited to 'src') diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 8806c8b..16ec83f 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -14,6 +14,7 @@ ,etag/1 ,ssl_opts/1]). +%% Exported for ct -export([store_etag_in_cache/2]). -include("rebar.hrl"). @@ -28,6 +29,9 @@ -type download_result() :: {bad_download, binary() | string()} | {fetch_fail, _, _} | cached_result(). +%%============================================================================== +%% Public API +%%============================================================================== -spec lock(AppDir, Source) -> Res when AppDir :: file:name(), Source :: tuple(), @@ -35,6 +39,12 @@ lock(_AppDir, Source) -> Source. +%%------------------------------------------------------------------------------ +%% @doc +%% Return true if the stored version of the pkg is older than the current +%% version. +%% @end +%%------------------------------------------------------------------------------ -spec needs_update(Dir, Pkg) -> Res when Dir :: file:name(), Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, @@ -48,6 +58,11 @@ needs_update(Dir, {pkg, _Name, Vsn, _Hash}) -> true end. +%%------------------------------------------------------------------------------ +%% @doc +%% Download the given pkg. +%% @end +%%------------------------------------------------------------------------------ -spec download(TmpDir, Pkg, State) -> Res when TmpDir :: file:name(), Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, @@ -56,6 +71,13 @@ needs_update(Dir, {pkg, _Name, Vsn, _Hash}) -> download(TmpDir, Pkg, State) -> download(TmpDir, Pkg, State, true). +%%------------------------------------------------------------------------------ +%% @doc +%% Download the given pkg. The etag belonging to the pkg file will be updated +%% only if the UpdateEtag is true and the ETag returned from the hexpm server +%% is different. +%% @end +%%------------------------------------------------------------------------------ -spec download(TmpDir, Pkg, State, UpdateETag) -> Res when TmpDir :: file:name(), Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, @@ -69,14 +91,113 @@ download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State, UpdateETag) -> ETagFile = binary_to_list(<>), CachePath = filename:join(PackageDir, Package), ETagPath = filename:join(PackageDir, ETagFile), - case rebar_utils:url_append_path(CDN, filename:join(?REMOTE_PACKAGE_DIR, Package)) of + case rebar_utils:url_append_path(CDN, filename:join(?REMOTE_PACKAGE_DIR, + Package)) of {ok, Url} -> - cached_download(TmpDir, CachePath, Pkg, Url, etag(ETagPath), State, ETagPath, UpdateETag); + cached_download(TmpDir, CachePath, Pkg, Url, etag(ETagPath), State, + ETagPath, UpdateETag); _ -> {fetch_fail, Name, Vsn} end. --spec cached_download(TmpDir, CachePath, Pkg, Url, ETag, State, ETagPath, UpdateETag) -> Res when +%%------------------------------------------------------------------------------ +%% @doc +%% Implementation of rebar_resource make_vsn callback. +%% Returns {error, string()} as this operation is not supported for pkg sources. +%% @end +%%------------------------------------------------------------------------------ +-spec make_vsn(Vsn) -> Res when + Vsn :: any(), + Res :: {'error',[1..255,...]}. +make_vsn(_) -> + {error, "Replacing version of type pkg not supported."}. + +%%------------------------------------------------------------------------------ +%% @doc +%% Download the pkg belonging to the given address. If the etag of the pkg +%% is the same what we stored in the etag file previously return {ok, cached}, +%% if the file has changed (so the etag is not the same anymore) return +%% {ok, Contents, NewEtag}, otherwise if some error occured return error. +%% @end +%%------------------------------------------------------------------------------ +-spec request(Url, ETag) -> Res when + Url :: string(), + ETag :: false | string(), + Res :: 'error' | {ok, cached} | {ok, any(), string()}. +request(Url, ETag) -> + HttpOptions = [{ssl, ssl_opts(Url)}, + {relaxed, true} | rebar_utils:get_proxy_auth()], + case httpc:request(get, {Url, [{"if-none-match", "\"" ++ ETag ++ "\""} + || ETag =/= false] ++ + [{"User-Agent", rebar_utils:user_agent()}]}, + HttpOptions, [{body_format, binary}], rebar) of + {ok, {{_Version, 200, _Reason}, Headers, Body}} -> + ?DEBUG("Successfully downloaded ~ts", [Url]), + {"etag", ETag1} = lists:keyfind("etag", 1, Headers), + {ok, Body, rebar_string:trim(ETag1, both, [$"])}; + {ok, {{_Version, 304, _Reason}, _Headers, _Body}} -> + ?DEBUG("Cached copy of ~ts still valid", [Url]), + {ok, cached}; + {ok, {{_Version, Code, _Reason}, _Headers, _Body}} -> + ?DEBUG("Request to ~p failed: status code ~p", [Url, Code]), + error; + {error, Reason} -> + ?DEBUG("Request to ~p failed: ~p", [Url, Reason]), + error + end. + +%%------------------------------------------------------------------------------ +%% @doc +%% Read the etag belonging to the pkg file from the cache directory. The etag +%% is stored in a separate file when the etag belonging to the package is +%% returned from the hexpm server. The name is package-vsn.etag. +%% @end +%%------------------------------------------------------------------------------ +-spec etag(Path) -> Res when + Path :: file:name(), + Res :: false | string(). +etag(Path) -> + case file:read_file(Path) of + {ok, Bin} -> + binary_to_list(Bin); + {error, _} -> + false + end. + +%%------------------------------------------------------------------------------ +%% @doc +%% Return the SSL options adequate for the project based on +%% its configuration, including for validation of certs. +%% @end +%%------------------------------------------------------------------------------ +-spec ssl_opts(Url) -> Res when + Url :: string(), + Res :: proplists:proplist(). +ssl_opts(Url) -> + case get_ssl_config() of + ssl_verify_enabled -> + ssl_opts(ssl_verify_enabled, Url); + ssl_verify_disabled -> + [{verify, verify_none}] + end. + +%%------------------------------------------------------------------------------ +%% @doc +%% Store the given etag in the .cache folder. The name is pakckage-vsn.etag. +%% @end +%%------------------------------------------------------------------------------ +-spec store_etag_in_cache(File, ETag) -> Res when + File :: file:name(), + ETag :: string(), + Res :: ok. +store_etag_in_cache(Path, ETag) -> + ok = file:write_file(Path, ETag). + +%%%============================================================================= +%%% Private functions +%%%============================================================================= +-spec cached_download(TmpDir, CachePath, Pkg, Url, ETag, State, ETagPath, + UpdateETag) -> Res when TmpDir :: file:name(), CachePath :: file:name(), Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, @@ -86,7 +207,8 @@ download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State, UpdateETag) -> ETagPath :: file:name(), UpdateETag :: boolean(), Res :: download_result(). -cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, Url, ETag, State, ETagPath, UpdateETag) -> +cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, Url, ETag, + State, ETagPath, UpdateETag) -> case request(Url, ETag) of {ok, cached} -> ?INFO("Version cached at ~ts is up to date, reusing it", [CachePath]), @@ -94,7 +216,8 @@ cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, Url, ETag, State {ok, Body, NewETag} -> ?INFO("Downloaded package, caching at ~ts", [CachePath]), maybe_store_etag_in_cache(UpdateETag, ETagPath, NewETag), - serve_from_download(TmpDir, CachePath, Pkg, NewETag, Body, State, ETagPath); + serve_from_download(TmpDir, CachePath, Pkg, NewETag, Body, State, + ETagPath); error when ETag =/= false -> store_etag_in_cache(ETagPath, ETag), ?INFO("Download error, using cached file at ~ts", [CachePath]), @@ -125,11 +248,13 @@ serve_from_cache(TmpDir, CachePath, Pkg, State) -> ?DEBUG("Checksums: registry: ~p, pkg: ~p", [_Reg, Chk]), {bad_registry_checksum, CachePath}; {_Hash, _Bin, _Reg, _Tar} -> - ?DEBUG("Checksums: expected: ~p, registry: ~p, pkg: ~p, meta: ~p", [_Hash, _Reg, _Bin, _Tar]), + ?DEBUG("Checksums: expected: ~p, registry: ~p, pkg: ~p, meta: ~p", + [_Hash, _Reg, _Bin, _Tar]), {bad_checksum, CachePath} end. --spec serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State, ETagPath) -> Res when +-spec serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State, + ETagPath) -> Res when TmpDir :: file:name(), CachePath :: file:name(), Package :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, @@ -145,7 +270,8 @@ serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State, ETagPath) - ETag -> serve_from_cache(TmpDir, CachePath, Package, State); FileETag -> - ?DEBUG("Downloaded file ~ts ETag ~ts doesn't match returned ETag ~ts", [CachePath, ETag, FileETag]), + ?DEBUG("Downloaded file ~ts ETag ~ts doesn't match returned ETag ~ts", + [CachePath, ETag, FileETag]), {bad_download, CachePath} end. @@ -180,69 +306,19 @@ extract(TmpDir, CachePath) -> checksums(Pkg={pkg, _Name, _Vsn, Hash}, Files, Contents, Version, Meta, State) -> Blob = <>, <> = crypto:hash(sha256, Blob), - BinChecksum = list_to_binary(rebar_string:uppercase(lists:flatten(io_lib:format("~64.16.0b", [X])))), + BinChecksum = list_to_binary( + rebar_string:uppercase( + lists:flatten(io_lib:format("~64.16.0b", [X])))), RegistryChecksum = rebar_packages:registry_checksum(Pkg, State), {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files), {Hash, BinChecksum, RegistryChecksum, TarChecksum}. --spec make_vsn(Vsn) -> Res when - Vsn :: any(), - Res :: {'error',[1..255,...]}. -make_vsn(_) -> - {error, "Replacing version of type pkg not supported."}. - --spec request(Url, ETag) -> Res when - Url :: string(), - ETag :: false | string(), - Res :: 'error' | {ok, cached} | {ok, any(), string()}. -request(Url, ETag) -> - HttpOptions = [{ssl, ssl_opts(Url)}, {relaxed, true} | rebar_utils:get_proxy_auth()], - case httpc:request(get, {Url, [{"if-none-match", "\"" ++ ETag ++ "\""} || ETag =/= false]++ - [{"User-Agent", rebar_utils:user_agent()}]}, - HttpOptions, - [{body_format, binary}], - rebar) of - {ok, {{_Version, 200, _Reason}, Headers, Body}} -> - ?DEBUG("Successfully downloaded ~ts", [Url]), - {"etag", ETag1} = lists:keyfind("etag", 1, Headers), - {ok, Body, rebar_string:trim(ETag1, both, [$"])}; - {ok, {{_Version, 304, _Reason}, _Headers, _Body}} -> - ?DEBUG("Cached copy of ~ts still valid", [Url]), - {ok, cached}; - {ok, {{_Version, Code, _Reason}, _Headers, _Body}} -> - ?DEBUG("Request to ~p failed: status code ~p", [Url, Code]), - error; - {error, Reason} -> - ?DEBUG("Request to ~p failed: ~p", [Url, Reason]), - error - end. - --spec etag(Path) -> Res when - Path :: file:name(), - Res :: false | string(). -etag(Path) -> - case file:read_file(Path) of - {ok, Bin} -> - binary_to_list(Bin); - {error, _} -> - false - end. - -%% @doc returns the SSL options adequate for the project based on -%% its configuration, including for validation of certs. --spec ssl_opts(Url) -> Res when - Url :: string(), - Res :: proplists:proplist(). -ssl_opts(Url) -> - case get_ssl_config() of - ssl_verify_enabled -> - ssl_opts(ssl_verify_enabled, Url); - ssl_verify_disabled -> - [{verify, verify_none}] - end. - -%% @doc returns the SSL options adequate for the project based on +%%------------------------------------------------------------------------------ +%% @doc +%% Return the SSL options adequate for the project based on %% its configuration, including for validation of certs. +%% @end +%%------------------------------------------------------------------------------ -spec ssl_opts(Enabled, Url) -> Res when Enabled :: atom(), Url :: string(), @@ -250,13 +326,16 @@ ssl_opts(Url) -> ssl_opts(ssl_verify_enabled, Url) -> case check_ssl_version() of true -> - {ok, {_, _, Hostname, _, _, _}} = http_uri:parse(rebar_utils:to_list(Url)), - VerifyFun = {fun ssl_verify_hostname:verify_fun/3, [{check_hostname, Hostname}]}, + {ok, {_, _, Hostname, _, _, _}} = + http_uri:parse(rebar_utils:to_list(Url)), + VerifyFun = {fun ssl_verify_hostname:verify_fun/3, + [{check_hostname, Hostname}]}, CACerts = certifi:cacerts(), - [{verify, verify_peer}, {depth, 2}, {cacerts, CACerts} - ,{partial_chain, fun partial_chain/1}, {verify_fun, VerifyFun}]; + [{verify, verify_peer}, {depth, 2}, {cacerts, CACerts}, + {partial_chain, fun partial_chain/1}, {verify_fun, VerifyFun}]; false -> - ?WARN("Insecure HTTPS request (peer verification disabled), please update to OTP 17.4 or later", []), + ?WARN("Insecure HTTPS request (peer verification disabled), " + "please update to OTP 17.4 or later", []), [{verify, verify_none}] end. @@ -267,9 +346,8 @@ partial_chain(Certs) -> Certs1 = [{Cert, public_key:pkix_decode_cert(Cert, otp)} || Cert <- Certs], CACerts = certifi:cacerts(), CACerts1 = [public_key:pkix_decode_cert(Cert, otp) || Cert <- CACerts], - case ec_lists:find(fun({_, Cert}) -> - check_cert(CACerts1, Cert) + check_cert(CACerts1, Cert) end, Certs1) of {ok, Trusted} -> {trusted_ca, element(1, Trusted)}; @@ -293,7 +371,7 @@ check_cert(CACerts, Cert) -> end, CACerts). -spec check_ssl_version() -> - boolean(). + boolean(). check_ssl_version() -> case application:get_key(ssl, vsn) of {ok, Vsn} -> @@ -340,10 +418,3 @@ maybe_store_etag_in_cache(false = _UpdateETag, _Path, _ETag) -> ok; maybe_store_etag_in_cache(true = _UpdateETag, Path, ETag) -> store_etag_in_cache(Path, ETag). - --spec store_etag_in_cache(File, ETag) -> Res when - File :: file:name(), - ETag :: string(), - Res :: ok. -store_etag_in_cache(Path, ETag) -> - ok = file:write_file(Path, ETag). -- cgit v1.1 From 01eeda6c2c46a83ac2b288c0602cfc320e849a88 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 20 Apr 2018 18:28:03 -0400 Subject: Handle Schemaless Proxy URLs in ENV vars We've had multiple tickets opened because of unclear PROXY settings when the scheme is missing form the URI. To be helpful, we instead add them dynamically whenever they're missing. Example issues: - https://github.com/erlang/rebar3/issues/1747 - https://github.com/erlang/rebar3/issues/1697 --- src/rebar_utils.erl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 9bbe54e..a911cc2 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -854,10 +854,18 @@ set_httpc_options(_, []) -> ok; set_httpc_options(Scheme, Proxy) -> - {ok, {_, UserInfo, Host, Port, _, _}} = http_uri:parse(Proxy), + URI = normalise_proxy(Scheme, Proxy), + {ok, {_, UserInfo, Host, Port, _, _}} = http_uri:parse(URI), httpc:set_options([{Scheme, {{Host, Port}, []}}], rebar), set_proxy_auth(UserInfo). +normalise_proxy(Scheme, URI) -> + case re:run(URI, "://", [unicode]) of + nomatch when Scheme =:= https_proxy -> "https://" ++ URI; + nomatch when Scheme =:= proxy -> "http://" ++ URI; + _ -> URI + end. + url_append_path(Url, ExtraPath) -> case http_uri:parse(Url) of {ok, {Scheme, UserInfo, Host, Port, Path, Query}} -> -- cgit v1.1 From b46f56a158b2a02b05881e4ac3b53be7f1d46011 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 20 Apr 2018 18:48:12 -0400 Subject: Do not die on a pkg etag cache write fail We can probably still move ahead without a cache if we must --- src/rebar_pkg_resource.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index f77a81a..2fb1868 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -231,4 +231,4 @@ maybe_store_etag_in_cache(true = _UpdateETag, Path, ETag) -> store_etag_in_cache(Path, ETag). store_etag_in_cache(Path, ETag) -> - ok = file:write_file(Path, ETag). + _ = file:write_file(Path, ETag). -- cgit v1.1 From 53af2354cc990d23908709ba3b439de5c196023f Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 20 Apr 2018 19:27:46 -0400 Subject: Fix various Dialyzer warnings Some still remain from erl_type calls from Dialyzer, but most of them are handled. Decided to just ignore rebar_alias since playing with the type specs of abstract code format is just a nightmare and hard to do cross-versions I guess. --- src/rebar_app_info.erl | 2 +- src/rebar_hg_resource.erl | 2 +- src/rebar_prv_escriptize.erl | 4 +--- src/rebar_prv_unlock.erl | 8 ++------ 4 files changed, 5 insertions(+), 11 deletions(-) (limited to 'src') diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index de497d5..88d6335 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -301,7 +301,7 @@ app_file(#app_info_t{app_file=AppFile}) -> AppFile. %% @doc sets the .app file for an app. --spec app_file(t(), file:filename_all()) -> t(). +-spec app_file(t(), file:filename_all() | undefined) -> t(). app_file(AppInfo=#app_info_t{}, AppFile) -> AppInfo#app_info_t{app_file=AppFile}. diff --git a/src/rebar_hg_resource.erl b/src/rebar_hg_resource.erl index 6d25783..abcca88 100644 --- a/src/rebar_hg_resource.erl +++ b/src/rebar_hg_resource.erl @@ -137,7 +137,7 @@ get_branch_ref(Dir, Branch) -> rebar_utils:sh("hg -R \"" ++ rebar_utils:escape_double_quotes(Dir) ++ "\" log --template \"{node}\n\" --rev " ++ rebar_utils:escape_chars(Branch), [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]), - rebar_string:strip(BranchRefString, both, "\n"). + rebar_string:trim(BranchRefString, both, "\n"). maybe_warn_local_url(Url) -> diff --git a/src/rebar_prv_escriptize.erl b/src/rebar_prv_escriptize.erl index 3d6e708..4d767fd 100644 --- a/src/rebar_prv_escriptize.erl +++ b/src/rebar_prv_escriptize.erl @@ -267,9 +267,7 @@ rm_newline(String) -> [C || C <- String, C =/= $\n]. write_windows_script(Target) -> - CmdPath = if is_binary(Target) -> <>; - is_list(Target) -> Target ++ ".cmd" - end, + CmdPath = Target ++ ".cmd", CmdScript= "@echo off\r\n" "setlocal\r\n" diff --git a/src/rebar_prv_unlock.erl b/src/rebar_prv_unlock.erl index 8480991..6fe8bd8 100644 --- a/src/rebar_prv_unlock.erl +++ b/src/rebar_prv_unlock.erl @@ -48,12 +48,8 @@ do(State) -> ?PRV_ERROR({file,Reason}); {ok, _} -> Locks = rebar_config:consult_lock_file(LockFile), - case handle_unlocks(State, Locks, LockFile) of - {ok, NewLocks} -> - {ok, rebar_state:set(State, {locks, default}, NewLocks)}; - {error, Reason} -> - ?PRV_ERROR({file,Reason}) - end + {ok, NewLocks} = handle_unlocks(State, Locks, LockFile), + {ok, rebar_state:set(State, {locks, default}, NewLocks)} end. -spec format_error(any()) -> iolist(). -- cgit v1.1 From 6ab00567f506de5b95ef6456a4048ed184f4d58e Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 20 Apr 2018 20:08:58 -0400 Subject: Display error when rebar.config.script fails This will at least display the script that failed with the stacktrace before failing as usual, but without altering the return value. This should make the common failure path more user-friendly without breaking any existing behaviour that may have relied on the script file working. If there's any unexpected side-effect, it will be visual only rather than blocking full builds if ?ABORT were used. --- src/rebar_config.erl | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src') diff --git a/src/rebar_config.erl b/src/rebar_config.erl index 7316d83..797dddc 100644 --- a/src/rebar_config.erl +++ b/src/rebar_config.erl @@ -290,6 +290,8 @@ consult_and_eval(File, Script) -> {ok, Term} -> {ok, [Term]}; Error -> + ?ERROR("Error evaluating configuration script at ~p:~n~p~n", + [Script, Error]), Error end. -- cgit v1.1 From 3630a06a0131f8ad87e5b727a94ff5ffe1f6a611 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 20 Apr 2018 20:11:13 -0400 Subject: Fix broken windows build Let's bundle this with another PR --- src/rebar_prv_escriptize.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_escriptize.erl b/src/rebar_prv_escriptize.erl index 4d767fd..fceb65e 100644 --- a/src/rebar_prv_escriptize.erl +++ b/src/rebar_prv_escriptize.erl @@ -267,7 +267,7 @@ rm_newline(String) -> [C || C <- String, C =/= $\n]. write_windows_script(Target) -> - CmdPath = Target ++ ".cmd", + CmdPath = unicode:characters_to_list(Target) ++ ".cmd", CmdScript= "@echo off\r\n" "setlocal\r\n" -- cgit v1.1 From 543fe579a6d7c71fb4ed6a898540b573f6255dd0 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 20 Apr 2018 21:03:05 -0400 Subject: Fix precedence rules of erl_opts for test profile When adding the 'TEST' macro to the test profile, we mistakenly sourced the erl_opts values from the base profile rather than the test profile itself. This means that in cases where the base profile set an option such as 'no_debug_info' and a profile overrode it with 'debug_info', the default options would get injected within the test profile, and broke the precedence rules, yielding incompatible values. This patch fixes things by adding the macro to the values sourced from the test profile itself, fixing the issue. --- src/rebar3.erl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar3.erl b/src/rebar3.erl index 80b4e4d..eb5ad58 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -389,7 +389,11 @@ state_from_global_config(Config, GlobalConfigFile) -> rebar_state:providers(rebar_state:new(GlobalConfig3, Config), GlobalPlugins). test_state(State) -> - ErlOpts = rebar_state:get(State, erl_opts, []), + %% Fetch the test profile's erl_opts only + Opts = rebar_state:opts(State), + Profiles = rebar_opts:get(Opts, profiles, []), + ProfileOpts = proplists:get_value(test, Profiles, []), + ErlOpts = proplists:get_value(erl_opts, ProfileOpts, []), TestOpts = safe_define_test_macro(ErlOpts), [{extra_src_dirs, ["test"]}, {erl_opts, TestOpts}]. -- cgit v1.1 From 8e165d0cc8b80cd3f5a9f31964254b9a9d5212f5 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 20 Apr 2018 22:35:52 -0400 Subject: Reload apps running in shell with new config This patch makes it so that whenever the rebar3 shell has a new configuration for an application that is already running and would be restarted (without risking the stability of the node or functionality of rebar_agent), we stop and restart the app. --- src/rebar_prv_shell.erl | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_shell.erl b/src/rebar_prv_shell.erl index 47e0366..2c48083 100644 --- a/src/rebar_prv_shell.erl +++ b/src/rebar_prv_shell.erl @@ -271,11 +271,11 @@ maybe_boot_apps(State) -> case find_apps_to_boot(State) of undefined -> %% try to read in sys.config file - ok = reread_config(State); + ok = reread_config([], State); Apps -> %% load apps, then check config, then boot them. load_apps(Apps), - ok = reread_config(State), + ok = reread_config(Apps, State), boot_apps(Apps) end. @@ -340,11 +340,36 @@ load_apps(Apps) -> not lists:keymember(App, 1, application:loaded_applications())], ok. -reread_config(State) -> +reread_config(AppsToStart, State) -> case find_config(State) of no_config -> ok; ConfigList -> + %% This allows people who use applications that are also + %% depended on by rebar3 or its plugins to change their + %% configuration at runtime based on the configuration files. + %% + %% To do this, we stop apps that are already started before + %% reloading their configuration. + %% + %% We make an exception for apps that: + %% - are not already running + %% - would not be restarted (and hence would break some + %% compatibility with rebar3) + %% - are not in the config files and would see no config + %% changes + %% - are not in a blacklist, where changing their config + %% would be risky to the shell or the rebar3 agent + %% functionality (i.e. changing inets may break proxy + %% settings, stopping `kernel' would break everything) + Running = [App || {App, _, _} <- application:which_applications()], + BlackList = [inets, stdlib, kernel, rebar], + _ = [application:stop(App) + || Config <- ConfigList, + {App, _} <- Config, + lists:member(App, Running), + lists:member(App, AppsToStart), + not lists:member(App, BlackList)], _ = rebar_utils:reread_config(ConfigList), ok end. -- cgit v1.1 From b7d7c0eb8bfd9f09af3f3ed02a1982f3d333d50a Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sat, 21 Apr 2018 09:26:58 -0400 Subject: Fix local upgrade etag handling Since packages store etags on disk directly, the local install feature can no longer depend on this; we instead port the etag feature back to local providers only. --- src/rebar_prv_local_upgrade.erl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_local_upgrade.erl b/src/rebar_prv_local_upgrade.erl index 1ac3adb..3b3c9cb 100644 --- a/src/rebar_prv_local_upgrade.erl +++ b/src/rebar_prv_local_upgrade.erl @@ -80,7 +80,7 @@ maybe_fetch_rebar3(Rebar3Md5) -> case rebar_pkg_resource:request("https://s3.amazonaws.com/rebar3/rebar3", Rebar3Md5) of {ok, Binary, ETag} -> file:write_file(TmpFile, Binary), - case rebar_pkg_resource:etag(TmpFile) of + case etag(TmpFile) of ETag -> {saved, TmpFile}; _ -> @@ -92,3 +92,12 @@ maybe_fetch_rebar3(Rebar3Md5) -> ?CONSOLE("No upgrade available", []), up_to_date end. + +etag(Path) -> + case file:read_file(Path) of + {ok, Binary} -> + <> = crypto:hash(md5, Binary), + rebar_string:lowercase(lists:flatten(io_lib:format("~32.16.0b", [X]))); + {error, _} -> + false + end. -- cgit v1.1 From b3f99aa42fcbeffa8751df308cb194a470cd0e80 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Sun, 22 Apr 2018 09:09:01 +0300 Subject: ensure Dest exists before copying to it --- src/rebar_file_utils.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index bb3ca71..0dbc40a 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -191,8 +191,8 @@ cp_r(Sources, Dest) -> {unix, _} -> EscSources = [rebar_utils:escape_chars(Src) || Src <- Sources], SourceStr = rebar_string:join(EscSources, " "), - {ok, []} = rebar_utils:sh(?FMT("cp -Rp ~ts \"~ts\"", - [SourceStr, rebar_utils:escape_double_quotes(Dest)]), + {ok, []} = rebar_utils:sh(?FMT("mkdir -p \"~ts\" && cp -Rp ~ts $_", + [rebar_utils:escape_double_quotes(Dest), SourceStr]), [{use_stdout, false}, abort_on_error]), ok; {win32, _} -> -- cgit v1.1 From 288d0c7484ba9214cc6598100000bcce0fefe462 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 23 Apr 2018 13:38:37 -0400 Subject: Clear all dialyzer errors - this is done by adding a HiPE dependency - also in this: adding a test for directories, and fixing some indent --- src/rebar.app.src | 1 + src/rebar_string.erl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 43ad620..c96f65c 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -8,6 +8,7 @@ {registered, []}, {applications, [kernel, stdlib, + hipe, sasl, compiler, crypto, diff --git a/src/rebar_string.erl b/src/rebar_string.erl index c1858f9..47cb15c 100644 --- a/src/rebar_string.erl +++ b/src/rebar_string.erl @@ -11,7 +11,7 @@ %% used. Instead we just adopt join/2 locally and hope it works %% for most unicode use cases anyway. join([], Sep) when is_list(Sep) -> - []; + []; join([H|T], Sep) -> H ++ lists:append([Sep ++ X || X <- T]). -- cgit v1.1 From c22fde17a7a30cee5e3e5e04fab187eed2ecfe42 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Mon, 23 Apr 2018 22:34:40 +0300 Subject: fix & test --- src/rebar_file_utils.erl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 0dbc40a..492d690 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -191,8 +191,12 @@ cp_r(Sources, Dest) -> {unix, _} -> EscSources = [rebar_utils:escape_chars(Src) || Src <- Sources], SourceStr = rebar_string:join(EscSources, " "), - {ok, []} = rebar_utils:sh(?FMT("mkdir -p \"~ts\" && cp -Rp ~ts $_", - [rebar_utils:escape_double_quotes(Dest), SourceStr]), + % ensure destination exists before copying files into it + {ok, []} = rebar_utils:sh(?FMT("mkdir -p ~ts", + [rebar_utils:escape_chars(Dest)]), + [{use_stdout, false}, abort_on_error]), + {ok, []} = rebar_utils:sh(?FMT("cp -Rp ~ts \"~ts\"", + [SourceStr, rebar_utils:escape_double_quotes(Dest)]), [{use_stdout, false}, abort_on_error]), ok; {win32, _} -> -- cgit v1.1 From df9c769afc4c3480305a1f204b37828531e10331 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 26 Apr 2018 21:55:30 -0400 Subject: Display warnings when cache dirs are read-only This should provide more help to users directly and avoid having them go through opening tickets for help. Fixes #1767 --- src/rebar_packages.erl | 7 ++++++- src/rebar_prv_local_install.erl | 11 ++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 50d4025..8cebeca 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -97,7 +97,12 @@ registry_dir(State) -> case rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN) of ?DEFAULT_CDN -> RegistryDir = filename:join([CacheDir, "hex", "default"]), - ok = filelib:ensure_dir(filename:join(RegistryDir, "placeholder")), + case filelib:ensure_dir(filename:join(RegistryDir, "placeholder")) of + ok -> ok; + {error, Posix} when Posix == eaccess; Posix == enoent -> + ?ABORT("Could not write to ~p. Please ensure the path is writeable.", + [RegistryDir]) + end, {ok, RegistryDir}; CDN -> case rebar_utils:url_append_path(CDN, ?REMOTE_PACKAGE_DIR) of diff --git a/src/rebar_prv_local_install.erl b/src/rebar_prv_local_install.erl index 282c548..bb019c4 100644 --- a/src/rebar_prv_local_install.erl +++ b/src/rebar_prv_local_install.erl @@ -12,6 +12,7 @@ -export([extract_escript/2]). -include("rebar.hrl"). +-include_lib("providers/include/providers.hrl"). -include_lib("kernel/include/file.hrl"). -define(PROVIDER, install). @@ -54,6 +55,9 @@ do(State) -> end. -spec format_error(any()) -> iolist(). +format_error({non_writeable, Dir}) -> + io_lib:format("Could not write to ~p. Please ensure the path is writeable.", + [Dir]); format_error(Reason) -> io_lib:format("~p", [Reason]). @@ -71,7 +75,12 @@ extract_escript(State, ScriptPath) -> %% And add a rebar3 bin script to ~/.cache/rebar3/bin Opts = rebar_state:opts(State), OutputDir = filename:join(rebar_dir:global_cache_dir(Opts), "lib"), - filelib:ensure_dir(filename:join(OutputDir, "empty")), + case filelib:ensure_dir(filename:join(OutputDir, "empty")) of + ok -> + ok; + {error, Posix} when Posix == eaccess; Posix == enoent -> + throw(?PRV_ERROR({non_writeable, OutputDir})) + end, ?INFO("Extracting rebar3 libs to ~ts...", [OutputDir]), zip:extract(Archive, [{cwd, OutputDir}]), -- cgit v1.1 From 775261dd0b24d74e48f5873ef1ffff56c0c4829a Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 27 Apr 2018 09:32:39 -0400 Subject: Prevent copying or symlink non-existing src_dirs This would cause crashes on linux and force people to have a src_dirs config that is strictly matching what is on the file system rather than acting as a specification of those that are valid. To compare, if lib_dirs worked the same, then any repo that did not both have apps/ and lib/ would crash, as the spec mentions both options as valid. --- src/rebar_prv_compile.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index 72320fb..1748118 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -236,7 +236,10 @@ copy_app_dirs(AppInfo, OldAppDir, AppDir) -> symlink_or_copy(OldAppDir, AppDir, Dir) -> Source = filename:join([OldAppDir, Dir]), Target = filename:join([AppDir, Dir]), - rebar_file_utils:symlink_or_copy(Source, Target). + case ec_file:is_dir(Source) of + true -> rebar_file_utils:symlink_or_copy(Source, Target); + false -> ok + end. copy(OldAppDir, AppDir, Dir) -> Source = filename:join([OldAppDir, Dir]), -- cgit v1.1 From 0af9aba244dd10d8fdcb8520057cf8a4dcdd90bf Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 27 Apr 2018 10:22:07 -0400 Subject: Fix symlink/copying logic in compiler for priv Priv and include dirs need the virtual symlink in order to preserve hook functionality in some edge cases. --- src/rebar_prv_compile.erl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index 1748118..0b4fa5f 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -225,7 +225,11 @@ copy_app_dirs(AppInfo, OldAppDir, AppDir) -> end, {SrcDirs, ExtraDirs} = resolve_src_dirs(rebar_app_info:opts(AppInfo)), %% link to src_dirs to be adjacent to ebin is needed for R15 use of cover/xref - [symlink_or_copy(OldAppDir, AppDir, Dir) || Dir <- ["priv", "include"] ++ SrcDirs], + %% priv/ and include/ are symlinked unconditionally to allow hooks + %% to write to them _after_ compilation has taken place when the + %% initial directory did not, and still work + [symlink_or_copy(OldAppDir, AppDir, Dir) || Dir <- ["priv", "include"]], + [symlink_or_copy_existing(OldAppDir, AppDir, Dir) || Dir <- SrcDirs], %% copy all extra_src_dirs as they build into themselves and linking means they %% are shared across profiles [copy(OldAppDir, AppDir, Dir) || Dir <- ExtraDirs]; @@ -236,6 +240,11 @@ copy_app_dirs(AppInfo, OldAppDir, AppDir) -> symlink_or_copy(OldAppDir, AppDir, Dir) -> Source = filename:join([OldAppDir, Dir]), Target = filename:join([AppDir, Dir]), + rebar_file_utils:symlink_or_copy(Source, Target). + +symlink_or_copy_existing(OldAppDir, AppDir, Dir) -> + Source = filename:join([OldAppDir, Dir]), + Target = filename:join([AppDir, Dir]), case ec_file:is_dir(Source) of true -> rebar_file_utils:symlink_or_copy(Source, Target); false -> ok -- cgit v1.1 From 5f9b4293bc029e2132d7c442cb5b4480915ea0e4 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 3 May 2018 07:12:11 -0400 Subject: Work around OTP-21 deprecation of get_stacktrace() Based off a macro by @okeuday at https://github.com/erlang/otp/pull/1783 --- src/rebar.hrl | 6 ++++++ src/rebar3.erl | 21 ++++++++++----------- src/rebar_agent.erl | 4 ++-- src/rebar_core.erl | 3 +-- src/rebar_dialyzer_format.erl | 4 ++-- src/rebar_fetch.erl | 4 ++-- src/rebar_plugins.erl | 4 ++-- src/rebar_prv_shell.erl | 8 ++++---- src/rebar_prv_update.erl | 4 ++-- src/rebar_state.erl | 6 +++--- 10 files changed, 34 insertions(+), 30 deletions(-) (limited to 'src') diff --git a/src/rebar.hrl b/src/rebar.hrl index ca44283..f461c70 100644 --- a/src/rebar.hrl +++ b/src/rebar.hrl @@ -52,6 +52,12 @@ -type rebar_set() :: set(). -endif. +-ifdef(fun_stacktrace). +-define(WITH_STACKTRACE(T, R, S), T:R -> S = erlang:get_stacktrace(),). +-else. +-define(WITH_STACKTRACE(T, R, S), T:R:S ->). +-endif. + -define(GRAPH_VSN, 2). -type v() :: {digraph:vertex(), term()} | 'false'. -type e() :: {digraph:vertex(), digraph:vertex()}. diff --git a/src/rebar3.erl b/src/rebar3.erl index eb5ad58..8e9d4b1 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -67,10 +67,10 @@ main(Args) -> {ok, _State} -> erlang:halt(0); Error -> - handle_error(Error) + handle_error(Error, []) catch - _:Error -> - handle_error(Error) + ?WITH_STACKTRACE(_,Error,Stacktrace) + handle_error(Error, Stacktrace) end. %% @doc Erlang-API entry point @@ -299,15 +299,15 @@ global_option_spec_list() -> %% @private translate unhandled errors and internal return codes into proper %% erroneous program exits. --spec handle_error(term()) -> no_return(). -handle_error(rebar_abort) -> +-spec handle_error(term(), term()) -> no_return(). +handle_error(rebar_abort, _) -> erlang:halt(1); -handle_error({error, rebar_abort}) -> +handle_error({error, rebar_abort}, _) -> erlang:halt(1); -handle_error({error, {Module, Reason}}) -> +handle_error({error, {Module, Reason}}, Stacktrace) -> case code:which(Module) of non_existing -> - ?CRASHDUMP("~p: ~p~n~p~n~n", [Module, Reason, erlang:get_stacktrace()]), + ?CRASHDUMP("~p: ~p~n~p~n~n", [Module, Reason, Stacktrace]), ?ERROR("Uncaught error in rebar_core. Run with DEBUG=1 to stacktrace or consult rebar3.crashdump", []), ?DEBUG("Uncaught error: ~p ~p", [Module, Reason]), ?INFO("When submitting a bug report, please include the output of `rebar3 report \"your command\"`", []); @@ -315,13 +315,12 @@ handle_error({error, {Module, Reason}}) -> ?ERROR("~ts", [Module:format_error(Reason)]) end, erlang:halt(1); -handle_error({error, Error}) when is_list(Error) -> +handle_error({error, Error}, _) when is_list(Error) -> ?ERROR("~ts", [Error]), erlang:halt(1); -handle_error(Error) -> +handle_error(Error, StackTrace) -> %% Nothing should percolate up from rebar_core; %% Dump this error to console - StackTrace = erlang:get_stacktrace(), ?CRASHDUMP("Error: ~p~n~p~n~n", [Error, StackTrace]), ?ERROR("Uncaught error in rebar_core. Run with DEBUG=1 to see stacktrace or consult rebar3.crashdump", []), ?DEBUG("Uncaught error: ~p", [Error]), diff --git a/src/rebar_agent.erl b/src/rebar_agent.erl index b5dcfcf..8d0f9cf 100644 --- a/src/rebar_agent.erl +++ b/src/rebar_agent.erl @@ -114,8 +114,8 @@ run(Namespace, Command, StrArgs, RState, Cwd) -> {{error, cwd_changed}, RState} end catch - Type:Reason -> - ?DEBUG("Agent Stacktrace: ~p", [erlang:get_stacktrace()]), + ?WITH_STACKTRACE(Type, Reason, Stacktrace) + ?DEBUG("Agent Stacktrace: ~p", [Stacktrace]), {{error, {Type, Reason}}, RState} end. diff --git a/src/rebar_core.erl b/src/rebar_core.erl index 6132a5e..6a1cdbf 100644 --- a/src/rebar_core.erl +++ b/src/rebar_core.erl @@ -157,8 +157,7 @@ do([ProviderName | Rest], State) -> {error, Error} -> {error, Error} catch - error:undef -> - Stack = erlang:get_stacktrace(), + ?WITH_STACKTRACE(error,undef,Stack) case Stack of [{ProviderName, do, [_], _}|_] -> %% This should really only happen if a plugin provider doesn't export do/1 diff --git a/src/rebar_dialyzer_format.erl b/src/rebar_dialyzer_format.erl index 5583633..cb0e958 100644 --- a/src/rebar_dialyzer_format.erl +++ b/src/rebar_dialyzer_format.erl @@ -53,9 +53,9 @@ format_warning_(Opts, Warning = {_Tag, {SrcFile, Line}, Msg}, {_LastFile, Acc}) String = message_to_string(Msg), {SrcFile, [lists:flatten(fmt("~n~ts~n~!c~4w~!!: ~ts", [F, Line, String])) | Acc]} catch - Error:Reason -> + ?WITH_STACKTRACE(Error, Reason, Stacktrace) ?DEBUG("Failed to pretty format warning: ~p:~p~n~p", - [Error, Reason, erlang:get_stacktrace()]), + [Error, Reason, Stacktrace]), {SrcFile, [dialyzer:format_warning(Warning, fullpath) | Acc]} end. diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index f68a54d..d2c7706 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -32,8 +32,8 @@ download_source(AppDir, Source, State) -> Error -> throw(?PRV_ERROR(Error)) catch - C:T -> - ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, erlang:get_stacktrace()]), + ?WITH_STACKTRACE(C,T,S) + ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, S]), throw(?PRV_ERROR({fetch_fail, Source})) end. diff --git a/src/rebar_plugins.erl b/src/rebar_plugins.erl index 57c34bc..bc6a1e0 100644 --- a/src/rebar_plugins.erl +++ b/src/rebar_plugins.erl @@ -114,8 +114,8 @@ handle_plugin(Profile, Plugin, State, Upgrade) -> {plugin_providers(Plugin), State4} catch - C:T -> - ?DEBUG("~p ~p ~p", [C, T, erlang:get_stacktrace()]), + ?WITH_STACKTRACE(C,T,S) + ?DEBUG("~p ~p ~p", [C, T, S]), ?WARN("Plugin ~p not available. It will not be used.", [Plugin]), {[], State} end. diff --git a/src/rebar_prv_shell.erl b/src/rebar_prv_shell.erl index 2c48083..d62f1bc 100644 --- a/src/rebar_prv_shell.erl +++ b/src/rebar_prv_shell.erl @@ -210,8 +210,8 @@ rewrite_leaders(OldUser, NewUser) -> %% reset the tty handler once more for remote shells error_logger:swap_handler(tty) catch - E:R -> % may fail with custom loggers - ?DEBUG("Logger changes failed for ~p:~p (~p)", [E,R,erlang:get_stacktrace()]), + ?WITH_STACKTRACE(E,R,S) % may fail with custom loggers + ?DEBUG("Logger changes failed for ~p:~p (~p)", [E,R,S]), hope_for_best end. @@ -235,9 +235,9 @@ maybe_run_script(State) -> File = filename:absname(RelFile), try run_script_file(File) catch - C:E -> + ?WITH_STACKTRACE(C,E,S) ?ABORT("Couldn't run shell escript ~p - ~p:~p~nStack: ~p", - [File, C, E, erlang:get_stacktrace()]) + [File, C, E, S]) end end. diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index 6dde024..1744631 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -73,8 +73,8 @@ do(State) -> ?PRV_ERROR({package_parse_cdn, CDN}) end catch - _E:C -> - ?DEBUG("Error creating package index: ~p ~p", [C, erlang:get_stacktrace()]), + ?WITH_STACKTRACE(_E, C, S) + ?DEBUG("Error creating package index: ~p ~p", [C, S]), throw(?PRV_ERROR(package_index_write)) end. diff --git a/src/rebar_state.erl b/src/rebar_state.erl index dd1f43f..3586dd6 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -418,9 +418,9 @@ create_logic_providers(ProviderModules, State0) -> end end, State0, ProviderModules) catch - C:T -> - ?DEBUG("~p: ~p ~p", [C, T, erlang:get_stacktrace()]), - ?CRASHDUMP("~p: ~p~n~p~n~n~p", [C, T, erlang:get_stacktrace(), State0]), + ?WITH_STACKTRACE(C,T,S) + ?DEBUG("~p: ~p ~p", [C, T, S]), + ?CRASHDUMP("~p: ~p~n~p~n~n~p", [C, T, S, State0]), throw({error, "Failed creating providers. Run with DEBUG=1 for stacktrace or consult rebar3.crashdump."}) end. -- cgit v1.1 From 50dfdfddc2c19fbf8ff44ff981ed7e0ed37b26d4 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 2 May 2018 23:40:35 -0400 Subject: Confirm lack of regressions on new logger in shell --- src/rebar_prv_shell.erl | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_shell.erl b/src/rebar_prv_shell.erl index d62f1bc..9a320ad 100644 --- a/src/rebar_prv_shell.erl +++ b/src/rebar_prv_shell.erl @@ -201,21 +201,28 @@ rewrite_leaders(OldUser, NewUser) -> lists:member(proplists:get_value(group_leader, erlang:process_info(Pid)), OldMasters)], try - %% enable error_logger's tty output - error_logger:swap_handler(tty), - %% disable the simple error_logger (which may have been added multiple - %% times). removes at most the error_logger added by init and the - %% error_logger added by the tty handler - remove_error_handler(3), - %% reset the tty handler once more for remote shells - error_logger:swap_handler(tty) + case erlang:function_exported(logger, module_info, 0) of + false -> + %% Old style logger had a lock-up issue and other problems related + %% to group leader handling. + %% enable error_logger's tty output + error_logger:swap_handler(tty), + %% disable the simple error_logger (which may have been added + %% multiple times). removes at most the error_logger added by + %% init and the error_logger added by the tty handler + remove_error_handler(3), + %% reset the tty handler once more for remote shells + error_logger:swap_handler(tty); + true -> + %% This is no longer a problem with the logger interface + ok + end catch ?WITH_STACKTRACE(E,R,S) % may fail with custom loggers ?DEBUG("Logger changes failed for ~p:~p (~p)", [E,R,S]), hope_for_best end. - setup_paths(State) -> %% Add deps to path code:add_pathsa(rebar_state:code_paths(State, all_deps)), -- cgit v1.1 From e321ca64981504f10a3be1715ce5d94c3cd10ae7 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 2 May 2018 23:53:29 -0400 Subject: Fix handling of loosely formatted OTP_VERSION file see https://github.com/erlang/rebar3/issues/1774 --- src/rebar_utils.erl | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index a911cc2..604abb8 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -506,11 +506,10 @@ otp_release1(Rel) -> %% It's fine to rely on the binary module here because we can %% be sure that it's available when the otp_release string does %% not begin with $R. - Size = byte_size(Vsn), %% The shortest vsn string consists of at least two digits %% followed by "\n". Therefore, it's safe to assume Size >= 3. - case binary:part(Vsn, {Size, -3}) of - <<"**\n">> -> + case binary:match(Vsn, <<"**">>) of + {Pos, _} -> %% The OTP documentation mentions that a system patched %% using the otp_patch_apply tool available to licensed %% customers will leave a '**' suffix in the version as a @@ -519,9 +518,9 @@ otp_release1(Rel) -> %% drop the suffix, given for all intents and purposes, we %% cannot obtain relevant information from it as far as %% tooling is concerned. - binary:bin_to_list(Vsn, {0, Size - 3}); - _ -> - binary:bin_to_list(Vsn, {0, Size - 1}) + binary:bin_to_list(Vsn, {0, Pos}); + nomatch -> + rebar_string:trim(binary:bin_to_list(Vsn), trailing, "\n") end end. -- cgit v1.1 From e4cffdb323f500d3016fa92ea80cefaff85dab76 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 3 May 2018 18:17:12 -0400 Subject: bump to 3.5.1 --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index c96f65c..7c346ab 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.5.1"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 9755fc740501e925e090cc7fc740d93d3c62e632 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 3 May 2018 18:20:30 -0400 Subject: Back to git-based versioning --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 7c346ab..c96f65c 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.5.1"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From dcf717ac2ef4fe2a27d1dd5fb036668a20e31b26 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 3 May 2018 19:35:24 -0400 Subject: Bump to 3.5.2 --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index c96f65c..42aecca 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.5.2"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From d457f2af6be3ff75fbe2de32127ce3958bca2c4f Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 3 May 2018 19:39:04 -0400 Subject: Return to git-based versioning --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 42aecca..c96f65c 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.5.2"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 254bb7be7c91aed2799061094c571b041038b5fb Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sat, 5 May 2018 10:39:25 -0400 Subject: Logger support in cth_readable; compact CT output cth_readable 1.4.0 supports the new logger interface from OTP-21, which likely breaks compatibility with R16 builds. It also includes a new compact interface, displaying output such as: ===> Running Common Test suites... %%% rebar_alias_SUITE: ...... %%% rebar_as_SUITE: ........... %%% rebar_compile_SUITE:................................ ............................ %%% rebar_compile_SUITE ==> test_name: SKIPPED %%% rebar_compile_SUITE ==> {tc_user_skip,"compile:env_compiler_options/0 available"} .. %%% rebar_cover_SUITE: ............. %%% rebar_ct_SUITE: .................................... Allowing to display more tests within less screen space. This mode has been added to the ct_readable option under the name 'compact' (now supporting true | false | compact), and has been made default for rebar3. --- src/rebar_prv_common_test.erl | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index f800610..9e71ee7 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -227,15 +227,20 @@ add_hooks(Opts, State) -> case {readable(State), lists:keyfind(ct_hooks, 1, Opts)} of {false, _} -> Opts; - {true, false} -> - [{ct_hooks, [cth_readable_failonly, cth_readable_shell, cth_retry]} | Opts]; - {true, {ct_hooks, Hooks}} -> + {Other, false} -> + [{ct_hooks, [cth_readable_failonly, readable_shell_type(Other), cth_retry]} | Opts]; + {Other, {ct_hooks, Hooks}} -> %% Make sure hooks are there once only. - ReadableHooks = [cth_readable_failonly, cth_readable_shell, cth_retry], - NewHooks = (Hooks -- ReadableHooks) ++ ReadableHooks, + ReadableHooks = [cth_readable_failonly, readable_shell_type(Other), cth_retry], + AllReadableHooks = [cth_readable_failonly, cth_retry, + cth_readable_shell, cth_readable_compact_shell], + NewHooks = (Hooks -- AllReadableHooks) ++ ReadableHooks, lists:keyreplace(ct_hooks, 1, Opts, {ct_hooks, NewHooks}) end. +readable_shell_type(true) -> cth_readable_shell; +readable_shell_type(compact) -> cth_readable_compact_shell. + select_tests(_, _, _, {error, _} = Error) -> Error; select_tests(State, ProjectApps, CmdOpts, CfgOpts) -> %% set application env if sys_config argument is provided @@ -425,20 +430,21 @@ append(A, B) -> A ++ B. add_transforms(CTOpts, State) when is_list(CTOpts) -> case readable(State) of - true -> - ReadableTransform = [{parse_transform, cth_readable_transform}], - (CTOpts -- ReadableTransform) ++ ReadableTransform; false -> - CTOpts + CTOpts; + Other when Other == true; Other == compact -> + ReadableTransform = [{parse_transform, cth_readable_transform}], + (CTOpts -- ReadableTransform) ++ ReadableTransform end; add_transforms({error, _} = Error, _State) -> Error. readable(State) -> {RawOpts, _} = rebar_state:command_parsed_args(State), case proplists:get_value(readable, RawOpts) of - true -> true; - false -> false; - undefined -> rebar_state:get(State, ct_readable, true) + "true" -> true; + "false" -> false; + "compact" -> compact; + undefined -> rebar_state:get(State, ct_readable, compact) end. test_dirs(State, Apps, Opts) -> @@ -762,7 +768,7 @@ ct_opts(_State) -> {scale_timetraps, undefined, "scale_timetraps", boolean, help(scale_timetraps)}, {create_priv_dir, undefined, "create_priv_dir", string, help(create_priv_dir)}, {include, undefined, "include", string, help(include)}, - {readable, undefined, "readable", boolean, help(readable)}, + {readable, undefined, "readable", string, help(readable)}, {verbose, $v, "verbose", boolean, help(verbose)}, {name, undefined, "name", atom, help(name)}, {sname, undefined, "sname", atom, help(sname)}, @@ -831,7 +837,7 @@ help(create_priv_dir) -> help(include) -> "Directories containing additional include files"; help(readable) -> - "Shows test case names and only displays logs to shell on failures"; + "Shows test case names and only displays logs to shell on failures (true | compact | false)"; help(verbose) -> "Verbose output"; help(name) -> -- cgit v1.1 From dc0f8b4d66d12c10ed4df85148b5e8ce13056f40 Mon Sep 17 00:00:00 2001 From: feng19 Date: Mon, 7 May 2018 21:54:50 +0800 Subject: add option(refresh_paths & refresh_paths_blacklist) for support set refresh_paths when recompile --- src/rebar_agent.erl | 125 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 101 insertions(+), 24 deletions(-) (limited to 'src') diff --git a/src/rebar_agent.erl b/src/rebar_agent.erl index 8d0f9cf..69e9b8e 100644 --- a/src/rebar_agent.erl +++ b/src/rebar_agent.erl @@ -131,40 +131,98 @@ maybe_show_warning(State) -> %% that makes sense. -spec refresh_paths(rebar_state:t()) -> ok. refresh_paths(RState) -> - ToRefresh = (rebar_state:code_paths(RState, all_deps) - ++ [filename:join([rebar_app_info:out_dir(App), "test"]) - || App <- rebar_state:project_apps(RState)] - %% make sure to never reload self; halt()s the VM - ) -- [filename:dirname(code:which(?MODULE))], + RefreshPaths = lists:usort( + application:get_env(rebar, refresh_paths, []) + ++ [all_deps, test]), + ToRefresh = parse_refresh_paths(RefreshPaths, RState), %% Modules from apps we can't reload without breaking functionality - Blacklist = [ec_cmd_log, providers, cf, cth_readable], + Blacklist = lists:usort( + application:get_env(rebar, refresh_paths_blacklist, []) + ++ [rebar, erlware_commons, providers, cf, cth_readable]), %% Similar to rebar_utils:update_code/1, but also forces a reload %% of used modules. Also forces to reload all of ebin/ instead %% of just the modules in the .app file, because 'extra_src_dirs' %% allows to load and compile files that are not to be kept %% in the app file. - lists:foreach(fun(Path) -> - Name = filename:basename(Path, "/ebin"), - Files = filelib:wildcard(filename:join([Path, "*.beam"])), - Modules = [list_to_atom(filename:basename(F, ".beam")) - || F <- Files], - App = list_to_atom(Name), + [refresh_path(Path, Blacklist) || Path <- ToRefresh], + ok. + +refresh_path(Path, Blacklist) -> + Name = filename:basename(Path, "/ebin"), + App = list_to_atom(Name), + case App of + test -> % skip + code:add_patha(Path), + ok; + _ -> application:load(App), case application:get_key(App, modules) of undefined -> - code:add_patha(Path), - ok; - {ok, Mods} -> - case {length(Mods), length(Mods -- Blacklist)} of - {X,X} -> - ?DEBUG("reloading ~p from ~ts", [Modules, Path]), - code:replace_path(App, Path), - reload_modules(Modules); - {_,_} -> + code:add_patha(Path); + {ok, _Mods} -> + case lists:member(App, Blacklist) of + false -> + refresh_path_do(Path, App); + true -> ?DEBUG("skipping app ~p, stable copy required", [App]) end end - end, ToRefresh). + end. +refresh_path_do(Path, App) -> + Files = filelib:wildcard(filename:join([Path, "*.beam"])), + Modules = [list_to_atom(filename:basename(F, ".beam")) + || F <- Files], + ?DEBUG("reloading ~p from ~ts", [Modules, Path]), + code:replace_path(App, Path), + reload_modules(Modules). + +%% @private parse refresh_paths option +%% no_deps means only project_apps's ebin path +%% no_test means no test path +%% OtherPath. +parse_refresh_paths(RefreshPaths0, RState) -> + RefreshPaths1 = + case lists:member(no_deps, RefreshPaths0) of + true -> + lists:usort([project_apps | lists:delete(all_deps, RefreshPaths0)]); + false -> + RefreshPaths0 + end, + RefreshPaths = + case lists:member(no_test, RefreshPaths1) of + true -> + lists:delete(test, RefreshPaths1); + false -> + RefreshPaths1 + end, + parse_refresh_paths(RefreshPaths, RState, []). +parse_refresh_paths([all_deps | RefreshPaths], RState, Acc) -> + Paths = rebar_state:code_paths(RState, all_deps), + parse_refresh_paths(RefreshPaths, RState, Paths ++ Acc); +parse_refresh_paths([project_apps | RefreshPaths], RState, Acc) -> + Paths = [filename:join([rebar_app_info:out_dir(App), "ebin"]) + || App <- rebar_state:project_apps(RState)], + parse_refresh_paths(RefreshPaths, RState, Paths ++ Acc); +parse_refresh_paths([test | RefreshPaths], RState, Acc) -> + Paths = [filename:join([rebar_app_info:out_dir(App), "test"]) + || App <- rebar_state:project_apps(RState)], + parse_refresh_paths(RefreshPaths, RState, Paths ++ Acc); +parse_refresh_paths([RefreshPath0 | RefreshPaths], RState, Acc) when is_list(RefreshPath0) -> + case filelib:is_dir(RefreshPath0) of + true -> + RefreshPath0 = + case filename:basename(RefreshPath0) of + "ebin" -> RefreshPath0; + _ -> filename:join([RefreshPath0, "ebin"]) + end, + parse_refresh_paths(RefreshPaths, RState, [RefreshPath0 | Acc]); + false -> + parse_refresh_paths(RefreshPaths, RState, Acc) + end; +parse_refresh_paths([_ | RefreshPaths], RState, Acc) -> + parse_refresh_paths(RefreshPaths, RState, Acc); +parse_refresh_paths([], _RState, Acc) -> + lists:usort(Acc). %% @private from a disk config, reload and reapply with the current %% profiles; used to find changes in the config from a prior run. @@ -179,8 +237,27 @@ refresh_state(RState, _Dir) -> %% @private takes a list of modules and reloads them -spec reload_modules([module()]) -> term(). reload_modules([]) -> noop; -reload_modules(Modules) -> - reload_modules(Modules, erlang:function_exported(code, prepare_loading, 1)). +reload_modules(Modules0) -> + Modules = [M || M <- Modules0, is_changed(M)], + reload_modules(Modules, erlang:function_exported(code, prepare_loading, 1)). + +%% @spec is_changed(atom()) -> boolean() +%% @doc true if the loaded module is a beam with a vsn attribute +%% and does not match the on-disk beam file, returns false otherwise. +is_changed(M) -> + try + module_vsn(M:module_info()) =/= module_vsn(code:get_object_code(M)) + catch _:_ -> + false + end. + +module_vsn({M, Beam, _Fn}) -> + {ok, {M, Vsn}} = beam_lib:version(Beam), + Vsn; +module_vsn(L) when is_list(L) -> + {_, Attrs} = lists:keyfind(attributes, 1, L), + {_, Vsn} = lists:keyfind(vsn, 1, Attrs), + Vsn. %% @private reloading modules, when there are modules to actually reload reload_modules(Modules, true) -> -- cgit v1.1 From c5ae9f3fed035d2ef8f072da53647c212f92db8b Mon Sep 17 00:00:00 2001 From: Grigory Starinkin Date: Tue, 8 May 2018 21:42:43 +0100 Subject: start at least one async thread in async thread pool fixes an issue when shell is terminated with an error "Bus error: 10", on attempt to run rebar shell with verbose logging --- src/rebar3.erl | 2 +- src/rebar_prv_local_install.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar3.erl b/src/rebar3.erl index 8e9d4b1..9a5486d 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -54,7 +54,7 @@ %% ==================================================================== %% @doc For running with: -%% erl +sbtu +A0 -noinput -mode minimal -boot start_clean -s rebar3 main -extra "$@" +%% erl +sbtu +A1 -noinput -mode minimal -boot start_clean -s rebar3 main -extra "$@" -spec main() -> no_return(). main() -> List = init:get_plain_arguments(), diff --git a/src/rebar_prv_local_install.erl b/src/rebar_prv_local_install.erl index bb019c4..c41812f 100644 --- a/src/rebar_prv_local_install.erl +++ b/src/rebar_prv_local_install.erl @@ -64,7 +64,7 @@ format_error(Reason) -> bin_contents(OutputDir) -> <<"#!/usr/bin/env sh -erl -pz ", (rebar_utils:to_binary(OutputDir))/binary,"/*/ebin +sbtu +A0 -noshell -boot start_clean -s rebar3 main $REBAR3_ERL_ARGS -extra \"$@\" +erl -pz ", (rebar_utils:to_binary(OutputDir))/binary,"/*/ebin +sbtu +A1 -noshell -boot start_clean -s rebar3 main $REBAR3_ERL_ARGS -extra \"$@\" ">>. extract_escript(State, ScriptPath) -> -- cgit v1.1 From 33eb505573170ff8bf77d6b4758c88a75bf772d8 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 9 May 2018 21:52:37 -0400 Subject: bump to 3.5.3 --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index c96f65c..cc5a0c9 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.5.3"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 966ba72f9098ba1034cb26149a85f41dca1c4cef Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 9 May 2018 21:59:04 -0400 Subject: return to git-based versioning --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index cc5a0c9..c96f65c 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.5.3"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 246b4baaca6996810e455085bb75a33a044efa6a Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 30 May 2018 19:50:28 -0400 Subject: Logger OTP-21-rc2 support Also small output fix in rebar3 shell --- src/rebar_prv_shell.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_shell.erl b/src/rebar_prv_shell.erl index 9a320ad..52b50a0 100644 --- a/src/rebar_prv_shell.erl +++ b/src/rebar_prv_shell.erl @@ -236,7 +236,7 @@ maybe_run_script(State) -> ?DEBUG("No script_file specified.", []), ok; "none" -> - ?DEBUG("Shell script execution skipped (--script none).", []), + ?DEBUG("Shell script execution skipped (--script_file none).", []), ok; RelFile -> File = filename:absname(RelFile), -- cgit v1.1 From ea50c24f61e084f0cd258d0bdb1b2117783a7dcb Mon Sep 17 00:00:00 2001 From: Alexander Petrovsky Date: Thu, 31 May 2018 17:32:33 +0300 Subject: Introduce support of add and del operations under erl_opts directive --- src/rebar_opts.erl | 83 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 57 insertions(+), 26 deletions(-) (limited to 'src') diff --git a/src/rebar_opts.erl b/src/rebar_opts.erl index 1a927ba..8195a77 100644 --- a/src/rebar_opts.erl +++ b/src/rebar_opts.erl @@ -73,36 +73,42 @@ filter_debug_info([H|T]) -> apply_overrides(Opts, Name, Overrides) -> %% Inefficient. We want the order we get here though. Opts1 = lists:foldl(fun({override, O}, OptsAcc) -> - lists:foldl(fun({deps, Value}, OptsAcc1) -> - set(OptsAcc1, {deps,default}, Value); - ({Key, Value}, OptsAcc1) -> - set(OptsAcc1, Key, Value) - end, OptsAcc, O); + override_opt(O, OptsAcc); (_, OptsAcc) -> OptsAcc - end, Opts, Overrides), - - Opts2 = lists:foldl(fun({override, N, O}, OptsAcc) when N =:= Name -> - lists:foldl(fun({deps, Value}, OptsAcc1) -> - set(OptsAcc1, {deps,default}, Value); - ({Key, Value}, OptsAcc1) -> - set(OptsAcc1, Key, Value) - end, OptsAcc, O); + end, Opts, Overrides), + + Opts2 = lists:foldl(fun({add, O}, OptsAcc) -> + add_opt(O, OptsAcc); + (_, OptsAcc) -> + OptsAcc + end, Opts1, Overrides), + + Opts3 = lists:foldl(fun({del, O}, OptsAcc) -> + del_opt(O, OptsAcc); (_, OptsAcc) -> OptsAcc - end, Opts1, Overrides), - - lists:foldl(fun({add, N, O}, OptsAcc) when N =:= Name -> - lists:foldl(fun({deps, Value}, OptsAcc1) -> - OldValue = ?MODULE:get(OptsAcc1, {deps,default}, []), - set(OptsAcc1, {deps,default}, Value++OldValue); - ({Key, Value}, OptsAcc1) -> - OldValue = ?MODULE:get(OptsAcc1, Key, []), - set(OptsAcc1, Key, Value++OldValue) - end, OptsAcc, O); - (_, OptsAcc) -> - OptsAcc - end, Opts2, Overrides). + end, Opts2, Overrides), + + Opts4 = lists:foldl(fun({override, N, O}, OptsAcc) when N =:= Name -> + override_opt(O, OptsAcc); + (_, OptsAcc) -> + OptsAcc + end, Opts3, Overrides), + + Opts5 = lists:foldl(fun({add, N, O}, OptsAcc) when N =:= Name -> + add_opt(O, OptsAcc); + (_, OptsAcc) -> + OptsAcc + end, Opts4, Overrides), + + Opts6 = lists:foldl(fun({del, N, O}, OptsAcc) when N =:= Name -> + del_opt(O, OptsAcc); + (_, OptsAcc) -> + OptsAcc + end, Opts5, Overrides), + + Opts6. add_to_profile(Opts, Profile, KVs) when is_atom(Profile), is_list(KVs) -> Profiles = ?MODULE:get(Opts, profiles, []), @@ -133,6 +139,31 @@ merge_opts(NewOpts, OldOpts) -> %% Internal functions +add_opt(Opts1, Opts2) -> + lists:foldl(fun({deps, Value}, OptsAcc) -> + OldValue = ?MODULE:get(OptsAcc, {deps,default}, []), + set(OptsAcc, {deps,default}, Value++OldValue); + ({Key, Value}, OptsAcc) -> + OldValue = ?MODULE:get(OptsAcc, Key, []), + set(OptsAcc, Key, Value++OldValue) + end, Opts2, Opts1). + +del_opt(Opts1, Opts2) -> + lists:foldl(fun({deps, Value}, OptsAcc) -> + OldValue = ?MODULE:get(OptsAcc, {deps,default}, []), + set(OptsAcc, {deps,default}, OldValue--Value); + ({Key, Value}, OptsAcc) -> + OldValue = ?MODULE:get(OptsAcc, Key, []), + set(OptsAcc, Key, OldValue--Value) + end, Opts2, Opts1). + +override_opt(Opts1, Opts2) -> + lists:foldl(fun({deps, Value}, OptsAcc) -> + set(OptsAcc, {deps,default}, Value); + ({Key, Value}, OptsAcc) -> + set(OptsAcc, Key, Value) + end, Opts2, Opts1). + %% %% Function for dict:merge/3 (in merge_opts/2) to merge options by priority. %% -- cgit v1.1 From 06147af85baa1064c95f5fe5eb2c2d41a67c21a8 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Tue, 5 Jun 2018 20:43:23 -0400 Subject: Add hooks to the upgrade command As requested in #1733 --- src/rebar_prv_upgrade.erl | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'src') diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl index cd75672..e4469cf 100644 --- a/src/rebar_prv_upgrade.erl +++ b/src/rebar_prv_upgrade.erl @@ -43,6 +43,19 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> + Cwd = rebar_state:dir(State), + Providers = rebar_state:providers(State), + rebar_hooks:run_project_and_app_hooks(Cwd, pre, ?PROVIDER, Providers, State), + case do_(State) of + {ok, NewState} -> + rebar_hooks:run_project_and_app_hooks(Cwd, post, ?PROVIDER, Providers, NewState), + {ok, NewState}; + Other -> + rebar_hooks:run_project_and_app_hooks(Cwd, post, ?PROVIDER, Providers, State), + Other + end. + +do_(State) -> {Args, _} = rebar_state:command_parsed_args(State), Locks = rebar_state:get(State, {locks, default}, []), %% We have 3 sources of dependencies to upgrade from: -- cgit v1.1 From 3762e085e9a9de86552edb95384d08abd54697b6 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Tue, 5 Jun 2018 20:57:02 -0400 Subject: Add --start-clean flag to rebar3 shell Allows override of apps specified in other on-disk options. Equivalent to `--apps=""` Fixes #1785 --- src/rebar_prv_shell.erl | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_shell.erl b/src/rebar_prv_shell.erl index 52b50a0..af8d99f 100644 --- a/src/rebar_prv_shell.erl +++ b/src/rebar_prv_shell.erl @@ -76,6 +76,9 @@ init(State) -> "shell. (E.g. --apps app1,app2,app3) Defaults " "to rebar.config {shell, [{apps, Apps}]} or " "relx apps if not specified."}, + {start_clean, undefined, "start-clean", boolean, + "Cancel any applications in the 'apps' list " + "or release."}, {user_drv_args, undefined, "user_drv_args", string, "Arguments passed to user_drv start function for " "creating custom shells."}]} @@ -236,7 +239,7 @@ maybe_run_script(State) -> ?DEBUG("No script_file specified.", []), ok; "none" -> - ?DEBUG("Shell script execution skipped (--script_file none).", []), + ?DEBUG("Shell script execution skipped (--script none).", []), ok; RelFile -> File = filename:absname(RelFile), @@ -311,7 +314,12 @@ find_apps_option(State) -> {Opts, _} = rebar_state:command_parsed_args(State), case debug_get_value(apps, Opts, no_value, "Found shell apps from command line option.") of - no_value -> no_value; + no_value -> + case debug_get_value(start_clean, Opts, false, + "Found start-clean argument to disable apps") of + false -> no_value; + true -> [] + end; AppsStr -> [ list_to_atom(AppStr) || AppStr <- rebar_string:lexemes(AppsStr, " ,:") ] -- cgit v1.1 From a18e5e3c8689954b1ff07c9e64f7575c0a466871 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gon=C3=A7alo=20Tom=C3=A1s?= Date: Fri, 8 Jun 2018 19:19:04 +0100 Subject: fix #1808 --- src/rebar_env.erl | 54 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 20 deletions(-) (limited to 'src') diff --git a/src/rebar_env.erl b/src/rebar_env.erl index eea47de..e9adafb 100644 --- a/src/rebar_env.erl +++ b/src/rebar_env.erl @@ -37,26 +37,40 @@ create_env(State) -> -spec create_env(rebar_state:t(), rebar_dict()) -> proplists:proplist(). create_env(State, Opts) -> BaseDir = rebar_dir:base_dir(State), - [ - {"REBAR_DEPS_DIR", filename:absname(rebar_dir:deps_dir(State))}, - {"REBAR_BUILD_DIR", filename:absname(rebar_dir:base_dir(State))}, - {"REBAR_ROOT_DIR", filename:absname(rebar_dir:root_dir(State))}, - {"REBAR_CHECKOUTS_DIR", filename:absname(rebar_dir:checkouts_dir(State))}, - {"REBAR_PLUGINS_DIR", filename:absname(rebar_dir:plugins_dir(State))}, - {"REBAR_GLOBAL_CONFIG_DIR", filename:absname(rebar_dir:global_config_dir(State))}, - {"REBAR_GLOBAL_CACHE_DIR", filename:absname(rebar_dir:global_cache_dir(Opts))}, - {"REBAR_TEMPLATE_DIR", filename:absname(rebar_dir:template_dir(State))}, - {"REBAR_APP_DIRS", join_dirs(BaseDir, rebar_dir:lib_dirs(State))}, - {"REBAR_SRC_DIRS", join_dirs(BaseDir, rebar_dir:all_src_dirs(Opts))}, - {"ERLANG_ERTS_VER", erlang:system_info(version)}, - {"ERLANG_ROOT_DIR", code:root_dir()}, - {"ERLANG_LIB_DIR_erl_interface", code:lib_dir(erl_interface)}, - {"ERLANG_LIB_VER_erl_interface", re_version(code:lib_dir(erl_interface))}, - {"ERL", filename:join([code:root_dir(), "bin", "erl"])}, - {"ERLC", filename:join([code:root_dir(), "bin", "erlc"])}, - {"ERLANG_ARCH" , rebar_api:wordsize()}, - {"ERLANG_TARGET", rebar_api:get_arch()} - ]. + EnvVars = [ + {"REBAR_DEPS_DIR", filename:absname(rebar_dir:deps_dir(State))}, + {"REBAR_BUILD_DIR", filename:absname(rebar_dir:base_dir(State))}, + {"REBAR_ROOT_DIR", filename:absname(rebar_dir:root_dir(State))}, + {"REBAR_CHECKOUTS_DIR", filename:absname(rebar_dir:checkouts_dir(State))}, + {"REBAR_PLUGINS_DIR", filename:absname(rebar_dir:plugins_dir(State))}, + {"REBAR_GLOBAL_CONFIG_DIR", filename:absname(rebar_dir:global_config_dir(State))}, + {"REBAR_GLOBAL_CACHE_DIR", filename:absname(rebar_dir:global_cache_dir(Opts))}, + {"REBAR_TEMPLATE_DIR", filename:absname(rebar_dir:template_dir(State))}, + {"REBAR_APP_DIRS", join_dirs(BaseDir, rebar_dir:lib_dirs(State))}, + {"REBAR_SRC_DIRS", join_dirs(BaseDir, rebar_dir:all_src_dirs(Opts))}, + {"ERLANG_ERTS_VER", erlang:system_info(version)}, + {"ERLANG_ROOT_DIR", code:root_dir()}, + {"ERL", filename:join([code:root_dir(), "bin", "erl"])}, + {"ERLC", filename:join([code:root_dir(), "bin", "erlc"])}, + {"ERLANG_ARCH" , rebar_api:wordsize()}, + {"ERLANG_TARGET", rebar_api:get_arch()} + ], + EInterfaceVars = create_erl_interface_env(), + lists:append([EnvVars, EInterfaceVars]). + +-spec create_erl_interface_env() -> list(). +create_erl_interface_env() -> + case code:lib_dir(erl_interface) of + {error, bad_name} -> + ?WARN("erl_interface is missing. ERLANG_LIB_DIR_erl_interface and " + "ERLANG_LIB_VER_erl_interface will not be added to the environment.", []), + []; + Dir -> + [ + {"ERLANG_LIB_DIR_erl_interface", Dir}, + {"ERLANG_LIB_VER_erl_interface", re_version(Dir)} + ] + end. %% ==================================================================== %% Internal functions -- cgit v1.1 From d1fc937515b312c1e5129e9e435e5b462f34b874 Mon Sep 17 00:00:00 2001 From: Bryan Paxton Date: Thu, 7 Jun 2018 11:11:11 -0500 Subject: Abort if erl_first_files is not a list of strings - resolves #1645 --- src/rebar_erlc_compiler.erl | 8 ++++++++ src/rebar_utils.erl | 11 ++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index c588a25..ebdd9dd 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -285,6 +285,7 @@ gather_src(Opts, BaseDirParts, [Dir|Rest], Srcs, CompileOpts) -> %% files, so that yet to be compiled parse transformations are excluded from it. erl_first_files(Opts, ErlOpts, Dir, NeededErlFiles) -> ErlFirstFilesConf = rebar_opts:get(Opts, erl_first_files, []), + valid_erl_first_conf(ErlFirstFilesConf), NeededSrcDirs = lists:usort(lists:map(fun filename:dirname/1, NeededErlFiles)), %% NOTE: order of files here is important! ErlFirstFiles = @@ -796,3 +797,10 @@ dir_recursive(Opts, Dir, CompileOpts) when is_list(CompileOpts) -> undefined -> rebar_dir:recursive(Opts, Dir); Recursive -> Recursive end. + +valid_erl_first_conf(FileList) -> + case rebar_utils:is_list_of_strings(FileList) of + true -> true; + false -> ?ABORT("An invalid file list (~p) was provided as part of your erl_files_first directive", + [FileList]) + end. diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 604abb8..2ded481 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -73,7 +73,8 @@ list_dir/1, user_agent/0, reread_config/1, - get_proxy_auth/0]). + get_proxy_auth/0, + is_list_of_strings/1]). %% for internal use only @@ -919,3 +920,11 @@ get_proxy_auth() -> undefined -> []; {ok, ProxyAuth} -> ProxyAuth end. + +-spec rebar_utils:is_list_of_strings(term()) -> boolean(). +is_list_of_strings(List) when not is_list(hd(List)) -> + false; +is_list_of_strings(List) when is_list(hd(List)) -> + true; +is_list_of_strings(List) when is_list(List) -> + true. -- cgit v1.1 From a74dc021cc9dd89b7448d894e262ebe5a82285cd Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 8 Jun 2018 20:57:48 -0600 Subject: run compile provider in default namespace from bare --- src/rebar_prv_bare_compile.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_bare_compile.erl b/src/rebar_prv_bare_compile.erl index c29a711..5d3e977 100644 --- a/src/rebar_prv_bare_compile.erl +++ b/src/rebar_prv_bare_compile.erl @@ -46,7 +46,9 @@ do(State) -> [AppInfo] = rebar_state:project_apps(State), AppInfo1 = rebar_app_info:out_dir(AppInfo, rebar_dir:get_cwd()), - rebar_prv_compile:compile(State, AppInfo1), + + %% run compile in the default namespace + rebar_prv_compile:compile(rebar_state:namespace(State, default), AppInfo1), rebar_utils:cleanup_code_path(OrigPath), -- cgit v1.1 From 70f063dc15d4003f0a5dc85523de02397d9cc250 Mon Sep 17 00:00:00 2001 From: Bryan Paxton Date: Sat, 9 Jun 2018 18:25:17 -0500 Subject: Finish adding specs for private funs in rebar3.erl - added type specs for following private functions: - run/1 - set_options/2 - test_state/1 - safe_define_test_macro/1 - test_defined/1 --- src/rebar3.erl | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'src') diff --git a/src/rebar3.erl b/src/rebar3.erl index 9a5486d..ec8e953 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -93,6 +93,7 @@ run(BaseState, Commands) -> %% arguments passed, if they have any relevance; used to translate %% from the escript call-site into a common one with the library %% usage. +-spec run([any(), ...]) -> {ok, rebar_state:t()} | {error, term()}. run(RawArgs) -> start_and_load_apps(command_line), @@ -239,6 +240,7 @@ parse_args([Task | RawRest]) -> {list_to_atom(Task), RawRest}. %% @private actually not too sure what this does anymore. +-spec set_options(rebar_state:t(),{[any()],[any()]}) -> {rebar_state:t(),[any()]}. set_options(State, {Options, NonOptArgs}) -> GlobalDefines = proplists:get_all_values(defines, Options), @@ -387,6 +389,7 @@ state_from_global_config(Config, GlobalConfigFile) -> GlobalConfig3 = rebar_state:set(GlobalConfig2, {plugins, global}, rebar_state:get(GlobalConfigThrowAway, plugins, [])), rebar_state:providers(rebar_state:new(GlobalConfig3, Config), GlobalPlugins). +-spec test_state(rebar_state:t()) -> [{'extra_src_dirs',[string()]} | {'erl_opts',[any()]}]. test_state(State) -> %% Fetch the test profile's erl_opts only Opts = rebar_state:opts(State), @@ -396,6 +399,7 @@ test_state(State) -> TestOpts = safe_define_test_macro(ErlOpts), [{extra_src_dirs, ["test"]}, {erl_opts, TestOpts}]. +-spec safe_define_test_macro([any()]) -> [any()] | [{'d',atom()} | any()]. safe_define_test_macro(Opts) -> %% defining a compile macro twice results in an exception so %% make sure 'TEST' is only defined once @@ -404,6 +408,7 @@ safe_define_test_macro(Opts) -> false -> [{d, 'TEST'}|Opts] end. +-spec test_defined([{d, atom()} | {d, atom(), term()} | term()]) -> boolean(). test_defined([{d, 'TEST'}|_]) -> true; test_defined([{d, 'TEST', true}|_]) -> true; test_defined([_|Rest]) -> test_defined(Rest); -- cgit v1.1 From 38865da7ba01e7d5e60316e970f01959e85759ee Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Tue, 19 Jun 2018 16:38:56 -0400 Subject: Bump to 3.6.0 --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index c96f65c..5c76f59 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.6.0"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From d778b6e361397a7b70fc1c962c26e5b936f84ae1 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Tue, 19 Jun 2018 16:43:09 -0400 Subject: Back to git-based versioning --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 5c76f59..c96f65c 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.6.0"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 9a72b89ad2cdfe341a65a13766106b37ab446371 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Mon, 25 Jun 2018 13:54:59 -0600 Subject: otp21 eperm fix for local install/upgrade, no need to set uid/gid --- src/rebar_prv_local_install.erl | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_local_install.erl b/src/rebar_prv_local_install.erl index c41812f..cd6a204 100644 --- a/src/rebar_prv_local_install.erl +++ b/src/rebar_prv_local_install.erl @@ -89,15 +89,9 @@ extract_escript(State, ScriptPath) -> BinFile = filename:join(BinDir, "rebar3"), filelib:ensure_dir(BinFile), - {ok, #file_info{mode = _, - uid = Uid, - gid = Gid}} = file:read_file_info(ScriptPath), - ?INFO("Writing rebar3 run script ~ts...", [BinFile]), file:write_file(BinFile, bin_contents(OutputDir)), - ok = file:write_file_info(BinFile, #file_info{mode=33277, - uid=Uid, - gid=Gid}), + ok = file:write_file_info(BinFile, #file_info{mode=33277}), ?INFO("Add to $PATH for use: export PATH=~ts:$PATH", [BinDir]), -- cgit v1.1 From 9ec901e47bc1259204896e4a102f23eca2c22953 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 25 Jun 2018 16:50:40 -0400 Subject: Bump to 3.6.1 -- fixing eperm issues --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index c96f65c..1a844ca 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.6.1"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 5a3c8e5b3d74e619d9ea82d8004cb01d65a21443 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 25 Jun 2018 16:52:59 -0400 Subject: Back to git-based versioning --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 1a844ca..c96f65c 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.6.1"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From a8426aba8a7b86891d6f47fa3b7323cf35a786b1 Mon Sep 17 00:00:00 2001 From: Bryan Paxton Date: Tue, 26 Jun 2018 23:30:11 -0500 Subject: Fix for atoms in erl_first_files --- src/rebar_erlc_compiler.erl | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index ebdd9dd..920c3b4 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -799,8 +799,27 @@ dir_recursive(Opts, Dir, CompileOpts) when is_list(CompileOpts) -> end. valid_erl_first_conf(FileList) -> - case rebar_utils:is_list_of_strings(FileList) of + Strs = filter_file_list(FileList), + case rebar_utils:is_list_of_strings(Strs) of true -> true; false -> ?ABORT("An invalid file list (~p) was provided as part of your erl_files_first directive", [FileList]) end. + +filter_file_list(FileList) -> + Atoms = lists:filter( fun(X) -> is_atom(X) end, FileList), + case Atoms of + [] -> + FileList; + _ -> + atoms_in_erl_first_files_warning(Atoms), + lists:filter( fun(X) -> not(is_atom(X)) end, FileList) + end. + +atoms_in_erl_first_files_warning(Atoms) -> + W = "You have provided atoms as file entries in erl_first_files; " + "erl_first_files only expects lists of filenames as strings. " + "The following modules (~p) may not work as expected and it is advised " + "that you change these entires to string format " + "(e.g., \"src/module.erl\") ", + ?WARN(W, [Atoms]). -- cgit v1.1 From 6c7e2e3826ae9f444c2c2243d7a451f53d86edd8 Mon Sep 17 00:00:00 2001 From: feng19 Date: Wed, 11 Jul 2018 15:19:30 +0800 Subject: remove some refresh_paths's options(no_deps and no_test) & use beam_lib:md5/1 to get the module's vsn --- src/rebar_agent.erl | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) (limited to 'src') diff --git a/src/rebar_agent.erl b/src/rebar_agent.erl index 69e9b8e..445ae54 100644 --- a/src/rebar_agent.erl +++ b/src/rebar_agent.erl @@ -131,10 +131,8 @@ maybe_show_warning(State) -> %% that makes sense. -spec refresh_paths(rebar_state:t()) -> ok. refresh_paths(RState) -> - RefreshPaths = lists:usort( - application:get_env(rebar, refresh_paths, []) - ++ [all_deps, test]), - ToRefresh = parse_refresh_paths(RefreshPaths, RState), + RefreshPaths = application:get_env(rebar, refresh_paths, [all_deps, test]), + ToRefresh = parse_refresh_paths(RefreshPaths, RState, []), %% Modules from apps we can't reload without breaking functionality Blacklist = lists:usort( application:get_env(rebar, refresh_paths_blacklist, []) @@ -180,22 +178,6 @@ refresh_path_do(Path, App) -> %% no_deps means only project_apps's ebin path %% no_test means no test path %% OtherPath. -parse_refresh_paths(RefreshPaths0, RState) -> - RefreshPaths1 = - case lists:member(no_deps, RefreshPaths0) of - true -> - lists:usort([project_apps | lists:delete(all_deps, RefreshPaths0)]); - false -> - RefreshPaths0 - end, - RefreshPaths = - case lists:member(no_test, RefreshPaths1) of - true -> - lists:delete(test, RefreshPaths1); - false -> - RefreshPaths1 - end, - parse_refresh_paths(RefreshPaths, RState, []). parse_refresh_paths([all_deps | RefreshPaths], RState, Acc) -> Paths = rebar_state:code_paths(RState, all_deps), parse_refresh_paths(RefreshPaths, RState, Paths ++ Acc); @@ -246,16 +228,18 @@ reload_modules(Modules0) -> %% and does not match the on-disk beam file, returns false otherwise. is_changed(M) -> try - module_vsn(M:module_info()) =/= module_vsn(code:get_object_code(M)) + module_vsn(M:module_info(attributes)) =/= module_vsn(code:get_object_code(M)) catch _:_ -> false end. module_vsn({M, Beam, _Fn}) -> - {ok, {M, Vsn}} = beam_lib:version(Beam), + % Because the vsn can set by -vsn(X) in module. + % So didn't use beam_lib:version/1 to get the vsn. + % So if set -vsn(X) in module, it will always reload the module. + {ok, {M, <>}} = beam_lib:md5(Beam), Vsn; -module_vsn(L) when is_list(L) -> - {_, Attrs} = lists:keyfind(attributes, 1, L), +module_vsn(Attrs) when is_list(Attrs) -> {_, Vsn} = lists:keyfind(vsn, 1, Attrs), Vsn. -- cgit v1.1 From f08e5f62ffa7824ddc02afee2e5af4de43eb3887 Mon Sep 17 00:00:00 2001 From: simonxu72 Date: Thu, 26 Jul 2018 10:53:52 +0800 Subject: add git clone --reference path support --- src/rebar_git_resource.erl | 43 +++++++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 12 deletions(-) (limited to 'src') diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl index 3aa875f..2b5f453 100644 --- a/src/rebar_git_resource.erl +++ b/src/rebar_git_resource.erl @@ -134,38 +134,44 @@ maybe_warn_local_url(Url) -> %% Use different git clone commands depending on git --version git_clone(branch,Vsn,Url,Dir,Branch) when Vsn >= {1,7,10}; Vsn =:= undefined -> - rebar_utils:sh(?FMT("git clone ~ts ~ts -b ~ts --single-branch", - [rebar_utils:escape_chars(Url), + rebar_utils:sh(?FMT("git clone ~ts ~ts ~ts -b ~ts --single-branch", + [git_clone_options(), + rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir)), rebar_utils:escape_chars(Branch)]), [{cd, filename:dirname(Dir)}]); git_clone(branch,_Vsn,Url,Dir,Branch) -> - rebar_utils:sh(?FMT("git clone ~ts ~ts -b ~ts", - [rebar_utils:escape_chars(Url), + rebar_utils:sh(?FMT("git clone ~ts ~ts ~ts -b ~ts", + [git_clone_options(), + rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir)), rebar_utils:escape_chars(Branch)]), [{cd, filename:dirname(Dir)}]); git_clone(tag,Vsn,Url,Dir,Tag) when Vsn >= {1,7,10}; Vsn =:= undefined -> - rebar_utils:sh(?FMT("git clone ~ts ~ts -b ~ts --single-branch", - [rebar_utils:escape_chars(Url), + rebar_utils:sh(?FMT("git clone ~ts ~ts ~ts -b ~ts --single-branch", + [git_clone_options(), + rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir)), rebar_utils:escape_chars(Tag)]), [{cd, filename:dirname(Dir)}]); git_clone(tag,_Vsn,Url,Dir,Tag) -> - rebar_utils:sh(?FMT("git clone ~ts ~ts -b ~ts", - [rebar_utils:escape_chars(Url), + rebar_utils:sh(?FMT("git clone ~ts ~ts ~ts -b ~ts", + [git_clone_options(), + rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir)), rebar_utils:escape_chars(Tag)]), [{cd, filename:dirname(Dir)}]); git_clone(ref,_Vsn,Url,Dir,Ref) -> - rebar_utils:sh(?FMT("git clone -n ~ts ~ts", - [rebar_utils:escape_chars(Url), + rebar_utils:sh(?FMT("git clone ~ts -n ~ts ~ts", + [git_clone_options(), + rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]), rebar_utils:sh(?FMT("git checkout -q ~ts", [Ref]), [{cd, Dir}]); git_clone(rev,_Vsn,Url,Dir,Rev) -> - rebar_utils:sh(?FMT("git clone -n ~ts ~ts", - [rebar_utils:escape_chars(Url), + rebar_utils:sh(?FMT("git clone ~ts -n ~ts ~ts", + [git_clone_options(), + rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]), rebar_utils:sh(?FMT("git checkout -q ~ts", [rebar_utils:escape_chars(Rev)]), @@ -282,3 +288,16 @@ parse_tags(Dir) -> end end end. + +git_clone_options() -> + Option = case os:getenv("REBAR_GIT_CLONE_OPTIONS") of + false -> "" ; %% env var not set + [] -> "" ; %% env var set to empty + Opt -> + Opt + end, + + ?DEBUG("Git clone Option = ~p",[Option]), + Option. + + -- cgit v1.1 From b81871c61809a9e5c09f54d6c8298908d18a760c Mon Sep 17 00:00:00 2001 From: simonxu72 Date: Thu, 26 Jul 2018 21:12:06 +0800 Subject: combine []/Opt case options into one @ get_git_options/0 --- src/rebar_git_resource.erl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl index 2b5f453..0286762 100644 --- a/src/rebar_git_resource.erl +++ b/src/rebar_git_resource.erl @@ -292,8 +292,7 @@ parse_tags(Dir) -> git_clone_options() -> Option = case os:getenv("REBAR_GIT_CLONE_OPTIONS") of false -> "" ; %% env var not set - [] -> "" ; %% env var set to empty - Opt -> + Opt -> %% env var set to empty or others Opt end, -- cgit v1.1 From 1d10f7ae8c7b73c427609bfbfbde4c81a48359c0 Mon Sep 17 00:00:00 2001 From: Shamis Shukoor Date: Fri, 3 Aug 2018 09:23:20 +0200 Subject: fixed typo erl_first_files --- src/rebar_erlc_compiler.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index 920c3b4..a398ddd 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -802,7 +802,7 @@ valid_erl_first_conf(FileList) -> Strs = filter_file_list(FileList), case rebar_utils:is_list_of_strings(Strs) of true -> true; - false -> ?ABORT("An invalid file list (~p) was provided as part of your erl_files_first directive", + false -> ?ABORT("An invalid file list (~p) was provided as part of your erl_first_files directive", [FileList]) end. -- cgit v1.1 From e157b6c23fcf86e76a87edf38283d5684df1307f Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 2 Aug 2018 19:43:57 -0400 Subject: Friendlier output on include_lib errors Turns output like this: ===> Compiling apps/c/src/c.erl failed apps/c/src/c.erl:3: can't find include lib "parse_trans/include/codegen.hrl" Into: ===> Compiling apps/c/src/c.erl failed apps/c/src/c.erl:3: can't find include lib "parse_trans/include/codegen.hrl"; Make sure parse_trans is in your app file's 'applications' list Which is likely going to help newcomers encountering issues. --- src/rebar_base_compiler.erl | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rebar_base_compiler.erl b/src/rebar_base_compiler.erl index 3f273f1..00b0533 100644 --- a/src/rebar_base_compiler.erl +++ b/src/rebar_base_compiler.erl @@ -264,14 +264,24 @@ report(Messages) -> -spec format_errors(_, Extra, [err_or_warn()]) -> [string()] when Extra :: string(). format_errors(_MainSource, Extra, Errors) -> - [begin - [format_error(Source, Extra, Desc) || Desc <- Descs] - end + [[format_error(Source, Extra, Desc) || Desc <- Descs] || {Source, Descs} <- Errors]. %% @private format compiler errors into proper outputtable strings -spec format_error(file:filename(), Extra, err_or_warn()) -> string() when Extra :: string(). +format_error(Source, Extra, {Line, Mod=epp, Desc={include,lib,File}}) -> + %% Special case for include file errors, overtaking the default one + BaseDesc = Mod:format_error(Desc), + Friendly = case filename:split(File) of + [Lib, "include", _] -> + io_lib:format("; Make sure ~s is in your app " + "file's 'applications' list", [Lib]); + _ -> + "" + end, + FriendlyDesc = BaseDesc ++ Friendly, + ?FMT("~ts:~w: ~ts~ts~n", [Source, Line, Extra, FriendlyDesc]); format_error(Source, Extra, {{Line, Column}, Mod, Desc}) -> ErrorDesc = Mod:format_error(Desc), ?FMT("~ts:~w:~w: ~ts~ts~n", [Source, Line, Column, Extra, ErrorDesc]); -- cgit v1.1 From 8f36dfdc65a8dc6c4fb07c346d753e142b2735bd Mon Sep 17 00:00:00 2001 From: Gerald Xv Date: Wed, 5 Sep 2018 04:55:17 +0000 Subject: add report in the effects_code_generation, as it is the short form for both report_errors and report_warnings --- src/rebar_erlc_compiler.erl | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index a398ddd..57e5398 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -363,6 +363,7 @@ effects_code_generation(Option) -> report_errors -> false; return_errors-> false; return_warnings-> false; + report -> false; warnings_as_errors -> false; binary -> false; verbose -> false; -- cgit v1.1 From 0303567d95f0769f3d76df35d890458d9741cf4d Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 10 Sep 2018 10:20:57 -0400 Subject: Reload logger config in shell This requires some fancy dynamic work since the logger is started as part of the kernel and we lost the sys.config from users when working from there. We start conservatively by making it an optional thing, turning it on only where we know it to be safe. The changes are applied _after_ having loaded the rest of configs so if an inoffensive error happens, the shell works (with a bad error message) rather than plain exploding. --- src/rebar_prv_shell.erl | 2 +- src/rebar_utils.erl | 53 +++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 52 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_shell.erl b/src/rebar_prv_shell.erl index af8d99f..5b8d789 100644 --- a/src/rebar_prv_shell.erl +++ b/src/rebar_prv_shell.erl @@ -385,7 +385,7 @@ reread_config(AppsToStart, State) -> lists:member(App, Running), lists:member(App, AppsToStart), not lists:member(App, BlackList)], - _ = rebar_utils:reread_config(ConfigList), + _ = rebar_utils:reread_config(ConfigList, [update_logger]), ok end. diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 2ded481..7e57d01 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -72,7 +72,7 @@ info_useless/2, list_dir/1, user_agent/0, - reread_config/1, + reread_config/1, reread_config/2, get_proxy_auth/0, is_list_of_strings/1]). @@ -436,6 +436,18 @@ user_agent() -> ?FMT("Rebar/~ts (OTP/~ts)", [Vsn, otp_release()]). reread_config(ConfigList) -> + %% Default to not re-configuring the logger for now; + %% this can leak logs in CT redirection when setting up hooks + %% for example. If we want to turn it on by default, we may + %% want to disable it in CT at the same time or figure out a + %% way to silence it. + %% The same pattern may apply to other tasks, so let's enable + %% case-by-case. + reread_config(ConfigList, []). + +reread_config(ConfigList, Opts) -> + UpdateLoggerConfig = erlang:function_exported(logger, module_info, 0) andalso + proplists:get_value(update_logger, Opts, false), %% NB: we attempt to mimic -config here, which survives app reload, %% hence {persistent, true}. SetEnv = case version_tuple(?MODULE:otp_release()) of @@ -445,15 +457,52 @@ reread_config(ConfigList) -> fun (App, Key, Val) -> application:set_env(App, Key, Val, [{persistent, true}]) end end, try + Res = [SetEnv(Application, Key, Val) || Config <- ConfigList, {Application, Items} <- Config, - {Key, Val} <- Items] + {Key, Val} <- Items], + case UpdateLoggerConfig of + true -> reread_logger_config(); + false -> ok + end, + Res catch _:_ -> ?ERROR("The configuration file submitted could not be read " "and will be ignored.", []) end. +%% @private since the kernel app is already booted, re-reading its config +%% requires doing some magic to dynamically patch running handlers to +%% deal with the current value. +reread_logger_config() -> + KernelCfg = application:get_all_env(kernel), + LogCfg = proplists:get_value(logger, KernelCfg), + case LogCfg of + undefined -> + ok; + _ -> + %% Extract and apply settings related to primary configuration + %% -- primary config is used for settings shared across handlers + LogLvlPrimary = proplists:get_value(logger_info, KernelCfg, all), + {FilterDefault, Filters} = + case lists:keyfind(filters, 1, KernelCfg) of + false -> {log, []}; + {filters, FoundDef, FoundFilter} -> {FoundDef, FoundFilter} + end, + Primary = #{level => LogLvlPrimary, + filter_default => FilterDefault, + filters => Filters}, + %% Load the correct handlers based on their individual config. + [case Id of + default -> logger:update_handler_config(Id, Cfg); + _ -> logger:add_handler(Id, Mod, Cfg) + end || {handler, Id, Mod, Cfg} <- LogCfg], + logger:set_primary_config(Primary), + ok + end. + + %% @doc Given env. variable `FOO' we want to expand all references to %% it in `InStr'. References can have two forms: `$FOO' and `${FOO}' %% The end of form `$FOO' is delimited with whitespace or EOL -- cgit v1.1 From 4ec1d288c0b90ce92dffc687bb6287fcf5a2776d Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 13 Sep 2018 18:15:46 -0400 Subject: Bump to 3.6.2 --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index c96f65c..cc6b11e 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.6.2"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 5e678754d7951a1700e2fcca4e1eb91ecc862e1a Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 13 Sep 2018 18:17:36 -0400 Subject: Back to git-based versioning --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index cc6b11e..c96f65c 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.6.2"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 56b7d88975aa8da6446857cfd92de0825024bf63 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 13 Sep 2018 19:36:00 -0600 Subject: support for hex v2, multiple repository fetching, private organizations (#1884) * update to hex_core for hex-v2 repo support (#1865) * update to hex_core for hex-v2 repo support This patch adds only single repo hex-v2 support through hex_core. Packages no longer filtered out by buildtool metadata and the package index is updated per-package instead of fetched as one large ets dump. * tell travis to also build hex_core branch * support list of repos for hex packages (#1866) * support list of repos for hex packages repos are defined under the hex key in rebar configs. They can be defined at the top level of a project or globally, but not in profiles and the repos configured in dependencies are also ignored. Searching for packages involves first checking for a match in the local repo index cache, in the order repos are defined. If not found each repo is checked through the hex api for any known versions of the package and the first repo with a version that fits the constraint is used. * add {repos, replace, []} for overriding the global & default repos * add hex auth handling for repos (#1874) auth token are kept in a hex.config file that is modified by the rebar3 hex plugin. Repo names that have a : separating a parent and child are considered organizations. The parent repo's auth will be included with the child. So an organization named hexpm:rebar3_test will include any hexpm auth tokens found in the rebar3_test organization's configuration. * move packages to top level of of hexpm cache dir (#1876) * move packages to top level of of hexpm cache dir * append organization name to parent's repo_url when parsing repos * only eval config scripts and apply overrides once per app (#1879) * only eval config scripts and apply overrides once per app * move new resource behaviour to rebar_resource_v2 and keep v1 * cleanup use of rebar_resource module and unused functions * cleanup error messages and unused code * when discovering apps support mix packages as unbuilt apps (#1882) * use hex_core tarball unpacking support in pkg resource (#1883) * use hex_core tarball unpacking support in pkg resource * ignore etag if package doesn't exist and delete if checksum fails * add back tests for bad package checksums * improve bad registry checksum error message --- src/rebar.app.src | 2 + src/rebar.hrl | 31 ++- src/rebar3.erl | 19 +- src/rebar_api.erl | 2 +- src/rebar_app_discover.erl | 49 ++-- src/rebar_app_info.erl | 70 +++--- src/rebar_app_utils.erl | 109 ++++----- src/rebar_config.erl | 2 +- src/rebar_fetch.erl | 108 ++++----- src/rebar_git_resource.erl | 69 ++++-- src/rebar_hex_repos.erl | 142 ++++++++++++ src/rebar_hg_resource.erl | 67 ++++-- src/rebar_otp_app.erl | 16 +- src/rebar_packages.erl | 480 ++++++++++++++++++++++++++++------------ src/rebar_pkg_resource.erl | 437 +++++++++++++----------------------- src/rebar_prv_deps.erl | 8 +- src/rebar_prv_deps_tree.erl | 6 +- src/rebar_prv_install_deps.erl | 70 +++--- src/rebar_prv_local_upgrade.erl | 28 ++- src/rebar_prv_lock.erl | 9 +- src/rebar_prv_packages.erl | 106 +++++---- src/rebar_prv_repos.erl | 47 ++++ src/rebar_prv_shell.erl | 2 + src/rebar_prv_update.erl | 238 ++------------------ src/rebar_prv_upgrade.erl | 35 ++- src/rebar_resource.erl | 44 +++- src/rebar_resource_v2.erl | 147 ++++++++++++ src/rebar_state.erl | 76 ++++--- src/rebar_string.erl | 5 +- src/rebar_utils.erl | 157 ++++++++++--- 30 files changed, 1520 insertions(+), 1061 deletions(-) create mode 100644 src/rebar_hex_repos.erl create mode 100644 src/rebar_prv_repos.erl create mode 100644 src/rebar_resource_v2.erl (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index c96f65c..530a79e 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -30,6 +30,7 @@ relx, cf, inets, + hex_core, eunit_formatters]}, {env, [ %% Default log level @@ -67,6 +68,7 @@ rebar_prv_release, rebar_prv_relup, rebar_prv_report, + rebar_prv_repos, rebar_prv_shell, rebar_prv_state, rebar_prv_tar, diff --git a/src/rebar.hrl b/src/rebar.hrl index f461c70..572cbe8 100644 --- a/src/rebar.hrl +++ b/src/rebar.hrl @@ -25,14 +25,35 @@ -define(CONFIG_VERSION, "1.1.0"). -define(DEFAULT_CDN, "https://repo.hex.pm/"). -define(REMOTE_PACKAGE_DIR, "tarballs"). --define(REMOTE_REGISTRY_FILE, "registry.ets.gz"). -define(LOCK_FILE, "rebar.lock"). -define(DEFAULT_COMPILER_SOURCE_FORMAT, relative). +-define(PACKAGE_INDEX_VERSION, 4). +-define(PACKAGE_TABLE, package_index_v4). +-define(INDEX_FILE, "packages-v4.idx"). +-define(HEX_AUTH_FILE, "hex.config"). +-define(PUBLIC_HEX_REPO, <<"hexpm">>). --define(PACKAGE_INDEX_VERSION, 3). --define(PACKAGE_TABLE, package_index). --define(INDEX_FILE, "packages.idx"). --define(REGISTRY_FILE, "registry"). +%% ignore this function in all modules +%% not every module that exports it and relies on it being called implements provider +-ignore_xref([{format_error, 1}]). + +%% the package record is used in a select match spec which upsets dialyzer +%% this is the suggested workaround from Tobias +%% http://erlang.org/pipermail/erlang-questions/2009-February/041445.html +-type ms_field() :: '$1' | '_'. + +%% TODO: change package and requirement keys to be required (:=) after dropping support for OTP-18 +-record(package, {key :: {unicode:unicode_binary() | ms_field(), unicode:unicode_binary() | ms_field(), + unicode:unicode_binary() | ms_field()}, + checksum :: binary() | ms_field(), + retired :: boolean() | ms_field(), + dependencies :: [#{package => unicode:unicode_binary(), + requirement => unicode:unicode_binary()}] | ms_field()}). + +-record(resource, {type :: atom(), + module :: module(), + state :: term(), + implementation :: rebar_resource | rebar_resource_v2}). -ifdef(namespaced_types). -type rebar_dict() :: dict:dict(). diff --git a/src/rebar3.erl b/src/rebar3.erl index ec8e953..e87cb19 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -103,7 +103,7 @@ run(RawArgs) -> case erlang:system_info(version) of "6.1" -> ?WARN("Due to a filelib bug in Erlang 17.1 it is recommended" - "you update to a newer release.", []); + "you update to a newer release.", []); _ -> ok end, @@ -139,8 +139,14 @@ run_aux(State, RawArgs) -> rebar_state:set(State1, rebar_packages_cdn, CDN) end, + %% TODO: this means use of REBAR_PROFILE=profile will replace the repos with + %% the repos defined in the profile. But it will not work with `as profile`. + %% Maybe it shouldn't work with either to be consistent? + Resources = application:get_env(rebar, resources, []), + State2_ = rebar_state:create_resources(Resources, State2), + %% bootstrap test profile - State3 = rebar_state:add_to_profile(State2, test, test_state(State1)), + State3 = rebar_state:add_to_profile(State2_, test, test_state(State1)), %% Process each command, resetting any state between each one BaseDir = rebar_state:get(State, base_dir, ?DEFAULT_BASE_DIR), @@ -375,7 +381,11 @@ state_from_global_config(Config, GlobalConfigFile) -> %% We don't want to worry about global plugin install state effecting later %% usage. So we throw away the global profile state used for plugin install. - GlobalConfigThrowAway = rebar_state:current_profiles(GlobalConfig, [global]), + GlobalConfigThrowAway0 = rebar_state:current_profiles(GlobalConfig, [global]), + + Resources = application:get_env(rebar, resources, []), + GlobalConfigThrowAway = rebar_state:create_resources(Resources, GlobalConfigThrowAway0), + GlobalState = case rebar_state:get(GlobalConfigThrowAway, plugins, []) of [] -> GlobalConfigThrowAway; @@ -386,7 +396,8 @@ state_from_global_config(Config, GlobalConfigFile) -> end, GlobalPlugins = rebar_state:providers(GlobalState), GlobalConfig2 = rebar_state:set(GlobalConfig, plugins, []), - GlobalConfig3 = rebar_state:set(GlobalConfig2, {plugins, global}, rebar_state:get(GlobalConfigThrowAway, plugins, [])), + GlobalConfig3 = rebar_state:set(GlobalConfig2, {plugins, global}, + rebar_state:get(GlobalConfigThrowAway, plugins, [])), rebar_state:providers(rebar_state:new(GlobalConfig3, Config), GlobalPlugins). -spec test_state(rebar_state:t()) -> [{'extra_src_dirs',[string()]} | {'erl_opts',[any()]}]. diff --git a/src/rebar_api.erl b/src/rebar_api.erl index 9d9071e..4dabe8a 100644 --- a/src/rebar_api.erl +++ b/src/rebar_api.erl @@ -88,4 +88,4 @@ processing_base_dir(State) -> %% its configuration, including for validation of certs. -spec ssl_opts(string()) -> [term()]. ssl_opts(Url) -> - rebar_pkg_resource:ssl_opts(Url). + rebar_utils:ssl_opts(Url). diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index 382b36b..e82403c 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -9,8 +9,7 @@ find_apps/2, find_apps/3, find_app/2, - find_app/3, - find_app/4]). + find_app/3]). -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). @@ -95,7 +94,7 @@ format_error({missing_module, Module}) -> merge_deps(AppInfo, State) -> %% These steps make sure that hooks and artifacts are run in the context of %% the application they are defined at. If an umbrella structure is used and - %% they are deifned at the top level they will instead run in the context of + %% they are defined at the top level they will instead run in the context of %% the State and at the top level, not as part of an application. CurrentProfiles = rebar_state:current_profiles(State), Default = reset_hooks(rebar_state:default(State), CurrentProfiles), @@ -205,7 +204,7 @@ reset_hooks(Opts, CurrentProfiles) -> -spec all_app_dirs([file:name()]) -> [{file:name(), [file:name()]}]. all_app_dirs(LibDirs) -> lists:flatmap(fun(LibDir) -> - SrcDirs = find_config_src(LibDir, ["src"]), + {_, SrcDirs} = find_config_src(LibDir, ["src"]), app_dirs(LibDir, SrcDirs) end, LibDirs). @@ -278,8 +277,9 @@ find_apps(LibDirs, SrcDirs, Validate) -> %% app info record. -spec find_app(file:filename_all(), valid | invalid | all) -> {true, rebar_app_info:t()} | false. find_app(AppDir, Validate) -> - SrcDirs = find_config_src(AppDir, ["src"]), - find_app(rebar_app_info:new(), AppDir, SrcDirs, Validate). + {Config, SrcDirs} = find_config_src(AppDir, ["src"]), + AppInfo = rebar_app_info:update_opts(rebar_app_info:new(), dict:new(), Config), + find_app_(AppInfo, AppDir, SrcDirs, Validate). %% @doc check that a given app in a directory is there, and whether it's %% valid or not based on the second argument. Returns the related @@ -291,7 +291,7 @@ find_app(AppInfo, AppDir, Validate) -> %% of src/ AppOpts = rebar_app_info:opts(AppInfo), SrcDirs = rebar_dir:src_dirs(AppOpts, ["src"]), - find_app(AppInfo, AppDir, SrcDirs, Validate). + find_app_(AppInfo, AppDir, SrcDirs, Validate). %% @doc check that a given app in a directory is there, and whether it's %% valid or not based on the second argument. The third argument includes @@ -301,6 +301,14 @@ find_app(AppInfo, AppDir, Validate) -> [file:filename_all()], valid | invalid | all) -> {true, rebar_app_info:t()} | false. find_app(AppInfo, AppDir, SrcDirs, Validate) -> + Config = rebar_config:consult(AppDir), + AppInfo1 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), Config), + find_app_(AppInfo1, AppDir, SrcDirs, Validate). + +-spec find_app_(rebar_app_info:t(), file:filename_all(), + [file:filename_all()], valid | invalid | all) -> + {true, rebar_app_info:t()} | false. +find_app_(AppInfo, AppDir, SrcDirs, Validate) -> AppFile = filelib:wildcard(filename:join([AppDir, "ebin", "*.app"])), AppSrcFile = lists:append( [filelib:wildcard(filename:join([AppDir, SrcDir, "*.app.src"])) @@ -331,17 +339,14 @@ create_app_info(AppInfo, AppDir, AppFile) -> AppInfo2 = rebar_app_info:applications( rebar_app_info:app_details(AppInfo1, AppDetails), IncludedApplications++Applications), - C = rebar_config:consult(AppDir), - AppInfo3 = rebar_app_info:update_opts(AppInfo2, - rebar_app_info:opts(AppInfo2), C), - Valid = case rebar_app_utils:validate_application_info(AppInfo3) =:= true - andalso rebar_app_info:has_all_artifacts(AppInfo3) =:= true of + Valid = case rebar_app_utils:validate_application_info(AppInfo2) =:= true + andalso rebar_app_info:has_all_artifacts(AppInfo2) =:= true of true -> true; _ -> false end, - rebar_app_info:dir(rebar_app_info:valid(AppInfo3, Valid), AppDir). + rebar_app_info:dir(rebar_app_info:valid(AppInfo2, Valid), AppDir). %% @doc Read in and parse the .app file if it is availabe. Do the same for %% the .app.src file if it exists. @@ -403,12 +408,20 @@ try_handle_app_file(_AppInfo, Other, _AppDir, _AppSrcFile, _, _Validate) -> AppFile :: file:filename(), AppDir :: file:filename(), AppSrcFile :: file:filename(). -try_handle_app_src_file(_AppInfo, _, _AppDir, [], _Validate) -> - false; +try_handle_app_src_file(AppInfo, _, _AppDir, [], _Validate) -> + %% if .app and .app.src are not found check for a mix config file + %% it is assumed a plugin will build the application, including + %% a .app after this step + case filelib:is_file(filename:join(rebar_app_info:dir(AppInfo), "mix.exs")) of + true -> + {true, AppInfo}; + false -> + false + end; try_handle_app_src_file(_AppInfo, _, _AppDir, _AppSrcFile, valid) -> false; try_handle_app_src_file(AppInfo, _, AppDir, [File], Validate) when Validate =:= invalid - ; Validate =:= all -> + ; Validate =:= all -> AppInfo1 = rebar_app_info:app_file(AppInfo, undefined), AppInfo2 = create_app_info(AppInfo1, AppDir, File), case filename:extension(File) of @@ -437,8 +450,8 @@ to_atom(Bin) -> find_config_src(AppDir, Default) -> case rebar_config:consult(AppDir) of [] -> - Default; + {[], Default}; Terms -> %% TODO: handle profiles I guess, but we don't have that info - proplists:get_value(src_dirs, Terms, Default) + {Terms, proplists:get_value(src_dirs, Terms, Default)} end. diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index 88d6335..56ae4c0 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -7,6 +7,7 @@ new/4, new/5, update_opts/3, + update_opts_deps/2, discover/1, name/1, name/2, @@ -43,8 +44,6 @@ get/2, get/3, set/3, - resource_type/1, - resource_type/2, source/1, source/2, is_lock/1, @@ -53,6 +52,8 @@ is_checkout/2, valid/1, valid/2, + is_available/1, + is_available/2, verify_otp_vsn/1, has_all_artifacts/1, @@ -72,7 +73,7 @@ app_file_src :: file:filename_all() | undefined, app_file_src_script:: file:filename_all() | undefined, app_file :: file:filename_all() | undefined, - original_vsn :: binary() | string() | undefined, + original_vsn :: binary() | undefined, parent=root :: binary() | root, app_details=[] :: list(), applications=[] :: list(), @@ -83,11 +84,11 @@ dep_level=0 :: integer(), dir :: file:name(), out_dir :: file:name(), - resource_type :: pkg | src | undefined, source :: string() | tuple() | checkout | undefined, is_lock=false :: boolean(), is_checkout=false :: boolean(), - valid :: boolean() | undefined}). + valid :: boolean() | undefined, + is_available=false :: boolean()}). %%============================================================================ %% types @@ -114,14 +115,14 @@ new(AppName) -> {ok, t()}. new(AppName, Vsn) -> {ok, #app_info_t{name=rebar_utils:to_binary(AppName), - original_vsn=Vsn}}. + original_vsn=rebar_utils:to_binary(Vsn)}}. %% @doc build a complete version of the app info with all fields set. -spec new(atom() | binary() | string(), binary() | string(), file:name()) -> {ok, t()}. new(AppName, Vsn, Dir) -> {ok, #app_info_t{name=rebar_utils:to_binary(AppName), - original_vsn=Vsn, + original_vsn=rebar_utils:to_binary(Vsn), dir=rebar_utils:to_list(Dir), out_dir=rebar_utils:to_list(Dir)}}. @@ -130,7 +131,7 @@ new(AppName, Vsn, Dir) -> {ok, t()}. new(AppName, Vsn, Dir, Deps) -> {ok, #app_info_t{name=rebar_utils:to_binary(AppName), - original_vsn=Vsn, + original_vsn=rebar_utils:to_binary(Vsn), dir=rebar_utils:to_list(Dir), out_dir=rebar_utils:to_list(Dir), deps=Deps}}. @@ -141,7 +142,7 @@ new(AppName, Vsn, Dir, Deps) -> new(Parent, AppName, Vsn, Dir, Deps) -> {ok, #app_info_t{name=rebar_utils:to_binary(AppName), parent=Parent, - original_vsn=Vsn, + original_vsn=rebar_utils:to_binary(Vsn), dir=rebar_utils:to_list(Dir), out_dir=rebar_utils:to_list(Dir), deps=Deps}}. @@ -150,10 +151,12 @@ new(Parent, AppName, Vsn, Dir, Deps) -> %% file for the app -spec update_opts(t(), rebar_dict(), [any()]) -> t(). update_opts(AppInfo, Opts, Config) -> - LockDeps = case resource_type(AppInfo) of - pkg -> - Deps = deps(AppInfo), - [{{locks, default}, Deps}, {{deps, default}, Deps}]; + LockDeps = case source(AppInfo) of + Tuple when is_tuple(Tuple) andalso element(1, Tuple) =:= pkg -> + %% Deps are set separate for packages + %% instead of making it seem we have no deps + %% don't set anything here. + []; _ -> deps_from_config(dir(AppInfo), Config) end, @@ -165,8 +168,18 @@ update_opts(AppInfo, Opts, Config) -> NewOpts = rebar_opts:merge_opts(LocalOpts, Opts), - AppInfo#app_info_t{opts=NewOpts - ,default=NewOpts}. + AppInfo#app_info_t{opts=NewOpts, + default=NewOpts}. + +%% @doc update the opts based on new deps, usually from an app's hex registry metadata +-spec update_opts_deps(t(), [any()]) -> t(). +update_opts_deps(AppInfo=#app_info_t{opts=Opts}, Deps) -> + LocalOpts = dict:from_list([{{locks, default}, Deps}, {{deps, default}, Deps}]), + NewOpts = rebar_opts:merge_opts(LocalOpts, Opts), + AppInfo#app_info_t{opts=NewOpts, + default=NewOpts, + deps=Deps}. + %% @private extract the deps for an app in `Dir' based on its config file data -spec deps_from_config(file:filename(), [any()]) -> [{tuple(), any()}, ...]. @@ -350,15 +363,15 @@ parent(AppInfo=#app_info_t{}, Parent) -> %% @doc returns the original version of the app (unevaluated if %% asking for a semver) --spec original_vsn(t()) -> string(). +-spec original_vsn(t()) -> binary(). original_vsn(#app_info_t{original_vsn=Vsn}) -> Vsn. %% @doc stores the original version of the app (unevaluated if %% asking for a semver) --spec original_vsn(t(), string()) -> t(). +-spec original_vsn(t(), binary() | string()) -> t(). original_vsn(AppInfo=#app_info_t{}, Vsn) -> - AppInfo#app_info_t{original_vsn=Vsn}. + AppInfo#app_info_t{original_vsn=rebar_utils:to_binary(Vsn)}. %% @doc returns the list of applications the app depends on. -spec applications(t()) -> list(). @@ -438,16 +451,6 @@ ebin_dir(#app_info_t{out_dir=OutDir}) -> priv_dir(#app_info_t{out_dir=OutDir}) -> rebar_utils:to_list(filename:join(OutDir, "priv")). -%% @doc returns whether the app is source app or a package app. --spec resource_type(t()) -> pkg | src. -resource_type(#app_info_t{resource_type=ResourceType}) -> - ResourceType. - -%% @doc sets whether the app is source app or a package app. --spec resource_type(t(), pkg | src) -> t(). -resource_type(AppInfo=#app_info_t{}, Type) -> - AppInfo#app_info_t{resource_type=Type}. - %% @doc finds the source specification for the app -spec source(t()) -> string() | tuple(). source(#app_info_t{source=Source}) -> @@ -478,6 +481,17 @@ is_checkout(#app_info_t{is_checkout=IsCheckout}) -> is_checkout(AppInfo=#app_info_t{}, IsCheckout) -> AppInfo#app_info_t{is_checkout=IsCheckout}. +%% @doc returns whether the app source exists in the deps dir +-spec is_available(t()) -> boolean(). +is_available(#app_info_t{is_available=IsAvailable}) -> + IsAvailable. + +%% @doc sets whether the app's source is available +%% only set if the app's source is found in the expected dep directory +-spec is_available(t(), boolean()) -> t(). +is_available(AppInfo=#app_info_t{}, IsAvailable) -> + AppInfo#app_info_t{is_available=IsAvailable}. + %% @doc returns whether the app is valid (built) or not -spec valid(t()) -> boolean(). valid(AppInfo=#app_info_t{valid=undefined}) -> diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index 1d7ef5b..35e908c 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -217,26 +217,23 @@ parse_dep(_, Dep, _, _, _) -> dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> CheckoutsDir = rebar_utils:to_list(rebar_dir:checkouts_dir(State, Name)), AppInfo = case rebar_app_info:discover(CheckoutsDir) of - {ok, App} -> - rebar_app_info:source(rebar_app_info:is_checkout(App, true), checkout); - not_found -> - Dir = rebar_utils:to_list(filename:join(DepsDir, Name)), - {ok, AppInfo0} = - case rebar_app_info:discover(Dir) of - {ok, App} -> - {ok, rebar_app_info:parent(App, Parent)}; - not_found -> - rebar_app_info:new(Parent, Name, Vsn, Dir, []) - end, - rebar_app_info:source(AppInfo0, Source) - end, - C = rebar_config:consult(rebar_app_info:dir(AppInfo)), - AppInfo1 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), C), - Overrides = rebar_state:get(State, overrides, []), - AppInfo2 = rebar_app_info:set(AppInfo1, overrides, rebar_app_info:get(AppInfo, overrides, [])++Overrides), - AppInfo3 = rebar_app_info:apply_overrides(rebar_app_info:get(AppInfo2, overrides, []), AppInfo2), - AppInfo4 = rebar_app_info:apply_profiles(AppInfo3, [default, prod]), - AppInfo5 = rebar_app_info:profiles(AppInfo4, [default]), + {ok, App} -> + rebar_app_info:source(rebar_app_info:is_checkout(App, true), checkout); + not_found -> + Dir = rebar_utils:to_list(filename:join(DepsDir, Name)), + {ok, AppInfo0} = + case rebar_app_info:discover(Dir) of + {ok, App} -> + {ok, rebar_app_info:is_available(rebar_app_info:parent(App, Parent), + true)}; + not_found -> + rebar_app_info:new(Parent, Name, Vsn, Dir, []) + end, + rebar_app_info:source(AppInfo0, Source) + end, + Overrides = rebar_app_info:get(AppInfo, overrides, []) ++ rebar_state:get(State, overrides, []), + AppInfo2 = rebar_app_info:set(AppInfo, overrides, Overrides), + AppInfo5 = rebar_app_info:profiles(AppInfo2, [default]), rebar_app_info:is_lock(AppInfo5, IsLock). %% @doc Takes a given application app_info record along with the project. @@ -250,52 +247,36 @@ expand_deps_sources(Dep, State) -> %% around version if required. -spec update_source(rebar_app_info:t(), Source, rebar_state:t()) -> rebar_app_info:t() when - Source :: tuple() | atom() | binary(). % TODO: meta to source() + Source :: rebar_resource_v2:source(). update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) -> - {PkgName1, PkgVsn1} = case PkgVsn of - undefined -> - get_package(PkgName, "0", State); - <<"~>", Vsn/binary>> -> - [Vsn1] = [X || X <- binary:split(Vsn, [<<" ">>], [global]), X =/= <<>>], - get_package(PkgName, Vsn1, State); - _ -> - {PkgName, PkgVsn} - end, - %% store the expected hash for the dependency - Hash1 = case Hash of - undefined -> % unknown, define the hash since we know the dep - fetch_checksum(PkgName1, PkgVsn1, Hash, State); - _ -> % keep as is - Hash - end, - AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName1, PkgVsn1, Hash1}), - Deps = rebar_packages:deps(PkgName1 - ,PkgVsn1 - ,State), - AppInfo2 = rebar_app_info:resource_type(rebar_app_info:deps(AppInfo1, Deps), pkg), - rebar_app_info:original_vsn(AppInfo2, PkgVsn1); + case rebar_packages:resolve_version(PkgName, PkgVsn, Hash, + ?PACKAGE_TABLE, State) of + {ok, Package, RepoConfig} -> + #package{key={_, PkgVsn1, _}, + checksum=Hash1, + dependencies=Deps} = Package, + AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName, PkgVsn1, Hash1, RepoConfig}), + AppInfo2 = rebar_app_info:update_opts_deps(AppInfo1, Deps), + rebar_app_info:original_vsn(AppInfo2, PkgVsn1); + not_found -> + throw(?PRV_ERROR({missing_package, PkgName, PkgVsn})); + {error, {invalid_vsn, InvalidVsn}} -> + throw(?PRV_ERROR({invalid_vsn, PkgName, InvalidVsn})) + end; update_source(AppInfo, Source, _State) -> rebar_app_info:source(AppInfo, Source). -%% @doc grab the checksum for a given package --spec fetch_checksum(atom(), string(), iodata() | undefined, rebar_state:t()) -> - iodata() | no_return(). -fetch_checksum(PkgName, PkgVsn, Hash, State) -> - try - rebar_packages:registry_checksum({pkg, PkgName, PkgVsn, Hash}, State) - catch - _:_ -> - ?INFO("Package ~ts-~ts not found. Fetching registry updates and trying again...", [PkgName, PkgVsn]), - {ok, _} = rebar_prv_update:do(State), - rebar_packages:registry_checksum({pkg, PkgName, PkgVsn, Hash}, State) - end. - %% @doc convert a given exception's payload into an io description. -spec format_error(any()) -> iolist(). -format_error({missing_package, Package}) -> - io_lib:format("Package not found in registry: ~ts", [Package]); +format_error({missing_package, Name, undefined}) -> + io_lib:format("Package not found in any repo: ~ts.", [rebar_utils:to_binary(Name)]); +format_error({missing_package, Name, Vsn}) -> + io_lib:format("Package not found in any repo: ~ts-~ts.", [rebar_utils:to_binary(Name), + rebar_utils:to_binary(Vsn)]); format_error({parse_dep, Dep}) -> io_lib:format("Failed parsing dep ~p", [Dep]); +format_error({invalid_vsn, Dep, InvalidVsn}) -> + io_lib:format("Dep ~ts has invalid version ~ts", [Dep, InvalidVsn]); format_error(Error) -> io_lib:format("~p", [Error]). @@ -303,18 +284,6 @@ format_error(Error) -> %% Internal functions %% =================================================================== -%% @private find the correct version of a package based on the version -%% and name passed in. --spec get_package(binary(), binary() | string(), rebar_state:t()) -> - term() | no_return(). -get_package(Dep, Vsn, State) -> - case rebar_packages:find_highest_matching(Dep, Vsn, ?PACKAGE_TABLE, State) of - {ok, HighestDepVsn} -> - {Dep, HighestDepVsn}; - none -> - throw(?PRV_ERROR({missing_package, rebar_utils:to_binary(Dep)})) - end. - %% @private checks that all the beam files have been properly %% created. -spec has_all_beams(file:filename_all(), [module()]) -> diff --git a/src/rebar_config.erl b/src/rebar_config.erl index 797dddc..2651ca1 100644 --- a/src/rebar_config.erl +++ b/src/rebar_config.erl @@ -74,7 +74,7 @@ consult_lock_file(File) -> read_attrs(beta, Locks, []); [{Vsn, Locks}|Attrs] when is_list(Locks) -> % versioned lock file %% Because this is the first version of rebar3 to introduce a lock - %% file, all versionned lock files with a different versions have + %% file, all versioned lock files with a different version have %% to be newer. case Vsn of ?CONFIG_VERSION -> diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index d2c7706..9c76e0e 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -7,104 +7,74 @@ %% ------------------------------------------------------------------- -module(rebar_fetch). --export([lock_source/3, - download_source/3, - needs_update/3]). +-export([lock_source/2, + download_source/2, + needs_update/2]). -export([format_error/1]). -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). --spec lock_source(file:filename_all(), rebar_resource:resource(), rebar_state:t()) -> - rebar_resource:resource() | {error, string()}. -lock_source(AppDir, Source, State) -> - Resources = rebar_state:resources(State), - Module = get_resource_type(Source, Resources), - Module:lock(AppDir, Source). +-spec lock_source(rebar_app_info:t(), rebar_state:t()) + -> rebar_resource_v2:source() | {error, string()}. +lock_source(AppInfo, State) -> + rebar_resource_v2:lock(AppInfo, State). --spec download_source(file:filename_all(), rebar_resource:resource(), rebar_state:t()) -> - true | {error, any()}. -download_source(AppDir, Source, State) -> - try download_source_(AppDir, Source, State) of - true -> - true; - Error -> - throw(?PRV_ERROR(Error)) +-spec download_source(rebar_app_info:t(), rebar_state:t()) + -> rebar_app_info:t() | {error, any()}. +download_source(AppInfo, State) -> + AppDir = rebar_app_info:dir(AppInfo), + try download_source_(AppInfo, State) of + ok -> + %% freshly downloaded, update the app info opts to reflect the new config + Config = rebar_config:consult(AppDir), + AppInfo1 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), Config), + case rebar_app_discover:find_app(AppInfo1, AppDir, all) of + {true, AppInfo2} -> + rebar_app_info:is_available(AppInfo2, true); + false -> + throw(?PRV_ERROR({dep_app_not_found, rebar_app_info:name(AppInfo1)})) + end; + {error, Reason} -> + throw(?PRV_ERROR(Reason)) catch + throw:{no_resource, Type, Location} -> + throw(?PRV_ERROR({no_resource, Location, Type})); ?WITH_STACKTRACE(C,T,S) ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, S]), - throw(?PRV_ERROR({fetch_fail, Source})) + throw(?PRV_ERROR({fetch_fail, rebar_app_info:source(AppInfo)})) end. -download_source_(AppDir, Source, State) -> - Resources = rebar_state:resources(State), - Module = get_resource_type(Source, Resources), +download_source_(AppInfo, State) -> + AppDir = rebar_app_info:dir(AppInfo), TmpDir = ec_file:insecure_mkdtemp(), AppDir1 = rebar_utils:to_list(AppDir), - case Module:download(TmpDir, Source, State) of - {ok, _} -> + case rebar_resource_v2:download(TmpDir, AppInfo, State) of + ok -> ec_file:mkdir_p(AppDir1), code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), ok = rebar_file_utils:rm_rf(filename:absname(AppDir1)), ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]), - ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)), - true; + rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)); Error -> Error end. --spec needs_update(file:filename_all(), rebar_resource:resource(), rebar_state:t()) -> boolean() | {error, string()}. -needs_update(AppDir, Source, State) -> - Resources = rebar_state:resources(State), - Module = get_resource_type(Source, Resources), +-spec needs_update(rebar_app_info:t(), rebar_state:t()) + -> boolean() | {error, string()}. +needs_update(AppInfo, State) -> try - Module:needs_update(AppDir, Source) + rebar_resource_v2:needs_update(AppInfo, State) catch _:_ -> true end. -format_error({bad_download, CachePath}) -> - io_lib:format("Download of package does not match md5sum from server: ~ts", [CachePath]); -format_error({unexpected_hash, CachePath, Expected, Found}) -> - io_lib:format("The checksum for package at ~ts (~ts) does not match the " - "checksum previously locked (~ts). Either unlock or " - "upgrade the package, or make sure you fetched it from " - "the same index from which it was initially fetched.", - [CachePath, Found, Expected]); -format_error({failed_extract, CachePath}) -> - io_lib:format("Failed to extract package: ~ts", [CachePath]); -format_error({bad_etag, Source}) -> - io_lib:format("MD5 Checksum comparison failed for: ~ts", [Source]); format_error({fetch_fail, Name, Vsn}) -> io_lib:format("Failed to fetch and copy dep: ~ts-~ts", [Name, Vsn]); format_error({fetch_fail, Source}) -> io_lib:format("Failed to fetch and copy dep: ~p", [Source]); -format_error({bad_checksum, File}) -> - io_lib:format("Checksum mismatch against tarball in ~ts", [File]); -format_error({bad_registry_checksum, File}) -> - io_lib:format("Checksum mismatch against registry in ~ts", [File]). - -get_resource_type({Type, Location}, Resources) -> - find_resource_module(Type, Location, Resources); -get_resource_type({Type, Location, _}, Resources) -> - find_resource_module(Type, Location, Resources); -get_resource_type({Type, _, _, Location}, Resources) -> - find_resource_module(Type, Location, Resources); -get_resource_type(_, _) -> - rebar_pkg_resource. - -find_resource_module(Type, Location, Resources) -> - case lists:keyfind(Type, 1, Resources) of - false -> - case code:which(Type) of - non_existing -> - {error, io_lib:format("Cannot handle dependency ~ts.~n" - " No module for resource type ~p", [Location, Type])}; - _ -> - Type - end; - {Type, Module} -> - Module - end. +format_error({dep_app_not_found, AppName}) -> + io_lib:format("Dependency failure: source for ~ts does not contain a " + "recognizable project and can not be built", [AppName]). diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl index 3aa875f..29c9ad7 100644 --- a/src/rebar_git_resource.erl +++ b/src/rebar_git_resource.erl @@ -2,21 +2,30 @@ %% ex: ts=4 sw=4 et -module(rebar_git_resource). --behaviour(rebar_resource). +-behaviour(rebar_resource_v2). --export([lock/2 - ,download/3 - ,needs_update/2 - ,make_vsn/1]). +-export([init/2, + lock/2, + download/4, + needs_update/2, + make_vsn/2]). -include("rebar.hrl"). %% Regex used for parsing scp style remote url -define(SCP_PATTERN, "\\A(?[^@]+)@(?[^:]+):(?.+)\\z"). -lock(AppDir, {git, Url, _}) -> - lock(AppDir, {git, Url}); -lock(AppDir, {git, Url}) -> +-spec init(atom(), rebar_state:t()) -> {ok, rebar_resource_v2:resource()}. +init(Type, _State) -> + Resource = rebar_resource_v2:new(Type, ?MODULE, #{}), + {ok, Resource}. + +lock(AppInfo, _) -> + lock_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). + +lock_(AppDir, {git, Url, _}) -> + lock_(AppDir, {git, Url}); +lock_(AppDir, {git, Url}) -> AbortMsg = lists:flatten(io_lib:format("Locking of git dependency failed in ~ts", [AppDir])), Dir = rebar_utils:escape_double_quotes(AppDir), {ok, VsnString} = @@ -33,14 +42,17 @@ lock(AppDir, {git, Url}) -> %% Return true if either the git url or tag/branch/ref is not the same as the currently %% checked out git repo for the dep -needs_update(Dir, {git, Url, {tag, Tag}}) -> +needs_update(AppInfo, _) -> + needs_update_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). + +needs_update_(Dir, {git, Url, {tag, Tag}}) -> {ok, Current} = rebar_utils:sh(?FMT("git describe --tags --exact-match", []), [{cd, Dir}]), Current1 = rebar_string:trim(rebar_string:trim(Current, both, "\n"), both, "\r"), ?DEBUG("Comparing git tag ~ts with ~ts", [Tag, Current1]), not ((Current1 =:= Tag) andalso compare_url(Dir, Url)); -needs_update(Dir, {git, Url, {branch, Branch}}) -> +needs_update_(Dir, {git, Url, {branch, Branch}}) -> %% Fetch remote so we can check if the branch has changed SafeBranch = rebar_utils:escape_chars(Branch), {ok, _} = rebar_utils:sh(?FMT("git fetch origin ~ts", [SafeBranch]), @@ -50,9 +62,9 @@ needs_update(Dir, {git, Url, {branch, Branch}}) -> [{cd, Dir}]), ?DEBUG("Checking git branch ~ts for updates", [Branch]), not ((Current =:= []) andalso compare_url(Dir, Url)); -needs_update(Dir, {git, Url, "master"}) -> - needs_update(Dir, {git, Url, {branch, "master"}}); -needs_update(Dir, {git, _, Ref}) -> +needs_update_(Dir, {git, Url, "master"}) -> + needs_update_(Dir, {git, Url, {branch, "master"}}); +needs_update_(Dir, {git, _, Ref}) -> {ok, Current} = rebar_utils:sh(?FMT("git rev-parse --short=7 -q HEAD", []), [{cd, Dir}]), Current1 = rebar_string:trim(rebar_string:trim(Current, both, "\n"), @@ -98,25 +110,35 @@ parse_git_url(not_scp, Url) -> {error, Reason} end. -download(Dir, {git, Url}, State) -> +download(TmpDir, AppInfo, State, _) -> + case download_(TmpDir, rebar_app_info:source(AppInfo), State) of + {ok, _} -> + ok; + {error, Reason} -> + {error, Reason}; + Error -> + {error, Error} + end. + +download_(Dir, {git, Url}, State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), - download(Dir, {git, Url, {branch, "master"}}, State); -download(Dir, {git, Url, ""}, State) -> + download_(Dir, {git, Url, {branch, "master"}}, State); +download_(Dir, {git, Url, ""}, State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), - download(Dir, {git, Url, {branch, "master"}}, State); -download(Dir, {git, Url, {branch, Branch}}, _State) -> + download_(Dir, {git, Url, {branch, "master"}}, State); +download_(Dir, {git, Url, {branch, Branch}}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), git_clone(branch, git_vsn(), Url, Dir, Branch); -download(Dir, {git, Url, {tag, Tag}}, _State) -> +download_(Dir, {git, Url, {tag, Tag}}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), git_clone(tag, git_vsn(), Url, Dir, Tag); -download(Dir, {git, Url, {ref, Ref}}, _State) -> +download_(Dir, {git, Url, {ref, Ref}}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), git_clone(ref, git_vsn(), Url, Dir, Ref); -download(Dir, {git, Url, Rev}, _State) -> +download_(Dir, {git, Url, Rev}, _State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), @@ -195,7 +217,10 @@ git_vsn_fetch() -> undefined end. -make_vsn(Dir) -> +make_vsn(AppInfo, _) -> + make_vsn_(rebar_app_info:dir(AppInfo)). + +make_vsn_(Dir) -> case collect_default_refcount(Dir) of Vsn={plain, _} -> Vsn; diff --git a/src/rebar_hex_repos.erl b/src/rebar_hex_repos.erl new file mode 100644 index 0000000..ebee191 --- /dev/null +++ b/src/rebar_hex_repos.erl @@ -0,0 +1,142 @@ +-module(rebar_hex_repos). + +-export([from_state/2, + get_repo_config/2, + auth_config/1, + update_auth_config/2, + format_error/1]). + +-ifdef(TEST). +%% exported for test purposes +-export([repos/1, merge_repos/1]). +-endif. + +-include("rebar.hrl"). +-include_lib("providers/include/providers.hrl"). + +-export_type([repo/0]). + +-type repo() :: #{name => unicode:unicode_binary(), + api_url => binary(), + api_key => binary(), + repo_url => binary(), + repo_public_key => binary(), + repo_verify => binary()}. + +from_state(BaseConfig, State) -> + HexConfig = rebar_state:get(State, hex, []), + Repos = repos(HexConfig), + %% auth is stored in a separate config file since the plugin generates and modifies it + Auth = ?MODULE:auth_config(State), + %% add base config entries that are specific to use by rebar3 and not overridable + Repos1 = merge_with_base_and_auth(Repos, BaseConfig, Auth), + %% merge organizations parent repo options into each oraganization repo + update_organizations(Repos1). + +-spec get_repo_config(unicode:unicode_binary(), rebar_state:t() | [repo()]) + -> {ok, repo()} | error. +get_repo_config(RepoName, Repos) when is_list(Repos) -> + case ec_lists:find(fun(#{name := N}) -> N =:= RepoName end, Repos) of + error -> + throw(?PRV_ERROR({repo_not_found, RepoName})); + {ok, RepoConfig} -> + {ok, RepoConfig} + end; +get_repo_config(RepoName, State) -> + Resources = rebar_state:resources(State), + #{repos := Repos} = rebar_resource_v2:find_resource_state(pkg, Resources), + get_repo_config(RepoName, Repos). + +merge_with_base_and_auth(Repos, BaseConfig, Auth) -> + [maps:merge(maps:get(maps:get(name, Repo), Auth, #{}), + maps:merge(Repo, BaseConfig)) || Repo <- Repos]. + +%% A user's list of repos are merged by name while keeping the order +%% intact. The order is based on the first use of a repo by name in the +%% list. The default repo is appended to the user's list. +repos(HexConfig) -> + HexDefaultConfig = default_repo(), + case [R || R <- HexConfig, element(1, R) =:= repos] of + [] -> + [HexDefaultConfig]; + %% we only care if the first element is a replace entry + [{repos, replace, Repos} | _]-> + merge_repos(Repos); + Repos -> + RepoList = repo_list(Repos), + merge_repos(RepoList ++ [HexDefaultConfig]) + end. + +-spec merge_repos([repo()]) -> [repo()]. +merge_repos(Repos) -> + lists:foldl(fun(R=#{name := Name}, ReposAcc) -> + %% private organizations include the parent repo before a : + case rebar_string:split(Name, <<":">>) of + [Repo, Org] -> + update_repo_list(R#{name => Name, + organization => Org, + parent => Repo}, ReposAcc); + _ -> + update_repo_list(R, ReposAcc) + end + end, [], Repos). + +update_organizations(Repos) -> + lists:map(fun(Repo=#{organization := Organization, + parent := ParentName}) -> + {ok, Parent} = get_repo_config(ParentName, Repos), + ParentRepoUrl = rebar_utils:to_list(maps:get(repo_url, Parent)), + {ok, RepoUrl} = + rebar_utils:url_append_path(ParentRepoUrl, + filename:join("repos", rebar_utils:to_list(Organization))), + %% still let the organization config override this constructed repo url + maps:merge(Parent#{repo_url => rebar_utils:to_binary(RepoUrl)}, Repo); + (Repo) -> + Repo + end, Repos). + +update_repo_list(R=#{name := N}, [H=#{name := HN} | Rest]) when N =:= HN -> + [maps:merge(R, H) | Rest]; +update_repo_list(R, [H | Rest]) -> + [H | update_repo_list(R, Rest)]; +update_repo_list(R, []) -> + [R]. + +default_repo() -> + HexDefaultConfig = hex_core:default_config(), + HexDefaultConfig#{name => ?PUBLIC_HEX_REPO}. + +repo_list([]) -> + []; +repo_list([{repos, Repos} | T]) -> + Repos ++ repo_list(T); +repo_list([{repos, replace, Repos} | T]) -> + Repos ++ repo_list(T). + +format_error({repo_not_found, RepoName}) -> + io_lib:format("The repo ~ts was not found in the configuration.", [RepoName]). + +%% auth functions + +%% authentication is in a separate config file because the hex plugin updates it + +-spec auth_config_file(rebar_state:t()) -> file:filename_all(). +auth_config_file(State) -> + filename:join(rebar_dir:global_config_dir(State), ?HEX_AUTH_FILE). + +-spec auth_config(rebar_state:t()) -> map(). +auth_config(State) -> + case file:consult(auth_config_file(State)) of + {ok, [Config]} -> + Config; + _ -> + #{} + end. + +-spec update_auth_config(map(), rebar_state:t()) -> ok. +update_auth_config(Updates, State) -> + Config = auth_config(State), + AuthConfigFile = auth_config_file(State), + ok = filelib:ensure_dir(AuthConfigFile), + NewConfig = iolist_to_binary([io_lib:print(maps:merge(Config, Updates)) | ".\n"]), + ok = file:write_file(AuthConfigFile, NewConfig). diff --git a/src/rebar_hg_resource.erl b/src/rebar_hg_resource.erl index abcca88..21d4b9d 100644 --- a/src/rebar_hg_resource.erl +++ b/src/rebar_hg_resource.erl @@ -2,39 +2,51 @@ %% ex: ts=4 sw=4 et -module(rebar_hg_resource). --behaviour(rebar_resource). +-behaviour(rebar_resource_v2). --export([lock/2 - ,download/3 - ,needs_update/2 - ,make_vsn/1]). +-export([init/2, + lock/2, + download/4, + needs_update/2, + make_vsn/2]). -include("rebar.hrl"). -lock(AppDir, {hg, Url, _}) -> - lock(AppDir, {hg, Url}); -lock(AppDir, {hg, Url}) -> +-spec init(atom(), rebar_state:t()) -> {ok, rebar_resource_v2:resource()}. +init(Type, _State) -> + Resource = rebar_resource_v2:new(Type, ?MODULE, #{}), + {ok, Resource}. + +lock(AppInfo, _) -> + lock_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). + +lock_(AppDir, {hg, Url, _}) -> + lock_(AppDir, {hg, Url}); +lock_(AppDir, {hg, Url}) -> Ref = get_ref(AppDir), {hg, Url, {ref, Ref}}. %% Return `true' if either the hg url or tag/branch/ref is not the same as %% the currently checked out repo for the dep -needs_update(Dir, {hg, Url, {tag, Tag}}) -> +needs_update(AppInfo, _) -> + needs_update_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). + +needs_update_(Dir, {hg, Url, {tag, Tag}}) -> Ref = get_ref(Dir), {ClosestTag, Distance} = get_tag_distance(Dir, Ref), ?DEBUG("Comparing hg tag ~ts with ref ~ts (closest tag is ~ts at distance ~ts)", [Tag, Ref, ClosestTag, Distance]), not ((Distance =:= "0") andalso (Tag =:= ClosestTag) andalso compare_url(Dir, Url)); -needs_update(Dir, {hg, Url, {branch, Branch}}) -> +needs_update_(Dir, {hg, Url, {branch, Branch}}) -> Ref = get_ref(Dir), BRef = get_branch_ref(Dir, Branch), not ((Ref =:= BRef) andalso compare_url(Dir, Url)); -needs_update(Dir, {hg, Url, "default"}) -> +needs_update_(Dir, {hg, Url, "default"}) -> Ref = get_ref(Dir), BRef = get_branch_ref(Dir, "default"), not ((Ref =:= BRef) andalso compare_url(Dir, Url)); -needs_update(Dir, {hg, Url, Ref}) -> +needs_update_(Dir, {hg, Url, Ref}) -> LocalRef = get_ref(Dir), TargetRef = case Ref of {ref, Ref1} -> @@ -48,13 +60,23 @@ needs_update(Dir, {hg, Url, Ref}) -> ?DEBUG("Comparing hg ref ~ts with ~ts", [Ref1, LocalRef]), not ((LocalRef =:= TargetRef) andalso compare_url(Dir, Url)). -download(Dir, {hg, Url}, State) -> +download(TmpDir, AppInfo, State, _) -> + case download_(TmpDir, rebar_app_info:source(AppInfo), State) of + {ok, _} -> + ok; + {error, Reason} -> + {error, Reason}; + Error -> + {error, Error} + end. + +download_(Dir, {hg, Url}, State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), - download(Dir, {hg, Url, {branch, "default"}}, State); -download(Dir, {hg, Url, ""}, State) -> + download_(Dir, {hg, Url, {branch, "default"}}, State); +download_(Dir, {hg, Url, ""}, State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), - download(Dir, {hg, Url, {branch, "default"}}, State); -download(Dir, {hg, Url, {branch, Branch}}, _State) -> + download_(Dir, {hg, Url, {branch, "default"}}, State); +download_(Dir, {hg, Url, {branch, Branch}}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), rebar_utils:sh(?FMT("hg clone -q -b ~ts ~ts ~ts", @@ -62,7 +84,7 @@ download(Dir, {hg, Url, {branch, Branch}}, _State) -> rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]); -download(Dir, {hg, Url, {tag, Tag}}, _State) -> +download_(Dir, {hg, Url, {tag, Tag}}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), rebar_utils:sh(?FMT("hg clone -q -u ~ts ~ts ~ts", @@ -70,7 +92,7 @@ download(Dir, {hg, Url, {tag, Tag}}, _State) -> rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]); -download(Dir, {hg, Url, {ref, Ref}}, _State) -> +download_(Dir, {hg, Url, {ref, Ref}}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), rebar_utils:sh(?FMT("hg clone -q -r ~ts ~ts ~ts", @@ -78,7 +100,7 @@ download(Dir, {hg, Url, {ref, Ref}}, _State) -> rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]); -download(Dir, {hg, Url, Rev}, _State) -> +download_(Dir, {hg, Url, Rev}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), rebar_utils:sh(?FMT("hg clone -q -r ~ts ~ts ~ts", @@ -87,7 +109,10 @@ download(Dir, {hg, Url, Rev}, _State) -> rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]). -make_vsn(Dir) -> +make_vsn(AppInfo, _) -> + make_vsn_(rebar_app_info:dir(AppInfo)). + +make_vsn_(Dir) -> BaseHg = "hg -R \"" ++ rebar_utils:escape_double_quotes(Dir) ++ "\" ", Ref = get_ref(Dir), Cmd = BaseHg ++ "log --template \"{latesttag}+build.{latesttagdistance}.rev.{node|short}\"" diff --git a/src/rebar_otp_app.erl b/src/rebar_otp_app.erl index f5bb9cf..1d854da 100644 --- a/src/rebar_otp_app.erl +++ b/src/rebar_otp_app.erl @@ -59,8 +59,8 @@ compile(State, App) -> format_error({missing_app_file, Filename}) -> io_lib:format("App file is missing: ~ts", [Filename]); -format_error({file_read, File, Reason}) -> - io_lib:format("Failed to read required file ~ts for processing: ~ts", [File, file:format_error(Reason)]); +format_error({file_read, AppName, File, Reason}) -> + io_lib:format("Failed to read required ~ts file for processing the application '~ts': ~ts", [File, AppName, file:format_error(Reason)]); format_error({invalid_name, File, AppName}) -> io_lib:format("Invalid ~ts: name of application (~p) must match filename.", [File, AppName]). @@ -79,7 +79,7 @@ validate_app(State, App) -> Error end; {error, Reason} -> - ?PRV_ERROR({file_read, AppFile, Reason}) + ?PRV_ERROR({file_read, rebar_app_info:name(App), ".app", Reason}) end. validate_app_modules(State, App, AppData) -> @@ -110,7 +110,7 @@ preprocess(State, AppInfo, AppSrcFile) -> A1 = apply_app_vars(AppVars, AppData), %% AppSrcFile may contain instructions for generating a vsn number - Vsn = app_vsn(AppData, AppSrcFile, State), + Vsn = app_vsn(AppInfo, AppData, AppSrcFile, State), A2 = lists:keystore(vsn, 1, A1, {vsn, Vsn}), %% systools:make_relup/4 fails with {missing_param, registered} @@ -131,7 +131,7 @@ preprocess(State, AppInfo, AppSrcFile) -> AppFile; {error, Reason} -> - throw(?PRV_ERROR({file_read, AppSrcFile, Reason})) + throw(?PRV_ERROR({file_read, rebar_app_info:name(AppInfo), ".app.src", Reason})) end. load_app_vars(State) -> @@ -226,10 +226,8 @@ consult_app_file(Filename) -> end end. -app_vsn(AppData, AppFile, State) -> - AppDir = filename:dirname(filename:dirname(AppFile)), - Resources = rebar_state:resources(State), - rebar_utils:vcs_vsn(get_value(vsn, AppData, AppFile), AppDir, Resources). +app_vsn(AppInfo, AppData, AppFile, State) -> + rebar_utils:vcs_vsn(AppInfo, get_value(vsn, AppData, AppFile), State). get_value(Key, AppInfo, AppFile) -> case proplists:get_value(Key, AppInfo) of diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 8cebeca..8a3ffea 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -1,18 +1,18 @@ -module(rebar_packages). --export([packages/1 - ,close_packages/0 - ,load_and_verify_version/1 - ,deps/3 +-export([get/2 + ,get_all_names/1 ,registry_dir/1 - ,package_dir/1 - ,registry_checksum/2 - ,find_highest_matching/6 - ,find_highest_matching/4 - ,find_highest_matching_/6 - ,find_all/3 + ,package_dir/2 + ,find_highest_matching/5 ,verify_table/1 - ,format_error/1]). + ,format_error/1 + ,update_package/3 + ,resolve_version/5]). + +-ifdef(TEST). +-export([new_package_table/0, find_highest_matching_/5, cmp_/4, cmpl_/4, valid_vsn/1]). +-endif. -export_type([package/0]). @@ -23,119 +23,133 @@ -type vsn() :: binary(). -type package() :: pkg_name() | {pkg_name(), vsn()}. --spec packages(rebar_state:t()) -> ets:tid(). -packages(State) -> - catch ets:delete(?PACKAGE_TABLE), - case load_and_verify_version(State) of - true -> - ok; - false -> - ?DEBUG("Error loading package index.", []), - handle_bad_index(State) +format_error({missing_package, Name, Vsn}) -> + io_lib:format("Package not found in any repo: ~ts-~ts.", [rebar_utils:to_binary(Name), + rebar_utils:to_binary(Vsn)]); +format_error({missing_package, Pkg}) -> + io_lib:format("Package not found in any repo: ~p.", [Pkg]). + +-spec get(rebar_hex_repos:repo(), binary()) -> {ok, map()} | {error, term()}. +get(Config, Name) -> + try hex_api_package:get(Config, Name) of + {ok, {200, _Headers, PkgInfo}} -> + {ok, PkgInfo}; + {ok, {404, _, _}} -> + {error, not_found}; + Error -> + ?DEBUG("Hex api request failed: ~p", [Error]), + {error, unknown} + catch + error:{badmatch, {error, {failed_connect, _}}} -> + {error, failed_to_connect}; + _:Exception -> + ?DEBUG("hex_api_package:get failed: ~p", [Exception]), + {error, unknown} end. -handle_bad_index(State) -> - ?ERROR("Bad packages index. Trying to fix by updating the registry.", []), - {ok, State1} = rebar_prv_update:do(State), - case load_and_verify_version(State1) of - true -> - ok; - false -> - %% Still unable to load after an update, create an empty registry - ets:new(?PACKAGE_TABLE, [named_table, public]) + +-spec get_all_names(rebar_state:t()) -> [binary()]. +get_all_names(State) -> + verify_table(State), + lists:usort(ets:select(?PACKAGE_TABLE, [{#package{key={'$1', '_', '_'}, + _='_'}, + [], ['$1']}])). + +-spec get_package_versions(unicode:unicode_binary(), unicode:unicode_binary(), ets:tid(), rebar_state:t()) + -> [vsn()]. +get_package_versions(Dep, Repo, Table, State) -> + ?MODULE:verify_table(State), + ets:select(Table, [{#package{key={Dep,'$1', Repo}, + _='_'}, + [], ['$1']}]). + + +get_package(Dep, Vsn, Hash, Repo, Table, State) -> + get_package(Dep, Vsn, Hash, false, [Repo], Table, State). + +-spec get_package(unicode:unicode_binary(), unicode:unicode_binary(), + binary() | undefined | '_', boolean() | '_', + unicode:unicode_binary() | '_' | list(), ets:tab(), rebar_state:t()) + -> {ok, #package{}} | not_found. +get_package(Dep, Vsn, undefined, Retired, Repo, Table, State) -> + get_package(Dep, Vsn, '_', Retired, Repo, Table, State); +get_package(Dep, Vsn, Hash, Retired, Repos, Table, State) -> + ?MODULE:verify_table(State), + case ets:select(Table, [{#package{key={Dep, Vsn, Repo}, + checksum=Hash, + retired=Retired, + _='_'}, [], ['$_']} || Repo <- Repos]) of + %% have to allow multiple matches in the list for cases that Repo is `_` + [Package | _] -> + {ok, Package}; + _ -> + not_found end. -close_packages() -> - catch ets:delete(?PACKAGE_TABLE). +new_package_table() -> + ?PACKAGE_TABLE = ets:new(?PACKAGE_TABLE, [named_table, public, ordered_set, {keypos, 2}]), + ets:insert(?PACKAGE_TABLE, {?PACKAGE_INDEX_VERSION, package_index_version}). load_and_verify_version(State) -> {ok, RegistryDir} = registry_dir(State), case ets:file2tab(filename:join(RegistryDir, ?INDEX_FILE)) of {ok, _} -> - case ets:lookup_element(?PACKAGE_TABLE, package_index_version, 2) of + case ets:lookup_element(?PACKAGE_TABLE, package_index_version, 1) of ?PACKAGE_INDEX_VERSION -> true; - _ -> + V -> + %% no reason to confuse the user since we just start fresh and they + %% shouldn't notice, so log as a debug message only + ?DEBUG("Package index version mismatch. Current version ~p, this rebar3 expecting ~p", + [V, ?PACKAGE_INDEX_VERSION]), (catch ets:delete(?PACKAGE_TABLE)), - rebar_prv_update:hex_to_index(State) + new_package_table() end; - _ -> - rebar_prv_update:hex_to_index(State) + _ -> + new_package_table() end. -deps(Name, Vsn, State) -> - try - deps_(Name, Vsn, State) - catch - _:_ -> - handle_missing_package({Name, Vsn}, State, fun(State1) -> deps_(Name, Vsn, State1) end) - end. - -deps_(Name, Vsn, State) -> - ?MODULE:verify_table(State), - ets:lookup_element(?PACKAGE_TABLE, {rebar_utils:to_binary(Name), rebar_utils:to_binary(Vsn)}, 2). - -handle_missing_package(Dep, State, Fun) -> - case Dep of - {Name, Vsn} -> - ?INFO("Package ~ts-~ts not found. Fetching registry updates and trying again...", [Name, Vsn]); - _ -> - ?INFO("Package ~p not found. Fetching registry updates and trying again...", [Dep]) - end, +handle_missing_package(PkgKey, Repo, State, Fun) -> + Name = + case PkgKey of + {N, Vsn, _Repo} -> + ?DEBUG("Package ~ts-~ts not found. Fetching registry updates for " + "package and trying again...", [N, Vsn]), + N; + _ -> + ?DEBUG("Package ~p not found. Fetching registry updates for " + "package and trying again...", [PkgKey]), + PkgKey + end, - {ok, State1} = rebar_prv_update:do(State), - try - Fun(State1) + update_package(Name, Repo, State), + try + Fun(State) catch _:_ -> %% Even after an update the package is still missing, time to error out - throw(?PRV_ERROR({missing_package, Dep})) + throw(?PRV_ERROR({missing_package, PkgKey})) end. registry_dir(State) -> CacheDir = rebar_dir:global_cache_dir(rebar_state:opts(State)), - case rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN) of - ?DEFAULT_CDN -> - RegistryDir = filename:join([CacheDir, "hex", "default"]), - case filelib:ensure_dir(filename:join(RegistryDir, "placeholder")) of - ok -> ok; - {error, Posix} when Posix == eaccess; Posix == enoent -> - ?ABORT("Could not write to ~p. Please ensure the path is writeable.", - [RegistryDir]) - end, - {ok, RegistryDir}; - CDN -> - case rebar_utils:url_append_path(CDN, ?REMOTE_PACKAGE_DIR) of - {ok, Parsed} -> - {ok, {_, _, Host, _, Path, _}} = http_uri:parse(Parsed), - CDNHostPath = lists:reverse(rebar_string:lexemes(Host, ".")), - CDNPath = tl(filename:split(Path)), - RegistryDir = filename:join([CacheDir, "hex"] ++ CDNHostPath ++ CDNPath), - ok = filelib:ensure_dir(filename:join(RegistryDir, "placeholder")), - {ok, RegistryDir}; - _ -> - {uri_parse_error, CDN} - end - end. + RegistryDir = filename:join([CacheDir, "hex"]), + case filelib:ensure_dir(filename:join(RegistryDir, "placeholder")) of + ok -> ok; + {error, Posix} when Posix == eaccess; Posix == enoent -> + ?ABORT("Could not write to ~p. Please ensure the path is writeable.", + [RegistryDir]) + end, + {ok, RegistryDir}. -package_dir(State) -> - case registry_dir(State) of - {ok, RegistryDir} -> - PackageDir = filename:join([RegistryDir, "packages"]), - ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")), - {ok, PackageDir}; - Error -> - Error - end. +-spec package_dir(rebar_hex_repos:repo(), rebar_state:t()) -> {ok, file:filename_all()}. +package_dir(Repo, State) -> + {ok, RegistryDir} = registry_dir(State), + RepoName = maps:get(name, Repo), + PackageDir = filename:join([RegistryDir, rebar_utils:to_list(RepoName), "packages"]), + ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")), + {ok, PackageDir}. -registry_checksum({pkg, Name, Vsn, _Hash}, State) -> - try - ?MODULE:verify_table(State), - ets:lookup_element(?PACKAGE_TABLE, {Name, Vsn}, 3) - catch - _:_ -> - throw(?PRV_ERROR({missing_package, rebar_utils:to_binary(Name), rebar_utils:to_binary(Vsn)})) - end. %% Hex supports use of ~> to specify the version required for a dependency. %% Since rebar3 requires exact versions to choose from we find the highest @@ -152,31 +166,28 @@ registry_checksum({pkg, Name, Vsn, _Hash}, State) -> %% `~> 2.1.3-dev` | `>= 2.1.3-dev and < 2.2.0` %% `~> 2.0` | `>= 2.0.0 and < 3.0.0` %% `~> 2.1` | `>= 2.1.0 and < 3.0.0` -find_highest_matching(Dep, Constraint, Table, State) -> - find_highest_matching(undefined, undefined, Dep, Constraint, Table, State). - -find_highest_matching(Pkg, PkgVsn, Dep, Constraint, Table, State) -> - try find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State) of +find_highest_matching(Dep, Constraint, Repo, Table, State) -> + try find_highest_matching_(Dep, Constraint, Repo, Table, State) of none -> - handle_missing_package(Dep, State, + handle_missing_package(Dep, Repo, State, fun(State1) -> - find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State1) + find_highest_matching_(Dep, Constraint, Repo, Table, State1) end); Result -> Result catch _:_ -> - handle_missing_package(Dep, State, + handle_missing_package(Dep, Repo, State, fun(State1) -> - find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State1) + find_highest_matching_(Dep, Constraint, Repo, Table, State1) end) end. -find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State) -> - try find_all(Dep, Table, State) of - {ok, [Vsn]} -> - handle_single_vsn(Pkg, PkgVsn, Dep, Vsn, Constraint); - {ok, Vsns} -> +find_highest_matching_(Dep, Constraint, #{name := Repo}, Table, State) -> + try get_package_versions(Dep, Repo, Table, State) of + [Vsn] -> + handle_single_vsn(Vsn, Constraint); + Vsns -> case handle_vsns(Constraint, Vsns) of none -> none; @@ -188,18 +199,6 @@ find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State) -> none end. -find_all(Dep, Table, State) -> - ?MODULE:verify_table(State), - try ets:lookup_element(Table, Dep, 2) of - [Vsns] when is_list(Vsns)-> - {ok, Vsns}; - Vsns -> - {ok, Vsns} - catch - error:badarg -> - none - end. - handle_vsns(Constraint, Vsns) -> lists:foldl(fun(Version, Highest) -> case ec_semver:pes(Version, Constraint) andalso @@ -211,26 +210,227 @@ handle_vsns(Constraint, Vsns) -> end end, none, Vsns). -handle_single_vsn(Pkg, PkgVsn, Dep, Vsn, Constraint) -> +handle_single_vsn(Vsn, Constraint) -> case ec_semver:pes(Vsn, Constraint) of true -> {ok, Vsn}; false -> - case {Pkg, PkgVsn} of - {undefined, undefined} -> - ?DEBUG("Only existing version of ~ts is ~ts which does not match constraint ~~> ~ts. " - "Using anyway, but it is not guaranteed to work.", [Dep, Vsn, Constraint]); - _ -> - ?DEBUG("[~ts:~ts] Only existing version of ~ts is ~ts which does not match constraint ~~> ~ts. " - "Using anyway, but it is not guaranteed to work.", [Pkg, PkgVsn, Dep, Vsn, Constraint]) - end, - {ok, Vsn} + none end. -format_error({missing_package, Name, Vsn}) -> - io_lib:format("Package not found in registry: ~ts-~ts.", [rebar_utils:to_binary(Name), rebar_utils:to_binary(Vsn)]); -format_error({missing_package, Dep}) -> - io_lib:format("Package not found in registry: ~p.", [Dep]). - verify_table(State) -> ets:info(?PACKAGE_TABLE, named_table) =:= true orelse load_and_verify_version(State). + +parse_deps(Deps) -> + [{maps:get(app, D, Name), {pkg, Name, Constraint, undefined}} + || D=#{package := Name, + requirement := Constraint} <- Deps]. + +parse_checksum(<>) -> + list_to_binary( + rebar_string:uppercase( + lists:flatten(io_lib:format("~64.16.0b", [Checksum])))); +parse_checksum(Checksum) -> + Checksum. + +update_package(Name, RepoConfig=#{name := Repo}, State) -> + ?MODULE:verify_table(State), + try hex_repo:get_package(RepoConfig#{repo_key => maps:get(read_key, RepoConfig, <<>>)}, Name) of + {ok, {200, _Headers, #{releases := Releases}}} -> + _ = insert_releases(Name, Releases, Repo, ?PACKAGE_TABLE), + {ok, RegistryDir} = rebar_packages:registry_dir(State), + PackageIndex = filename:join(RegistryDir, ?INDEX_FILE), + ok = ets:tab2file(?PACKAGE_TABLE, PackageIndex); + {ok, {403, _Headers, <<>>}} -> + not_found; + {ok, {404, _Headers, _}} -> + not_found; + Error -> + ?DEBUG("Hex get_package request failed: ~p", [Error]), + %% TODO: add better log message. hex_core should export a format_error + ?WARN("Failed to update package from repo ~ts", [Repo]), + fail + catch + _:Exception -> + ?DEBUG("hex_repo:get_package failed for package ~p: ~p", [Name, Exception]), + fail + end. + +insert_releases(Name, Releases, Repo, Table) -> + [true = ets:insert(Table, + #package{key={Name, Version, Repo}, + checksum=parse_checksum(Checksum), + retired=maps:get(retired, Release, false), + dependencies=parse_deps(Dependencies)}) + || Release=#{checksum := Checksum, + version := Version, + dependencies := Dependencies} <- Releases]. + +-spec resolve_version(unicode:unicode_binary(), unicode:unicode_binary() | undefined, + binary() | undefined, + ets:tab(), rebar_state:t()) + -> {error, {invalid_vsn, unicode:unicode_binary()}} | + not_found | + {ok, #package{}, map()}. +%% if checksum is defined search for any matching repo matching pkg-vsn and checksum +resolve_version(Dep, DepVsn, Hash, HexRegistry, State) when is_binary(Hash) -> + Resources = rebar_state:resources(State), + #{repos := RepoConfigs} = rebar_resource_v2:find_resource_state(pkg, Resources), + RepoNames = [RepoName || #{name := RepoName} <- RepoConfigs], + + %% allow retired packages when we have a checksum + case get_package(Dep, DepVsn, Hash, '_', RepoNames, HexRegistry, State) of + {ok, Package=#package{key={_, _, RepoName}}} -> + {ok, RepoConfig} = rebar_hex_repos:get_repo_config(RepoName, RepoConfigs), + {ok, Package, RepoConfig}; + _ -> + Fun = fun(Repo) -> + case resolve_version_(Dep, DepVsn, Repo, HexRegistry, State) of + none -> + not_found; + {ok, Vsn} -> + get_package(Dep, Vsn, Hash, Repo, HexRegistry, State) + end + end, + handle_missing_no_exception(Fun, Dep, State) + end; +resolve_version(Dep, undefined, Hash, HexRegistry, State) -> + Fun = fun(Repo) -> + case highest_matching(Dep, "0", Repo, HexRegistry, State) of + none -> + not_found; + {ok, Vsn} -> + get_package(Dep, Vsn, Hash, Repo, HexRegistry, State) + end + end, + handle_missing_no_exception(Fun, Dep, State); +resolve_version(Dep, DepVsn, Hash, HexRegistry, State) -> + case valid_vsn(DepVsn) of + false -> + {error, {invalid_vsn, DepVsn}}; + _ -> + Fun = fun(Repo) -> + case resolve_version_(Dep, DepVsn, Repo, HexRegistry, State) of + none -> + not_found; + {ok, Vsn} -> + get_package(Dep, Vsn, Hash, Repo, HexRegistry, State) + end + end, + handle_missing_no_exception(Fun, Dep, State) + end. + +check_all_repos(Fun, RepoConfigs) -> + ec_lists:search(fun(#{name := R}) -> + Fun(R) + end, RepoConfigs). + +handle_missing_no_exception(Fun, Dep, State) -> + Resources = rebar_state:resources(State), + #{repos := RepoConfigs} = rebar_resource_v2:find_resource_state(pkg, Resources), + + %% first check all repos in order for a local match + %% if none is found then we step through checking after updating the repo registry + case check_all_repos(Fun, RepoConfigs) of + not_found -> + ec_lists:search(fun(Config=#{name := R}) -> + case ?MODULE:update_package(Dep, Config, State) of + ok -> + Fun(R); + _ -> + not_found + end + end, RepoConfigs); + Result -> + Result + end. + +resolve_version_(Dep, DepVsn, Repo, HexRegistry, State) -> + case DepVsn of + <<"~>", Vsn/binary>> -> + highest_matching(Dep, rm_ws(Vsn), Repo, HexRegistry, State); + <<">=", Vsn/binary>> -> + cmp(Dep, rm_ws(Vsn), Repo, HexRegistry, State, fun ec_semver:gte/2); + <<">", Vsn/binary>> -> + cmp(Dep, rm_ws(Vsn), Repo, HexRegistry, State, fun ec_semver:gt/2); + <<"<=", Vsn/binary>> -> + cmpl(Dep, rm_ws(Vsn), Repo, HexRegistry, State, fun ec_semver:lte/2); + <<"<", Vsn/binary>> -> + cmpl(Dep, rm_ws(Vsn), Repo, HexRegistry, State, fun ec_semver:lt/2); + <<"==", Vsn/binary>> -> + {ok, Vsn}; + Vsn -> + {ok, Vsn} + end. + +rm_ws(<<" ", R/binary>>) -> + rm_ws(R); +rm_ws(R) -> + R. + +valid_vsn(Vsn) -> + %% Regepx from https://github.com/sindresorhus/semver-regex/blob/master/index.js + SemVerRegExp = "v?(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)(\\.(0|[1-9][0-9]*))?" + "(-[0-9a-z-]+(\\.[0-9a-z-]+)*)?(\\+[0-9a-z-]+(\\.[0-9a-z-]+)*)?", + SupportedVersions = "^(>=?|<=?|~>|==)?\\s*" ++ SemVerRegExp ++ "$", + re:run(Vsn, SupportedVersions, [unicode]) =/= nomatch. + +highest_matching(Dep, Vsn, Repo, HexRegistry, State) -> + find_highest_matching_(Dep, Vsn, #{name => Repo}, HexRegistry, State). + +cmp(Dep, Vsn, Repo, HexRegistry, State, CmpFun) -> + case get_package_versions(Dep, Repo, HexRegistry, State) of + [] -> + none; + Vsns -> + cmp_(undefined, Vsn, Vsns, CmpFun) + end. + +cmp_(undefined, MinVsn, [], _CmpFun) -> + {ok, MinVsn}; +cmp_(HighestDepVsn, _MinVsn, [], _CmpFun) -> + {ok, HighestDepVsn}; + +cmp_(BestMatch, MinVsn, [Vsn | R], CmpFun) -> + case CmpFun(Vsn, MinVsn) of + true -> + cmp_(Vsn, Vsn, R, CmpFun); + false -> + cmp_(BestMatch, MinVsn, R, CmpFun) + end. + +%% We need to treat this differently since we want a version that is LOWER but +%% the higest possible one. +cmpl(Dep, Vsn, Repo, HexRegistry, State, CmpFun) -> + case get_package_versions(Dep, Repo, HexRegistry, State) of + [] -> + none; + Vsns -> + cmpl_(undefined, Vsn, Vsns, CmpFun) + end. + +cmpl_(undefined, MaxVsn, [], _CmpFun) -> + {ok, MaxVsn}; +cmpl_(HighestDepVsn, _MaxVsn, [], _CmpFun) -> + {ok, HighestDepVsn}; + +cmpl_(undefined, MaxVsn, [Vsn | R], CmpFun) -> + case CmpFun(Vsn, MaxVsn) of + true -> + cmpl_(Vsn, MaxVsn, R, CmpFun); + false -> + cmpl_(undefined, MaxVsn, R, CmpFun) + end; + +cmpl_(BestMatch, MaxVsn, [Vsn | R], CmpFun) -> + case CmpFun(Vsn, MaxVsn) of + true -> + case ec_semver:gte(Vsn, BestMatch) of + true -> + cmpl_(Vsn, MaxVsn, R, CmpFun); + false -> + cmpl_(BestMatch, MaxVsn, R, CmpFun) + end; + false -> + cmpl_(BestMatch, MaxVsn, R, CmpFun) + end. diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 2cf167e..97eabb6 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -2,42 +2,51 @@ %% ex: ts=4 sw=4 et -module(rebar_pkg_resource). --behaviour(rebar_resource). +-behaviour(rebar_resource_v2). --export([lock/2 - ,download/3 - ,download/4 - ,needs_update/2 - ,make_vsn/1]). +-export([init/2, + lock/2, + download/4, + download/5, + needs_update/2, + make_vsn/2, + format_error/1]). --export([request/2 - ,etag/1 - ,ssl_opts/1]). - -%% Exported for ct +-ifdef(TEST). +%% exported for test purposes -export([store_etag_in_cache/2]). +-endif. -include("rebar.hrl"). --include_lib("public_key/include/OTP-PUB-KEY.hrl"). - --type cached_result() :: {'bad_checksum',string()} | - {'bad_registry_checksum',string()} | - {'failed_extract',string()} | - {'ok','true'} | - {'unexpected_hash',string(),_,binary()}. +-include_lib("providers/include/providers.hrl"). --type download_result() :: {bad_download, binary() | string()} | - {fetch_fail, _, _} | cached_result(). +-type package() :: {pkg, binary(), binary(), binary(), rebar_hex_repos:repo()}. %%============================================================================== %% Public API %%============================================================================== --spec lock(AppDir, Source) -> Res when - AppDir :: file:name(), - Source :: tuple(), - Res :: {atom(), string(), any()}. -lock(_AppDir, Source) -> - Source. + +-spec init(atom(), rebar_state:t()) -> {ok, rebar_resource_v2:resource()}. +init(Type, State) -> + {ok, Vsn} = application:get_key(rebar, vsn), + BaseConfig = #{http_adapter => hex_http_httpc, + http_user_agent_fragment => + <<"(rebar3/", (list_to_binary(Vsn))/binary, ") (httpc)">>, + http_adapter_config => #{profile => rebar}}, + Repos = rebar_hex_repos:from_state(BaseConfig, State), + Resource = rebar_resource_v2:new(Type, ?MODULE, #{repos => Repos, + base_config => BaseConfig}), + {ok, Resource}. + + + +-spec lock(AppInfo, ResourceState) -> Res when + AppInfo :: rebar_app_info:t(), + ResourceState :: rebar_resource_v2:resource_state(), + Res :: {atom(), string(), any(), binary()}. +lock(AppInfo, _) -> + {pkg, Name, Vsn, Hash, _RepoConfig} = rebar_app_info:source(AppInfo), + {pkg, Name, Vsn, Hash}. %%------------------------------------------------------------------------------ %% @doc @@ -45,13 +54,13 @@ lock(_AppDir, Source) -> %% version. %% @end %%------------------------------------------------------------------------------ --spec needs_update(Dir, Pkg) -> Res when - Dir :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, +-spec needs_update(AppInfo, ResourceState) -> Res when + AppInfo :: rebar_app_info:t(), + ResourceState :: rebar_resource_v2:resource_state(), Res :: boolean(). -needs_update(Dir, {pkg, _Name, Vsn, _Hash}) -> - [AppInfo] = rebar_app_discover:find_apps([Dir], all), - case rebar_app_info:original_vsn(AppInfo) =:= rebar_utils:to_list(Vsn) of +needs_update(AppInfo, _) -> + {pkg, _Name, Vsn, _Hash, _} = rebar_app_info:source(AppInfo), + case rebar_app_info:original_vsn(AppInfo) =:= rebar_utils:to_binary(Vsn) of true -> false; false -> @@ -63,13 +72,19 @@ needs_update(Dir, {pkg, _Name, Vsn, _Hash}) -> %% Download the given pkg. %% @end %%------------------------------------------------------------------------------ --spec download(TmpDir, Pkg, State) -> Res when +-spec download(TmpDir, AppInfo, State, ResourceState) -> Res when TmpDir :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + AppInfo :: rebar_app_info:t(), + ResourceState :: rebar_resource_v2:resource_state(), State :: rebar_state:t(), - Res :: {'error',_} | {'ok',_} | {'tarball',binary() | string()}. -download(TmpDir, Pkg, State) -> - download(TmpDir, Pkg, State, true). + Res :: ok | {error,_}. +download(TmpDir, AppInfo, State, ResourceState) -> + case download(TmpDir, rebar_app_info:source(AppInfo), State, ResourceState, true) of + ok -> + ok; + Error -> + {error, Error} + end. %%------------------------------------------------------------------------------ %% @doc @@ -78,26 +93,28 @@ download(TmpDir, Pkg, State) -> %% is different. %% @end %%------------------------------------------------------------------------------ --spec download(TmpDir, Pkg, State, UpdateETag) -> Res when +-spec download(TmpDir, Pkg, State, ResourceState, UpdateETag) -> Res when TmpDir :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + Pkg :: package(), State :: rebar_state:t(), + ResourceState:: rebar_resource_v2:resource_state(), UpdateETag :: boolean(), - Res :: download_result(). -download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State, UpdateETag) -> - CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), - {ok, PackageDir} = rebar_packages:package_dir(State), + Res :: ok | {error,_} | {unexpected_hash, string(), integer(), integer()} | + {fetch_fail, binary(), binary()}. +download(TmpDir, Pkg={pkg, Name, Vsn, _Hash, Repo}, State, _ResourceState, UpdateETag) -> + {ok, PackageDir} = rebar_packages:package_dir(Repo, State), Package = binary_to_list(<>), ETagFile = binary_to_list(<>), CachePath = filename:join(PackageDir, Package), ETagPath = filename:join(PackageDir, ETagFile), - case rebar_utils:url_append_path(CDN, filename:join(?REMOTE_PACKAGE_DIR, - Package)) of - {ok, Url} -> - cached_download(TmpDir, CachePath, Pkg, Url, etag(ETagPath), State, - ETagPath, UpdateETag); - _ -> - {fetch_fail, Name, Vsn} + case cached_download(TmpDir, CachePath, Pkg, etag(CachePath, ETagPath), ETagPath, UpdateETag) of + {bad_registry_checksum, Expected, Found} -> + %% checksum comparison failed. in case this is from a modified cached package + %% overwrite the etag if it exists so it is not relied on again + store_etag_in_cache(ETagPath, <<>>), + ?PRV_ERROR({bad_registry_checksum, Name, Vsn, Expected, Found}); + Result -> + Result end. %%------------------------------------------------------------------------------ @@ -106,12 +123,19 @@ download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State, UpdateETag) -> %% Returns {error, string()} as this operation is not supported for pkg sources. %% @end %%------------------------------------------------------------------------------ --spec make_vsn(Vsn) -> Res when - Vsn :: any(), - Res :: {'error',[1..255,...]}. -make_vsn(_) -> +-spec make_vsn(AppInfo, ResourceState) -> Res when + AppInfo :: rebar_app_info:t(), + ResourceState :: rebar_resource_v2:resource_state(), + Res :: {'error', string()}. +make_vsn(_, _) -> {error, "Replacing version of type pkg not supported."}. +format_error({bad_registry_checksum, Name, Vsn, Expected, Found}) -> + io_lib:format("The checksum for package at ~ts-~ts (~ts) does not match the " + "checksum expected from the registry (~ts). " + "Run `rebar3 do unlock ~ts, update` and then try again.", + [Name, Vsn, Found, Expected, Name]). + %%------------------------------------------------------------------------------ %% @doc %% Download the pkg belonging to the given address. If the etag of the pkg @@ -120,29 +144,24 @@ make_vsn(_) -> %% {ok, Contents, NewEtag}, otherwise if some error occured return error. %% @end %%------------------------------------------------------------------------------ --spec request(Url, ETag) -> Res when - Url :: string(), - ETag :: false | string(), - Res :: 'error' | {ok, cached} | {ok, any(), string()}. -request(Url, ETag) -> - HttpOptions = [{ssl, ssl_opts(Url)}, - {relaxed, true} | rebar_utils:get_proxy_auth()], - case httpc:request(get, {Url, [{"if-none-match", "\"" ++ ETag ++ "\""} - || ETag =/= false] ++ - [{"User-Agent", rebar_utils:user_agent()}]}, - HttpOptions, [{body_format, binary}], rebar) of - {ok, {{_Version, 200, _Reason}, Headers, Body}} -> - ?DEBUG("Successfully downloaded ~ts", [Url]), - {"etag", ETag1} = lists:keyfind("etag", 1, Headers), - {ok, Body, rebar_string:trim(ETag1, both, [$"])}; - {ok, {{_Version, 304, _Reason}, _Headers, _Body}} -> - ?DEBUG("Cached copy of ~ts still valid", [Url]), +-spec request(rebar_hex_repos:repo(), binary(), binary(), false | binary()) + -> {ok, cached} | {ok, binary(), binary()} | error. +request(Config, Name, Version, ETag) -> + Config1 = Config#{http_etag => ETag}, + try hex_repo:get_tarball(Config1, Name, Version) of + {ok, {200, #{<<"etag">> := ETag1}, Tarball}} -> + {ok, Tarball, ETag1}; + {ok, {304, _Headers, _}} -> {ok, cached}; - {ok, {{_Version, Code, _Reason}, _Headers, _Body}} -> - ?DEBUG("Request to ~p failed: status code ~p", [Url, Code]), + {ok, {Code, _Headers, _Body}} -> + ?DEBUG("Request for package ~s-~s failed: status code ~p", [Name, Version, Code]), error; {error, Reason} -> - ?DEBUG("Request to ~p failed: ~p", [Url, Reason]), + ?DEBUG("Request for package ~s-~s failed: ~p", [Name, Version, Reason]), + error + catch + _:Exception -> + ?DEBUG("hex_repo:get_tarball failed: ~p", [Exception]), error end. @@ -153,32 +172,23 @@ request(Url, ETag) -> %% returned from the hexpm server. The name is package-vsn.etag. %% @end %%------------------------------------------------------------------------------ --spec etag(Path) -> Res when - Path :: file:name(), - Res :: false | string(). -etag(Path) -> - case file:read_file(Path) of +-spec etag(PackagePath, ETagPath) -> Res when + PackagePath :: file:name(), + ETagPath :: file:name(), + Res :: binary(). +etag(PackagePath, ETagPath) -> + case file:read_file(ETagPath) of {ok, Bin} -> - binary_to_list(Bin); + %% just in case a user deleted a cached package but not its etag + %% verify the package is also there, and if not, ignore the etag + case filelib:is_file(PackagePath) of + true -> + Bin; + false -> + <<>> + end; {error, _} -> - false - end. - -%%------------------------------------------------------------------------------ -%% @doc -%% Return the SSL options adequate for the project based on -%% its configuration, including for validation of certs. -%% @end -%%------------------------------------------------------------------------------ --spec ssl_opts(Url) -> Res when - Url :: string(), - Res :: proplists:proplist(). -ssl_opts(Url) -> - case get_ssl_config() of - ssl_verify_enabled -> - ssl_opts(ssl_verify_enabled, Url); - ssl_verify_disabled -> - [{verify, verify_none}] + <<>> end. %%------------------------------------------------------------------------------ @@ -188,7 +198,7 @@ ssl_opts(Url) -> %%------------------------------------------------------------------------------ -spec store_etag_in_cache(File, ETag) -> Res when File :: file:name(), - ETag :: string(), + ETag :: binary(), Res :: ok. store_etag_in_cache(Path, ETag) -> _ = file:write_file(Path, ETag). @@ -196,223 +206,74 @@ store_etag_in_cache(Path, ETag) -> %%%============================================================================= %%% Private functions %%%============================================================================= --spec cached_download(TmpDir, CachePath, Pkg, Url, ETag, State, ETagPath, - UpdateETag) -> Res when +-spec cached_download(TmpDir, CachePath, Pkg, ETag, ETagPath, UpdateETag) -> Res when TmpDir :: file:name(), CachePath :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, - Url :: string(), - ETag :: false | string(), - State :: rebar_state:t(), + Pkg :: package(), + ETag :: binary(), ETagPath :: file:name(), UpdateETag :: boolean(), - Res :: download_result(). -cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, Url, ETag, - State, ETagPath, UpdateETag) -> - case request(Url, ETag) of + Res :: ok | {unexpected_hash, integer(), integer()} | {fetch_fail, binary(), binary()}. +cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash, RepoConfig}, ETag, + ETagPath, UpdateETag) -> + case request(RepoConfig, Name, Vsn, ETag) of {ok, cached} -> ?INFO("Version cached at ~ts is up to date, reusing it", [CachePath]), - serve_from_cache(TmpDir, CachePath, Pkg, State); + serve_from_cache(TmpDir, CachePath, Pkg); {ok, Body, NewETag} -> ?INFO("Downloaded package, caching at ~ts", [CachePath]), maybe_store_etag_in_cache(UpdateETag, ETagPath, NewETag), - serve_from_download(TmpDir, CachePath, Pkg, NewETag, Body, State, - ETagPath); - error when ETag =/= false -> + serve_from_download(TmpDir, CachePath, Pkg, Body); + error when ETag =/= <<>> -> store_etag_in_cache(ETagPath, ETag), ?INFO("Download error, using cached file at ~ts", [CachePath]), - serve_from_cache(TmpDir, CachePath, Pkg, State); + serve_from_cache(TmpDir, CachePath, Pkg); error -> {fetch_fail, Name, Vsn} end. --spec serve_from_cache(TmpDir, CachePath, Pkg, State) -> Res when +-spec serve_from_cache(TmpDir, CachePath, Pkg) -> Res when TmpDir :: file:name(), CachePath :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, - State :: rebar_state:t(), - Res :: cached_result(). -serve_from_cache(TmpDir, CachePath, Pkg, State) -> - {Files, Contents, Version, Meta} = extract(TmpDir, CachePath), - case checksums(Pkg, Files, Contents, Version, Meta, State) of - {Chk, Chk, Chk, Chk} -> - ok = erl_tar:extract({binary, Contents}, [{cwd, TmpDir}, compressed]), - {ok, true}; - {_Hash, Chk, Chk, Chk} -> - ?DEBUG("Expected hash ~p does not match checksums ~p", [_Hash, Chk]), - {unexpected_hash, CachePath, _Hash, Chk}; - {Chk, _Bin, Chk, Chk} -> - ?DEBUG("Checksums: registry: ~p, pkg: ~p", [Chk, _Bin]), - {failed_extract, CachePath}; - {Chk, Chk, _Reg, Chk} -> - ?DEBUG("Checksums: registry: ~p, pkg: ~p", [_Reg, Chk]), - {bad_registry_checksum, CachePath}; - {_Hash, _Bin, _Reg, _Tar} -> - ?DEBUG("Checksums: expected: ~p, registry: ~p, pkg: ~p, meta: ~p", - [_Hash, _Reg, _Bin, _Tar]), - {bad_checksum, CachePath} + Pkg :: package(), + Res :: ok | {error,_} | {bad_registry_checksum, integer(), integer()}. +serve_from_cache(TmpDir, CachePath, Pkg) -> + {ok, Binary} = file:read_file(CachePath), + serve_from_memory(TmpDir, Binary, Pkg). + +-spec serve_from_memory(TmpDir, Tarball, Package) -> Res when + TmpDir :: file:name(), + Tarball :: binary(), + Package :: package(), + Res :: ok | {error,_} | {bad_registry_checksum, integer(), integer()}. +serve_from_memory(TmpDir, Binary, {pkg, _Name, _Vsn, Hash, _RepoConfig}) -> + RegistryChecksum = list_to_integer(binary_to_list(Hash), 16), + case hex_tarball:unpack(Binary, TmpDir) of + {ok, #{checksum := <>}} when RegistryChecksum =/= Checksum -> + ?DEBUG("Expected hash ~64.16.0B does not match checksum of fetched package ~64.16.0B", + [RegistryChecksum, Checksum]), + {bad_registry_checksum, RegistryChecksum, Checksum}; + {ok, #{checksum := <>}} -> + ok; + {error, Reason} -> + {error, {hex_tarball, Reason}} end. --spec serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State, - ETagPath) -> Res when +-spec serve_from_download(TmpDir, CachePath, Package, Binary) -> Res when TmpDir :: file:name(), CachePath :: file:name(), - Package :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, - ETag :: string(), + Package :: package(), Binary :: binary(), - State :: rebar_state:t(), - ETagPath :: file:name(), - Res :: download_result(). -serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State, ETagPath) -> + Res :: ok | {error,_}. +serve_from_download(TmpDir, CachePath, Package, Binary) -> ?DEBUG("Writing ~p to cache at ~ts", [Package, CachePath]), file:write_file(CachePath, Binary), - case etag(ETagPath) of - ETag -> - serve_from_cache(TmpDir, CachePath, Package, State); - FileETag -> - ?DEBUG("Downloaded file ~ts ETag ~ts doesn't match returned ETag ~ts", - [CachePath, ETag, FileETag]), - {bad_download, CachePath} - end. - --spec extract(TmpDir, CachePath) -> Res when - TmpDir :: file:name(), - CachePath :: file:name(), - Res :: {Files, Contents, Version, Meta}, - Files :: list({file:name(), binary()}), - Contents :: binary(), - Version :: binary(), - Meta :: binary(). -extract(TmpDir, CachePath) -> - ec_file:mkdir_p(TmpDir), - {ok, Files} = erl_tar:extract(CachePath, [memory]), - {"contents.tar.gz", Contents} = lists:keyfind("contents.tar.gz", 1, Files), - {"VERSION", Version} = lists:keyfind("VERSION", 1, Files), - {"metadata.config", Meta} = lists:keyfind("metadata.config", 1, Files), - {Files, Contents, Version, Meta}. - --spec checksums(Pkg, Files, Contents, Version, Meta, State) -> Res when - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, - Files :: list({file:name(), binary()}), - Contents :: binary(), - Version :: binary(), - Meta :: binary(), - State :: rebar_state:t(), - Res :: {Hash, BinChecksum, RegistryChecksum, TarChecksum}, - Hash :: binary(), - BinChecksum :: binary(), - RegistryChecksum :: any(), - TarChecksum :: binary(). -checksums(Pkg={pkg, _Name, _Vsn, Hash}, Files, Contents, Version, Meta, State) -> - Blob = <>, - <> = crypto:hash(sha256, Blob), - BinChecksum = list_to_binary( - rebar_string:uppercase( - lists:flatten(io_lib:format("~64.16.0b", [X])))), - RegistryChecksum = rebar_packages:registry_checksum(Pkg, State), - {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files), - {Hash, BinChecksum, RegistryChecksum, TarChecksum}. - -%%------------------------------------------------------------------------------ -%% @doc -%% Return the SSL options adequate for the project based on -%% its configuration, including for validation of certs. -%% @end -%%------------------------------------------------------------------------------ --spec ssl_opts(Enabled, Url) -> Res when - Enabled :: atom(), - Url :: string(), - Res :: proplists:proplist(). -ssl_opts(ssl_verify_enabled, Url) -> - case check_ssl_version() of - true -> - {ok, {_, _, Hostname, _, _, _}} = - http_uri:parse(rebar_utils:to_list(Url)), - VerifyFun = {fun ssl_verify_hostname:verify_fun/3, - [{check_hostname, Hostname}]}, - CACerts = certifi:cacerts(), - [{verify, verify_peer}, {depth, 2}, {cacerts, CACerts}, - {partial_chain, fun partial_chain/1}, {verify_fun, VerifyFun}]; - false -> - ?WARN("Insecure HTTPS request (peer verification disabled), " - "please update to OTP 17.4 or later", []), - [{verify, verify_none}] - end. - --spec partial_chain(Certs) -> Res when - Certs :: list(any()), - Res :: unknown_ca | {trusted_ca, any()}. -partial_chain(Certs) -> - Certs1 = [{Cert, public_key:pkix_decode_cert(Cert, otp)} || Cert <- Certs], - CACerts = certifi:cacerts(), - CACerts1 = [public_key:pkix_decode_cert(Cert, otp) || Cert <- CACerts], - case ec_lists:find(fun({_, Cert}) -> - check_cert(CACerts1, Cert) - end, Certs1) of - {ok, Trusted} -> - {trusted_ca, element(1, Trusted)}; - _ -> - unknown_ca - end. - --spec extract_public_key_info(Cert) -> Res when - Cert :: #'OTPCertificate'{tbsCertificate::#'OTPTBSCertificate'{}}, - Res :: any(). -extract_public_key_info(Cert) -> - ((Cert#'OTPCertificate'.tbsCertificate)#'OTPTBSCertificate'.subjectPublicKeyInfo). - --spec check_cert(CACerts, Cert) -> Res when - CACerts :: list(any()), - Cert :: any(), - Res :: boolean(). -check_cert(CACerts, Cert) -> - lists:any(fun(CACert) -> - extract_public_key_info(CACert) == extract_public_key_info(Cert) - end, CACerts). - --spec check_ssl_version() -> - boolean(). -check_ssl_version() -> - case application:get_key(ssl, vsn) of - {ok, Vsn} -> - parse_vsn(Vsn) >= {5, 3, 6}; - _ -> - false - end. - --spec get_ssl_config() -> - ssl_verify_disabled | ssl_verify_enabled. -get_ssl_config() -> - GlobalConfigFile = rebar_dir:global_config(), - Config = rebar_config:consult_file(GlobalConfigFile), - case proplists:get_value(ssl_verify, Config, []) of - false -> - ssl_verify_disabled; - _ -> - ssl_verify_enabled - end. - --spec parse_vsn(Vsn) -> Res when - Vsn :: string(), - Res :: {integer(), integer(), integer()}. -parse_vsn(Vsn) -> - version_pad(rebar_string:lexemes(Vsn, ".-")). - --spec version_pad(list(nonempty_string())) -> Res when - Res :: {integer(), integer(), integer()}. -version_pad([Major]) -> - {list_to_integer(Major), 0, 0}; -version_pad([Major, Minor]) -> - {list_to_integer(Major), list_to_integer(Minor), 0}; -version_pad([Major, Minor, Patch]) -> - {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}; -version_pad([Major, Minor, Patch | _]) -> - {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}. + serve_from_memory(TmpDir, Binary, Package). -spec maybe_store_etag_in_cache(UpdateETag, Path, ETag) -> Res when UpdateETag :: boolean(), Path :: file:name(), - ETag :: string(), + ETag :: binary(), Res :: ok. maybe_store_etag_in_cache(false = _UpdateETag, _Path, _ETag) -> ok; diff --git a/src/rebar_prv_deps.erl b/src/rebar_prv_deps.erl index a88b014..577a859 100644 --- a/src/rebar_prv_deps.erl +++ b/src/rebar_prv_deps.erl @@ -97,10 +97,11 @@ display_dep(_State, {Name, _Vsn, Source}) when is_tuple(Source) -> display_dep(_State, {Name, _Vsn, Source, _Opts}) when is_tuple(Source) -> ?CONSOLE("~ts* (~ts source)", [rebar_utils:to_binary(Name), type(Source)]); %% Locked -display_dep(State, {Name, Source={pkg, _, Vsn}, Level}) when is_integer(Level) -> +display_dep(State, {Name, _Source={pkg, _, Vsn}, Level}) when is_integer(Level) -> DepsDir = rebar_dir:deps_dir(State), AppDir = filename:join([DepsDir, rebar_utils:to_binary(Name)]), - NeedsUpdate = case rebar_fetch:needs_update(AppDir, Source, State) of + {ok, AppInfo} = rebar_app_info:discover(AppDir), + NeedsUpdate = case rebar_fetch:needs_update(AppInfo, State) of true -> "*"; false -> "" end, @@ -108,7 +109,8 @@ display_dep(State, {Name, Source={pkg, _, Vsn}, Level}) when is_integer(Level) - display_dep(State, {Name, Source, Level}) when is_tuple(Source), is_integer(Level) -> DepsDir = rebar_dir:deps_dir(State), AppDir = filename:join([DepsDir, rebar_utils:to_binary(Name)]), - NeedsUpdate = case rebar_fetch:needs_update(AppDir, Source, State) of + {ok, AppInfo} = rebar_app_info:discover(AppDir), + NeedsUpdate = case rebar_fetch:needs_update(AppInfo, State) of true -> "*"; false -> "" end, diff --git a/src/rebar_prv_deps_tree.erl b/src/rebar_prv_deps_tree.erl index 07c7972..d7b49c5 100644 --- a/src/rebar_prv_deps_tree.erl +++ b/src/rebar_prv_deps_tree.erl @@ -39,18 +39,16 @@ format_error(Reason) -> %% Internal functions print_deps_tree(SrcDeps, Verbose, State) -> - Resources = rebar_state:resources(State), D = lists:foldl(fun(App, Dict) -> Name = rebar_app_info:name(App), Vsn = rebar_app_info:original_vsn(App), - AppDir = rebar_app_info:dir(App), - Vsn1 = rebar_utils:vcs_vsn(Vsn, AppDir, Resources), + Vsn1 = rebar_utils:vcs_vsn(App, Vsn, State), Source = rebar_app_info:source(App), Parent = rebar_app_info:parent(App), dict:append_list(Parent, [{Name, Vsn1, Source}], Dict) end, dict:new(), SrcDeps), ProjectAppNames = [{rebar_app_info:name(App) - ,rebar_utils:vcs_vsn(rebar_app_info:original_vsn(App), rebar_app_info:dir(App), Resources) + ,rebar_utils:vcs_vsn(App, rebar_app_info:original_vsn(App), State) ,project} || App <- rebar_state:project_apps(State)], case dict:find(root, D) of {ok, Children} -> diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index b735ed0..bad5af4 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -277,10 +277,8 @@ update_unseen_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Loc -spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer()) -> {rebar_app_info:t(), [rebar_app_info:t()], rebar_state:t()}. handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) -> Name = rebar_app_info:name(AppInfo), - C = rebar_config:consult(rebar_app_info:dir(AppInfo)), - AppInfo0 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), C), - AppInfo1 = rebar_app_info:apply_overrides(rebar_app_info:get(AppInfo, overrides, []), AppInfo0), + AppInfo1 = rebar_app_info:apply_overrides(rebar_app_info:get(AppInfo, overrides, []), AppInfo), AppInfo2 = rebar_app_info:apply_profiles(AppInfo1, [default, prod]), Plugins = rebar_app_info:get(AppInfo2, plugins, []), @@ -297,34 +295,33 @@ handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) -> AppInfo4 = rebar_app_info:deps(AppInfo3, rebar_state:deps_names(Deps)), %% Keep all overrides from the global config and this dep when parsing its deps - Overrides = rebar_app_info:get(AppInfo0, overrides, []), + Overrides = rebar_app_info:get(AppInfo, overrides, []), Deps1 = rebar_app_utils:parse_deps(Name, DepsDir, Deps, rebar_state:set(State, overrides, Overrides) ,Locks, Level+1), {AppInfo4, Deps1, State1}. -spec maybe_fetch(rebar_app_info:t(), atom(), boolean(), - sets:set(binary()), rebar_state:t()) -> {boolean(), rebar_app_info:t()}. + sets:set(binary()), rebar_state:t()) -> {ok, rebar_app_info:t()}. maybe_fetch(AppInfo, Profile, Upgrade, Seen, State) -> AppDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)), %% Don't fetch dep if it exists in the _checkouts dir case rebar_app_info:is_checkout(AppInfo) of true -> - {false, AppInfo}; + {ok, AppInfo}; false -> - case rebar_app_discover:find_app(AppInfo, AppDir, all) of + case rebar_app_info:is_available(AppInfo) of false -> - true = fetch_app(AppInfo, AppDir, State), - maybe_symlink_default(State, Profile, AppDir, AppInfo), - {true, rebar_app_info:valid(update_app_info(AppDir, AppInfo), false)}; - {true, AppInfo1} -> - case sets:is_element(rebar_app_info:name(AppInfo1), Seen) of + AppInfo1 = fetch_app(AppInfo, State), + maybe_symlink_default(State, Profile, AppDir, AppInfo1), + {ok, rebar_app_info:is_available(rebar_app_info:valid(AppInfo1, false), true)}; + true -> + case sets:is_element(rebar_app_info:name(AppInfo), Seen) of true -> - {false, AppInfo1}; + {ok, AppInfo}; false -> - maybe_symlink_default(State, Profile, AppDir, AppInfo1), - MaybeUpgrade = maybe_upgrade(AppInfo, AppDir, Upgrade, State), - AppInfo2 = update_app_info(AppDir, AppInfo1), - {MaybeUpgrade, AppInfo2} + maybe_symlink_default(State, Profile, AppDir, AppInfo), + AppInfo1 = maybe_upgrade(AppInfo, AppDir, Upgrade, State), + {ok, AppInfo1} end end end. @@ -372,52 +369,37 @@ make_relative_to_root(State, Path) when is_list(Path) -> Root = rebar_dir:root_dir(State), rebar_dir:make_relative_path(Path, Root). -fetch_app(AppInfo, AppDir, State) -> +fetch_app(AppInfo, State) -> ?INFO("Fetching ~ts (~p)", [rebar_app_info:name(AppInfo), - format_source(rebar_app_info:source(AppInfo))]), - Source = rebar_app_info:source(AppInfo), - true = rebar_fetch:download_source(AppDir, Source, State). - -format_source({pkg, Name, Vsn, _Hash}) -> {pkg, Name, Vsn}; -format_source(Source) -> Source. - -%% This is called after the dep has been downloaded and unpacked, if it hadn't been already. -%% So this is the first time for newly downloaded apps that its .app/.app.src data can -%% be read in an parsed. -update_app_info(AppDir, AppInfo) -> - case rebar_app_discover:find_app(AppInfo, AppDir, all) of - {true, AppInfo1} -> - AppInfo1; - false -> - throw(?PRV_ERROR({dep_app_not_found, AppDir, rebar_app_info:name(AppInfo)})) - end. + rebar_resource_v2:format_source(rebar_app_info:source(AppInfo))]), + rebar_fetch:download_source(AppInfo, State). -maybe_upgrade(AppInfo, AppDir, Upgrade, State) -> - Source = rebar_app_info:source(AppInfo), +maybe_upgrade(AppInfo, _AppDir, Upgrade, State) -> case Upgrade orelse rebar_app_info:is_lock(AppInfo) of true -> - case rebar_fetch:needs_update(AppDir, Source, State) of + case rebar_fetch:needs_update(AppInfo, State) of true -> - ?INFO("Upgrading ~ts (~p)", [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)]), - true = rebar_fetch:download_source(AppDir, Source, State); + ?INFO("Upgrading ~ts (~p)", [rebar_app_info:name(AppInfo), + rebar_resource_v2:format_source(rebar_app_info:source(AppInfo))]), + rebar_fetch:download_source(AppInfo, State); false -> case Upgrade of true -> ?INFO("No upgrade needed for ~ts", [rebar_app_info:name(AppInfo)]), - false; + AppInfo; false -> - false + AppInfo end end; false -> - false + AppInfo end. warn_skip_deps(AppInfo, State) -> Msg = "Skipping ~ts (from ~p) as an app of the same name " "has already been fetched", Args = [rebar_app_info:name(AppInfo), - rebar_app_info:source(AppInfo)], + rebar_resource_v2:format_source(rebar_app_info:source(AppInfo))], case rebar_state:get(State, deps_error_on_conflict, false) of false -> case rebar_state:get(State, deps_warning_on_conflict, true) of diff --git a/src/rebar_prv_local_upgrade.erl b/src/rebar_prv_local_upgrade.erl index 3b3c9cb..1931d65 100644 --- a/src/rebar_prv_local_upgrade.erl +++ b/src/rebar_prv_local_upgrade.erl @@ -77,7 +77,7 @@ get_md5(Rebar3Path) -> maybe_fetch_rebar3(Rebar3Md5) -> TmpDir = ec_file:insecure_mkdtemp(), TmpFile = filename:join(TmpDir, "rebar3"), - case rebar_pkg_resource:request("https://s3.amazonaws.com/rebar3/rebar3", Rebar3Md5) of + case request("https://s3.amazonaws.com/rebar3/rebar3", Rebar3Md5) of {ok, Binary, ETag} -> file:write_file(TmpFile, Binary), case etag(TmpFile) of @@ -101,3 +101,29 @@ etag(Path) -> {error, _} -> false end. + +-spec request(Url, ETag) -> Res when + Url :: string(), + ETag :: false | string(), + Res :: 'error' | {ok, cached} | {ok, any(), string()}. +request(Url, ETag) -> + HttpOptions = [{ssl, rebar_utils:ssl_opts(Url)}, + {relaxed, true} | rebar_utils:get_proxy_auth()], + case httpc:request(get, {Url, [{"if-none-match", "\"" ++ ETag ++ "\""} + || ETag =/= false] ++ + [{"User-Agent", rebar_utils:user_agent()}]}, + HttpOptions, [{body_format, binary}], rebar) of + {ok, {{_Version, 200, _Reason}, Headers, Body}} -> + ?DEBUG("Successfully downloaded ~ts", [Url]), + {"etag", ETag1} = lists:keyfind("etag", 1, Headers), + {ok, Body, rebar_string:trim(ETag1, both, [$"])}; + {ok, {{_Version, 304, _Reason}, _Headers, _Body}} -> + ?DEBUG("Cached copy of ~ts still valid", [Url]), + {ok, cached}; + {ok, {{_Version, Code, _Reason}, _Headers, _Body}} -> + ?DEBUG("Request to ~p failed: status code ~p", [Url, Code]), + error; + {error, Reason} -> + ?DEBUG("Request to ~p failed: ~p", [Url, Reason]), + error + end. diff --git a/src/rebar_prv_lock.erl b/src/rebar_prv_lock.erl index cbe8dfe..570c03f 100644 --- a/src/rebar_prv_lock.erl +++ b/src/rebar_prv_lock.erl @@ -54,12 +54,9 @@ format_error(Reason) -> build_locks(State) -> AllDeps = rebar_state:lock(State), [begin - Dir = rebar_app_info:dir(Dep), - Source = rebar_app_info:source(Dep), - %% If source is tuple it is a source dep %% e.g. {git, "git://github.com/ninenines/cowboy.git", "master"} - {rebar_app_info:name(Dep) - ,rebar_fetch:lock_source(Dir, Source, State) - ,rebar_app_info:dep_level(Dep)} + {rebar_app_info:name(Dep), + rebar_fetch:lock_source(Dep, State), + rebar_app_info:dep_level(Dep)} end || Dep <- AllDeps, not(rebar_app_info:is_checkout(Dep))]. diff --git a/src/rebar_prv_packages.erl b/src/rebar_prv_packages.erl index 6e8e683..3e54cdc 100644 --- a/src/rebar_prv_packages.erl +++ b/src/rebar_prv_packages.erl @@ -15,53 +15,75 @@ -spec init(rebar_state:t()) -> {ok, rebar_state:t()}. init(State) -> - State1 = rebar_state:add_provider(State, providers:create([{name, ?PROVIDER}, - {module, ?MODULE}, - {bare, true}, - {deps, ?DEPS}, - {example, "rebar3 pkgs"}, - {short_desc, "List available packages."}, - {desc, info("List available packages")}, - {opts, []}])), + State1 = rebar_state:add_provider(State, + providers:create([{name, ?PROVIDER}, + {module, ?MODULE}, + {bare, true}, + {deps, ?DEPS}, + {example, "rebar3 pkgs elli"}, + {short_desc, "List information for a package."}, + {desc, info("List information for a package")}, + {opts, [{package, undefined, undefined, string, + "Package to fetch information for."}]}])), {ok, State1}. -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - rebar_packages:packages(State), - case rebar_state:command_args(State) of - [Name] -> - print_packages(get_packages(rebar_utils:to_binary(Name))); - _ -> - print_packages(sort_packages()) - end, - {ok, State}. + {Args, _} = rebar_state:command_parsed_args(State), + case proplists:get_value(package, Args, undefined) of + undefined -> + ?PRV_ERROR(no_package_arg); + Name -> + Resources = rebar_state:resources(State), + #{repos := Repos} = rebar_resource_v2:find_resource_state(pkg, Resources), + Results = get_package(rebar_utils:to_binary(Name), Repos), + case lists:all(fun({_, {error, not_found}}) -> true; (_) -> false end, Results) of + true -> + ?PRV_ERROR({not_found, Name}); + false -> + [print_packages(Result) || Result <- Results], + {ok, State} + end + end. --spec format_error(any()) -> iolist(). -format_error(load_registry_fail) -> - "Failed to load package regsitry. Try running 'rebar3 update' to fix". +-spec get_package(binary(), [map()]) -> [{binary(), {ok, map()} | {error, term()}}]. +get_package(Name, Repos) -> + lists:foldl(fun(RepoConfig, Acc) -> + [{maps:get(name, RepoConfig), rebar_packages:get(RepoConfig, Name)} | Acc] + end, [], Repos). -print_packages(Pkgs) -> - orddict:map(fun(Name, Vsns) -> - SortedVsns = lists:sort(fun(A, B) -> - ec_semver:lte(ec_semver:parse(A) - ,ec_semver:parse(B)) - end, Vsns), - VsnStr = join(SortedVsns, <<", ">>), - ?CONSOLE("~ts:~n Versions: ~ts~n", [Name, VsnStr]) - end, Pkgs). -sort_packages() -> - ets:foldl(fun({package_index_version, _}, Acc) -> - Acc; - ({Pkg, Vsns}, Acc) -> - orddict:store(Pkg, Vsns, Acc); - (_, Acc) -> - Acc - end, orddict:new(), ?PACKAGE_TABLE). +-spec format_error(any()) -> iolist(). +format_error(no_package_arg) -> + "Missing package argument to `rebar3 pkgs` command."; +format_error({not_found, Name}) -> + io_lib:format("Package ~ts not found in any repo.", [Name]); +format_error(unknown) -> + "Something went wrong with fetching package metadata.". -get_packages(Name) -> - ets:lookup(?PACKAGE_TABLE, Name). +print_packages({RepoName, {error, not_found}}) -> + ?CONSOLE("~ts: Package not found in this repo.~n", [RepoName]); +print_packages({RepoName, {error, _}}) -> + ?CONSOLE("~ts: Error fetching from this repo.~n", [RepoName]); +print_packages({RepoName, {ok, #{<<"name">> := Name, + <<"meta">> := Meta, + <<"releases">> := Releases}}}) -> + Description = maps:get(<<"description">>, Meta, ""), + Licenses = join(maps:get(<<"licenses">>, Meta, []), <<", ">>), + Links = join_map(maps:get(<<"links">>, Meta, []), <<"\n ">>), + Maintainers = join(maps:get(<<"maintainers">>, Meta, []), <<", ">>), + Versions = [V || #{<<"version">> := V} <- Releases], + VsnStr = join(Versions, <<", ">>), + ?CONSOLE("~ts:~n" + " Name: ~ts~n" + " Description: ~ts~n" + " Licenses: ~ts~n" + " Maintainers: ~ts~n" + " Links:~n ~ts~n" + " Versions: ~ts~n", [RepoName, Name, Description, Licenses, Maintainers, Links, VsnStr]); +print_packages(_) -> + ok. -spec join([binary()], binary()) -> binary(). join([Bin], _Sep) -> @@ -69,6 +91,14 @@ join([Bin], _Sep) -> join([Bin | T], Sep) -> <>. +-spec join_map(map(), binary()) -> binary(). +join_map(Map, Sep) -> + join_tuple_list(maps:to_list(Map), Sep). + +join_tuple_list([{K, V}], _Sep) -> + <>; +join_tuple_list([{K, V} | T], Sep) -> + <>. info(Description) -> io_lib:format("~ts.~n", [Description]). diff --git a/src/rebar_prv_repos.erl b/src/rebar_prv_repos.erl new file mode 100644 index 0000000..0515910 --- /dev/null +++ b/src/rebar_prv_repos.erl @@ -0,0 +1,47 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et + +-module(rebar_prv_repos). + +-behaviour(provider). + +-export([init/1, + do/1, + format_error/1]). + +-include("rebar.hrl"). + +-define(PROVIDER, repos). +-define(DEPS, []). + +%% =================================================================== +%% Public API +%% =================================================================== + +-spec init(rebar_state:t()) -> {ok, rebar_state:t()}. +init(State) -> + Provider = providers:create( + [{name, ?PROVIDER}, + {module, ?MODULE}, + {bare, false}, + {deps, ?DEPS}, + {example, "rebar3 repos"}, + {short_desc, "Print current package repository configuration"}, + {desc, "Display repository configuration for debugging purpose"}, + {opts, []}]), + State1 = rebar_state:add_provider(State, Provider), + {ok, State1}. + +-spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. +do(State) -> + Resources = rebar_state:resources(State), + #{repos := Repos} = rebar_resource_v2:find_resource_state(pkg, Resources), + + ?CONSOLE("Repos:", []), + %%TODO: do some formatting + ?CONSOLE("~p", [Repos]), + {ok, State}. + +-spec format_error(any()) -> iolist(). +format_error(Reason) -> + io_lib:format("~p", [Reason]). diff --git a/src/rebar_prv_shell.erl b/src/rebar_prv_shell.erl index 5b8d789..760f0d8 100644 --- a/src/rebar_prv_shell.erl +++ b/src/rebar_prv_shell.erl @@ -40,6 +40,8 @@ -define(PROVIDER, shell). -define(DEPS, [compile]). +-dialyzer({nowarn_function, rewrite_leaders/2}). + %% =================================================================== %% Public API %% =================================================================== diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index 1744631..4c820c5 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -9,12 +9,6 @@ do/1, format_error/1]). --export([hex_to_index/1]). - --ifdef(TEST). --export([cmp_/6, cmpl_/6, valid_vsn/1]). --endif. - -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). @@ -39,44 +33,13 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - try - case rebar_packages:registry_dir(State) of - {ok, RegistryDir} -> - filelib:ensure_dir(filename:join(RegistryDir, "dummy")), - HexFile = filename:join(RegistryDir, "registry"), - ?INFO("Updating package registry...", []), - TmpDir = ec_file:insecure_mkdtemp(), - TmpFile = filename:join(TmpDir, "packages.gz"), - - CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), - case rebar_utils:url_append_path(CDN, ?REMOTE_REGISTRY_FILE) of - {ok, Url} -> - HttpOptions = [{relaxed, true} | rebar_utils:get_proxy_auth()], - ?DEBUG("Fetching registry from ~p", [Url]), - case httpc:request(get, {Url, [{"User-Agent", rebar_utils:user_agent()}]}, - HttpOptions, [{stream, TmpFile}, {sync, true}], - rebar) of - {ok, saved_to_file} -> - {ok, Data} = file:read_file(TmpFile), - Unzipped = zlib:gunzip(Data), - ok = file:write_file(HexFile, Unzipped), - ?INFO("Writing registry to ~ts", [HexFile]), - hex_to_index(State), - {ok, State}; - _ -> - ?PRV_ERROR(package_index_download) - end; - _ -> - ?PRV_ERROR({package_parse_cdn, CDN}) - end; - {uri_parse_error, CDN} -> - ?PRV_ERROR({package_parse_cdn, CDN}) - end - catch - ?WITH_STACKTRACE(_E, C, S) - ?DEBUG("Error creating package index: ~p ~p", [C, S]), - throw(?PRV_ERROR(package_index_write)) - end. + Names = rebar_packages:get_all_names(State), + Resources = rebar_state:resources(State), + #{repos := RepoConfigs} = rebar_resource_v2:find_resource_state(pkg, Resources), + [[update_package(Name, RepoConfig, State) + || Name <- Names] + || RepoConfig <- RepoConfigs], + {ok, State}. -spec format_error(any()) -> iolist(). format_error({package_parse_cdn, Uri}) -> @@ -86,186 +49,11 @@ format_error(package_index_download) -> format_error(package_index_write) -> "Failed to write package index.". -is_supported(<<"make">>) -> true; -is_supported(<<"rebar">>) -> true; -is_supported(<<"rebar3">>) -> true; -is_supported(_) -> false. - -hex_to_index(State) -> - {ok, RegistryDir} = rebar_packages:registry_dir(State), - HexFile = filename:join(RegistryDir, "registry"), - try ets:file2tab(HexFile) of - {ok, Registry} -> - try - PackageIndex = filename:join(RegistryDir, "packages.idx"), - ?INFO("Generating package index...", []), - (catch ets:delete(?PACKAGE_TABLE)), - ets:new(?PACKAGE_TABLE, [named_table, public]), - ets:foldl(fun({{Pkg, PkgVsn}, [Deps, Checksum, BuildTools | _]}, _) when is_list(BuildTools) -> - case lists:any(fun is_supported/1, BuildTools) of - true -> - DepsList = update_deps_list(Pkg, PkgVsn, Deps, Registry, State), - HashedDeps = update_deps_hashes(DepsList), - ets:insert(?PACKAGE_TABLE, {{Pkg, PkgVsn}, HashedDeps, Checksum}); - false -> - true - end; - (_, _) -> - true - end, true, Registry), - - ets:foldl(fun({Pkg, [[]]}, _) when is_binary(Pkg) -> - true; - ({Pkg, [Vsns=[_Vsn | _Rest]]}, _) when is_binary(Pkg) -> - %% Verify the package is of the right build tool by checking if the first - %% version exists in the table from the foldl above - case [V || V <- Vsns, ets:member(?PACKAGE_TABLE, {Pkg, V})] of - [] -> - true; - Vsns1 -> - ets:insert(?PACKAGE_TABLE, {Pkg, Vsns1}) - end; - (_, _) -> - true - end, true, Registry), - ets:insert(?PACKAGE_TABLE, {package_index_version, ?PACKAGE_INDEX_VERSION}), - ?INFO("Writing index to ~ts", [PackageIndex]), - ets:tab2file(?PACKAGE_TABLE, PackageIndex), - true - after - catch ets:delete(Registry) - end; - {error, Reason} -> - ?DEBUG("Error loading package registry: ~p", [Reason]), - false - catch - _:_ -> - fail - end. - -update_deps_list(Pkg, PkgVsn, Deps, HexRegistry, State) -> - lists:foldl(fun([Dep, DepVsn, false, AppName | _], DepsListAcc) -> - Dep1 = {Pkg, PkgVsn, Dep, AppName}, - case {valid_vsn(DepVsn), DepVsn} of - %% Those are all not perfectly implemented! - %% and doubled since spaces seem not to be - %% enforced - {false, Vsn} -> - ?DEBUG("[~ts:~ts], Bad dependency version for ~ts: ~ts.", - [Pkg, PkgVsn, Dep, Vsn]), - DepsListAcc; - {_, <<"~>", Vsn/binary>>} -> - highest_matching(Dep1, rm_ws(Vsn), HexRegistry, - State, DepsListAcc); - {_, <<">=", Vsn/binary>>} -> - cmp(Dep1, rm_ws(Vsn), HexRegistry, State, - DepsListAcc, fun ec_semver:gte/2); - {_, <<">", Vsn/binary>>} -> - cmp(Dep1, rm_ws(Vsn), HexRegistry, State, - DepsListAcc, fun ec_semver:gt/2); - {_, <<"<=", Vsn/binary>>} -> - cmpl(Dep1, rm_ws(Vsn), HexRegistry, State, - DepsListAcc, fun ec_semver:lte/2); - {_, <<"<", Vsn/binary>>} -> - cmpl(Dep1, rm_ws(Vsn), HexRegistry, State, - DepsListAcc, fun ec_semver:lt/2); - {_, <<"==", Vsn/binary>>} -> - [{AppName, {pkg, Dep, Vsn, undefined}} | DepsListAcc]; - {_, Vsn} -> - [{AppName, {pkg, Dep, Vsn, undefined}} | DepsListAcc] - end; - ([_Dep, _DepVsn, true, _AppName | _], DepsListAcc) -> - DepsListAcc - end, [], Deps). - -update_deps_hashes(List) -> - [{Name, {pkg, PkgName, Vsn, lookup_hash(PkgName, Vsn, Hash)}} - || {Name, {pkg, PkgName, Vsn, Hash}} <- List]. - -lookup_hash(Name, Vsn, undefined) -> - try - ets:lookup_element(?PACKAGE_TABLE, {Name, Vsn}, 3) - catch - _:_ -> - undefined - end; -lookup_hash(_, _, Hash) -> - Hash. - - -rm_ws(<<" ", R/binary>>) -> - rm_ws(R); -rm_ws(R) -> - R. - -valid_vsn(Vsn) -> - %% Regepx from https://github.com/sindresorhus/semver-regex/blob/master/index.js - SemVerRegExp = "v?(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)(\\.(0|[1-9][0-9]*))?" - "(-[0-9a-z-]+(\\.[0-9a-z-]+)*)?(\\+[0-9a-z-]+(\\.[0-9a-z-]+)*)?", - SupportedVersions = "^(>=?|<=?|~>|==)?\\s*" ++ SemVerRegExp ++ "$", - re:run(Vsn, SupportedVersions, [unicode]) =/= nomatch. - -highest_matching({Pkg, PkgVsn, Dep, App}, Vsn, HexRegistry, State, DepsListAcc) -> - case rebar_packages:find_highest_matching_(Pkg, PkgVsn, Dep, Vsn, HexRegistry, State) of - {ok, HighestDepVsn} -> - [{App, {pkg, Dep, HighestDepVsn, undefined}} | DepsListAcc]; - none -> - ?DEBUG("[~ts:~ts] Missing registry entry for package ~ts. Try to fix with `rebar3 update`", - [Pkg, PkgVsn, Dep]), - DepsListAcc - end. - -cmp({_Pkg, _PkgVsn, Dep, _App} = Dep1, Vsn, HexRegistry, State, DepsListAcc, CmpFun) -> - {ok, Vsns} = rebar_packages:find_all(Dep, HexRegistry, State), - cmp_(undefined, Vsn, Vsns, DepsListAcc, Dep1, CmpFun). - - -cmp_(undefined, _MinVsn, [], DepsListAcc, {Pkg, PkgVsn, Dep, _App}, _CmpFun) -> - ?DEBUG("[~ts:~ts] Missing registry entry for package ~ts. Try to fix with `rebar3 update`", - [Pkg, PkgVsn, Dep]), - DepsListAcc; -cmp_(HighestDepVsn, _MinVsn, [], DepsListAcc, {_Pkg, _PkgVsn, Dep, App}, _CmpFun) -> - [{App, {pkg, Dep, HighestDepVsn, undefined}} | DepsListAcc]; - -cmp_(BestMatch, MinVsn, [Vsn | R], DepsListAcc, Dep, CmpFun) -> - case CmpFun(Vsn, MinVsn) of - true -> - cmp_(Vsn, Vsn, R, DepsListAcc, Dep, CmpFun); - false -> - cmp_(BestMatch, MinVsn, R, DepsListAcc, Dep, CmpFun) - end. - -%% We need to treat this differently since we want a version that is LOWER but -%% the higest possible one. -cmpl({_Pkg, _PkgVsn, Dep, _App} = Dep1, Vsn, HexRegistry, State, DepsListAcc, CmpFun) -> - {ok, Vsns} = rebar_packages:find_all(Dep, HexRegistry, State), - cmpl_(undefined, Vsn, Vsns, DepsListAcc, Dep1, CmpFun). - -cmpl_(undefined, _MaxVsn, [], DepsListAcc, {Pkg, PkgVsn, Dep, _App}, _CmpFun) -> - ?DEBUG("[~ts:~ts] Missing registry entry for package ~ts. Try to fix with `rebar3 update`", - [Pkg, PkgVsn, Dep]), - DepsListAcc; - -cmpl_(HighestDepVsn, _MaxVsn, [], DepsListAcc, {_Pkg, _PkgVsn, Dep, App}, _CmpFun) -> - [{App, {pkg, Dep, HighestDepVsn, undefined}} | DepsListAcc]; - -cmpl_(undefined, MaxVsn, [Vsn | R], DepsListAcc, Dep, CmpFun) -> - case CmpFun(Vsn, MaxVsn) of - true -> - cmpl_(Vsn, MaxVsn, R, DepsListAcc, Dep, CmpFun); - false -> - cmpl_(undefined, MaxVsn, R, DepsListAcc, Dep, CmpFun) - end; -cmpl_(BestMatch, MaxVsn, [Vsn | R], DepsListAcc, Dep, CmpFun) -> - case CmpFun(Vsn, MaxVsn) of - true -> - case ec_semver:gte(Vsn, BestMatch) of - true -> - cmpl_(Vsn, MaxVsn, R, DepsListAcc, Dep, CmpFun); - false -> - cmpl_(BestMatch, MaxVsn, R, DepsListAcc, Dep, CmpFun) - end; - false -> - cmpl_(BestMatch, MaxVsn, R, DepsListAcc, Dep, CmpFun) +update_package(Name, RepoConfig, State) -> + case rebar_packages:update_package(Name, RepoConfig, State) of + fail -> + ?WARN("Failed to fetch updates for package ~ts from repo ~ts", [Name, maps:get(name, RepoConfig)]); + _ -> + ok end. diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl index e4469cf..b1b1b16 100644 --- a/src/rebar_prv_upgrade.erl +++ b/src/rebar_prv_upgrade.erl @@ -82,17 +82,22 @@ do_(State) -> Deps = [Dep || Dep <- TopDeps ++ ProfileDeps, % TopDeps > ProfileDeps is_atom(Dep) orelse is_atom(element(1, Dep))], Names = parse_names(rebar_utils:to_binary(proplists:get_value(package, Args, <<"">>)), Locks), + DepsDict = deps_dict(rebar_state:all_deps(State)), AltDeps = find_non_default_deps(Deps, State), FilteredNames = cull_default_names_if_profiles(Names, Deps, State), case prepare_locks(FilteredNames, Deps, Locks, [], DepsDict, AltDeps) of {error, Reason} -> {error, Reason}; - {Locks0, _Unlocks0} -> + {Locks0, Unlocks0} -> Deps0 = top_level_deps(Deps, Locks), State1 = rebar_state:set(State, {deps, default}, Deps0), DepsDir = rebar_prv_install_deps:profile_dep_dir(State, default), D = rebar_app_utils:parse_deps(root, DepsDir, Deps0, State1, Locks0, 0), + + %% first update the package index for the packages to be upgraded + update_pkg_deps(Unlocks0, D, State1), + State2 = rebar_state:set(State1, {parsed_deps, default}, D), State3 = rebar_state:set(State2, {locks, default}, Locks0), State4 = rebar_state:set(State3, upgrade, true), @@ -121,6 +126,34 @@ format_error({transitive_dependency, Name}) -> format_error(Reason) -> io_lib:format("~p", [Reason]). +%% fetch updates for package deps that have been unlocked for upgrade +update_pkg_deps([], _, _) -> + ok; +update_pkg_deps([{Name, _, _} | Rest], AppInfos, State) -> + case rebar_app_utils:find(Name, AppInfos) of + {ok, AppInfo} -> + case element(1, rebar_app_info:source(AppInfo)) of + pkg -> + Resources = rebar_state:resources(State), + #{repos := RepoConfigs} = rebar_resource_v2:find_resource_state(pkg, Resources), + [update_package(Name, RepoConfig, State) || RepoConfig <- RepoConfigs]; + _ -> + skip + end; + _ -> + %% this should be impossible... + skip + end, + update_pkg_deps(Rest, AppInfos, State). + +update_package(Name, RepoConfig, State) -> + case rebar_packages:update_package(Name, RepoConfig, State) of + fail -> + ?WARN("Failed to fetch updates for package ~ts from repo ~ts", [Name, maps:get(name, RepoConfig)]); + _ -> + ok + end. + parse_names(Bin, Locks) -> case lists:usort(re:split(Bin, <<" *, *">>, [trim, unicode])) of %% Nothing submitted, use *all* apps diff --git a/src/rebar_resource.erl b/src/rebar_resource.erl index cdce7a8..a3a8edb 100644 --- a/src/rebar_resource.erl +++ b/src/rebar_resource.erl @@ -2,23 +2,53 @@ %% ex: ts=4 sw=4 et -module(rebar_resource). --export([]). +-export([new/3, + lock/2, + download/4, + needs_update/2, + make_vsn/2]). --export_type([resource/0 - ,type/0 - ,location/0 - ,ref/0]). +-export_type([source/0, + type/0, + location/0, + ref/0]). --type resource() :: {type(), location(), ref()}. +-include("rebar.hrl"). + +-type source() :: {type(), location(), ref()} | {type(), location(), ref(), binary()}. -type type() :: atom(). -type location() :: string(). -type ref() :: any(). -callback lock(file:filename_all(), tuple()) -> - rebar_resource:resource(). + source(). -callback download(file:filename_all(), tuple(), rebar_state:t()) -> {tarball, file:filename_all()} | {ok, any()} | {error, any()}. -callback needs_update(file:filename_all(), tuple()) -> boolean(). -callback make_vsn(file:filename_all()) -> {plain, string()} | {error, string()}. + +-spec new(type(), module(), term()) -> rebar_resource_v2:resource(). +new(Type, Module, State) -> + #resource{type=Type, + module=Module, + state=State, + implementation=?MODULE}. + +lock(Module, AppInfo) -> + Module:lock(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). + +download(Module, TmpDir, AppInfo, State) -> + case Module:download(TmpDir, rebar_app_info:source(AppInfo), State) of + {ok, _} -> + ok; + Error -> + Error + end. + +needs_update(Module, AppInfo) -> + Module:needs_update(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). + +make_vsn(Module, AppInfo) -> + Module:make_vsn(rebar_app_info:dir(AppInfo)). diff --git a/src/rebar_resource_v2.erl b/src/rebar_resource_v2.erl new file mode 100644 index 0000000..f032f6e --- /dev/null +++ b/src/rebar_resource_v2.erl @@ -0,0 +1,147 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +-module(rebar_resource_v2). + +-export([new/3, + find_resource_state/2, + format_source/1, + lock/2, + download/3, + needs_update/2, + make_vsn/3, + format_error/1]). + +-export_type([resource/0, + source/0, + type/0, + location/0, + ref/0, + resource_state/0]). + +-include("rebar.hrl"). +-include_lib("providers/include/providers.hrl"). + +-type resource() :: #resource{}. +-type source() :: {type(), location(), ref()} | {type(), location(), ref(), binary()}. +-type type() :: atom(). +-type location() :: string(). +-type ref() :: any(). +-type resource_state() :: term(). + +-callback init(type(), rebar_state:t()) -> {ok, resource()}. +-callback lock(rebar_app_info:t(), resource_state()) -> source(). +-callback download(file:filename_all(), rebar_app_info:t(), resource_state(), rebar_state:t()) -> + ok | {error, any()}. +-callback needs_update(rebar_app_info:t(), resource_state()) -> boolean(). +-callback make_vsn(rebar_app_info:t(), resource_state()) -> + {plain, string()} | {error, string()}. + +-spec new(type(), module(), term()) -> resource(). +new(Type, Module, State) -> + #resource{type=Type, + module=Module, + state=State, + implementation=?MODULE}. + +-spec find_resource(type(), [resource()]) -> {ok, resource()} | {error, not_found}. +find_resource(Type, Resources) -> + case ec_lists:find(fun(#resource{type=T}) -> T =:= Type end, Resources) of + error when is_atom(Type) -> + case code:which(Type) of + non_existing -> + {error, not_found}; + _ -> + {ok, rebar_resource:new(Type, Type, #{})} + end; + error -> + {error, not_found}; + {ok, Resource} -> + {ok, Resource} + end. + +find_resource_state(Type, Resources) -> + case lists:keyfind(Type, #resource.type, Resources) of + false -> + {error, not_found}; + #resource{state=State} -> + State + end. + +format_source({pkg, Name, Vsn, _Hash, _}) -> {pkg, Name, Vsn}; +format_source(Source) -> Source. + +lock(AppInfo, State) -> + resource_run(lock, rebar_app_info:source(AppInfo), [AppInfo], State). + +resource_run(Function, Source, Args, State) -> + Resources = rebar_state:resources(State), + case get_resource_type(Source, Resources) of + {ok, #resource{type=_, + module=Module, + state=ResourceState, + implementation=?MODULE}} -> + erlang:apply(Module, Function, Args++[ResourceState]); + {ok, #resource{type=_, + module=Module, + state=_, + implementation=rebar_resource}} -> + erlang:apply(rebar_resource, Function, [Module | Args]) + end. + +download(TmpDir, AppInfo, State) -> + resource_run(download, rebar_app_info:source(AppInfo), [TmpDir, AppInfo, State], State). + +needs_update(AppInfo, State) -> + resource_run(needs_update, rebar_app_info:source(AppInfo), [AppInfo], State). + +%% this is a special case since it is used for project apps as well, not just deps +make_vsn(AppInfo, VcsType, State) -> + Resources = rebar_state:resources(State), + case is_resource_type(VcsType, Resources) of + true -> + case find_resource(VcsType, Resources) of + {ok, #resource{type=_, + module=Module, + state=ResourceState, + implementation=?MODULE}} -> + Module:make_vsn(AppInfo, ResourceState); + {ok, #resource{type=_, + module=Module, + state=_, + implementation=rebar_resource}} -> + rebar_resource:make_vsn(Module, AppInfo) + end; + false -> + unknown + end. + +format_error({no_resource, Location, Type}) -> + io_lib:format("Cannot handle dependency ~ts.~n" + " No module found for resource type ~p.", [Location, Type]); +format_error({no_resource, Source}) -> + io_lib:format("Cannot handle dependency ~ts.~n" + " No module found for unknown resource type.", [Source]). + +is_resource_type(Type, Resources) -> + lists:any(fun(#resource{type=T}) -> T =:= Type end, Resources). + +-spec get_resource_type(term(), [resource()]) -> {ok, resource()}. +get_resource_type({Type, Location}, Resources) -> + get_resource(Type, Location, Resources); +get_resource_type({Type, Location, _}, Resources) -> + get_resource(Type, Location, Resources); +get_resource_type({Type, _, _, Location}, Resources) -> + get_resource(Type, Location, Resources); +get_resource_type(Location={Type, _, _, _, _}, Resources) -> + get_resource(Type, Location, Resources); +get_resource_type(Source, _) -> + throw(?PRV_ERROR({no_resource, Source})). + +-spec get_resource(type(), term(), [resource()]) -> {ok, resource()}. +get_resource(Type, Location, Resources) -> + case find_resource(Type, Resources) of + {error, not_found} -> + throw(?PRV_ERROR({no_resource, Location, Type})); + {ok, Resource} -> + {ok, Resource} + end. diff --git a/src/rebar_state.erl b/src/rebar_state.erl index 3586dd6..ec74eea 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -38,6 +38,7 @@ to_list/1, + create_resources/2, resources/1, resources/2, add_resource/2, providers/1, providers/2, add_provider/2, allow_provider_overrides/1, allow_provider_overrides/2 @@ -75,26 +76,24 @@ -spec new() -> t(). new() -> - BaseState = base_state(), + BaseState = base_state(dict:new()), BaseState#state_t{dir = rebar_dir:get_cwd()}. -spec new(list()) -> t(). new(Config) when is_list(Config) -> - BaseState = base_state(), Opts = base_opts(Config), - BaseState#state_t { dir = rebar_dir:get_cwd(), - default = Opts, - opts = Opts }. + BaseState = base_state(Opts), + BaseState#state_t{dir=rebar_dir:get_cwd(), + default=Opts}. -spec new(t() | atom(), list()) -> t(). new(Profile, Config) when is_atom(Profile) , is_list(Config) -> - BaseState = base_state(), Opts = base_opts(Config), - BaseState#state_t { dir = rebar_dir:get_cwd(), - current_profiles = [Profile], - default = Opts, - opts = Opts }; + BaseState = base_state(Opts), + BaseState#state_t{dir = rebar_dir:get_cwd(), + current_profiles = [Profile], + default = Opts}; new(ParentState=#state_t{}, Config) -> %% Load terms from rebar.config, if it exists Dir = rebar_dir:get_cwd(), @@ -129,20 +128,15 @@ deps_from_config(Dir, Config) -> [{{locks, default}, D}, {{deps, default}, Deps}] end. -base_state() -> - case application:get_env(rebar, resources) of - undefined -> - Resources = []; - {ok, Resources} -> - Resources - end, - #state_t{resources=Resources}. +base_state(Opts) -> + #state_t{opts=Opts}. base_opts(Config) -> Deps = proplists:get_value(deps, Config, []), Plugins = proplists:get_value(plugins, Config, []), ProjectPlugins = proplists:get_value(project_plugins, Config, []), - Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins}, {{project_plugins, default}, ProjectPlugins} | Config], + Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins}, + {{project_plugins, default}, ProjectPlugins} | Config], true = rebar_config:verify_config_format(Terms), dict:from_list(Terms). @@ -359,18 +353,50 @@ namespace(#state_t{namespace=Namespace}) -> namespace(State=#state_t{}, Namespace) -> State#state_t{namespace=Namespace}. --spec resources(t()) -> [{rebar_resource:type(), module()}]. +-spec resources(t()) -> [{rebar_resource_v2:type(), module()}]. resources(#state_t{resources=Resources}) -> Resources. --spec resources(t(), [{rebar_resource:type(), module()}]) -> t(). +-spec resources(t(), [{rebar_resource_v2:type(), module()}]) -> t(). resources(State, NewResources) -> - State#state_t{resources=NewResources}. - --spec add_resource(t(), {rebar_resource:type(), module()}) -> t(). -add_resource(State=#state_t{resources=Resources}, Resource) -> + lists:foldl(fun(Resource, StateAcc) -> + add_resource(StateAcc, Resource) + end, State, NewResources). + +-spec add_resource(t(), {rebar_resource_v2:type(), module()}) -> t(). +add_resource(State=#state_t{resources=Resources}, {ResourceType, ResourceModule}) -> + _ = code:ensure_loaded(ResourceModule), + Resource = case erlang:function_exported(ResourceModule, init, 2) of + true -> + case ResourceModule:init(ResourceType, State) of + {ok, R=#resource{}} -> + R; + _ -> + %% init didn't return a resource + %% must be an old resource + warn_old_resource(ResourceModule), + rebar_resource:new(ResourceType, + ResourceModule, + #{}) + end; + false -> + %% no init, must be initial implementation + warn_old_resource(ResourceModule), + rebar_resource:new(ResourceType, + ResourceModule, + #{}) + end, State#state_t{resources=[Resource | Resources]}. +warn_old_resource(ResourceModule) -> + ?WARN("Using custom resource ~s that implements a deprecated api. " + "It should be upgraded to rebar_resource_v2.", [ResourceModule]). + +create_resources(Resources, State) -> + lists:foldl(fun(R, StateAcc) -> + add_resource(StateAcc, R) + end, State, Resources). + providers(#state_t{providers=Providers}) -> Providers. diff --git a/src/rebar_string.erl b/src/rebar_string.erl index 47cb15c..d03b14e 100644 --- a/src/rebar_string.erl +++ b/src/rebar_string.erl @@ -1,7 +1,7 @@ %%% @doc Compatibility module for string functionality %%% for pre- and post-unicode support. -module(rebar_string). --export([join/2, lexemes/2, trim/3, uppercase/1, lowercase/1, chr/2]). +-export([join/2, split/2, lexemes/2, trim/3, uppercase/1, lowercase/1, chr/2]). -ifdef(unicode_str). @@ -15,6 +15,7 @@ join([], Sep) when is_list(Sep) -> join([H|T], Sep) -> H ++ lists:append([Sep ++ X || X <- T]). +split(Str, SearchPattern) -> string:split(Str, SearchPattern). lexemes(Str, SepList) -> string:lexemes(Str, SepList). trim(Str, Direction, Cluster=[_]) -> string:trim(Str, Direction, Cluster). uppercase(Str) -> string:uppercase(Str). @@ -27,6 +28,8 @@ chr([], _C, _I) -> 0. -else. join(Strings, Separator) -> string:join(Strings, Separator). +split(Str, SearchPattern) when is_list(Str) -> string:split(Str, SearchPattern); +split(Str, SearchPattern) when is_binary(Str) -> binary:split(Str, SearchPattern). lexemes(Str, SepList) -> string:tokens(Str, SepList). trim(Str, Direction, [Char]) -> Dir = case Direction of diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 7e57d01..995d212 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -74,13 +74,15 @@ user_agent/0, reread_config/1, reread_config/2, get_proxy_auth/0, - is_list_of_strings/1]). + is_list_of_strings/1, + ssl_opts/1]). %% for internal use only -export([otp_release/0]). -include("rebar.hrl"). +-include_lib("public_key/include/OTP-PUB-KEY.hrl"). -define(ONE_LEVEL_INDENT, " "). -define(APP_NAME_INDEX, 2). @@ -710,12 +712,12 @@ escript_foldl(Fun, Acc, File) -> Error end. -vcs_vsn(Vcs, Dir, Resources) -> - case vcs_vsn_cmd(Vcs, Dir, Resources) of +vcs_vsn(AppInfo, Vcs, State) -> + case vcs_vsn_cmd(AppInfo, Vcs, State) of {plain, VsnString} -> VsnString; {cmd, CmdString} -> - vcs_vsn_invoke(CmdString, Dir); + vcs_vsn_invoke(CmdString, rebar_app_info:dir(AppInfo)); unknown -> ?ABORT("vcs_vsn: Unknown vsn format: ~p", [Vcs]); {error, Reason} -> @@ -723,23 +725,18 @@ vcs_vsn(Vcs, Dir, Resources) -> end. %% Temp work around for repos like relx that use "semver" -vcs_vsn_cmd(Vsn, _, _) when is_binary(Vsn) -> +vcs_vsn_cmd(_AppInfo, Vsn, _) when is_binary(Vsn) -> {plain, Vsn}; -vcs_vsn_cmd(VCS, Dir, Resources) when VCS =:= semver ; VCS =:= "semver" -> - vcs_vsn_cmd(git, Dir, Resources); -vcs_vsn_cmd({cmd, _Cmd}=Custom, _, _) -> +vcs_vsn_cmd(AppInfo, VCS, State) when VCS =:= semver ; VCS =:= "semver" -> + vcs_vsn_cmd(AppInfo, git, State); +vcs_vsn_cmd(_AppInfo, {cmd, _Cmd}=Custom, _) -> Custom; -vcs_vsn_cmd(VCS, Dir, Resources) when is_atom(VCS) -> - case find_resource_module(VCS, Resources) of - {ok, Module} -> - Module:make_vsn(Dir); - {error, _} -> - unknown - end; -vcs_vsn_cmd(VCS, Dir, Resources) when is_list(VCS) -> +vcs_vsn_cmd(AppInfo, VCS, State) when is_atom(VCS) -> + rebar_resource_v2:make_vsn(AppInfo, VCS, State); +vcs_vsn_cmd(AppInfo, VCS, State) when is_list(VCS) -> try list_to_existing_atom(VCS) of AVCS -> - case vcs_vsn_cmd(AVCS, Dir, Resources) of + case vcs_vsn_cmd(AppInfo, AVCS, State) of unknown -> {plain, VCS}; Other -> Other end @@ -754,19 +751,6 @@ vcs_vsn_invoke(Cmd, Dir) -> {ok, VsnString} = rebar_utils:sh(Cmd, [{cd, Dir}, {use_stdout, false}]), rebar_string:trim(VsnString, trailing, "\n"). -find_resource_module(Type, Resources) -> - case lists:keyfind(Type, 1, Resources) of - false -> - case code:which(Type) of - non_existing -> - {error, unknown}; - _ -> - {ok, Type} - end; - {Type, Module} -> - {ok, Module} - end. - %% @doc ident to the level specified -spec indent(non_neg_integer()) -> iolist(). indent(Amount) when erlang:is_integer(Amount) -> @@ -977,3 +961,116 @@ is_list_of_strings(List) when is_list(hd(List)) -> true; is_list_of_strings(List) when is_list(List) -> true. + +%%------------------------------------------------------------------------------ +%% @doc +%% Return the SSL options adequate for the project based on +%% its configuration, including for validation of certs. +%% @end +%%------------------------------------------------------------------------------ +-spec ssl_opts(Url) -> Res when + Url :: string(), + Res :: proplists:proplist(). +ssl_opts(Url) -> + case get_ssl_config() of + ssl_verify_enabled -> + ssl_opts(ssl_verify_enabled, Url); + ssl_verify_disabled -> + [{verify, verify_none}] + end. + +%%------------------------------------------------------------------------------ +%% @doc +%% Return the SSL options adequate for the project based on +%% its configuration, including for validation of certs. +%% @end +%%------------------------------------------------------------------------------ +-spec ssl_opts(Enabled, Url) -> Res when + Enabled :: atom(), + Url :: string(), + Res :: proplists:proplist(). +ssl_opts(ssl_verify_enabled, Url) -> + case check_ssl_version() of + true -> + {ok, {_, _, Hostname, _, _, _}} = + http_uri:parse(rebar_utils:to_list(Url)), + VerifyFun = {fun ssl_verify_hostname:verify_fun/3, + [{check_hostname, Hostname}]}, + CACerts = certifi:cacerts(), + [{verify, verify_peer}, {depth, 2}, {cacerts, CACerts}, + {partial_chain, fun partial_chain/1}, {verify_fun, VerifyFun}]; + false -> + ?WARN("Insecure HTTPS request (peer verification disabled), " + "please update to OTP 17.4 or later", []), + [{verify, verify_none}] + end. + +-spec partial_chain(Certs) -> Res when + Certs :: list(any()), + Res :: unknown_ca | {trusted_ca, any()}. +partial_chain(Certs) -> + Certs1 = [{Cert, public_key:pkix_decode_cert(Cert, otp)} || Cert <- Certs], + CACerts = certifi:cacerts(), + CACerts1 = [public_key:pkix_decode_cert(Cert, otp) || Cert <- CACerts], + case ec_lists:find(fun({_, Cert}) -> + check_cert(CACerts1, Cert) + end, Certs1) of + {ok, Trusted} -> + {trusted_ca, element(1, Trusted)}; + _ -> + unknown_ca + end. + +-spec extract_public_key_info(Cert) -> Res when + Cert :: #'OTPCertificate'{tbsCertificate::#'OTPTBSCertificate'{}}, + Res :: any(). +extract_public_key_info(Cert) -> + ((Cert#'OTPCertificate'.tbsCertificate)#'OTPTBSCertificate'.subjectPublicKeyInfo). + +-spec check_cert(CACerts, Cert) -> Res when + CACerts :: list(any()), + Cert :: any(), + Res :: boolean(). +check_cert(CACerts, Cert) -> + lists:any(fun(CACert) -> + extract_public_key_info(CACert) == extract_public_key_info(Cert) + end, CACerts). + +-spec check_ssl_version() -> + boolean(). +check_ssl_version() -> + case application:get_key(ssl, vsn) of + {ok, Vsn} -> + parse_vsn(Vsn) >= {5, 3, 6}; + _ -> + false + end. + +-spec get_ssl_config() -> + ssl_verify_disabled | ssl_verify_enabled. +get_ssl_config() -> + GlobalConfigFile = rebar_dir:global_config(), + Config = rebar_config:consult_file(GlobalConfigFile), + case proplists:get_value(ssl_verify, Config, []) of + false -> + ssl_verify_disabled; + _ -> + ssl_verify_enabled + end. + +-spec parse_vsn(Vsn) -> Res when + Vsn :: string(), + Res :: {integer(), integer(), integer()}. +parse_vsn(Vsn) -> + version_pad(rebar_string:lexemes(Vsn, ".-")). + +-spec version_pad(list(nonempty_string())) -> Res when + Res :: {integer(), integer(), integer()}. +version_pad([Major]) -> + {list_to_integer(Major), 0, 0}; +version_pad([Major, Minor]) -> + {list_to_integer(Major), list_to_integer(Minor), 0}; +version_pad([Major, Minor, Patch]) -> + {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}; +version_pad([Major, Minor, Patch | _]) -> + {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}. -- cgit v1.1 From 5889a6e32bf09e5c50b49be330229bd157b5fae9 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 13 Sep 2018 20:13:59 -0600 Subject: Bump to 3.7.0-rc1 --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 530a79e..c122e04 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.7.0-rc1"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 221cbf5cac0204f3d2456552c08c38325ff592e2 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 13 Sep 2018 20:26:47 -0600 Subject: Back to git-based versioning --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index c122e04..530a79e 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.7.0-rc1"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From cd858a45e37f6563f22bf3aebee5911521f3c497 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Mon, 17 Sep 2018 09:02:15 -0600 Subject: reset default profile app plugins to empty if at the top level (#1885) top level default profile plugins are installed in the run_aux function. This commit removes the plugins from the plugin opts of an application that is also at the top level and skips the default profile when installing top level plugins in project_apps_install so they aren't handled twice. Additionally, before handling a plugin the state's list of known plugin apps is checked and the plugin is skipped if it has already been handled. --- src/rebar_app_discover.erl | 35 +++++++++++++++++++++-------------- src/rebar_plugins.erl | 33 ++++++++++++++++++++++++--------- 2 files changed, 45 insertions(+), 23 deletions(-) (limited to 'src') diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index e82403c..9b1346d 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -98,8 +98,7 @@ merge_deps(AppInfo, State) -> %% the State and at the top level, not as part of an application. CurrentProfiles = rebar_state:current_profiles(State), Default = reset_hooks(rebar_state:default(State), CurrentProfiles), - {C, State1} = project_app_config(AppInfo, State), - AppInfo0 = rebar_app_info:update_opts(AppInfo, Default, C), + {AppInfo0, State1} = project_app_config(AppInfo, Default, State), Name = rebar_app_info:name(AppInfo0), @@ -154,28 +153,36 @@ parse_profile_deps(Profile, Name, Deps, Opts, State) -> %% @doc Find the app-level config and return the state updated %% with the relevant app-level data. --spec project_app_config(rebar_app_info:t(), rebar_state:t()) -> +-spec project_app_config(rebar_app_info:t(), rebar_dict(), rebar_state:t()) -> {Config, rebar_state:t()} when Config :: [any()]. -project_app_config(AppInfo, State) -> +project_app_config(AppInfo, Default, State) -> C = rebar_config:consult(rebar_app_info:dir(AppInfo)), + AppInfo1 = rebar_app_info:update_opts(AppInfo, Default, C), + {AppInfo2, State1} = maybe_reset_hooks_plugins(AppInfo1, State), + {AppInfo2, State1}. + +-spec maybe_reset_hooks_plugins(AppInfo, State) -> {AppInfo, State} when + AppInfo :: rebar_app_info:t(), + State :: rebar_state:t(). +maybe_reset_hooks_plugins(AppInfo, State) -> Dir = rebar_app_info:dir(AppInfo), - Opts = maybe_reset_hooks(Dir, rebar_state:opts(State), State), - {C, rebar_state:opts(State, Opts)}. - -%% @private Check if the app is at the root of the project. -%% If it is, then drop the hooks from the config so they aren't run twice --spec maybe_reset_hooks(file:filename(), Opts, rebar_state:t()) -> Opts when - Opts :: rebar_dict(). -maybe_reset_hooks(Dir, Opts, State) -> case ec_file:real_dir_path(rebar_dir:root_dir(State)) of Dir -> CurrentProfiles = rebar_state:current_profiles(State), - reset_hooks(Opts, CurrentProfiles); + Opts = reset_hooks(rebar_state:opts(State), CurrentProfiles), + State1 = rebar_state:opts(State, Opts), + + %% set plugins to empty since this is an app at the top level + %% and top level plugins are installed in run_aux + AppInfo1 = rebar_app_info:set(rebar_app_info:set(AppInfo, {plugins,default}, []), plugins, []), + + {AppInfo1, State1}; _ -> - Opts + {AppInfo, State} end. + %% @doc make the hooks empty for a given set of options -spec reset_hooks(Opts, Profiles) -> Opts when diff --git a/src/rebar_plugins.erl b/src/rebar_plugins.erl index bc6a1e0..d58675d 100644 --- a/src/rebar_plugins.erl +++ b/src/rebar_plugins.erl @@ -39,13 +39,18 @@ project_apps_install(State) -> Profiles = rebar_state:current_profiles(State), ProjectApps = rebar_state:project_apps(State), lists:foldl(fun(Profile, StateAcc) -> - Plugins = rebar_state:get(State, {plugins, Profile}, []), - StateAcc1 = handle_plugins(Profile, Plugins, StateAcc), + StateAcc1 = case Profile of + default -> + %% default profile top level plugins + %% are installed in run_aux + StateAcc; + _ -> + Plugins = rebar_state:get(State, {plugins, Profile}, []), + handle_plugins(Profile, Plugins, StateAcc) + end, lists:foldl(fun(AppInfo, StateAcc2) -> - C = rebar_config:consult(rebar_app_info:dir(AppInfo)), - AppInfo0 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), C), - Plugins2 = rebar_app_info:get(AppInfo0, {plugins, Profile}, []), + Plugins2 = rebar_app_info:get(AppInfo, {plugins, Profile}, []), handle_plugins(Profile, Plugins2, StateAcc2) end, StateAcc1, ProjectApps) end, State, Profiles). @@ -62,12 +67,25 @@ install(State, AppInfo) -> State2 = lists:foldl(fun(Profile, StateAcc) -> Plugins = rebar_app_info:get(AppInfo, {plugins, Profile}, []), - handle_plugins(Profile, Plugins, StateAcc) + Plugins1 = filter_existing_plugins(Plugins, StateAcc), + handle_plugins(Profile, Plugins1, StateAcc) end, State1, Profiles), %% Reset the overrides after processing the dep rebar_state:set(State2, overrides, StateOverrides). +filter_existing_plugins(Plugins, State) -> + PluginNames = lists:zip(Plugins, rebar_state:deps_names(Plugins)), + AllPlugins = rebar_state:all_plugin_deps(State), + lists:filtermap(fun({Plugin, PluginName}) -> + case rebar_app_utils:find(PluginName, AllPlugins) of + {ok, _} -> + false; + _ -> + {true, Plugin} + end + end, PluginNames). + handle_plugins(Profile, Plugins, State) -> handle_plugins(Profile, Plugins, State, false). @@ -76,7 +94,6 @@ handle_plugins(Profile, Plugins, State, Upgrade) -> Locks = rebar_state:lock(State), DepsDir = rebar_state:get(State, deps_dir, ?DEFAULT_DEPS_DIR), State1 = rebar_state:set(State, deps_dir, ?DEFAULT_PLUGINS_DIR), - %% Install each plugin individually so if one fails to install it doesn't effect the others {_PluginProviders, State2} = lists:foldl(fun(Plugin, {PluginAcc, StateAcc}) -> @@ -122,8 +139,6 @@ handle_plugin(Profile, Plugin, State, Upgrade) -> build_plugin(AppInfo, Apps, State) -> Providers = rebar_state:providers(State), - %Providers1 = rebar_state:providers(rebar_app_info:state(AppInfo)), - %rebar_app_info:state_or_new(State, AppInfo) S = rebar_state:all_deps(State, Apps), S1 = rebar_state:set(S, deps_dir, ?DEFAULT_PLUGINS_DIR), rebar_prv_compile:compile(S1, Providers, AppInfo). -- cgit v1.1 From 43769ae87f7c60a9b6cb3bf08085a4a4206961b7 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Mon, 17 Sep 2018 09:02:30 -0600 Subject: only print skip warning if version is different (#1886) --- src/rebar_prv_install_deps.erl | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index bad5af4..068c4c8 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -259,9 +259,21 @@ update_seen_dep(AppInfo, _Profile, _Level, Deps, Apps, State, Upgrade, Seen, Loc %% If seen from lock file or user requested an upgrade %% don't print warning about skipping case lists:keymember(Name, 1, Locks) of - false when Upgrade -> ok; - false when not Upgrade -> warn_skip_deps(AppInfo, State); - true -> ok + false when Upgrade -> + ok; + false when not Upgrade -> + {ok, SeenApp} = rebar_app_utils:find(Name, Apps), + Source = rebar_app_info:source(AppInfo), + case rebar_app_info:source(SeenApp) of + Source -> + %% dep is the same version and checksum as the one we already saw. + %% meaning there is no conflict, so don't warn about it. + skip; + _ -> + warn_skip_deps(Name, Source, State) + end; + true -> + ok end, {Deps, Apps, State, Seen}. @@ -395,11 +407,11 @@ maybe_upgrade(AppInfo, _AppDir, Upgrade, State) -> AppInfo end. -warn_skip_deps(AppInfo, State) -> +warn_skip_deps(Name, Source, State) -> Msg = "Skipping ~ts (from ~p) as an app of the same name " "has already been fetched", - Args = [rebar_app_info:name(AppInfo), - rebar_resource_v2:format_source(rebar_app_info:source(AppInfo))], + Args = [Name, + rebar_resource_v2:format_source(Source)], case rebar_state:get(State, deps_error_on_conflict, false) of false -> case rebar_state:get(State, deps_warning_on_conflict, true) of -- cgit v1.1 From 8e0ef83de48968fed80114cc4518004ed59fa649 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Wed, 19 Sep 2018 08:24:38 -0600 Subject: add option to compiler provider to only build dependencies (#1888) --- src/rebar_prv_compile.erl | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index 0b4fa5f..edb2d82 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -30,22 +30,41 @@ init(State) -> {example, "rebar3 compile"}, {short_desc, "Compile apps .app.src and .erl files."}, {desc, "Compile apps .app.src and .erl files."}, - {opts, []}])), + {opts, [{deps_only, $d, "deps_only", undefined, + "Only compile dependencies, no project apps will be built."}]}])), {ok, State1}. -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> + IsDepsOnly = is_deps_only(State), DepsPaths = rebar_state:code_paths(State, all_deps), PluginDepsPaths = rebar_state:code_paths(State, all_plugin_deps), rebar_utils:remove_from_code_path(PluginDepsPaths), code:add_pathsa(DepsPaths), - ProjectApps = rebar_state:project_apps(State), Providers = rebar_state:providers(State), Deps = rebar_state:deps_to_build(State), - Cwd = rebar_state:dir(State), - copy_and_build_apps(State, Providers, Deps), + + State1 = case IsDepsOnly of + true -> + State; + false -> + handle_project_apps(DepsPaths, Providers, State) + end, + + rebar_utils:cleanup_code_path(rebar_state:code_paths(State1, default) + ++ rebar_state:code_paths(State, all_plugin_deps)), + + {ok, State1}. + +is_deps_only(State) -> + {Args, _} = rebar_state:command_parsed_args(State), + proplists:get_value(deps_only, Args, false). + +handle_project_apps(DepsPaths, Providers, State) -> + Cwd = rebar_state:dir(State), + ProjectApps = rebar_state:project_apps(State), {ok, ProjectApps1} = rebar_digraph:compile_order(ProjectApps), %% Run top level hooks *before* project apps compiled but *after* deps are @@ -66,10 +85,9 @@ do(State) -> true -> true end, - rebar_utils:cleanup_code_path(rebar_state:code_paths(State3, default) - ++ rebar_state:code_paths(State, all_plugin_deps)), - {ok, State3}. + State3. + -spec format_error(any()) -> iolist(). format_error({missing_artifact, File}) -> -- cgit v1.1 From 2dfba204e4dea5d1c3821fd26d22bd7201595f6c Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 21 Sep 2018 10:32:57 -0600 Subject: properly support top level app erl_opts from REBAR_CONFIG os var (#1889) When REBAR_CONFIG was set it would not effect the top level app's configuration because app_discover was rereading the top level rebar.config which ignored REBAR_CONFIG. Instead this patch has it use the existing configuration from REBAR_CONFIG. --- src/rebar_app_discover.erl | 74 ++++++++++++++++++++++++--------------------- src/rebar_app_info.erl | 14 +++++++-- src/rebar_erlc_compiler.erl | 1 + src/rebar_prv_plugins.erl | 4 +-- 4 files changed, 54 insertions(+), 39 deletions(-) (limited to 'src') diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index 9b1346d..224539b 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -7,7 +7,7 @@ find_unbuilt_apps/1, find_apps/1, find_apps/2, - find_apps/3, + find_apps/4, find_app/2, find_app/3]). @@ -23,7 +23,7 @@ do(State, LibDirs) -> Dirs = [filename:join(BaseDir, LibDir) || LibDir <- LibDirs], RebarOpts = rebar_state:opts(State), SrcDirs = rebar_dir:src_dirs(RebarOpts, ["src"]), - Apps = find_apps(Dirs, SrcDirs, all), + Apps = find_apps(Dirs, SrcDirs, all, State), ProjectDeps = rebar_state:deps_names(State), DepsDir = rebar_dir:deps_dir(State), CurrentProfiles = rebar_state:current_profiles(State), @@ -52,7 +52,7 @@ do(State, LibDirs) -> Name = rebar_app_info:name(AppInfo), case enable(State, AppInfo) of true -> - {AppInfo1, StateAcc1} = merge_deps(AppInfo, StateAcc), + {AppInfo1, StateAcc1} = merge_opts(AppInfo, StateAcc), OutDir = filename:join(DepsDir, Name), AppInfo2 = rebar_app_info:out_dir(AppInfo1, OutDir), ProjectDeps1 = lists:delete(Name, ProjectDeps), @@ -87,33 +87,34 @@ format_error({module_list, File}) -> format_error({missing_module, Module}) -> io_lib:format("Module defined in app file missing: ~p~n", [Module]). -%% @doc handles the merging and application of profiles and overrides -%% for a given application, within its own context. --spec merge_deps(rebar_app_info:t(), rebar_state:t()) -> +%% @doc merges configuration of a project app and the top level state +%% some configuration like erl_opts must be merged into a subapp's opts +%% while plugins and hooks need to be kept defined to only either the +%% top level state or an individual application. +-spec merge_opts(rebar_app_info:t(), rebar_state:t()) -> {rebar_app_info:t(), rebar_state:t()}. -merge_deps(AppInfo, State) -> +merge_opts(AppInfo, State) -> %% These steps make sure that hooks and artifacts are run in the context of %% the application they are defined at. If an umbrella structure is used and %% they are defined at the top level they will instead run in the context of %% the State and at the top level, not as part of an application. CurrentProfiles = rebar_state:current_profiles(State), - Default = reset_hooks(rebar_state:default(State), CurrentProfiles), - {AppInfo0, State1} = project_app_config(AppInfo, Default, State), + {AppInfo1, State1} = maybe_reset_hooks_plugins(AppInfo, State), - Name = rebar_app_info:name(AppInfo0), + Name = rebar_app_info:name(AppInfo1), %% We reset the opts here to default so no profiles are applied multiple times - AppInfo1 = rebar_app_info:apply_overrides(rebar_state:get(State1, overrides, []), AppInfo0), - AppInfo2 = rebar_app_info:apply_profiles(AppInfo1, CurrentProfiles), + AppInfo2 = rebar_app_info:apply_overrides(rebar_state:get(State1, overrides, []), AppInfo1), + AppInfo3 = rebar_app_info:apply_profiles(AppInfo2, CurrentProfiles), %% Will throw an exception if checks fail - rebar_app_info:verify_otp_vsn(AppInfo2), + rebar_app_info:verify_otp_vsn(AppInfo3), State2 = lists:foldl(fun(Profile, StateAcc) -> - handle_profile(Profile, Name, AppInfo2, StateAcc) + handle_profile(Profile, Name, AppInfo3, StateAcc) end, State1, lists:reverse(CurrentProfiles)), - {AppInfo2, State2}. + {AppInfo3, State2}. %% @doc Applies a given profile for an app, ensuring the deps %% match the context it will require. @@ -151,25 +152,15 @@ parse_profile_deps(Profile, Name, Deps, Opts, State) -> ,Locks ,1). -%% @doc Find the app-level config and return the state updated -%% with the relevant app-level data. --spec project_app_config(rebar_app_info:t(), rebar_dict(), rebar_state:t()) -> - {Config, rebar_state:t()} when - Config :: [any()]. -project_app_config(AppInfo, Default, State) -> - C = rebar_config:consult(rebar_app_info:dir(AppInfo)), - AppInfo1 = rebar_app_info:update_opts(AppInfo, Default, C), - {AppInfo2, State1} = maybe_reset_hooks_plugins(AppInfo1, State), - {AppInfo2, State1}. - +%% reset the State hooks if there is a top level application -spec maybe_reset_hooks_plugins(AppInfo, State) -> {AppInfo, State} when AppInfo :: rebar_app_info:t(), State :: rebar_state:t(). maybe_reset_hooks_plugins(AppInfo, State) -> Dir = rebar_app_info:dir(AppInfo), + CurrentProfiles = rebar_state:current_profiles(State), case ec_file:real_dir_path(rebar_dir:root_dir(State)) of Dir -> - CurrentProfiles = rebar_state:current_profiles(State), Opts = reset_hooks(rebar_state:opts(State), CurrentProfiles), State1 = rebar_state:opts(State, Opts), @@ -179,7 +170,11 @@ maybe_reset_hooks_plugins(AppInfo, State) -> {AppInfo1, State1}; _ -> - {AppInfo, State} + %% if not in the top root directory then we need to merge in the + %% default state opts to this subapp's opts + Default = reset_hooks(rebar_state:default(State), CurrentProfiles), + AppInfo1 = rebar_app_info:update_opts(AppInfo, Default), + {AppInfo1, State} end. @@ -211,8 +206,8 @@ reset_hooks(Opts, CurrentProfiles) -> -spec all_app_dirs([file:name()]) -> [{file:name(), [file:name()]}]. all_app_dirs(LibDirs) -> lists:flatmap(fun(LibDir) -> - {_, SrcDirs} = find_config_src(LibDir, ["src"]), - app_dirs(LibDir, SrcDirs) + {_, SrcDirs} = find_config_src(LibDir, ["src"]), + app_dirs(LibDir, SrcDirs) end, LibDirs). %% @private find the directories for all apps based on their source dirs @@ -270,11 +265,11 @@ find_apps(LibDirs, Validate) -> %% @doc for each directory passed, with the configured source directories, %% find all apps according to the validity rule passed in. %% Returns all the related app info records. --spec find_apps([file:filename_all()], [file:filename_all()], valid | invalid | all) -> [rebar_app_info:t()]. -find_apps(LibDirs, SrcDirs, Validate) -> +-spec find_apps([file:filename_all()], [file:filename_all()], valid | invalid | all, rebar_state:t()) -> [rebar_app_info:t()]. +find_apps(LibDirs, SrcDirs, Validate, State) -> rebar_utils:filtermap( fun({AppDir, AppSrcDirs}) -> - find_app(rebar_app_info:new(), AppDir, AppSrcDirs, Validate) + find_app(rebar_app_info:new(), AppDir, AppSrcDirs, Validate, State) end, all_app_dirs(LibDirs, SrcDirs) ). @@ -305,8 +300,19 @@ find_app(AppInfo, AppDir, Validate) -> %% the directories where source files can be located. Returns the related %% app info record. -spec find_app(rebar_app_info:t(), file:filename_all(), - [file:filename_all()], valid | invalid | all) -> + [file:filename_all()], valid | invalid | all, rebar_state:t()) -> {true, rebar_app_info:t()} | false. +find_app(AppInfo, AppDir, SrcDirs, Validate, State) -> + AppInfo1 = case ec_file:real_dir_path(rebar_dir:root_dir(State)) of + AppDir -> + Opts = rebar_state:opts(State), + rebar_app_info:default(rebar_app_info:opts(AppInfo, Opts), Opts); + _ -> + Config = rebar_config:consult(AppDir), + rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), Config) + end, + find_app_(AppInfo1, AppDir, SrcDirs, Validate). + find_app(AppInfo, AppDir, SrcDirs, Validate) -> Config = rebar_config:consult(AppDir), AppInfo1 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), Config), diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index 56ae4c0..eb95311 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -7,6 +7,7 @@ new/4, new/5, update_opts/3, + update_opts/2, update_opts_deps/2, discover/1, name/1, @@ -158,7 +159,7 @@ update_opts(AppInfo, Opts, Config) -> %% don't set anything here. []; _ -> - deps_from_config(dir(AppInfo), Config) + deps_from_config(dir(AppInfo), proplists:get_value(deps, Config, [])) end, Plugins = proplists:get_value(plugins, Config, []), @@ -171,6 +172,13 @@ update_opts(AppInfo, Opts, Config) -> AppInfo#app_info_t{opts=NewOpts, default=NewOpts}. +%% @doc update current app info opts by merging in a new dict of opts +-spec update_opts(t(), rebar_dict()) -> t(). +update_opts(AppInfo=#app_info_t{opts=LocalOpts}, Opts) -> + NewOpts = rebar_opts:merge_opts(LocalOpts, Opts), + AppInfo#app_info_t{opts=NewOpts, + default=NewOpts}. + %% @doc update the opts based on new deps, usually from an app's hex registry metadata -spec update_opts_deps(t(), [any()]) -> t(). update_opts_deps(AppInfo=#app_info_t{opts=Opts}, Deps) -> @@ -183,10 +191,10 @@ update_opts_deps(AppInfo=#app_info_t{opts=Opts}, Deps) -> %% @private extract the deps for an app in `Dir' based on its config file data -spec deps_from_config(file:filename(), [any()]) -> [{tuple(), any()}, ...]. -deps_from_config(Dir, Config) -> +deps_from_config(Dir, ConfigDeps) -> case rebar_config:consult_lock_file(filename:join(Dir, ?LOCK_FILE)) of [] -> - [{{deps, default}, proplists:get_value(deps, Config, [])}]; + [{{deps, default}, ConfigDeps}]; D -> %% We want the top level deps only from the lock file. %% This ensures deterministic overrides for configs. diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index 57e5398..c63ca5b 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -99,6 +99,7 @@ compile(AppInfo, CompileOpts) when element(1, AppInfo) == app_info_t -> {recursive, dir_recursive(RebarOpts, "src", CompileOpts)}], MibsOpts = [check_last_mod, {recursive, dir_recursive(RebarOpts, "mibs", CompileOpts)}], + rebar_base_compiler:run(RebarOpts, check_files([filename:join(Dir, File) || File <- rebar_opts:get(RebarOpts, xrl_first_files, [])]), diff --git a/src/rebar_prv_plugins.erl b/src/rebar_prv_plugins.erl index 4bea3b3..d66b645 100644 --- a/src/rebar_prv_plugins.erl +++ b/src/rebar_prv_plugins.erl @@ -36,7 +36,7 @@ do(State) -> GlobalPlugins = rebar_state:get(GlobalConfig, plugins, []), GlobalSrcDirs = rebar_state:get(GlobalConfig, src_dirs, ["src"]), GlobalPluginsDir = filename:join([rebar_dir:global_cache_dir(rebar_state:opts(State)), "plugins", "*"]), - GlobalApps = rebar_app_discover:find_apps([GlobalPluginsDir], GlobalSrcDirs, all), + GlobalApps = rebar_app_discover:find_apps([GlobalPluginsDir], GlobalSrcDirs, all, State), display_plugins("Global plugins", GlobalApps, GlobalPlugins), RebarOpts = rebar_state:opts(State), @@ -44,7 +44,7 @@ do(State) -> Plugins = rebar_state:get(State, plugins, []), PluginsDirs = filelib:wildcard(filename:join(rebar_dir:plugins_dir(State), "*")), CheckoutsDirs = filelib:wildcard(filename:join(rebar_dir:checkouts_dir(State), "*")), - Apps = rebar_app_discover:find_apps(CheckoutsDirs++PluginsDirs, SrcDirs, all), + Apps = rebar_app_discover:find_apps(CheckoutsDirs++PluginsDirs, SrcDirs, all, State), display_plugins("Local plugins", Apps, Plugins), {ok, State}. -- cgit v1.1 From 8bcb7da1dd294f13bc8d2040ec88f0b41b941e1b Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Mon, 1 Oct 2018 15:41:34 -0600 Subject: warn if the matched package is retired, skip prerelease (#1897) retired packages are now used the same as any other but a warning will be printed when it is resolved. prerelease versions are skipped unless explicitly given as the version in the constraint or lock file. --- src/rebar.hrl | 6 +++--- src/rebar_app_utils.erl | 41 +++++++++++++++++++++++++++++++++++------ src/rebar_packages.erl | 44 ++++++++++++++++++++------------------------ 3 files changed, 58 insertions(+), 33 deletions(-) (limited to 'src') diff --git a/src/rebar.hrl b/src/rebar.hrl index 572cbe8..f11302d 100644 --- a/src/rebar.hrl +++ b/src/rebar.hrl @@ -27,9 +27,9 @@ -define(REMOTE_PACKAGE_DIR, "tarballs"). -define(LOCK_FILE, "rebar.lock"). -define(DEFAULT_COMPILER_SOURCE_FORMAT, relative). --define(PACKAGE_INDEX_VERSION, 4). --define(PACKAGE_TABLE, package_index_v4). --define(INDEX_FILE, "packages-v4.idx"). +-define(PACKAGE_INDEX_VERSION, 5). +-define(PACKAGE_TABLE, package_index). +-define(INDEX_FILE, "packages.idx"). -define(HEX_AUTH_FILE, "hex.config"). -define(PUBLIC_HEX_REPO, <<"hexpm">>). diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index 35e908c..fb08ea9 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -254,10 +254,13 @@ update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) -> {ok, Package, RepoConfig} -> #package{key={_, PkgVsn1, _}, checksum=Hash1, - dependencies=Deps} = Package, - AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName, PkgVsn1, Hash1, RepoConfig}), + dependencies=Deps, + retired=Retired} = Package, + maybe_warn_retired(PkgName, PkgVsn1, Hash, Retired), + PkgVsn2 = list_to_binary(lists:flatten(ec_semver:format(PkgVsn1))), + AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName, PkgVsn2, Hash1, RepoConfig}), AppInfo2 = rebar_app_info:update_opts_deps(AppInfo1, Deps), - rebar_app_info:original_vsn(AppInfo2, PkgVsn1); + rebar_app_info:original_vsn(AppInfo2, PkgVsn2); not_found -> throw(?PRV_ERROR({missing_package, PkgName, PkgVsn})); {error, {invalid_vsn, InvalidVsn}} -> @@ -269,10 +272,10 @@ update_source(AppInfo, Source, _State) -> %% @doc convert a given exception's payload into an io description. -spec format_error(any()) -> iolist(). format_error({missing_package, Name, undefined}) -> - io_lib:format("Package not found in any repo: ~ts.", [rebar_utils:to_binary(Name)]); + io_lib:format("Package not found in any repo: ~ts", [rebar_utils:to_binary(Name)]); format_error({missing_package, Name, Vsn}) -> - io_lib:format("Package not found in any repo: ~ts-~ts.", [rebar_utils:to_binary(Name), - rebar_utils:to_binary(Vsn)]); + io_lib:format("Package not found in any repo: ~ts ~ts", [rebar_utils:to_binary(Name), + rebar_utils:to_binary(Vsn)]); format_error({parse_dep, Dep}) -> io_lib:format("Failed parsing dep ~p", [Dep]); format_error({invalid_vsn, Dep, InvalidVsn}) -> @@ -284,6 +287,32 @@ format_error(Error) -> %% Internal functions %% =================================================================== +maybe_warn_retired(_, _, _, false) -> + ok; +maybe_warn_retired(_, _, Hash, _) when is_binary(Hash) -> + %% don't warn if this is a lock + ok; +maybe_warn_retired(Name, Vsn, _, R=#{reason := Reason}) -> + Message = maps:get(message, R, ""), + ?WARN("Warning: package ~s-~s is retired: (~s) ~s", + [Name, ec_semver:format(Vsn), retire_reason(Reason), Message]); +maybe_warn_retired(_, _, _, _) -> + ok. + +%% TODO: move to hex_core +retire_reason('RETIRED_OTHER') -> + "other"; +retire_reason('RETIRED_INVALID') -> + "invalid"; +retire_reason('RETIRED_SECURITY') -> + "security"; +retire_reason('RETIRED_DEPRECATED') -> + "deprecated"; +retire_reason('RETIRED_RENAMED') -> + "renamed"; +retire_reason(_Other) -> + "other". + %% @private checks that all the beam files have been properly %% created. -spec has_all_beams(file:filename_all(), [module()]) -> diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 8a3ffea..7533a44 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -24,10 +24,10 @@ -type package() :: pkg_name() | {pkg_name(), vsn()}. format_error({missing_package, Name, Vsn}) -> - io_lib:format("Package not found in any repo: ~ts-~ts.", [rebar_utils:to_binary(Name), - rebar_utils:to_binary(Vsn)]); + io_lib:format("Package not found in any repo: ~ts ~ts", [rebar_utils:to_binary(Name), + rebar_utils:to_binary(Vsn)]); format_error({missing_package, Pkg}) -> - io_lib:format("Package not found in any repo: ~p.", [Pkg]). + io_lib:format("Package not found in any repo: ~p", [Pkg]). -spec get(rebar_hex_repos:repo(), binary()) -> {ok, map()} | {error, term()}. get(Config, Name) -> @@ -55,29 +55,25 @@ get_all_names(State) -> _='_'}, [], ['$1']}])). --spec get_package_versions(unicode:unicode_binary(), unicode:unicode_binary(), ets:tid(), rebar_state:t()) - -> [vsn()]. +-spec get_package_versions(unicode:unicode_binary(), unicode:unicode_binary(), + ets:tid(), rebar_state:t()) -> [vsn()]. get_package_versions(Dep, Repo, Table, State) -> ?MODULE:verify_table(State), - ets:select(Table, [{#package{key={Dep,'$1', Repo}, - _='_'}, - [], ['$1']}]). - - -get_package(Dep, Vsn, Hash, Repo, Table, State) -> - get_package(Dep, Vsn, Hash, false, [Repo], Table, State). + AllowPreRelease = rebar_state:get(State, deps_allow_prerelease, false), + ets:select(Table, [{#package{key={Dep,{'$1', '$2'}, Repo}, + _='_'}, + [{'==', '$2', {{[],[]}}} || not AllowPreRelease], [{{'$1', '$2'}}]}]). -spec get_package(unicode:unicode_binary(), unicode:unicode_binary(), - binary() | undefined | '_', boolean() | '_', - unicode:unicode_binary() | '_' | list(), ets:tab(), rebar_state:t()) + binary() | undefined | '_', + [unicode:unicode_binary()] | ['_'], ets:tab(), rebar_state:t()) -> {ok, #package{}} | not_found. -get_package(Dep, Vsn, undefined, Retired, Repo, Table, State) -> - get_package(Dep, Vsn, '_', Retired, Repo, Table, State); -get_package(Dep, Vsn, Hash, Retired, Repos, Table, State) -> +get_package(Dep, Vsn, undefined, Repos, Table, State) -> + get_package(Dep, Vsn, '_', Repos, Table, State); +get_package(Dep, Vsn, Hash, Repos, Table, State) -> ?MODULE:verify_table(State), - case ets:select(Table, [{#package{key={Dep, Vsn, Repo}, + case ets:select(Table, [{#package{key={Dep, ec_semver:parse(Vsn), Repo}, checksum=Hash, - retired=Retired, _='_'}, [], ['$_']} || Repo <- Repos]) of %% have to allow multiple matches in the list for cases that Repo is `_` [Package | _] -> @@ -258,7 +254,7 @@ update_package(Name, RepoConfig=#{name := Repo}, State) -> insert_releases(Name, Releases, Repo, Table) -> [true = ets:insert(Table, - #package{key={Name, Version, Repo}, + #package{key={Name, ec_semver:parse(Version), Repo}, checksum=parse_checksum(Checksum), retired=maps:get(retired, Release, false), dependencies=parse_deps(Dependencies)}) @@ -279,7 +275,7 @@ resolve_version(Dep, DepVsn, Hash, HexRegistry, State) when is_binary(Hash) -> RepoNames = [RepoName || #{name := RepoName} <- RepoConfigs], %% allow retired packages when we have a checksum - case get_package(Dep, DepVsn, Hash, '_', RepoNames, HexRegistry, State) of + case get_package(Dep, DepVsn, Hash, RepoNames, HexRegistry, State) of {ok, Package=#package{key={_, _, RepoName}}} -> {ok, RepoConfig} = rebar_hex_repos:get_repo_config(RepoName, RepoConfigs), {ok, Package, RepoConfig}; @@ -289,7 +285,7 @@ resolve_version(Dep, DepVsn, Hash, HexRegistry, State) when is_binary(Hash) -> none -> not_found; {ok, Vsn} -> - get_package(Dep, Vsn, Hash, Repo, HexRegistry, State) + get_package(Dep, Vsn, Hash, [Repo], HexRegistry, State) end end, handle_missing_no_exception(Fun, Dep, State) @@ -300,7 +296,7 @@ resolve_version(Dep, undefined, Hash, HexRegistry, State) -> none -> not_found; {ok, Vsn} -> - get_package(Dep, Vsn, Hash, Repo, HexRegistry, State) + get_package(Dep, Vsn, Hash, [Repo], HexRegistry, State) end end, handle_missing_no_exception(Fun, Dep, State); @@ -314,7 +310,7 @@ resolve_version(Dep, DepVsn, Hash, HexRegistry, State) -> none -> not_found; {ok, Vsn} -> - get_package(Dep, Vsn, Hash, Repo, HexRegistry, State) + get_package(Dep, Vsn, Hash, [Repo], HexRegistry, State) end end, handle_missing_no_exception(Fun, Dep, State) -- cgit v1.1 From 6ea0a600b4f560565ca963b69c86b9e9cb0ea0b8 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 5 Oct 2018 07:58:36 -0600 Subject: upgrade relx and set base_dir in overlay_vars (#1902) --- src/rebar_relx.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_relx.erl b/src/rebar_relx.erl index 4548761..431e1bc 100644 --- a/src/rebar_relx.erl +++ b/src/rebar_relx.erl @@ -40,7 +40,8 @@ do(Module, Command, Provider, State) -> ,{caller, api} ,{log_level, LogLevel} | output_dir(OutputDir, Options)] ++ ErlOpts, AllOptions); Config -> - Config1 = merge_overlays(Config), + Config1 = [{overlay_vars, [{base_dir, rebar_dir:base_dir(State)}]} + | merge_overlays(Config)], relx:main([{lib_dirs, LibDirs} ,{config, Config1} ,{caller, api} -- cgit v1.1 From dec484643c233fda9c17d38c1854ba7f3f37547b Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 5 Oct 2018 08:29:07 -0600 Subject: compiler behaviour (#1893) * add compile type for dynamic project compilation * new rebar_compiler abstraction for running multiple compilers rebar_compiler is a new behaviour that a plugin can implement to be called on any ues of the compile provider to compile source files and keep track of their dependencies. * fix check that modules in .app modules list are from src_dirs * use project_type to find module for building projects * allow plugins to add project builders and compilers --- src/rebar.app.src | 3 + src/rebar3.erl | 5 +- src/rebar_app_discover.erl | 2 +- src/rebar_app_info.erl | 43 ++- src/rebar_app_utils.erl | 5 +- src/rebar_base_compiler.erl | 33 +- src/rebar_compiler.erl | 304 ++++++++++++++++ src/rebar_compiler_erl.erl | 368 ++++++++++++++++++++ src/rebar_compiler_mib.erl | 70 ++++ src/rebar_compiler_xrl.erl | 50 +++ src/rebar_compiler_yrl.erl | 49 +++ src/rebar_dir.erl | 5 +- src/rebar_erlc_compiler.erl | 827 -------------------------------------------- src/rebar_file_utils.erl | 11 +- src/rebar_otp_app.erl | 38 +- src/rebar_packages.erl | 2 +- src/rebar_prv_clean.erl | 3 +- src/rebar_prv_compile.erl | 43 ++- src/rebar_state.erl | 37 ++ src/rebar_utils.erl | 87 ++++- 20 files changed, 1080 insertions(+), 905 deletions(-) create mode 100644 src/rebar_compiler.erl create mode 100644 src/rebar_compiler_erl.erl create mode 100644 src/rebar_compiler_mib.erl create mode 100644 src/rebar_compiler_xrl.erl create mode 100644 src/rebar_compiler_yrl.erl delete mode 100644 src/rebar_erlc_compiler.erl (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 530a79e..724f921 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -40,6 +40,9 @@ {pkg, rebar_pkg_resource}, {hg, rebar_hg_resource}]}, + {app_compilers, [{erl, rebar_erlc_compiler}]}, + {compilers, [rebar_compiler_xrl, rebar_compiler_yrl, rebar_compiler_mib, rebar_compiler_erl]}, + {providers, [rebar_prv_app_discovery, rebar_prv_as, rebar_prv_bare_compile, diff --git a/src/rebar3.erl b/src/rebar3.erl index e87cb19..c931a85 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -139,11 +139,14 @@ run_aux(State, RawArgs) -> rebar_state:set(State1, rebar_packages_cdn, CDN) end, + Compilers = application:get_env(rebar, compilers, []), + State0 = rebar_state:compilers(State2, Compilers), + %% TODO: this means use of REBAR_PROFILE=profile will replace the repos with %% the repos defined in the profile. But it will not work with `as profile`. %% Maybe it shouldn't work with either to be consistent? Resources = application:get_env(rebar, resources, []), - State2_ = rebar_state:create_resources(Resources, State2), + State2_ = rebar_state:create_resources(Resources, State0), %% bootstrap test profile State3 = rebar_state:add_to_profile(State2_, test, test_state(State1)), diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index 224539b..74681c7 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -427,7 +427,7 @@ try_handle_app_src_file(AppInfo, _, _AppDir, [], _Validate) -> %% a .app after this step case filelib:is_file(filename:join(rebar_app_info:dir(AppInfo), "mix.exs")) of true -> - {true, AppInfo}; + {true, rebar_app_info:project_type(AppInfo, mix)}; false -> false end; diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index eb95311..b1c6243 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -24,7 +24,6 @@ parent/2, original_vsn/1, original_vsn/2, - ebin_dir/1, priv_dir/1, applications/1, applications/2, @@ -38,6 +37,8 @@ dir/2, out_dir/1, out_dir/2, + ebin_dir/1, + ebin_dir/2, default/1, default/2, opts/1, @@ -47,6 +48,8 @@ set/3, source/1, source/2, + project_type/1, + project_type/2, is_lock/1, is_lock/2, is_checkout/1, @@ -68,7 +71,10 @@ -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). --export_type([t/0]). +-export_type([t/0, + project_type/0]). + +-type project_type() :: rebar3 | mix | undefined. -record(app_info_t, {name :: binary() | undefined, app_file_src :: file:filename_all() | undefined, @@ -85,10 +91,12 @@ dep_level=0 :: integer(), dir :: file:name(), out_dir :: file:name(), + ebin_dir :: file:name(), source :: string() | tuple() | checkout | undefined, is_lock=false :: boolean(), is_checkout=false :: boolean(), valid :: boolean() | undefined, + project_type :: project_type(), is_available=false :: boolean()}). %%============================================================================ @@ -125,7 +133,8 @@ new(AppName, Vsn, Dir) -> {ok, #app_info_t{name=rebar_utils:to_binary(AppName), original_vsn=rebar_utils:to_binary(Vsn), dir=rebar_utils:to_list(Dir), - out_dir=rebar_utils:to_list(Dir)}}. + out_dir=rebar_utils:to_list(Dir), + ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin")}}. %% @doc build a complete version of the app info with all fields set. -spec new(atom() | binary() | string(), binary() | string(), file:name(), list()) -> @@ -135,6 +144,7 @@ new(AppName, Vsn, Dir, Deps) -> original_vsn=rebar_utils:to_binary(Vsn), dir=rebar_utils:to_list(Dir), out_dir=rebar_utils:to_list(Dir), + ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin"), deps=Deps}}. %% @doc build a complete version of the app info with all fields set. @@ -146,6 +156,7 @@ new(Parent, AppName, Vsn, Dir, Deps) -> original_vsn=rebar_utils:to_binary(Vsn), dir=rebar_utils:to_list(Dir), out_dir=rebar_utils:to_list(Dir), + ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin"), deps=Deps}}. %% @doc update the opts based on the contents of a config @@ -447,12 +458,21 @@ out_dir(#app_info_t{out_dir=OutDir}) -> %% should go -spec out_dir(t(), file:name()) -> t(). out_dir(AppInfo=#app_info_t{}, OutDir) -> - AppInfo#app_info_t{out_dir=rebar_utils:to_list(OutDir)}. + AppInfo#app_info_t{out_dir=rebar_utils:to_list(OutDir), + ebin_dir=filename:join(rebar_utils:to_list(OutDir), "ebin")}. %% @doc gets the directory where ebin files for the app should go -spec ebin_dir(t()) -> file:name(). -ebin_dir(#app_info_t{out_dir=OutDir}) -> - rebar_utils:to_list(filename:join(OutDir, "ebin")). +ebin_dir(#app_info_t{ebin_dir=undefined, + out_dir=OutDir}) -> + filename:join(rebar_utils:to_list(OutDir), "ebin"); +ebin_dir(#app_info_t{ebin_dir=EbinDir}) -> + EbinDir. + +%% @doc sets the directory where beam files should go +-spec ebin_dir(t(), file:name()) -> t(). +ebin_dir(AppInfo, EbinDir) -> + AppInfo#app_info_t{ebin_dir=EbinDir}. %% @doc gets the directory where private files for the app should go -spec priv_dir(t()) -> file:name(). @@ -500,6 +520,17 @@ is_available(#app_info_t{is_available=IsAvailable}) -> is_available(AppInfo=#app_info_t{}, IsAvailable) -> AppInfo#app_info_t{is_available=IsAvailable}. +%% @doc +-spec project_type(t()) -> atom(). +project_type(#app_info_t{project_type=ProjectType}) -> + ProjectType. + +%% @doc +-spec project_type(t(), atom()) -> t(). +project_type(AppInfo=#app_info_t{}, ProjectType) -> + AppInfo#app_info_t{project_type=ProjectType}. + + %% @doc returns whether the app is valid (built) or not -spec valid(t()) -> boolean(). valid(AppInfo=#app_info_t{valid=undefined}) -> diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index fb08ea9..605944e 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -273,9 +273,8 @@ update_source(AppInfo, Source, _State) -> -spec format_error(any()) -> iolist(). format_error({missing_package, Name, undefined}) -> io_lib:format("Package not found in any repo: ~ts", [rebar_utils:to_binary(Name)]); -format_error({missing_package, Name, Vsn}) -> - io_lib:format("Package not found in any repo: ~ts ~ts", [rebar_utils:to_binary(Name), - rebar_utils:to_binary(Vsn)]); +format_error({missing_package, Name, Constraint}) -> + io_lib:format("Package not found in any repo: ~ts ~ts", [Name, Constraint]); format_error({parse_dep, Dep}) -> io_lib:format("Failed parsing dep ~p", [Dep]); format_error({invalid_vsn, Dep, InvalidVsn}) -> diff --git a/src/rebar_base_compiler.erl b/src/rebar_base_compiler.erl index 00b0533..2fb3a12 100644 --- a/src/rebar_base_compiler.erl +++ b/src/rebar_base_compiler.erl @@ -33,6 +33,8 @@ run/8, ok_tuple/2, error_tuple/4, + report/1, + maybe_report/1, format_error_source/2]). -type desc() :: term(). @@ -94,14 +96,14 @@ run(Config, FirstFiles, SourceDir, SourceExt, TargetDir, TargetExt, TargetDir :: file:filename(), SourceExt :: string(), TargetExt :: string(). -run(Config, FirstFiles, SourceDir, SourceExt, TargetDir, TargetExt, +run(Config, FirstFiles, SourceDirs, SourceExt, TargetDir, TargetExt, Compile3Fn, Opts) -> %% Convert simple extension to proper regex SourceExtRe = "^(?!\\._).*\\" ++ SourceExt ++ [$$], Recursive = proplists:get_value(recursive, Opts, true), %% Find all possible source files - FoundFiles = rebar_utils:find_files(SourceDir, SourceExtRe, Recursive), + FoundFiles = rebar_utils:find_files_in_dirs(SourceDirs, SourceExtRe, Recursive), %% Remove first files from found files RestFiles = [Source || Source <- FoundFiles, not lists:member(Source, FirstFiles)], @@ -111,7 +113,7 @@ run(Config, FirstFiles, SourceDir, SourceExt, TargetDir, TargetExt, run(Config, FirstFiles, RestFiles, fun(S, C) -> - Target = target_file(S, SourceDir, SourceExt, + Target = target_file(S, SourceExt, TargetDir, TargetExt), simple_compile_wrapper(S, Target, Compile3Fn, C, CheckLastMod) end). @@ -160,32 +162,15 @@ simple_compile_wrapper(Source, Target, Compile3Fn, Config, true) -> %% @private take a basic source set of file fragments and a target location, %% create a file path and name for a compile artifact. --spec target_file(SourceFile, SourceDir, SourceExt, TargetDir, TargetExt) -> File when +-spec target_file(SourceFile, SourceExt, TargetDir, TargetExt) -> File when SourceFile :: file:filename(), - SourceDir :: file:filename(), TargetDir :: file:filename(), SourceExt :: string(), TargetExt :: string(), File :: file:filename(). -target_file(SourceFile, SourceDir, SourceExt, TargetDir, TargetExt) -> - BaseFile = remove_common_path(SourceFile, SourceDir), - filename:join([TargetDir, filename:basename(BaseFile, SourceExt) ++ TargetExt]). - -%% @private removes the common prefix between two file paths. -%% The remainder of the first file path passed will have its ending returned -%% when either path starts diverging. --spec remove_common_path(file:filename(), file:filename()) -> file:filename(). -remove_common_path(Fname, Path) -> - remove_common_path1(filename:split(Fname), filename:split(Path)). - -%% @private given two lists of file fragments, discard the identical -%% prefixed sections, and return the final bit of the first operand -%% as a filename. --spec remove_common_path1([string()], [string()]) -> file:filename(). -remove_common_path1([Part | RestFilename], [Part | RestPath]) -> - remove_common_path1(RestFilename, RestPath); -remove_common_path1(FilenameParts, _) -> - filename:join(FilenameParts). +target_file(SourceFile, SourceExt, TargetDir, TargetExt) -> + %% BaseFile = remove_common_path(SourceFile, SourceDir), + filename:join([TargetDir, filename:basename(SourceFile, SourceExt) ++ TargetExt]). %% @private runs the compile function `CompileFn' on every file %% passed internally, along with the related project configuration. diff --git a/src/rebar_compiler.erl b/src/rebar_compiler.erl new file mode 100644 index 0000000..c2c514a --- /dev/null +++ b/src/rebar_compiler.erl @@ -0,0 +1,304 @@ +-module(rebar_compiler). + +-export([compile_all/2, + clean/2, + + ok_tuple/2, + error_tuple/4, + maybe_report/1, + format_error_source/2, + report/1]). + +-include("rebar.hrl"). + +-type extension() :: string(). +-type out_mappings() :: [{extension(), file:filename()}]. + +-callback context(rebar_app_info:t()) -> #{src_dirs => [file:dirname()], + include_dirs => [file:dirname()], + src_ext => extension(), + out_mappings => out_mappings()}. +-callback needed_files(digraph:graph(), [file:filename()], rebar_app_info:t()) -> [file:filename()]. +-callback dependencies(file:filename(), file:dirname(), [file:dirname()]) -> [file:filename()]. +-callback compile(file:filename(), out_mappings(), rebar_dict(), list()) -> + ok | {ok, [string()]} | {ok, [string()], [string()]}. + +-define(DAG_VSN, 2). +-define(DAG_FILE, "source.dag"). +-type dag_v() :: {digraph:vertex(), term()} | 'false'. +-type dag_e() :: {digraph:vertex(), digraph:vertex()}. +-type dag() :: {list(dag_v()), list(dag_e()), list(string())}. +-record(dag, {vsn = ?DAG_VSN :: pos_integer(), + info = {[], [], []} :: dag()}). + +-define(RE_PREFIX, "^(?!\\._)"). + +compile_all(Compilers, AppInfo) -> + OutDir = rebar_utils:to_list(rebar_app_info:out_dir(AppInfo)), + + %% Make sure that outdir is on the path + ok = rebar_file_utils:ensure_dir(OutDir), + true = code:add_patha(filename:absname(OutDir)), + + %% necessary for erlang:function_exported/3 to work as expected + %% called here for clarity as it's required by both opts_changed/2 + %% and erl_compiler_opts_set/0 in needed_files + _ = code:ensure_loaded(compile), + + lists:foreach(fun(CompilerMod) -> + run(CompilerMod, AppInfo), + run_on_extra_src_dirs(CompilerMod, AppInfo, fun run/2) + end, Compilers), + ok. + +run(CompilerMod, AppInfo) -> + #{src_dirs := SrcDirs, + include_dirs := InclDirs, + src_ext := SrcExt, + out_mappings := Mappings} = CompilerMod:context(AppInfo), + + BaseDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)), + EbinDir = rebar_utils:to_list(rebar_app_info:ebin_dir(AppInfo)), + + BaseOpts = rebar_app_info:opts(AppInfo), + AbsInclDirs = [filename:join(BaseDir, InclDir) || InclDir <- InclDirs], + FoundFiles = find_source_files(BaseDir, SrcExt, SrcDirs, BaseOpts), + + OutDir = rebar_app_info:out_dir(AppInfo), + AbsSrcDirs = [filename:join(BaseDir, SrcDir) || SrcDir <- SrcDirs], + G = init_dag(CompilerMod, AbsInclDirs, AbsSrcDirs, FoundFiles, OutDir, EbinDir), + {{FirstFiles, FirstFileOpts}, {RestFiles, Opts}} = CompilerMod:needed_files(G, FoundFiles, AppInfo), + true = digraph:delete(G), + + compile_each(FirstFiles, FirstFileOpts, BaseOpts, Mappings, CompilerMod), + compile_each(RestFiles, Opts, BaseOpts, Mappings, CompilerMod). + +compile_each([], _Opts, _Config, _Outs, _CompilerMod) -> + ok; +compile_each([Source | Rest], Opts, Config, Outs, CompilerMod) -> + case CompilerMod:compile(Source, Outs, Config, Opts) of + ok -> + ?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]); + {ok, Warnings} -> + report(Warnings), + ?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]); + skipped -> + ?DEBUG("~tsSkipped ~ts", [rebar_utils:indent(1), filename:basename(Source)]); + Error -> + NewSource = format_error_source(Source, Config), + ?ERROR("Compiling ~ts failed", [NewSource]), + maybe_report(Error), + ?DEBUG("Compilation failed: ~p", [Error]), + ?FAIL + end, + compile_each(Rest, Opts, Config, Outs, CompilerMod). + +%% @doc remove compiled artifacts from an AppDir. +-spec clean([module()], rebar_app_info:t()) -> 'ok'. +clean(Compilers, AppInfo) -> + lists:foreach(fun(CompilerMod) -> + clean_(CompilerMod, AppInfo), + run_on_extra_src_dirs(CompilerMod, AppInfo, fun clean_/2) + end, Compilers). + +clean_(CompilerMod, AppInfo) -> + #{src_dirs := SrcDirs, + src_ext := SrcExt} = CompilerMod:context(AppInfo), + BaseDir = rebar_app_info:dir(AppInfo), + Opts = rebar_app_info:opts(AppInfo), + EbinDir = rebar_app_info:ebin_dir(AppInfo), + + FoundFiles = find_source_files(BaseDir, SrcExt, SrcDirs, Opts), + CompilerMod:clean(FoundFiles, AppInfo), + rebar_file_utils:rm_rf(dag_file(CompilerMod, EbinDir)). + + +run_on_extra_src_dirs(CompilerMod, AppInfo, Fun) -> + ExtraDirs = rebar_dir:extra_src_dirs(rebar_app_info:opts(AppInfo), []), + run_on_extra_src_dirs(ExtraDirs, CompilerMod, AppInfo, Fun). + +run_on_extra_src_dirs([], _CompilerMod, _AppInfo, _Fun) -> + ok; +run_on_extra_src_dirs([Dir | Rest], CompilerMod, AppInfo, Fun) -> + case filelib:is_dir(filename:join(rebar_app_info:dir(AppInfo), Dir)) of + true -> + EbinDir = filename:join(rebar_app_info:out_dir(AppInfo), Dir), + AppInfo1 = rebar_app_info:ebin_dir(AppInfo, EbinDir), + AppInfo2 = rebar_app_info:set(AppInfo1, src_dirs, [Dir]), + AppInfo3 = rebar_app_info:set(AppInfo2, extra_src_dirs, ["src"]), + Fun(CompilerMod, AppInfo3); + _ -> + ok + end, + run_on_extra_src_dirs(Rest, CompilerMod, AppInfo, Fun). + +%% These functions are here for the ultimate goal of getting rid of +%% rebar_base_compiler. This can't be done because of existing plugins. + +ok_tuple(Source, Ws) -> + rebar_base_compiler:ok_tuple(Source, Ws). + +error_tuple(Source, Es, Ws, Opts) -> + rebar_base_compiler:error_tuple(Source, Es, Ws, Opts). + +maybe_report(Reportable) -> + rebar_base_compiler:maybe_report(Reportable). + +format_error_source(Path, Opts) -> + rebar_base_compiler:format_error_source(Path, Opts). + +report(Messages) -> + rebar_base_compiler:report(Messages). + +%% private functions + +find_source_files(BaseDir, SrcExt, SrcDirs, Opts) -> + SourceExtRe = "^(?!\\._).*\\" ++ SrcExt ++ [$$], + lists:flatmap(fun(SrcDir) -> + Recursive = rebar_dir:recursive(Opts, SrcDir), + rebar_utils:find_files_in_dirs([filename:join(BaseDir, SrcDir)], SourceExtRe, Recursive) + end, SrcDirs). + +dag_file(CompilerMod, Dir) -> + filename:join([rebar_dir:local_cache_dir(Dir), CompilerMod, ?DAG_FILE]). + +%% private graph functions + +%% Get dependency graph of given Erls files and their dependencies (header files, +%% parse transforms, behaviours etc.) located in their directories or given +%% InclDirs. Note that last modification times stored in vertices already respect +%% dependencies induced by given graph G. +init_dag(Compiler, InclDirs, SrcDirs, Erls, Dir, EbinDir) -> + G = digraph:new([acyclic]), + try restore_dag(Compiler, G, InclDirs, Dir) + catch + _:_ -> + ?WARN("Failed to restore ~ts file. Discarding it.~n", [dag_file(Compiler, Dir)]), + file:delete(dag_file(Compiler, Dir)) + end, + Dirs = lists:usort(InclDirs ++ SrcDirs), + %% A source file may have been renamed or deleted. Remove it from the graph + %% and remove any beam file for that source if it exists. + Modified = maybe_rm_beams_and_edges(G, EbinDir, Erls), + Modified1 = lists:foldl(update_dag_fun(G, Compiler, Dirs), Modified, Erls), + if Modified1 -> store_dag(Compiler, G, InclDirs, Dir); not Modified1 -> ok end, + G. + +maybe_rm_beams_and_edges(G, Dir, Files) -> + Vertices = digraph:vertices(G), + case lists:filter(fun(File) -> + case filename:extension(File) =:= ".erl" of + true -> + maybe_rm_beam_and_edge(G, Dir, File); + false -> + false + end + end, lists:sort(Vertices) -- lists:sort(Files)) of + [] -> + false; + _ -> + true + end. + +maybe_rm_beam_and_edge(G, OutDir, Source) -> + %% This is NOT a double check it is the only check that the source file is actually gone + case filelib:is_regular(Source) of + true -> + %% Actually exists, don't delete + false; + false -> + Target = target_base(OutDir, Source) ++ ".beam", + ?DEBUG("Source ~ts is gone, deleting previous beam file if it exists ~ts", [Source, Target]), + file:delete(Target), + digraph:del_vertex(G, Source), + true + end. + + +target_base(OutDir, Source) -> + filename:join(OutDir, filename:basename(Source, ".erl")). + +restore_dag(Compiler, G, InclDirs, Dir) -> + case file:read_file(dag_file(Compiler, Dir)) of + {ok, Data} -> + % Since externally passed InclDirs can influence dependency graph (see + % modify_dag), we have to check here that they didn't change. + #dag{vsn=?DAG_VSN, info={Vs, Es, InclDirs}} = + binary_to_term(Data), + lists:foreach( + fun({V, LastUpdated}) -> + digraph:add_vertex(G, V, LastUpdated) + end, Vs), + lists:foreach( + fun({_, V1, V2, _}) -> + digraph:add_edge(G, V1, V2) + end, Es); + {error, _} -> + ok + end. + +store_dag(Compiler, G, InclDirs, Dir) -> + Vs = lists:map(fun(V) -> digraph:vertex(G, V) end, digraph:vertices(G)), + Es = lists:map(fun(E) -> digraph:edge(G, E) end, digraph:edges(G)), + File = dag_file(Compiler, Dir), + ok = filelib:ensure_dir(File), + Data = term_to_binary(#dag{info={Vs, Es, InclDirs}}, [{compressed, 2}]), + file:write_file(File, Data). + +update_dag(G, Compiler, Dirs, Source) -> + case digraph:vertex(G, Source) of + {_, LastUpdated} -> + case filelib:last_modified(Source) of + 0 -> + %% The file doesn't exist anymore, + %% erase it from the graph. + %% All the edges will be erased automatically. + digraph:del_vertex(G, Source), + modified; + LastModified when LastUpdated < LastModified -> + modify_dag(G, Compiler, Source, LastModified, filename:dirname(Source), Dirs); + _ -> + Modified = lists:foldl( + update_dag_fun(G, Compiler, Dirs), + false, digraph:out_neighbours(G, Source)), + MaxModified = update_max_modified_deps(G, Source), + case Modified orelse MaxModified > LastUpdated of + true -> modified; + false -> unmodified + end + end; + false -> + modify_dag(G, Compiler, Source, filelib:last_modified(Source), filename:dirname(Source), Dirs) + end. + +modify_dag(G, Compiler, Source, LastModified, SourceDir, Dirs) -> + AbsIncls = Compiler:dependencies(Source, SourceDir, Dirs), + digraph:add_vertex(G, Source, LastModified), + digraph:del_edges(G, digraph:out_edges(G, Source)), + lists:foreach( + fun(Incl) -> + update_dag(G, Compiler, Dirs, Incl), + digraph:add_edge(G, Source, Incl) + end, AbsIncls), + modified. + +update_dag_fun(G, Compiler, Dirs) -> + fun(Erl, Modified) -> + case update_dag(G, Compiler, Dirs, Erl) of + modified -> true; + unmodified -> Modified + end + end. + +update_max_modified_deps(G, Source) -> + MaxModified = + lists:foldl(fun(File, Acc) -> + case digraph:vertex(G, File) of + {_, MaxModified} when MaxModified > Acc -> + MaxModified; + _ -> + Acc + end + end, 0, [Source | digraph:out_neighbours(G, Source)]), + digraph:add_vertex(G, Source, MaxModified), + MaxModified. diff --git a/src/rebar_compiler_erl.erl b/src/rebar_compiler_erl.erl new file mode 100644 index 0000000..d9bc69b --- /dev/null +++ b/src/rebar_compiler_erl.erl @@ -0,0 +1,368 @@ +-module(rebar_compiler_erl). + +-behaviour(rebar_compiler). + +-export([context/1, + needed_files/3, + dependencies/3, + compile/4, + clean/2]). + +-include("rebar.hrl"). + +context(AppInfo) -> + EbinDir = rebar_app_info:ebin_dir(AppInfo), + Mappings = [{".beam", EbinDir}], + + OutDir = rebar_app_info:dir(AppInfo), + SrcDirs = rebar_dir:src_dirs(rebar_app_info:opts(AppInfo), ["src"]), + ExistingSrcDirs = lists:filter(fun(D) -> + ec_file:is_dir(filename:join(OutDir, D)) + end, SrcDirs), + + RebarOpts = rebar_app_info:opts(AppInfo), + ErlOpts = rebar_opts:erl_opts(RebarOpts), + ErlOptIncludes = proplists:get_all_values(i, ErlOpts), + InclDirs = lists:map(fun(Incl) -> filename:absname(Incl) end, ErlOptIncludes), + + #{src_dirs => ExistingSrcDirs, + include_dirs => [filename:join([OutDir, "include"]) | InclDirs], + src_ext => ".erl", + out_mappings => Mappings}. + + +needed_files(Graph, FoundFiles, AppInfo) -> + OutDir = rebar_app_info:out_dir(AppInfo), + Dir = rebar_app_info:dir(AppInfo), + EbinDir = rebar_app_info:ebin_dir(AppInfo), + RebarOpts = rebar_app_info:opts(AppInfo), + ErlOpts = rebar_opts:erl_opts(RebarOpts), + ?DEBUG("erlopts ~p", [ErlOpts]), + ?DEBUG("files to compile ~p", [FoundFiles]), + + %% Make sure that the ebin dir is on the path + ok = rebar_file_utils:ensure_dir(EbinDir), + true = code:add_patha(filename:absname(EbinDir)), + + {ParseTransforms, Rest} = split_source_files(FoundFiles, ErlOpts), + NeededErlFiles = case needed_files(Graph, ErlOpts, RebarOpts, OutDir, EbinDir, ParseTransforms) of + [] -> + needed_files(Graph, ErlOpts, RebarOpts, OutDir, EbinDir, Rest); + _ -> + %% at least one parse transform in the opts needs updating, so recompile all + FoundFiles + end, + {ErlFirstFiles, ErlOptsFirst} = erl_first_files(RebarOpts, ErlOpts, Dir, NeededErlFiles), + SubGraph = digraph_utils:subgraph(Graph, NeededErlFiles), + DepErlsOrdered = digraph_utils:topsort(SubGraph), + OtherErls = lists:reverse(DepErlsOrdered), + + PrivIncludes = [{i, filename:join(OutDir, Src)} + || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])], + AdditionalOpts = PrivIncludes ++ [{i, filename:join(OutDir, "include")}, {i, OutDir}, return], + + true = digraph:delete(SubGraph), + + {{ErlFirstFiles, ErlOptsFirst ++ AdditionalOpts}, + {[Erl || Erl <- OtherErls, + not lists:member(Erl, ErlFirstFiles)], ErlOpts ++ AdditionalOpts}}. + +dependencies(Source, SourceDir, Dirs) -> + {ok, Fd} = file:open(Source, [read]), + Incls = parse_attrs(Fd, [], SourceDir), + AbsIncls = expand_file_names(Incls, Dirs), + ok = file:close(Fd), + AbsIncls. + +compile(Source, [{_, OutDir}], Config, ErlOpts) -> + case compile:file(Source, [{outdir, OutDir} | ErlOpts]) of + {ok, _Mod} -> + ok; + {ok, _Mod, []} -> + ok; + {ok, _Mod, Ws} -> + FormattedWs = format_error_sources(Ws, Config), + rebar_compiler:ok_tuple(Source, FormattedWs); + {error, Es, Ws} -> + error_tuple(Source, Es, Ws, Config, ErlOpts); + error -> + error + end. + +clean(Files, AppInfo) -> + EbinDir = rebar_app_info:ebin_dir(AppInfo), + [begin + Source = filename:basename(File, ".erl"), + Target = target_base(EbinDir, Source) ++ ".beam", + file:delete(Target) + end || File <- Files]. + +%% + +error_tuple(Module, Es, Ws, AllOpts, Opts) -> + FormattedEs = format_error_sources(Es, AllOpts), + FormattedWs = format_error_sources(Ws, AllOpts), + rebar_compiler:error_tuple(Module, FormattedEs, FormattedWs, Opts). + +format_error_sources(Es, Opts) -> + [{rebar_compiler:format_error_source(Src, Opts), Desc} + || {Src, Desc} <- Es]. + +%% Get files which need to be compiled first, i.e. those specified in erl_first_files +%% and parse_transform options. Also produce specific erl_opts for these first +%% files, so that yet to be compiled parse transformations are excluded from it. +erl_first_files(Opts, ErlOpts, Dir, NeededErlFiles) -> + ErlFirstFilesConf = rebar_opts:get(Opts, erl_first_files, []), + valid_erl_first_conf(ErlFirstFilesConf), + NeededSrcDirs = lists:usort(lists:map(fun filename:dirname/1, NeededErlFiles)), + %% NOTE: order of files here is important! + ErlFirstFiles = + [filename:join(Dir, File) || File <- ErlFirstFilesConf, + lists:member(filename:join(Dir, File), NeededErlFiles)], + {ParseTransforms, ParseTransformsErls} = + lists:unzip(lists:flatmap( + fun(PT) -> + PTerls = [filename:join(D, module_to_erl(PT)) || D <- NeededSrcDirs], + [{PT, PTerl} || PTerl <- PTerls, lists:member(PTerl, NeededErlFiles)] + end, proplists:get_all_values(parse_transform, ErlOpts))), + ErlOptsFirst = lists:filter(fun({parse_transform, PT}) -> + not lists:member(PT, ParseTransforms); + (_) -> + true + end, ErlOpts), + {ErlFirstFiles ++ ParseTransformsErls, ErlOptsFirst}. + +split_source_files(SourceFiles, ErlOpts) -> + ParseTransforms = proplists:get_all_values(parse_transform, ErlOpts), + lists:partition(fun(Source) -> + lists:member(filename_to_atom(Source), ParseTransforms) + end, SourceFiles). + +filename_to_atom(F) -> list_to_atom(filename:rootname(filename:basename(F))). + +%% Get subset of SourceFiles which need to be recompiled, respecting +%% dependencies induced by given graph G. +needed_files(Graph, ErlOpts, RebarOpts, Dir, OutDir, SourceFiles) -> + lists:filter(fun(Source) -> + TargetBase = target_base(OutDir, Source), + Target = TargetBase ++ ".beam", + PrivIncludes = [{i, filename:join(Dir, Src)} + || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])], + AllOpts = [{outdir, filename:dirname(Target)} + ,{i, filename:join(Dir, "include")} + ,{i, Dir}] ++ PrivIncludes ++ ErlOpts, + digraph:vertex(Graph, Source) > {Source, filelib:last_modified(Target)} + orelse opts_changed(AllOpts, TargetBase) + orelse erl_compiler_opts_set() + end, SourceFiles). + +target_base(OutDir, Source) -> + filename:join(OutDir, filename:basename(Source, ".erl")). + +opts_changed(NewOpts, Target) -> + TotalOpts = case erlang:function_exported(compile, env_compiler_options, 0) of + true -> NewOpts ++ compile:env_compiler_options(); + false -> NewOpts + end, + case compile_info(Target) of + {ok, Opts} -> lists:any(fun effects_code_generation/1, lists:usort(TotalOpts) -- lists:usort(Opts)); + _ -> true + end. + +effects_code_generation(Option) -> + case Option of + beam -> false; + report_warnings -> false; + report_errors -> false; + return_errors-> false; + return_warnings-> false; + report -> false; + warnings_as_errors -> false; + binary -> false; + verbose -> false; + {cwd,_} -> false; + {outdir, _} -> false; + _ -> true + end. + +compile_info(Target) -> + case beam_lib:chunks(Target, [compile_info]) of + {ok, {_mod, Chunks}} -> + CompileInfo = proplists:get_value(compile_info, Chunks, []), + {ok, proplists:get_value(options, CompileInfo, [])}; + {error, beam_lib, Reason} -> + ?WARN("Couldn't read debug info from ~p for reason: ~p", [Target, Reason]), + {error, Reason} + end. + +erl_compiler_opts_set() -> + EnvSet = case os:getenv("ERL_COMPILER_OPTIONS") of + false -> false; + _ -> true + end, + %% return false if changed env opts would have been caught in opts_changed/2 + EnvSet andalso not erlang:function_exported(compile, env_compiler_options, 0). + +valid_erl_first_conf(FileList) -> + Strs = filter_file_list(FileList), + case rebar_utils:is_list_of_strings(Strs) of + true -> true; + false -> ?ABORT("An invalid file list (~p) was provided as part of your erl_first_files directive", + [FileList]) + end. + +filter_file_list(FileList) -> + Atoms = lists:filter( fun(X) -> is_atom(X) end, FileList), + case Atoms of + [] -> + FileList; + _ -> + atoms_in_erl_first_files_warning(Atoms), + lists:filter( fun(X) -> not(is_atom(X)) end, FileList) + end. + +atoms_in_erl_first_files_warning(Atoms) -> + W = "You have provided atoms as file entries in erl_first_files; " + "erl_first_files only expects lists of filenames as strings. " + "The following modules (~p) may not work as expected and it is advised " + "that you change these entires to string format " + "(e.g., \"src/module.erl\") ", + ?WARN(W, [Atoms]). + +module_to_erl(Mod) -> + atom_to_list(Mod) ++ ".erl". + +parse_attrs(Fd, Includes, Dir) -> + case io:parse_erl_form(Fd, "") of + {ok, Form, _Line} -> + case erl_syntax:type(Form) of + attribute -> + NewIncludes = process_attr(Form, Includes, Dir), + parse_attrs(Fd, NewIncludes, Dir); + _ -> + parse_attrs(Fd, Includes, Dir) + end; + {eof, _} -> + Includes; + _Err -> + parse_attrs(Fd, Includes, Dir) + end. + +process_attr(Form, Includes, Dir) -> + AttrName = erl_syntax:atom_value(erl_syntax:attribute_name(Form)), + process_attr(AttrName, Form, Includes, Dir). + +process_attr(import, Form, Includes, _Dir) -> + case erl_syntax_lib:analyze_import_attribute(Form) of + {Mod, _Funs} -> + [module_to_erl(Mod)|Includes]; + Mod -> + [module_to_erl(Mod)|Includes] + end; +process_attr(file, Form, Includes, _Dir) -> + {File, _} = erl_syntax_lib:analyze_file_attribute(Form), + [File|Includes]; +process_attr(include, Form, Includes, _Dir) -> + [FileNode] = erl_syntax:attribute_arguments(Form), + File = erl_syntax:string_value(FileNode), + [File|Includes]; +process_attr(include_lib, Form, Includes, Dir) -> + [FileNode] = erl_syntax:attribute_arguments(Form), + RawFile = erl_syntax:string_value(FileNode), + maybe_expand_include_lib_path(RawFile, Dir) ++ Includes; +process_attr(behavior, Form, Includes, _Dir) -> + process_attr(behaviour, Form, Includes, _Dir); +process_attr(behaviour, Form, Includes, _Dir) -> + [FileNode] = erl_syntax:attribute_arguments(Form), + File = module_to_erl(erl_syntax:atom_value(FileNode)), + [File|Includes]; +process_attr(compile, Form, Includes, _Dir) -> + [Arg] = erl_syntax:attribute_arguments(Form), + case erl_syntax:concrete(Arg) of + {parse_transform, Mod} -> + [module_to_erl(Mod)|Includes]; + {core_transform, Mod} -> + [module_to_erl(Mod)|Includes]; + L when is_list(L) -> + lists:foldl( + fun({parse_transform, Mod}, Acc) -> + [module_to_erl(Mod)|Acc]; + ({core_transform, Mod}, Acc) -> + [module_to_erl(Mod)|Acc]; + (_, Acc) -> + Acc + end, Includes, L); + _ -> + Includes + end; +process_attr(_, _Form, Includes, _Dir) -> + Includes. + +%% NOTE: If, for example, one of the entries in Files, refers to +%% gen_server.erl, that entry will be dropped. It is dropped because +%% such an entry usually refers to the beam file, and we don't pass a +%% list of OTP src dirs for finding gen_server.erl's full path. Also, +%% if gen_server.erl was modified, it's not rebar's task to compile a +%% new version of the beam file. Therefore, it's reasonable to drop +%% such entries. Also see process_attr(behaviour, Form, Includes). +-spec expand_file_names([file:filename()], + [file:filename()]) -> [file:filename()]. +expand_file_names(Files, Dirs) -> + %% We check if Files exist by itself or within the directories + %% listed in Dirs. + %% Return the list of files matched. + lists:flatmap( + fun(Incl) -> + case filelib:is_regular(Incl) of + true -> + [Incl]; + false -> + lists:flatmap( + fun(Dir) -> + FullPath = filename:join(Dir, Incl), + case filelib:is_regular(FullPath) of + true -> + [FullPath]; + false -> + [] + end + end, Dirs) + end + end, Files). + +%% Given a path like "stdlib/include/erl_compile.hrl", return +%% "OTP_INSTALL_DIR/lib/erlang/lib/stdlib-x.y.z/include/erl_compile.hrl". +%% Usually a simple [Lib, SubDir, File1] = filename:split(File) should +%% work, but to not crash when an unusual include_lib path is used, +%% utilize more elaborate logic. +maybe_expand_include_lib_path(File, Dir) -> + File1 = filename:basename(File), + case filename:split(filename:dirname(File)) of + [_] -> + warn_and_find_path(File, Dir); + [Lib | SubDir] -> + case code:lib_dir(list_to_atom(Lib), list_to_atom(filename:join(SubDir))) of + {error, bad_name} -> + warn_and_find_path(File, Dir); + AppDir -> + [filename:join(AppDir, File1)] + end + end. + +%% The use of -include_lib was probably incorrect by the user but lets try to make it work. +%% We search in the outdir and outdir/../include to see if the header exists. +warn_and_find_path(File, Dir) -> + SrcHeader = filename:join(Dir, File), + case filelib:is_regular(SrcHeader) of + true -> + [SrcHeader]; + false -> + IncludeDir = filename:join(rebar_utils:droplast(filename:split(Dir))++["include"]), + IncludeHeader = filename:join(IncludeDir, File), + case filelib:is_regular(IncludeHeader) of + true -> + [filename:join(IncludeDir, File)]; + false -> + [] + end + end. diff --git a/src/rebar_compiler_mib.erl b/src/rebar_compiler_mib.erl new file mode 100644 index 0000000..32516bf --- /dev/null +++ b/src/rebar_compiler_mib.erl @@ -0,0 +1,70 @@ +-module(rebar_compiler_mib). + +-behaviour(rebar_compiler). + +-export([context/1, + needed_files/3, + dependencies/3, + compile/4, + clean/2]). + +-include("rebar.hrl"). +-include_lib("stdlib/include/erl_compile.hrl"). + +context(AppInfo) -> + Dir = rebar_app_info:dir(AppInfo), + Mappings = [{".bin", filename:join([Dir, "priv", "mibs"])}, + {".hrl", filename:join(Dir, "include")}], + + #{src_dirs => ["mibs"], + include_dirs => [], + src_ext => ".mib", + out_mappings => Mappings}. + +needed_files(_, FoundFiles, AppInfo) -> + FirstFiles = [], + + %% Remove first files from found files + RestFiles = [Source || Source <- FoundFiles, not lists:member(Source, FirstFiles)], + + Opts = rebar_opts:get(rebar_app_info:opts(AppInfo), mib_opts, []), + {{FirstFiles, Opts}, {RestFiles, Opts}}. + +dependencies(_, _, _) -> + []. + +compile(Source, OutDirs, _, Opts) -> + {_, BinOut} = lists:keyfind(".bin", 1, OutDirs), + {_, HrlOut} = lists:keyfind(".hrl", 1, OutDirs), + + ok = rebar_file_utils:ensure_dir(BinOut), + ok = rebar_file_utils:ensure_dir(HrlOut), + Mib = filename:join(BinOut, filename:basename(Source, ".mib")), + HrlFilename = Mib ++ ".hrl", + + AllOpts = [{outdir, BinOut}, {i, [BinOut]}] ++ Opts, + + case snmpc:compile(Source, AllOpts) of + {ok, _} -> + MibToHrlOpts = + case proplists:get_value(verbosity, AllOpts, undefined) of + undefined -> + #options{specific = [], + cwd = rebar_dir:get_cwd()}; + Verbosity -> + #options{specific = [{verbosity, Verbosity}], + cwd = rebar_dir:get_cwd()} + end, + ok = snmpc:mib_to_hrl(Mib, Mib, MibToHrlOpts), + rebar_file_utils:mv(HrlFilename, HrlOut), + ok; + {error, compilation_failed} -> + ?FAIL + end. + +clean(MibFiles, AppInfo) -> + AppDir = rebar_app_info:dir(AppInfo), + MIBs = [filename:rootname(filename:basename(MIB)) || MIB <- MibFiles], + rebar_file_utils:delete_each( + [filename:join([AppDir, "include", MIB++".hrl"]) || MIB <- MIBs]), + ok = rebar_file_utils:rm_rf(filename:join([AppDir, "priv/mibs/*.bin"])). diff --git a/src/rebar_compiler_xrl.erl b/src/rebar_compiler_xrl.erl new file mode 100644 index 0000000..5c023f0 --- /dev/null +++ b/src/rebar_compiler_xrl.erl @@ -0,0 +1,50 @@ +-module(rebar_compiler_xrl). + +-behaviour(rebar_compiler). + +-export([context/1, + needed_files/3, + dependencies/3, + compile/4, + clean/2]). + +context(AppInfo) -> + Dir = rebar_app_info:dir(AppInfo), + Mappings = [{".erl", filename:join([Dir, "src"])}], + #{src_dirs => ["src"], + include_dirs => [], + src_ext => ".xrl", + out_mappings => Mappings}. + +needed_files(_, FoundFiles, AppInfo) -> + FirstFiles = [], + + %% Remove first files from found files + RestFiles = [Source || Source <- FoundFiles, not lists:member(Source, FirstFiles)], + + Opts = rebar_opts:get(rebar_app_info:opts(AppInfo), xrl_opts, []), + + {{FirstFiles, Opts}, {RestFiles, Opts}}. + +dependencies(_, _, _) -> + []. + +compile(Source, [{_, OutDir}], _, Opts) -> + BaseName = filename:basename(Source), + Target = filename:join([OutDir, BaseName]), + AllOpts = [{parserfile, Target} | Opts], + AllOpts1 = [{includefile, filename:join(OutDir, I)} || {includefile, I} <- AllOpts, + filename:pathtype(I) =:= relative], + case leex:file(Source, AllOpts1 ++ [{return, true}]) of + {ok, _} -> + ok; + {ok, _Mod, Ws} -> + rebar_compiler:ok_tuple(Source, Ws); + {error, Es, Ws} -> + rebar_compiler:error_tuple(Source, Es, Ws, AllOpts1) + end. + +clean(XrlFiles, _AppInfo) -> + rebar_file_utils:delete_each( + [rebar_utils:to_list(re:replace(F, "\\.xrl$", ".erl", [unicode])) + || F <- XrlFiles]). diff --git a/src/rebar_compiler_yrl.erl b/src/rebar_compiler_yrl.erl new file mode 100644 index 0000000..765ea36 --- /dev/null +++ b/src/rebar_compiler_yrl.erl @@ -0,0 +1,49 @@ +-module(rebar_compiler_yrl). + +-behaviour(rebar_compiler). + +-export([context/1, + needed_files/3, + dependencies/3, + compile/4, + clean/2]). + +context(AppInfo) -> + Dir = rebar_app_info:dir(AppInfo), + Mappings = [{".erl", filename:join([Dir, "src"])}], + #{src_dirs => ["src"], + include_dirs => [], + src_ext => ".yrl", + out_mappings => Mappings}. + +needed_files(_, FoundFiles, AppInfo) -> + FirstFiles = [], + + %% Remove first files from found files + RestFiles = [Source || Source <- FoundFiles, not lists:member(Source, FirstFiles)], + + Opts = rebar_opts:get(rebar_app_info:opts(AppInfo), yrl_opts, []), + {{FirstFiles, Opts}, {RestFiles, Opts}}. + +dependencies(_, _, _) -> + []. + +compile(Source, [{_, OutDir}], _, Opts) -> + BaseName = filename:basename(Source), + Target = filename:join([OutDir, BaseName]), + AllOpts = [{parserfile, Target} | Opts], + AllOpts1 = [{includefile, filename:join(OutDir, I)} || {includefile, I} <- AllOpts, + filename:pathtype(I) =:= relative], + case yeec:file(Source, AllOpts1 ++ [{return, true}]) of + {ok, _} -> + ok; + {ok, _Mod, Ws} -> + rebar_compiler:ok_tuple(Source, Ws); + {error, Es, Ws} -> + rebar_compiler:error_tuple(Source, Es, Ws, AllOpts1) + end. + +clean(YrlFiles, _AppInfo) -> + rebar_file_utils:delete_each( + [rebar_utils:to_list(re:replace(F, "\\.yrl$", ".erl", [unicode])) + || F <- YrlFiles]). diff --git a/src/rebar_dir.erl b/src/rebar_dir.erl index d7be423..17bc48e 100644 --- a/src/rebar_dir.erl +++ b/src/rebar_dir.erl @@ -301,9 +301,8 @@ all_src_dirs(Opts, SrcDefault, ExtraDefault) -> src_dir_opts(Opts, Dir) -> RawSrcDirs = raw_src_dirs(src_dirs, Opts, []), RawExtraSrcDirs = raw_src_dirs(extra_src_dirs, Opts, []), - AllOpts = [Opt || {D,Opt} <- RawSrcDirs++RawExtraSrcDirs, - D==Dir], - lists:ukeysort(1,proplists:unfold(lists:append(AllOpts))). + AllOpts = [Opt || {D, Opt} <- RawSrcDirs++RawExtraSrcDirs, D==Dir], + lists:ukeysort(1, proplists:unfold(lists:append(AllOpts))). %%% @doc %%% Return the value of the 'recursive' option for the given directory. diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl deleted file mode 100644 index c63ca5b..0000000 --- a/src/rebar_erlc_compiler.erl +++ /dev/null @@ -1,827 +0,0 @@ -%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- -%% ex: ts=4 sw=4 et -%% ------------------------------------------------------------------- -%% -%% rebar: Erlang Build Tools -%% -%% Copyright (c) 2009, 2010 Dave Smith (dizzyd@dizzyd.com) -%% -%% Permission is hereby granted, free of charge, to any person obtaining a copy -%% of this software and associated documentation files (the "Software"), to deal -%% in the Software without restriction, including without limitation the rights -%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -%% copies of the Software, and to permit persons to whom the Software is -%% furnished to do so, subject to the following conditions: -%% -%% The above copyright notice and this permission notice shall be included in -%% all copies or substantial portions of the Software. -%% -%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -%% THE SOFTWARE. -%% ------------------------------------------------------------------- --module(rebar_erlc_compiler). - --export([compile/1, compile/2, compile/3, - compile_dir/3, compile_dir/4, - compile_dirs/5, - clean/1]). - --include("rebar.hrl"). --include_lib("stdlib/include/erl_compile.hrl"). - --define(ERLCINFO_VSN, 2). --define(ERLCINFO_FILE, "erlcinfo"). --type erlc_info_v() :: {digraph:vertex(), term()} | 'false'. --type erlc_info_e() :: {digraph:vertex(), digraph:vertex()}. --type erlc_info() :: {list(erlc_info_v()), list(erlc_info_e()), list(string())}. --record(erlcinfo, { - vsn = ?ERLCINFO_VSN :: pos_integer(), - info = {[], [], []} :: erlc_info() -}). - --type compile_opts() :: [compile_opt()]. --type compile_opt() :: {recursive, boolean()}. - --define(DEFAULT_OUTDIR, "ebin"). --define(RE_PREFIX, "^(?!\\._)"). - -%% =================================================================== -%% Public API -%% =================================================================== - -%% Supported configuration variables: -%% -%% * erl_opts - Erlang list of options passed to compile:file/2 -%% It is also possible to specify platform specific -%% options by specifying a pair or a triplet where the -%% first string is a regex that is checked against the -%% string -%% -%% OtpRelease ++ "-" ++ SysArch ++ "-" ++ Words. -%% -%% where -%% -%% OtpRelease = erlang:system_info(otp_release). -%% SysArch = erlang:system_info(system_architecture). -%% Words = integer_to_list(8 * -%% erlang:system_info({wordsize, external})). -%% -%% E.g. to define HAVE_SENDFILE only on systems with -%% sendfile(), to define BACKLOG on Linux/FreeBSD as 128, -%% and to define 'old_inets' for R13 OTP release do: -%% -%% {erl_opts, [{platform_define, -%% "(linux|solaris|freebsd|darwin)", -%% 'HAVE_SENDFILE'}, -%% {platform_define, "(linux|freebsd)", -%% 'BACKLOG', 128}, -%% {platform_define, "R13", -%% 'old_inets'}]}. -%% - -%% @equiv compile(AppInfo, []) --spec compile(rebar_app_info:t()) -> ok. -compile(AppInfo) when element(1, AppInfo) == app_info_t -> - compile(AppInfo, []). - -%% @doc compile an individual application. --spec compile(rebar_app_info:t(), compile_opts()) -> ok. -compile(AppInfo, CompileOpts) when element(1, AppInfo) == app_info_t -> - Dir = rebar_utils:to_list(rebar_app_info:out_dir(AppInfo)), - RebarOpts = rebar_app_info:opts(AppInfo), - - SrcOpts = [check_last_mod, - {recursive, dir_recursive(RebarOpts, "src", CompileOpts)}], - MibsOpts = [check_last_mod, - {recursive, dir_recursive(RebarOpts, "mibs", CompileOpts)}], - - rebar_base_compiler:run(RebarOpts, - check_files([filename:join(Dir, File) - || File <- rebar_opts:get(RebarOpts, xrl_first_files, [])]), - filename:join(Dir, "src"), ".xrl", filename:join(Dir, "src"), ".erl", - fun compile_xrl/3, SrcOpts), - rebar_base_compiler:run(RebarOpts, - check_files([filename:join(Dir, File) - || File <- rebar_opts:get(RebarOpts, yrl_first_files, [])]), - filename:join(Dir, "src"), ".yrl", filename:join(Dir, "src"), ".erl", - fun compile_yrl/3, SrcOpts), - rebar_base_compiler:run(RebarOpts, - check_files([filename:join(Dir, File) - || File <- rebar_opts:get(RebarOpts, mib_first_files, [])]), - filename:join(Dir, "mibs"), ".mib", filename:join([Dir, "priv", "mibs"]), ".bin", - compile_mib(AppInfo), MibsOpts), - - SrcDirs = lists:map(fun(SrcDir) -> filename:join(Dir, SrcDir) end, - rebar_dir:src_dirs(RebarOpts, ["src"])), - OutDir = filename:join(Dir, outdir(RebarOpts)), - compile_dirs(RebarOpts, Dir, SrcDirs, OutDir, CompileOpts), - - ExtraDirs = rebar_dir:extra_src_dirs(RebarOpts), - F = fun(D) -> - case ec_file:is_dir(filename:join([Dir, D])) of - true -> compile_dirs(RebarOpts, Dir, [D], D, CompileOpts); - false -> ok - end - end, - lists:foreach(F, lists:map(fun(SrcDir) -> filename:join(Dir, SrcDir) end, ExtraDirs)). - -%% @hidden -%% these are kept for backwards compatibility but they're bad functions with -%% bad interfaces you probably shouldn't use -%% State/RebarOpts have to have src_dirs set and BaseDir must be the parent -%% directory of those src_dirs - --spec compile(rebar_dict() | rebar_state:t(), file:name(), file:name()) -> ok. -compile(State, BaseDir, OutDir) when element(1, State) == state_t -> - compile(rebar_state:opts(State), BaseDir, OutDir, [{recursive, false}]); -compile(RebarOpts, BaseDir, OutDir) -> - compile(RebarOpts, BaseDir, OutDir, [{recursive, false}]). - -%% @hidden - --spec compile(rebar_dict() | rebar_state:t(), file:name(), file:name(), compile_opts()) -> ok. -compile(State, BaseDir, OutDir, CompileOpts) when element(1, State) == state_t -> - compile(rebar_state:opts(State), BaseDir, OutDir, CompileOpts); -compile(RebarOpts, BaseDir, OutDir, CompileOpts) -> - SrcDirs = lists:map(fun(SrcDir) -> filename:join(BaseDir, SrcDir) end, - rebar_dir:src_dirs(RebarOpts, ["src"])), - compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, CompileOpts), - - ExtraDirs = rebar_dir:extra_src_dirs(RebarOpts), - F = fun(D) -> - case ec_file:is_dir(filename:join([BaseDir, D])) of - true -> compile_dirs(RebarOpts, BaseDir, [D], D, CompileOpts); - false -> ok - end - end, - lists:foreach(F, lists:map(fun(SrcDir) -> filename:join(BaseDir, SrcDir) end, ExtraDirs)). - -%% @equiv compile_dirs(Context, BaseDir, [Dir], Dir, [{recursive, false}]) --spec compile_dir(rebar_dict() | rebar_state:t(), file:name(), file:name()) -> ok. -compile_dir(State, BaseDir, Dir) when element(1, State) == state_t -> - compile_dir(rebar_state:opts(State), BaseDir, Dir, [{recursive, false}]); -compile_dir(RebarOpts, BaseDir, Dir) -> - compile_dir(RebarOpts, BaseDir, Dir, [{recursive, false}]). - -%% @equiv compile_dirs(Context, BaseDir, [Dir], Dir, Opts) --spec compile_dir(rebar_dict() | rebar_state:t(), file:name(), file:name(), compile_opts()) -> ok. -compile_dir(State, BaseDir, Dir, Opts) when element(1, State) == state_t -> - compile_dirs(rebar_state:opts(State), BaseDir, [Dir], Dir, Opts); -compile_dir(RebarOpts, BaseDir, Dir, Opts) -> - compile_dirs(RebarOpts, BaseDir, [Dir], Dir, Opts). - -%% @doc compile a list of directories with the given opts. --spec compile_dirs(rebar_dict() | rebar_state:t(), - file:filename(), - [file:filename()], - file:filename(), - compile_opts()) -> ok. -compile_dirs(State, BaseDir, Dirs, OutDir, CompileOpts) when element(1, State) == state_t -> - compile_dirs(rebar_state:opts(State), BaseDir, Dirs, OutDir, CompileOpts); -compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, CompileOpts) -> - ErlOpts = rebar_opts:erl_opts(RebarOpts), - ?DEBUG("erlopts ~p", [ErlOpts]), - AllErlFiles = gather_src(RebarOpts, BaseDir, SrcDirs, CompileOpts), - ?DEBUG("files to compile ~p", [AllErlFiles]), - - %% Make sure that outdir is on the path - ok = filelib:ensure_dir(filename:join(OutDir, "dummy.beam")), - true = code:add_patha(filename:absname(OutDir)), - - G = init_erlcinfo(include_abs_dirs(ErlOpts, BaseDir), AllErlFiles, BaseDir, OutDir), - - {ParseTransforms, Rest} = split_source_files(AllErlFiles, ErlOpts), - NeededErlFiles = case needed_files(G, ErlOpts, RebarOpts, BaseDir, OutDir, ParseTransforms) of - [] -> needed_files(G, ErlOpts, RebarOpts, BaseDir, OutDir, Rest); - %% at least one parse transform in the opts needs updating, so recompile all - _ -> AllErlFiles - end, - - {ErlFirstFiles, ErlOptsFirst} = erl_first_files(RebarOpts, ErlOpts, BaseDir, NeededErlFiles), - {DepErls, OtherErls} = lists:partition( - fun(Source) -> digraph:in_degree(G, Source) > 0 end, - [File || File <- NeededErlFiles, not lists:member(File, ErlFirstFiles)]), - SubGraph = digraph_utils:subgraph(G, DepErls), - DepErlsOrdered = digraph_utils:topsort(SubGraph), - FirstErls = ErlFirstFiles ++ lists:reverse(DepErlsOrdered), - try - rebar_base_compiler:run( - RebarOpts, FirstErls, OtherErls, - fun(S, C) -> - ErlOpts1 = case lists:member(S, ErlFirstFiles) of - true -> ErlOptsFirst; - false -> ErlOpts - end, - internal_erl_compile(C, BaseDir, S, OutDir, ErlOpts1, RebarOpts) - end) - after - true = digraph:delete(SubGraph), - true = digraph:delete(G) - end, - ok. - -%% @doc remove compiled artifacts from an AppDir. --spec clean(rebar_app_info:t()) -> 'ok'. -clean(AppInfo) -> - AppDir = rebar_app_info:out_dir(AppInfo), - - MibFiles = rebar_utils:find_files(filename:join([AppDir, "mibs"]), ?RE_PREFIX".*\\.mib\$"), - MIBs = [filename:rootname(filename:basename(MIB)) || MIB <- MibFiles], - rebar_file_utils:delete_each( - [filename:join([AppDir, "include",MIB++".hrl"]) || MIB <- MIBs]), - ok = rebar_file_utils:rm_rf(filename:join([AppDir, "priv/mibs/*.bin"])), - - YrlFiles = rebar_utils:find_files(filename:join([AppDir, "src"]), ?RE_PREFIX".*\\.[x|y]rl\$"), - rebar_file_utils:delete_each( - [rebar_utils:to_list(re:replace(F, "\\.[x|y]rl$", ".erl", [unicode])) - || F <- YrlFiles]), - - BinDirs = ["ebin"|rebar_dir:extra_src_dirs(rebar_app_info:opts(AppInfo))], - ok = clean_dirs(AppDir, BinDirs), - - %% Delete the build graph, if any - rebar_file_utils:rm_rf(erlcinfo_file(AppDir)). - -clean_dirs(_AppDir, []) -> ok; -clean_dirs(AppDir, [Dir|Rest]) -> - ok = rebar_file_utils:rm_rf(filename:join([AppDir, Dir, "*.beam"])), - %% Erlang compilation is recursive, so it's possible that we have a nested - %% directory structure in ebin with .beam files within. As such, we want - %% to scan whatever is left in the app's out_dir directory for sub-dirs which - %% satisfy our criteria. - BeamFiles = rebar_utils:find_files(filename:join([AppDir, Dir]), ?RE_PREFIX".*\\.beam\$"), - rebar_file_utils:delete_each(BeamFiles), - lists:foreach(fun(D) -> delete_dir(D, dirs(D)) end, dirs(filename:join([AppDir, Dir]))), - clean_dirs(AppDir, Rest). - - -%% =================================================================== -%% Internal functions -%% =================================================================== - -gather_src(Opts, BaseDir, Dirs, CompileOpts) -> - gather_src(Opts, filename:split(BaseDir), Dirs, [], CompileOpts). - -gather_src(_Opts, _BaseDirParts, [], Srcs, _CompileOpts) -> Srcs; -gather_src(Opts, BaseDirParts, [Dir|Rest], Srcs, CompileOpts) -> - DirParts = filename:split(Dir), - RelDir = case lists:prefix(BaseDirParts,DirParts) of - true -> - case lists:nthtail(length(BaseDirParts),DirParts) of - [] -> "."; - RestParts -> filename:join(RestParts) - end; - false -> Dir - end, - DirRecursive = dir_recursive(Opts, RelDir, CompileOpts), - gather_src(Opts, BaseDirParts, Rest, Srcs ++ rebar_utils:find_files(Dir, ?RE_PREFIX".*\\.erl\$", DirRecursive), CompileOpts). - -%% Get files which need to be compiled first, i.e. those specified in erl_first_files -%% and parse_transform options. Also produce specific erl_opts for these first -%% files, so that yet to be compiled parse transformations are excluded from it. -erl_first_files(Opts, ErlOpts, Dir, NeededErlFiles) -> - ErlFirstFilesConf = rebar_opts:get(Opts, erl_first_files, []), - valid_erl_first_conf(ErlFirstFilesConf), - NeededSrcDirs = lists:usort(lists:map(fun filename:dirname/1, NeededErlFiles)), - %% NOTE: order of files here is important! - ErlFirstFiles = - [filename:join(Dir, File) || File <- ErlFirstFilesConf, - lists:member(filename:join(Dir, File), NeededErlFiles)], - {ParseTransforms, ParseTransformsErls} = - lists:unzip(lists:flatmap( - fun(PT) -> - PTerls = [filename:join(D, module_to_erl(PT)) || D <- NeededSrcDirs], - [{PT, PTerl} || PTerl <- PTerls, lists:member(PTerl, NeededErlFiles)] - end, proplists:get_all_values(parse_transform, ErlOpts))), - ErlOptsFirst = lists:filter(fun({parse_transform, PT}) -> - not lists:member(PT, ParseTransforms); - (_) -> - true - end, ErlOpts), - {ErlFirstFiles ++ ParseTransformsErls, ErlOptsFirst}. - -split_source_files(SourceFiles, ErlOpts) -> - ParseTransforms = proplists:get_all_values(parse_transform, ErlOpts), - lists:partition(fun(Source) -> - lists:member(filename_to_atom(Source), ParseTransforms) - end, SourceFiles). - -filename_to_atom(F) -> list_to_atom(filename:rootname(filename:basename(F))). - -%% Get subset of SourceFiles which need to be recompiled, respecting -%% dependencies induced by given graph G. -needed_files(G, ErlOpts, RebarOpts, Dir, OutDir, SourceFiles) -> - lists:filter(fun(Source) -> - TargetBase = target_base(OutDir, Source), - Target = TargetBase ++ ".beam", - PrivIncludes = [{i, filename:join(Dir, Src)} - || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])], - AllOpts = [{outdir, filename:dirname(Target)} - ,{i, filename:join(Dir, "include")} - ,{i, Dir}] ++ PrivIncludes ++ ErlOpts, - %% necessary for erlang:function_exported/3 to work as expected - %% called here for clarity as it's required by both opts_changed/2 - %% and erl_compiler_opts_set/0 - _ = code:ensure_loaded(compile), - digraph:vertex(G, Source) > {Source, filelib:last_modified(Target)} - orelse opts_changed(AllOpts, TargetBase) - orelse erl_compiler_opts_set() - end, SourceFiles). - -maybe_rm_beam_and_edge(G, OutDir, Source) -> - %% This is NOT a double check it is the only check that the source file is actually gone - case filelib:is_regular(Source) of - true -> - %% Actually exists, don't delete - false; - false -> - Target = target_base(OutDir, Source) ++ ".beam", - ?DEBUG("Source ~ts is gone, deleting previous beam file if it exists ~ts", [Source, Target]), - file:delete(Target), - digraph:del_vertex(G, Source), - true - end. - -opts_changed(NewOpts, Target) -> - TotalOpts = case erlang:function_exported(compile, env_compiler_options, 0) of - true -> NewOpts ++ compile:env_compiler_options(); - false -> NewOpts - end, - case compile_info(Target) of - {ok, Opts} -> lists:any(fun effects_code_generation/1, lists:usort(TotalOpts) -- lists:usort(Opts)); - _ -> true - end. - -effects_code_generation(Option) -> - case Option of - beam -> false; - report_warnings -> false; - report_errors -> false; - return_errors-> false; - return_warnings-> false; - report -> false; - warnings_as_errors -> false; - binary -> false; - verbose -> false; - {cwd,_} -> false; - {outdir, _} -> false; - _ -> true - end. - -compile_info(Target) -> - case beam_lib:chunks(Target, [compile_info]) of - {ok, {_mod, Chunks}} -> - CompileInfo = proplists:get_value(compile_info, Chunks, []), - {ok, proplists:get_value(options, CompileInfo, [])}; - {error, beam_lib, Reason} -> - ?WARN("Couldn't read debug info from ~p for reason: ~p", [Target, Reason]), - {error, Reason} - end. - -erl_compiler_opts_set() -> - EnvSet = case os:getenv("ERL_COMPILER_OPTIONS") of - false -> false; - _ -> true - end, - %% return false if changed env opts would have been caught in opts_changed/2 - EnvSet andalso not erlang:function_exported(compile, env_compiler_options, 0). - -erlcinfo_file(Dir) -> - filename:join(rebar_dir:local_cache_dir(Dir), ?ERLCINFO_FILE). - -%% Get dependency graph of given Erls files and their dependencies (header files, -%% parse transforms, behaviours etc.) located in their directories or given -%% InclDirs. Note that last modification times stored in vertices already respect -%% dependencies induced by given graph G. -init_erlcinfo(InclDirs, Erls, Dir, OutDir) -> - G = digraph:new([acyclic]), - try restore_erlcinfo(G, InclDirs, Dir) - catch - _:_ -> - ?WARN("Failed to restore ~ts file. Discarding it.~n", [erlcinfo_file(Dir)]), - file:delete(erlcinfo_file(Dir)) - end, - Dirs = source_and_include_dirs(InclDirs, Erls), - %% A source file may have been renamed or deleted. Remove it from the graph - %% and remove any beam file for that source if it exists. - Modified = maybe_rm_beams_and_edges(G, OutDir, Erls), - Modified1 = lists:foldl(update_erlcinfo_fun(G, Dirs), Modified, Erls), - if Modified1 -> store_erlcinfo(G, InclDirs, Dir); not Modified1 -> ok end, - G. - -maybe_rm_beams_and_edges(G, Dir, Files) -> - Vertices = digraph:vertices(G), - case lists:filter(fun(File) -> - case filename:extension(File) =:= ".erl" of - true -> - maybe_rm_beam_and_edge(G, Dir, File); - false -> - false - end - end, lists:sort(Vertices) -- lists:sort(Files)) of - [] -> - false; - _ -> - true - end. - -source_and_include_dirs(InclDirs, Erls) -> - SourceDirs = lists:map(fun filename:dirname/1, Erls), - lists:usort(InclDirs ++ SourceDirs). - -update_erlcinfo(G, Dirs, Source) -> - case digraph:vertex(G, Source) of - {_, LastUpdated} -> - case filelib:last_modified(Source) of - 0 -> - %% The file doesn't exist anymore, - %% erase it from the graph. - %% All the edges will be erased automatically. - digraph:del_vertex(G, Source), - modified; - LastModified when LastUpdated < LastModified -> - modify_erlcinfo(G, Source, LastModified, filename:dirname(Source), Dirs); - _ -> - Modified = lists:foldl( - update_erlcinfo_fun(G, Dirs), - false, digraph:out_neighbours(G, Source)), - MaxModified = update_max_modified_deps(G, Source), - case Modified orelse MaxModified > LastUpdated of - true -> modified; - false -> unmodified - end - end; - false -> - modify_erlcinfo(G, Source, filelib:last_modified(Source), filename:dirname(Source), Dirs) - end. - -update_erlcinfo_fun(G, Dirs) -> - fun(Erl, Modified) -> - case update_erlcinfo(G, Dirs, Erl) of - modified -> true; - unmodified -> Modified - end - end. - -update_max_modified_deps(G, Source) -> - MaxModified = lists:max(lists:map( - fun(File) -> {_, MaxModified} = digraph:vertex(G, File), MaxModified end, - [Source|digraph:out_neighbours(G, Source)])), - digraph:add_vertex(G, Source, MaxModified), - MaxModified. - -modify_erlcinfo(G, Source, LastModified, Dir, Dirs) -> - {ok, Fd} = file:open(Source, [read]), - Incls = parse_attrs(Fd, [], Dir), - AbsIncls = expand_file_names(Incls, Dirs), - ok = file:close(Fd), - digraph:add_vertex(G, Source, LastModified), - digraph:del_edges(G, digraph:out_edges(G, Source)), - lists:foreach( - fun(Incl) -> - update_erlcinfo(G, Dirs, Incl), - digraph:add_edge(G, Source, Incl) - end, AbsIncls), - modified. - -restore_erlcinfo(G, InclDirs, Dir) -> - case file:read_file(erlcinfo_file(Dir)) of - {ok, Data} -> - % Since externally passed InclDirs can influence erlcinfo graph (see - % modify_erlcinfo), we have to check here that they didn't change. - #erlcinfo{vsn=?ERLCINFO_VSN, info={Vs, Es, InclDirs}} = - binary_to_term(Data), - lists:foreach( - fun({V, LastUpdated}) -> - digraph:add_vertex(G, V, LastUpdated) - end, Vs), - lists:foreach( - fun({_, V1, V2, _}) -> - digraph:add_edge(G, V1, V2) - end, Es); - {error, _} -> - ok - end. - -store_erlcinfo(G, InclDirs, Dir) -> - Vs = lists:map(fun(V) -> digraph:vertex(G, V) end, digraph:vertices(G)), - Es = lists:map(fun(E) -> digraph:edge(G, E) end, digraph:edges(G)), - File = erlcinfo_file(Dir), - ok = filelib:ensure_dir(File), - Data = term_to_binary(#erlcinfo{info={Vs, Es, InclDirs}}, [{compressed, 2}]), - file:write_file(File, Data). - -%% NOTE: If, for example, one of the entries in Files, refers to -%% gen_server.erl, that entry will be dropped. It is dropped because -%% such an entry usually refers to the beam file, and we don't pass a -%% list of OTP src dirs for finding gen_server.erl's full path. Also, -%% if gen_server.erl was modified, it's not rebar's task to compile a -%% new version of the beam file. Therefore, it's reasonable to drop -%% such entries. Also see process_attr(behaviour, Form, Includes). --spec expand_file_names([file:filename()], - [file:filename()]) -> [file:filename()]. -expand_file_names(Files, Dirs) -> - %% We check if Files exist by itself or within the directories - %% listed in Dirs. - %% Return the list of files matched. - lists:flatmap( - fun(Incl) -> - case filelib:is_regular(Incl) of - true -> - [Incl]; - false -> - lists:flatmap( - fun(Dir) -> - FullPath = filename:join(Dir, Incl), - case filelib:is_regular(FullPath) of - true -> - [FullPath]; - false -> - [] - end - end, Dirs) - end - end, Files). - --spec internal_erl_compile(rebar_dict(), file:filename(), file:filename(), - file:filename(), list(), rebar_dict()) -> - ok | {ok, any()} | {error, any(), any()}. -internal_erl_compile(Opts, Dir, Module, OutDir, ErlOpts, RebarOpts) -> - Target = target_base(OutDir, Module) ++ ".beam", - ok = filelib:ensure_dir(Target), - PrivIncludes = [{i, filename:join(Dir, Src)} - || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])], - AllOpts = [{outdir, filename:dirname(Target)}] ++ ErlOpts ++ PrivIncludes ++ - [{i, filename:join(Dir, "include")}, {i, Dir}, return], - case compile:file(Module, AllOpts) of - {ok, _Mod} -> - ok; - {ok, _Mod, Ws} -> - FormattedWs = format_error_sources(Ws, Opts), - rebar_base_compiler:ok_tuple(Module, FormattedWs); - {error, Es, Ws} -> - error_tuple(Module, Es, Ws, AllOpts, Opts) - end. - -error_tuple(Module, Es, Ws, AllOpts, Opts) -> - FormattedEs = format_error_sources(Es, Opts), - FormattedWs = format_error_sources(Ws, Opts), - rebar_base_compiler:error_tuple(Module, FormattedEs, FormattedWs, AllOpts). - -format_error_sources(Es, Opts) -> - [{rebar_base_compiler:format_error_source(Src, Opts), Desc} - || {Src, Desc} <- Es]. - -target_base(OutDir, Source) -> - filename:join(OutDir, filename:basename(Source, ".erl")). - --spec compile_mib(rebar_app_info:t()) -> - fun((file:filename(), file:filename(), rebar_dict()) -> 'ok'). -compile_mib(AppInfo) -> - fun(Source, Target, Opts) -> - Dir = filename:dirname(Target), - Mib = filename:rootname(Target), - HrlFilename = Mib ++ ".hrl", - - AppInclude = filename:join([rebar_app_info:dir(AppInfo), "include"]), - - ok = filelib:ensure_dir(Target), - ok = filelib:ensure_dir(filename:join([AppInclude, "dummy.hrl"])), - - AllOpts = [{outdir, Dir} - ,{i, [Dir]}] ++ - rebar_opts:get(Opts, mib_opts, []), - - case snmpc:compile(Source, AllOpts) of - {ok, _} -> - MibToHrlOpts = - case proplists:get_value(verbosity, AllOpts, undefined) of - undefined -> - #options{specific = [], - cwd = rebar_dir:get_cwd()}; - Verbosity -> - #options{specific = [{verbosity, Verbosity}], - cwd = rebar_dir:get_cwd()} - end, - ok = snmpc:mib_to_hrl(Mib, Mib, MibToHrlOpts), - rebar_file_utils:mv(HrlFilename, AppInclude), - ok; - {error, compilation_failed} -> - ?FAIL - end - end. - --spec compile_xrl(file:filename(), file:filename(), - rebar_dict()) -> 'ok'. -compile_xrl(Source, Target, Opts) -> - AllOpts = [{scannerfile, Target} | rebar_opts:get(Opts, xrl_opts, [])], - compile_xrl_yrl(Opts, Source, Target, AllOpts, leex). - --spec compile_yrl(file:filename(), file:filename(), - rebar_dict()) -> 'ok'. -compile_yrl(Source, Target, Opts) -> - AllOpts = [{parserfile, Target} | rebar_opts:get(Opts, yrl_opts, [])], - compile_xrl_yrl(Opts, Source, Target, AllOpts, yecc). - --spec compile_xrl_yrl(rebar_dict(), file:filename(), - file:filename(), list(), module()) -> 'ok'. -compile_xrl_yrl(_Opts, Source, Target, AllOpts, Mod) -> - %% FIX ME: should be the outdir or something - Dir = filename:dirname(filename:dirname(Target)), - AllOpts1 = [{includefile, filename:join(Dir, I)} || {includefile, I} <- AllOpts, - filename:pathtype(I) =:= relative], - case needs_compile(Source, Target) of - true -> - case Mod:file(Source, AllOpts1 ++ [{return, true}]) of - {ok, _} -> - ok; - {ok, _Mod, Ws} -> - rebar_base_compiler:ok_tuple(Source, Ws); - {error, Es, Ws} -> - rebar_base_compiler:error_tuple(Source, - Es, Ws, AllOpts1) - end; - false -> - skipped - end. - -needs_compile(Source, Target) -> - filelib:last_modified(Source) > filelib:last_modified(Target). - - - --spec dirs(file:filename()) -> [file:filename()]. -dirs(Dir) -> - [F || F <- filelib:wildcard(filename:join([Dir, "*"])), filelib:is_dir(F)]. - --spec delete_dir(file:filename(), [string()]) -> 'ok' | {'error', atom()}. -delete_dir(Dir, []) -> - file:del_dir(Dir); -delete_dir(Dir, Subdirs) -> - lists:foreach(fun(D) -> delete_dir(D, dirs(D)) end, Subdirs), - file:del_dir(Dir). - -parse_attrs(Fd, Includes, Dir) -> - case io:parse_erl_form(Fd, "") of - {ok, Form, _Line} -> - case erl_syntax:type(Form) of - attribute -> - NewIncludes = process_attr(Form, Includes, Dir), - parse_attrs(Fd, NewIncludes, Dir); - _ -> - parse_attrs(Fd, Includes, Dir) - end; - {eof, _} -> - Includes; - _Err -> - parse_attrs(Fd, Includes, Dir) - end. - -process_attr(Form, Includes, Dir) -> - AttrName = erl_syntax:atom_value(erl_syntax:attribute_name(Form)), - process_attr(AttrName, Form, Includes, Dir). - -process_attr(import, Form, Includes, _Dir) -> - case erl_syntax_lib:analyze_import_attribute(Form) of - {Mod, _Funs} -> - [module_to_erl(Mod)|Includes]; - Mod -> - [module_to_erl(Mod)|Includes] - end; -process_attr(file, Form, Includes, _Dir) -> - {File, _} = erl_syntax_lib:analyze_file_attribute(Form), - [File|Includes]; -process_attr(include, Form, Includes, _Dir) -> - [FileNode] = erl_syntax:attribute_arguments(Form), - File = erl_syntax:string_value(FileNode), - [File|Includes]; -process_attr(include_lib, Form, Includes, Dir) -> - [FileNode] = erl_syntax:attribute_arguments(Form), - RawFile = erl_syntax:string_value(FileNode), - maybe_expand_include_lib_path(RawFile, Dir) ++ Includes; -process_attr(behavior, Form, Includes, _Dir) -> - process_attr(behaviour, Form, Includes, _Dir); -process_attr(behaviour, Form, Includes, _Dir) -> - [FileNode] = erl_syntax:attribute_arguments(Form), - File = module_to_erl(erl_syntax:atom_value(FileNode)), - [File|Includes]; -process_attr(compile, Form, Includes, _Dir) -> - [Arg] = erl_syntax:attribute_arguments(Form), - case erl_syntax:concrete(Arg) of - {parse_transform, Mod} -> - [module_to_erl(Mod)|Includes]; - {core_transform, Mod} -> - [module_to_erl(Mod)|Includes]; - L when is_list(L) -> - lists:foldl( - fun({parse_transform, Mod}, Acc) -> - [module_to_erl(Mod)|Acc]; - ({core_transform, Mod}, Acc) -> - [module_to_erl(Mod)|Acc]; - (_, Acc) -> - Acc - end, Includes, L); - _ -> - Includes - end; -process_attr(_, _Form, Includes, _Dir) -> - Includes. - -module_to_erl(Mod) -> - atom_to_list(Mod) ++ ".erl". - -%% Given a path like "stdlib/include/erl_compile.hrl", return -%% "OTP_INSTALL_DIR/lib/erlang/lib/stdlib-x.y.z/include/erl_compile.hrl". -%% Usually a simple [Lib, SubDir, File1] = filename:split(File) should -%% work, but to not crash when an unusual include_lib path is used, -%% utilize more elaborate logic. -maybe_expand_include_lib_path(File, Dir) -> - File1 = filename:basename(File), - case filename:split(filename:dirname(File)) of - [_] -> - warn_and_find_path(File, Dir); - [Lib | SubDir] -> - case code:lib_dir(list_to_atom(Lib), list_to_atom(filename:join(SubDir))) of - {error, bad_name} -> - warn_and_find_path(File, Dir); - AppDir -> - [filename:join(AppDir, File1)] - end - end. - -%% The use of -include_lib was probably incorrect by the user but lets try to make it work. -%% We search in the outdir and outdir/../include to see if the header exists. -warn_and_find_path(File, Dir) -> - SrcHeader = filename:join(Dir, File), - case filelib:is_regular(SrcHeader) of - true -> - [SrcHeader]; - false -> - IncludeDir = filename:join(rebar_utils:droplast(filename:split(Dir))++["include"]), - IncludeHeader = filename:join(IncludeDir, File), - case filelib:is_regular(IncludeHeader) of - true -> - [filename:join(IncludeDir, File)]; - false -> - [] - end - end. - -%% -%% Ensure all files in a list are present and abort if one is missing -%% --spec check_files([file:filename()]) -> [file:filename()]. -check_files(FileList) -> - [check_file(F) || F <- FileList]. - -check_file(File) -> - case filelib:is_regular(File) of - false -> ?ABORT("File ~p is missing, aborting\n", [File]); - true -> File - end. - -outdir(RebarOpts) -> - ErlOpts = rebar_opts:erl_opts(RebarOpts), - proplists:get_value(outdir, ErlOpts, ?DEFAULT_OUTDIR). - -include_abs_dirs(ErlOpts, BaseDir) -> - ErlOptIncludes = proplists:get_all_values(i, ErlOpts), - InclDirs = lists:map(fun(Incl) -> filename:absname(Incl) end, ErlOptIncludes), - [filename:join([BaseDir, "include"])|InclDirs]. - -dir_recursive(Opts, Dir, CompileOpts) when is_list(CompileOpts) -> - case proplists:get_value(recursive,CompileOpts) of - undefined -> rebar_dir:recursive(Opts, Dir); - Recursive -> Recursive - end. - -valid_erl_first_conf(FileList) -> - Strs = filter_file_list(FileList), - case rebar_utils:is_list_of_strings(Strs) of - true -> true; - false -> ?ABORT("An invalid file list (~p) was provided as part of your erl_first_files directive", - [FileList]) - end. - -filter_file_list(FileList) -> - Atoms = lists:filter( fun(X) -> is_atom(X) end, FileList), - case Atoms of - [] -> - FileList; - _ -> - atoms_in_erl_first_files_warning(Atoms), - lists:filter( fun(X) -> not(is_atom(X)) end, FileList) - end. - -atoms_in_erl_first_files_warning(Atoms) -> - W = "You have provided atoms as file entries in erl_first_files; " - "erl_first_files only expects lists of filenames as strings. " - "The following modules (~p) may not work as expected and it is advised " - "that you change these entires to string format " - "(e.g., \"src/module.erl\") ", - ?WARN(W, [Atoms]). diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 492d690..a51a557 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -43,7 +43,8 @@ path_from_ancestor/2, canonical_path/1, resolve_link/1, - split_dirname/1]). + split_dirname/1, + ensure_dir/1]). -include("rebar.hrl"). @@ -386,7 +387,7 @@ reset_dir(Path) -> %% delete the directory if it exists _ = ec_file:remove(Path, [recursive]), %% recreate the directory - filelib:ensure_dir(filename:join([Path, "dummy.beam"])). + ensure_dir(Path). %% Linux touch but using erlang functions to work in bot *nix os and @@ -440,6 +441,10 @@ resolve_link(Path) -> split_dirname(Path) -> {filename:dirname(Path), filename:basename(Path)}. +-spec ensure_dir(filelib:dirname_all()) -> ok | {error, file:posix()}. +ensure_dir(Path) -> + filelib:ensure_dir(filename:join(Path, "fake_file")). + %% =================================================================== %% Internal functions %% =================================================================== @@ -505,7 +510,7 @@ cp_r_win32({true, SourceDir}, {false, DestDir}) -> false -> %% Specifying a target directory that doesn't currently exist. %% So let's attempt to create this directory - case filelib:ensure_dir(filename:join(DestDir, "dummy")) of + case ensure_dir(DestDir) of ok -> ok = xcopy_win32(SourceDir, DestDir); {error, Reason} -> diff --git a/src/rebar_otp_app.erl b/src/rebar_otp_app.erl index 1d854da..e14975f 100644 --- a/src/rebar_otp_app.erl +++ b/src/rebar_otp_app.erl @@ -60,7 +60,8 @@ compile(State, App) -> format_error({missing_app_file, Filename}) -> io_lib:format("App file is missing: ~ts", [Filename]); format_error({file_read, AppName, File, Reason}) -> - io_lib:format("Failed to read required ~ts file for processing the application '~ts': ~ts", [File, AppName, file:format_error(Reason)]); + io_lib:format("Failed to read required ~ts file for processing the application '~ts': ~ts", + [File, AppName, file:format_error(Reason)]); format_error({invalid_name, File, AppName}) -> io_lib:format("Invalid ~ts: name of application (~p) must match filename.", [File, AppName]). @@ -125,7 +126,7 @@ preprocess(State, AppInfo, AppSrcFile) -> %% Setup file .app filename and write new contents EbinDir = rebar_app_info:ebin_dir(AppInfo), - filelib:ensure_dir(filename:join(EbinDir, "dummy.beam")), + rebar_file_utils:ensure_dir(EbinDir), AppFile = rebar_app_utils:app_src_to_app(OutDir, AppSrcFile), ok = rebar_file_utils:write_file_if_contents_differ(AppFile, Spec, utf8), @@ -163,32 +164,17 @@ validate_name(AppName, File) -> ebin_modules(AppInfo, Dir) -> Beams = lists:sort(rebar_utils:beams(filename:join(Dir, "ebin"))), - ExtraDirs = extra_dirs(AppInfo), - F = fun(Beam) -> not in_extra_dir(AppInfo, Beam, ExtraDirs) end, - Filtered = lists:filter(F, Beams), + SrcDirs = rebar_dir:src_dirs(rebar_app_info:opts(AppInfo), ["src"]), + FindSourceRules = [{".beam", ".erl", + [{"ebin", SrcDir} || SrcDir <- SrcDirs]}], + Filtered = lists:filter(fun(Beam) -> + rebar_utils:find_source(filename:basename(Beam), + filename:dirname(Beam), + FindSourceRules) + =/= {error, not_found} + end, Beams), [rebar_utils:beam_to_mod(N) || N <- Filtered]. -extra_dirs(State) -> - Extras = rebar_dir:extra_src_dirs(rebar_app_info:opts(State)), - SrcDirs = rebar_dir:src_dirs(rebar_app_info:opts(State), ["src"]), - %% remove any dirs that are defined in `src_dirs` from `extra_src_dirs` - Extras -- SrcDirs. - -in_extra_dir(AppInfo, Beam, Dirs) -> - lists:any(fun(Dir) -> lists:prefix(filename:join([rebar_app_info:out_dir(AppInfo), Dir]), - beam_src(Beam)) end, - Dirs). - -beam_src(Beam) -> - case beam_lib:chunks(Beam, [compile_info]) of - {ok, {_mod, Chunks}} -> - CompileInfo = proplists:get_value(compile_info, Chunks, []), - proplists:get_value(source, CompileInfo, []); - {error, beam_lib, Reason} -> - ?WARN("Couldn't read debug info from ~p for reason: ~p", [Beam, Reason]), - [] - end. - ensure_registered(AppData) -> case lists:keyfind(registered, 1, AppData) of false -> diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 7533a44..7676213 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -60,7 +60,7 @@ get_all_names(State) -> get_package_versions(Dep, Repo, Table, State) -> ?MODULE:verify_table(State), AllowPreRelease = rebar_state:get(State, deps_allow_prerelease, false), - ets:select(Table, [{#package{key={Dep,{'$1', '$2'}, Repo}, + ets:select(Table, [{#package{key={Dep, {'$1', '$2'}, Repo}, _='_'}, [{'==', '$2', {{[],[]}}} || not AllowPreRelease], [{{'$1', '$2'}}]}]). diff --git a/src/rebar_prv_clean.erl b/src/rebar_prv_clean.erl index 4da0a64..3c8a0c3 100644 --- a/src/rebar_prv_clean.erl +++ b/src/rebar_prv_clean.erl @@ -67,11 +67,12 @@ format_error(Reason) -> %% =================================================================== clean_apps(State, Providers, Apps) -> + Compilers = rebar_state:compilers(State), [begin ?INFO("Cleaning out ~ts...", [rebar_app_info:name(AppInfo)]), AppDir = rebar_app_info:dir(AppInfo), AppInfo1 = rebar_hooks:run_all_hooks(AppDir, pre, ?PROVIDER, Providers, AppInfo, State), - rebar_erlc_compiler:clean(AppInfo1), + rebar_compiler:clean(Compilers, AppInfo1), rebar_hooks:run_all_hooks(AppDir, post, ?PROVIDER, Providers, AppInfo1, State) end || AppInfo <- Apps]. diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index edb2d82..54bdee2 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -92,6 +92,12 @@ handle_project_apps(DepsPaths, Providers, State) -> -spec format_error(any()) -> iolist(). format_error({missing_artifact, File}) -> io_lib:format("Missing artifact ~ts", [File]); +format_error({bad_project_builder, Name, Type, Module}) -> + io_lib:format("Error building application ~s:~n Required project builder ~s function " + "~s:build/1 not found", [Name, Type, Module]); +format_error({unknown_project_type, Name, Type}) -> + io_lib:format("Error building application ~s:~n " + "No project builder is configured for type ~s", [Name, Type]); format_error(Reason) -> io_lib:format("~p", [Reason]). @@ -135,10 +141,19 @@ build_extra_dir(State, Dir) -> true -> BaseDir = filename:join([rebar_dir:base_dir(State), "extras"]), OutDir = filename:join([BaseDir, Dir]), - filelib:ensure_dir(filename:join([OutDir, "dummy.beam"])), + rebar_file_utils:ensure_dir(OutDir), copy(rebar_state:dir(State), BaseDir, Dir), - rebar_erlc_compiler:compile_dir(State, BaseDir, OutDir); - false -> ok + + Compilers = rebar_state:compilers(State), + FakeApp = rebar_app_info:new(), + FakeApp1 = rebar_app_info:out_dir(FakeApp, BaseDir), + FakeApp2 = rebar_app_info:ebin_dir(FakeApp1, OutDir), + Opts = rebar_state:opts(State), + FakeApp3 = rebar_app_info:opts(FakeApp2, Opts), + FakeApp4 = rebar_app_info:set(FakeApp3, src_dirs, [OutDir]), + rebar_compiler:compile_all(Compilers, FakeApp4); + false -> + ok end. compile(State, AppInfo) -> @@ -150,7 +165,9 @@ compile(State, Providers, AppInfo) -> AppInfo1 = rebar_hooks:run_all_hooks(AppDir, pre, ?PROVIDER, Providers, AppInfo, State), AppInfo2 = rebar_hooks:run_all_hooks(AppDir, pre, ?ERLC_HOOK, Providers, AppInfo1, State), - rebar_erlc_compiler:compile(AppInfo2), + + build_app(AppInfo2, State), + AppInfo3 = rebar_hooks:run_all_hooks(AppDir, post, ?ERLC_HOOK, Providers, AppInfo2, State), AppInfo4 = rebar_hooks:run_all_hooks(AppDir, pre, ?APP_HOOK, Providers, AppInfo3, State), @@ -179,6 +196,24 @@ compile(State, Providers, AppInfo) -> %% Internal functions %% =================================================================== +build_app(AppInfo, State) -> + case rebar_app_info:project_type(AppInfo) of + Type when Type =:= rebar3 ; Type =:= undefined -> + Compilers = rebar_state:compilers(State), + rebar_compiler:compile_all(Compilers, AppInfo); + Type -> + ProjectBuilders = rebar_state:project_builders(State), + case lists:keyfind(Type, 1, ProjectBuilders) of + {_, Module} -> + %% load plugins since thats where project builders would be + PluginDepsPaths = rebar_state:code_paths(State, all_plugin_deps), + code:add_pathsa(PluginDepsPaths), + Module:build(AppInfo), + rebar_utils:remove_from_code_path(PluginDepsPaths); + _ -> + throw(?PRV_ERROR({unknown_project_type, rebar_app_info:name(AppInfo), Type})) + end + end. update_code_paths(State, ProjectApps, DepsPaths) -> ProjAppsPaths = paths_for_apps(ProjectApps), ExtrasPaths = paths_for_extras(State, ProjectApps), diff --git a/src/rebar_state.erl b/src/rebar_state.erl index ec74eea..c8a9400 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -38,6 +38,11 @@ to_list/1, + compilers/1, compilers/2, + prepend_compilers/2, append_compilers/2, + + project_builders/1, add_project_builder/3, + create_resources/2, resources/1, resources/2, add_resource/2, providers/1, providers/2, add_provider/2, @@ -66,6 +71,8 @@ all_plugin_deps = [] :: [rebar_app_info:t()], all_deps = [] :: [rebar_app_info:t()], + compilers = [] :: [{compiler_type(), extension(), extension(), compile_fun()}], + project_builders = [] :: [{rebar_app_info:project_type(), module()}], resources = [], providers = [], allow_provider_overrides = false :: boolean()}). @@ -74,6 +81,10 @@ -type t() :: #state_t{}. +-type compiler_type() :: atom(). +-type extension() :: string(). +-type compile_fun() :: fun(([file:filename()], rebar_app_info:t(), list()) -> ok). + -spec new() -> t(). new() -> BaseState = base_state(dict:new()), @@ -392,6 +403,32 @@ warn_old_resource(ResourceModule) -> ?WARN("Using custom resource ~s that implements a deprecated api. " "It should be upgraded to rebar_resource_v2.", [ResourceModule]). +compilers(#state_t{compilers=Compilers}) -> + Compilers. + +prepend_compilers(State=#state_t{compilers=Compilers}, NewCompilers) -> + State#state_t{compilers=NewCompilers++Compilers}. + +append_compilers(State=#state_t{compilers=Compilers}, NewCompilers) -> + State#state_t{compilers=Compilers++NewCompilers}. + +compilers(State, Compilers) -> + State#state_t{compilers=Compilers}. + +project_builders(#state_t{project_builders=ProjectBuilders}) -> + ProjectBuilders. + +add_project_builder(State=#state_t{project_builders=ProjectBuilders}, Type, Module) -> + _ = code:ensure_loaded(Module), + case erlang:function_exported(Module, build, 1) of + true -> + State#state_t{project_builders=[{Type, Module} | ProjectBuilders]}; + false -> + ?WARN("Unable to add project builder for type ~s, required function ~s:build/1 not found.", + [Type, Module]), + State + end. + create_resources(Resources, State) -> lists:foldl(fun(R, StateAcc) -> add_resource(StateAcc, R) diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 995d212..419802a 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -37,8 +37,9 @@ escript_foldl/3, find_files/2, find_files/3, + find_files_in_dirs/3, + find_source/3, beam_to_mod/1, - beam_to_mod/2, erl_to_mod/1, beams/1, find_executable/1, @@ -206,6 +207,12 @@ sh(Command0, Options0) -> find_files(Dir, Regex) -> find_files(Dir, Regex, true). +find_files_in_dirs([], _Regex, _Recursive) -> + []; +find_files_in_dirs([Dir | T], Regex, Recursive) -> + find_files(Dir, Regex, Recursive) ++ find_files_in_dirs(T, Regex, Recursive). + + find_files(Dir, Regex, Recursive) -> filelib:fold_files(Dir, Regex, Recursive, fun(F, Acc) -> [F | Acc] end, []). @@ -671,10 +678,6 @@ sh_loop(Port, Fun, Acc) -> end end. -beam_to_mod(Dir, Filename) -> - [Dir | Rest] = filename:split(Filename), - list_to_atom(filename:basename(rebar_string:join(Rest, "."), ".beam")). - beam_to_mod(Filename) -> list_to_atom(filename:basename(Filename, ".beam")). @@ -1074,3 +1077,77 @@ version_pad([Major, Minor, Patch]) -> {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}; version_pad([Major, Minor, Patch | _]) -> {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}. + + +-ifdef(filelib_find_source). +find_source(Filename, Dir, Rules) -> + filelib:find_source(Filename, Dir, Rules). +-else. +%% Looks for a file relative to a given directory + +-type find_file_rule() :: {ObjDirSuffix::string(), SrcDirSuffix::string()}. + +%% Looks for a source file relative to the object file name and directory + +-type find_source_rule() :: {ObjExtension::string(), SrcExtension::string(), + [find_file_rule()]}. + +keep_suffix_search_rules(Rules) -> + [T || {_,_,_}=T <- Rules]. + +-spec find_source(file:filename(), file:filename(), [find_source_rule()]) -> + {ok, file:filename()} | {error, not_found}. +find_source(Filename, Dir, Rules) -> + try_suffix_rules(keep_suffix_search_rules(Rules), Filename, Dir). + +try_suffix_rules(Rules, Filename, Dir) -> + Ext = filename:extension(Filename), + try_suffix_rules(Rules, filename:rootname(Filename, Ext), Dir, Ext). + +try_suffix_rules([{Ext,Src,Rules}|Rest], Root, Dir, Ext) + when is_list(Src), is_list(Rules) -> + case try_dir_rules(add_local_search(Rules), Root ++ Src, Dir) of + {ok, File} -> {ok, File}; + _Other -> + try_suffix_rules(Rest, Root, Dir, Ext) + end; +try_suffix_rules([_|Rest], Root, Dir, Ext) -> + try_suffix_rules(Rest, Root, Dir, Ext); +try_suffix_rules([], _Root, _Dir, _Ext) -> + {error, not_found}. + +%% ensuring we check the directory of the object file before any other directory +add_local_search(Rules) -> + Local = {"",""}, + [Local] ++ lists:filter(fun (X) -> X =/= Local end, Rules). + +try_dir_rules([{From, To}|Rest], Filename, Dir) + when is_list(From), is_list(To) -> + case try_dir_rule(Dir, Filename, From, To) of + {ok, File} -> {ok, File}; + error -> try_dir_rules(Rest, Filename, Dir) + end; +try_dir_rules([], _Filename, _Dir) -> + {error, not_found}. + +try_dir_rule(Dir, Filename, From, To) -> + case lists:suffix(From, Dir) of + true -> + NewDir = lists:sublist(Dir, 1, length(Dir)-length(From))++To, + Src = filename:join(NewDir, Filename), + case filelib:is_regular(Src) of + true -> {ok, Src}; + false -> find_regular_file(filelib:wildcard(Src)) + end; + false -> + error + end. + +find_regular_file([]) -> + error; +find_regular_file([File|Files]) -> + case filelib:is_regular(File) of + true -> {ok, File}; + false -> find_regular_file(Files) + end. +-endif. -- cgit v1.1 From d59b5fdd7b7adc8d814e36f731e98d3cd5eea7b9 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 5 Oct 2018 09:01:57 -0600 Subject: Bump to 3.7.0-rc2 --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 724f921..f17f6a2 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.7.0-rc2"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 21f0a4ebc233120f6634f010d082e71b97a8dd90 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 5 Oct 2018 09:03:29 -0600 Subject: Back to git-based versioning --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index f17f6a2..724f921 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.7.0-rc2"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 4d92f3c040680213127fc3faa9638e1cd34b50fa Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sat, 6 Oct 2018 10:45:06 -0600 Subject: remove unused app_compilers app env setting (#1903) --- src/rebar.app.src | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 724f921..6058efc 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -40,8 +40,8 @@ {pkg, rebar_pkg_resource}, {hg, rebar_hg_resource}]}, - {app_compilers, [{erl, rebar_erlc_compiler}]}, - {compilers, [rebar_compiler_xrl, rebar_compiler_yrl, rebar_compiler_mib, rebar_compiler_erl]}, + {compilers, [rebar_compiler_xrl, rebar_compiler_yrl, + rebar_compiler_mib, rebar_compiler_erl]}, {providers, [rebar_prv_app_discovery, rebar_prv_as, -- cgit v1.1 From 763fd05a0f6be023a4f646c55ddd72ae193e350b Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sat, 6 Oct 2018 18:52:20 -0600 Subject: fix yrl compiler and add test (#1906) --- src/rebar_compiler_yrl.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_compiler_yrl.erl b/src/rebar_compiler_yrl.erl index 765ea36..41d93b1 100644 --- a/src/rebar_compiler_yrl.erl +++ b/src/rebar_compiler_yrl.erl @@ -34,7 +34,7 @@ compile(Source, [{_, OutDir}], _, Opts) -> AllOpts = [{parserfile, Target} | Opts], AllOpts1 = [{includefile, filename:join(OutDir, I)} || {includefile, I} <- AllOpts, filename:pathtype(I) =:= relative], - case yeec:file(Source, AllOpts1 ++ [{return, true}]) of + case yecc:file(Source, AllOpts1 ++ [{return, true}]) of {ok, _} -> ok; {ok, _Mod, Ws} -> -- cgit v1.1 From 2986105bc2a5d4579d3cf174023917f147a6f6da Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sun, 7 Oct 2018 19:35:31 -0400 Subject: Re-add erlc compiler; plugins need it However, we add a once-printed deprecation warning for the new interface. --- src/rebar_erlc_compiler.erl | 841 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 841 insertions(+) create mode 100644 src/rebar_erlc_compiler.erl (limited to 'src') diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl new file mode 100644 index 0000000..e52791c --- /dev/null +++ b/src/rebar_erlc_compiler.erl @@ -0,0 +1,841 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +%% ------------------------------------------------------------------- +%% +%% rebar: Erlang Build Tools +%% +%% Copyright (c) 2009, 2010 Dave Smith (dizzyd@dizzyd.com) +%% +%% Permission is hereby granted, free of charge, to any person obtaining a copy +%% of this software and associated documentation files (the "Software"), to deal +%% in the Software without restriction, including without limitation the rights +%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%% copies of the Software, and to permit persons to whom the Software is +%% furnished to do so, subject to the following conditions: +%% +%% The above copyright notice and this permission notice shall be included in +%% all copies or substantial portions of the Software. +%% +%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%% THE SOFTWARE. +%% ------------------------------------------------------------------- +-module(rebar_erlc_compiler). + +-export([compile/1, compile/2, compile/3, + compile_dir/3, compile_dir/4, + compile_dirs/5, + clean/1]). + +-include("rebar.hrl"). +-include_lib("stdlib/include/erl_compile.hrl"). + +-define(ERLCINFO_VSN, 2). +-define(ERLCINFO_FILE, "erlcinfo"). +-type erlc_info_v() :: {digraph:vertex(), term()} | 'false'. +-type erlc_info_e() :: {digraph:vertex(), digraph:vertex()}. +-type erlc_info() :: {list(erlc_info_v()), list(erlc_info_e()), list(string())}. +-record(erlcinfo, { + vsn = ?ERLCINFO_VSN :: pos_integer(), + info = {[], [], []} :: erlc_info() +}). + +-type compile_opts() :: [compile_opt()]. +-type compile_opt() :: {recursive, boolean()}. + +-define(DEFAULT_OUTDIR, "ebin"). +-define(RE_PREFIX, "^(?!\\._)"). + +%% =================================================================== +%% Public API +%% =================================================================== + +%% Supported configuration variables: +%% +%% * erl_opts - Erlang list of options passed to compile:file/2 +%% It is also possible to specify platform specific +%% options by specifying a pair or a triplet where the +%% first string is a regex that is checked against the +%% string +%% +%% OtpRelease ++ "-" ++ SysArch ++ "-" ++ Words. +%% +%% where +%% +%% OtpRelease = erlang:system_info(otp_release). +%% SysArch = erlang:system_info(system_architecture). +%% Words = integer_to_list(8 * +%% erlang:system_info({wordsize, external})). +%% +%% E.g. to define HAVE_SENDFILE only on systems with +%% sendfile(), to define BACKLOG on Linux/FreeBSD as 128, +%% and to define 'old_inets' for R13 OTP release do: +%% +%% {erl_opts, [{platform_define, +%% "(linux|solaris|freebsd|darwin)", +%% 'HAVE_SENDFILE'}, +%% {platform_define, "(linux|freebsd)", +%% 'BACKLOG', 128}, +%% {platform_define, "R13", +%% 'old_inets'}]}. +%% + +%% @equiv compile(AppInfo, []) +-spec compile(rebar_app_info:t()) -> ok. +compile(AppInfo) when element(1, AppInfo) == app_info_t -> + compile(AppInfo, []). + +%% @doc compile an individual application. +-spec compile(rebar_app_info:t(), compile_opts()) -> ok. +compile(AppInfo, CompileOpts) when element(1, AppInfo) == app_info_t -> + warn_deprecated(), + Dir = rebar_utils:to_list(rebar_app_info:out_dir(AppInfo)), + RebarOpts = rebar_app_info:opts(AppInfo), + + SrcOpts = [check_last_mod, + {recursive, dir_recursive(RebarOpts, "src", CompileOpts)}], + MibsOpts = [check_last_mod, + {recursive, dir_recursive(RebarOpts, "mibs", CompileOpts)}], + + rebar_base_compiler:run(RebarOpts, + check_files([filename:join(Dir, File) + || File <- rebar_opts:get(RebarOpts, xrl_first_files, [])]), + filename:join(Dir, "src"), ".xrl", filename:join(Dir, "src"), ".erl", + fun compile_xrl/3, SrcOpts), + rebar_base_compiler:run(RebarOpts, + check_files([filename:join(Dir, File) + || File <- rebar_opts:get(RebarOpts, yrl_first_files, [])]), + filename:join(Dir, "src"), ".yrl", filename:join(Dir, "src"), ".erl", + fun compile_yrl/3, SrcOpts), + rebar_base_compiler:run(RebarOpts, + check_files([filename:join(Dir, File) + || File <- rebar_opts:get(RebarOpts, mib_first_files, [])]), + filename:join(Dir, "mibs"), ".mib", filename:join([Dir, "priv", "mibs"]), ".bin", + compile_mib(AppInfo), MibsOpts), + + SrcDirs = lists:map(fun(SrcDir) -> filename:join(Dir, SrcDir) end, + rebar_dir:src_dirs(RebarOpts, ["src"])), + OutDir = filename:join(Dir, outdir(RebarOpts)), + compile_dirs(RebarOpts, Dir, SrcDirs, OutDir, CompileOpts), + + ExtraDirs = rebar_dir:extra_src_dirs(RebarOpts), + F = fun(D) -> + case ec_file:is_dir(filename:join([Dir, D])) of + true -> compile_dirs(RebarOpts, Dir, [D], D, CompileOpts); + false -> ok + end + end, + lists:foreach(F, lists:map(fun(SrcDir) -> filename:join(Dir, SrcDir) end, ExtraDirs)). + +%% @hidden +%% these are kept for backwards compatibility but they're bad functions with +%% bad interfaces you probably shouldn't use +%% State/RebarOpts have to have src_dirs set and BaseDir must be the parent +%% directory of those src_dirs + +-spec compile(rebar_dict() | rebar_state:t(), file:name(), file:name()) -> ok. +compile(State, BaseDir, OutDir) when element(1, State) == state_t -> + compile(rebar_state:opts(State), BaseDir, OutDir, [{recursive, false}]); +compile(RebarOpts, BaseDir, OutDir) -> + compile(RebarOpts, BaseDir, OutDir, [{recursive, false}]). + +%% @hidden + +-spec compile(rebar_dict() | rebar_state:t(), file:name(), file:name(), compile_opts()) -> ok. +compile(State, BaseDir, OutDir, CompileOpts) when element(1, State) == state_t -> + compile(rebar_state:opts(State), BaseDir, OutDir, CompileOpts); +compile(RebarOpts, BaseDir, OutDir, CompileOpts) -> + warn_deprecated(), + SrcDirs = lists:map(fun(SrcDir) -> filename:join(BaseDir, SrcDir) end, + rebar_dir:src_dirs(RebarOpts, ["src"])), + compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, CompileOpts), + + ExtraDirs = rebar_dir:extra_src_dirs(RebarOpts), + F = fun(D) -> + case ec_file:is_dir(filename:join([BaseDir, D])) of + true -> compile_dirs(RebarOpts, BaseDir, [D], D, CompileOpts); + false -> ok + end + end, + lists:foreach(F, lists:map(fun(SrcDir) -> filename:join(BaseDir, SrcDir) end, ExtraDirs)). + +%% @equiv compile_dirs(Context, BaseDir, [Dir], Dir, [{recursive, false}]) +-spec compile_dir(rebar_dict() | rebar_state:t(), file:name(), file:name()) -> ok. +compile_dir(State, BaseDir, Dir) when element(1, State) == state_t -> + compile_dir(rebar_state:opts(State), BaseDir, Dir, [{recursive, false}]); +compile_dir(RebarOpts, BaseDir, Dir) -> + compile_dir(RebarOpts, BaseDir, Dir, [{recursive, false}]). + +%% @equiv compile_dirs(Context, BaseDir, [Dir], Dir, Opts) +-spec compile_dir(rebar_dict() | rebar_state:t(), file:name(), file:name(), compile_opts()) -> ok. +compile_dir(State, BaseDir, Dir, Opts) when element(1, State) == state_t -> + compile_dirs(rebar_state:opts(State), BaseDir, [Dir], Dir, Opts); +compile_dir(RebarOpts, BaseDir, Dir, Opts) -> + compile_dirs(RebarOpts, BaseDir, [Dir], Dir, Opts). + +%% @doc compile a list of directories with the given opts. +-spec compile_dirs(rebar_dict() | rebar_state:t(), + file:filename(), + [file:filename()], + file:filename(), + compile_opts()) -> ok. +compile_dirs(State, BaseDir, Dirs, OutDir, CompileOpts) when element(1, State) == state_t -> + compile_dirs(rebar_state:opts(State), BaseDir, Dirs, OutDir, CompileOpts); +compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, CompileOpts) -> + ErlOpts = rebar_opts:erl_opts(RebarOpts), + ?DEBUG("erlopts ~p", [ErlOpts]), + AllErlFiles = gather_src(RebarOpts, BaseDir, SrcDirs, CompileOpts), + ?DEBUG("files to compile ~p", [AllErlFiles]), + + %% Make sure that outdir is on the path + ok = filelib:ensure_dir(filename:join(OutDir, "dummy.beam")), + true = code:add_patha(filename:absname(OutDir)), + + G = init_erlcinfo(include_abs_dirs(ErlOpts, BaseDir), AllErlFiles, BaseDir, OutDir), + + {ParseTransforms, Rest} = split_source_files(AllErlFiles, ErlOpts), + NeededErlFiles = case needed_files(G, ErlOpts, RebarOpts, BaseDir, OutDir, ParseTransforms) of + [] -> needed_files(G, ErlOpts, RebarOpts, BaseDir, OutDir, Rest); + %% at least one parse transform in the opts needs updating, so recompile all + _ -> AllErlFiles + end, + + {ErlFirstFiles, ErlOptsFirst} = erl_first_files(RebarOpts, ErlOpts, BaseDir, NeededErlFiles), + {DepErls, OtherErls} = lists:partition( + fun(Source) -> digraph:in_degree(G, Source) > 0 end, + [File || File <- NeededErlFiles, not lists:member(File, ErlFirstFiles)]), + SubGraph = digraph_utils:subgraph(G, DepErls), + DepErlsOrdered = digraph_utils:topsort(SubGraph), + FirstErls = ErlFirstFiles ++ lists:reverse(DepErlsOrdered), + try + rebar_base_compiler:run( + RebarOpts, FirstErls, OtherErls, + fun(S, C) -> + ErlOpts1 = case lists:member(S, ErlFirstFiles) of + true -> ErlOptsFirst; + false -> ErlOpts + end, + internal_erl_compile(C, BaseDir, S, OutDir, ErlOpts1, RebarOpts) + end) + after + true = digraph:delete(SubGraph), + true = digraph:delete(G) + end, + ok. + +%% @doc remove compiled artifacts from an AppDir. +-spec clean(rebar_app_info:t()) -> 'ok'. +clean(AppInfo) -> + AppDir = rebar_app_info:out_dir(AppInfo), + + MibFiles = rebar_utils:find_files(filename:join([AppDir, "mibs"]), ?RE_PREFIX".*\\.mib\$"), + MIBs = [filename:rootname(filename:basename(MIB)) || MIB <- MibFiles], + rebar_file_utils:delete_each( + [filename:join([AppDir, "include",MIB++".hrl"]) || MIB <- MIBs]), + ok = rebar_file_utils:rm_rf(filename:join([AppDir, "priv/mibs/*.bin"])), + + YrlFiles = rebar_utils:find_files(filename:join([AppDir, "src"]), ?RE_PREFIX".*\\.[x|y]rl\$"), + rebar_file_utils:delete_each( + [rebar_utils:to_list(re:replace(F, "\\.[x|y]rl$", ".erl", [unicode])) + || F <- YrlFiles]), + + BinDirs = ["ebin"|rebar_dir:extra_src_dirs(rebar_app_info:opts(AppInfo))], + ok = clean_dirs(AppDir, BinDirs), + + %% Delete the build graph, if any + rebar_file_utils:rm_rf(erlcinfo_file(AppDir)). + +clean_dirs(_AppDir, []) -> ok; +clean_dirs(AppDir, [Dir|Rest]) -> + ok = rebar_file_utils:rm_rf(filename:join([AppDir, Dir, "*.beam"])), + %% Erlang compilation is recursive, so it's possible that we have a nested + %% directory structure in ebin with .beam files within. As such, we want + %% to scan whatever is left in the app's out_dir directory for sub-dirs which + %% satisfy our criteria. + BeamFiles = rebar_utils:find_files(filename:join([AppDir, Dir]), ?RE_PREFIX".*\\.beam\$"), + rebar_file_utils:delete_each(BeamFiles), + lists:foreach(fun(D) -> delete_dir(D, dirs(D)) end, dirs(filename:join([AppDir, Dir]))), + clean_dirs(AppDir, Rest). + + +%% =================================================================== +%% Internal functions +%% =================================================================== + +gather_src(Opts, BaseDir, Dirs, CompileOpts) -> + gather_src(Opts, filename:split(BaseDir), Dirs, [], CompileOpts). + +gather_src(_Opts, _BaseDirParts, [], Srcs, _CompileOpts) -> Srcs; +gather_src(Opts, BaseDirParts, [Dir|Rest], Srcs, CompileOpts) -> + DirParts = filename:split(Dir), + RelDir = case lists:prefix(BaseDirParts,DirParts) of + true -> + case lists:nthtail(length(BaseDirParts),DirParts) of + [] -> "."; + RestParts -> filename:join(RestParts) + end; + false -> Dir + end, + DirRecursive = dir_recursive(Opts, RelDir, CompileOpts), + gather_src(Opts, BaseDirParts, Rest, Srcs ++ rebar_utils:find_files(Dir, ?RE_PREFIX".*\\.erl\$", DirRecursive), CompileOpts). + +%% Get files which need to be compiled first, i.e. those specified in erl_first_files +%% and parse_transform options. Also produce specific erl_opts for these first +%% files, so that yet to be compiled parse transformations are excluded from it. +erl_first_files(Opts, ErlOpts, Dir, NeededErlFiles) -> + ErlFirstFilesConf = rebar_opts:get(Opts, erl_first_files, []), + valid_erl_first_conf(ErlFirstFilesConf), + NeededSrcDirs = lists:usort(lists:map(fun filename:dirname/1, NeededErlFiles)), + %% NOTE: order of files here is important! + ErlFirstFiles = + [filename:join(Dir, File) || File <- ErlFirstFilesConf, + lists:member(filename:join(Dir, File), NeededErlFiles)], + {ParseTransforms, ParseTransformsErls} = + lists:unzip(lists:flatmap( + fun(PT) -> + PTerls = [filename:join(D, module_to_erl(PT)) || D <- NeededSrcDirs], + [{PT, PTerl} || PTerl <- PTerls, lists:member(PTerl, NeededErlFiles)] + end, proplists:get_all_values(parse_transform, ErlOpts))), + ErlOptsFirst = lists:filter(fun({parse_transform, PT}) -> + not lists:member(PT, ParseTransforms); + (_) -> + true + end, ErlOpts), + {ErlFirstFiles ++ ParseTransformsErls, ErlOptsFirst}. + +split_source_files(SourceFiles, ErlOpts) -> + ParseTransforms = proplists:get_all_values(parse_transform, ErlOpts), + lists:partition(fun(Source) -> + lists:member(filename_to_atom(Source), ParseTransforms) + end, SourceFiles). + +filename_to_atom(F) -> list_to_atom(filename:rootname(filename:basename(F))). + +%% Get subset of SourceFiles which need to be recompiled, respecting +%% dependencies induced by given graph G. +needed_files(G, ErlOpts, RebarOpts, Dir, OutDir, SourceFiles) -> + lists:filter(fun(Source) -> + TargetBase = target_base(OutDir, Source), + Target = TargetBase ++ ".beam", + PrivIncludes = [{i, filename:join(Dir, Src)} + || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])], + AllOpts = [{outdir, filename:dirname(Target)} + ,{i, filename:join(Dir, "include")} + ,{i, Dir}] ++ PrivIncludes ++ ErlOpts, + %% necessary for erlang:function_exported/3 to work as expected + %% called here for clarity as it's required by both opts_changed/2 + %% and erl_compiler_opts_set/0 + _ = code:ensure_loaded(compile), + digraph:vertex(G, Source) > {Source, filelib:last_modified(Target)} + orelse opts_changed(AllOpts, TargetBase) + orelse erl_compiler_opts_set() + end, SourceFiles). + +maybe_rm_beam_and_edge(G, OutDir, Source) -> + %% This is NOT a double check it is the only check that the source file is actually gone + case filelib:is_regular(Source) of + true -> + %% Actually exists, don't delete + false; + false -> + Target = target_base(OutDir, Source) ++ ".beam", + ?DEBUG("Source ~ts is gone, deleting previous beam file if it exists ~ts", [Source, Target]), + file:delete(Target), + digraph:del_vertex(G, Source), + true + end. + +opts_changed(NewOpts, Target) -> + TotalOpts = case erlang:function_exported(compile, env_compiler_options, 0) of + true -> NewOpts ++ compile:env_compiler_options(); + false -> NewOpts + end, + case compile_info(Target) of + {ok, Opts} -> lists:any(fun effects_code_generation/1, lists:usort(TotalOpts) -- lists:usort(Opts)); + _ -> true + end. + +effects_code_generation(Option) -> + case Option of + beam -> false; + report_warnings -> false; + report_errors -> false; + return_errors-> false; + return_warnings-> false; + report -> false; + warnings_as_errors -> false; + binary -> false; + verbose -> false; + {cwd,_} -> false; + {outdir, _} -> false; + _ -> true + end. + +compile_info(Target) -> + case beam_lib:chunks(Target, [compile_info]) of + {ok, {_mod, Chunks}} -> + CompileInfo = proplists:get_value(compile_info, Chunks, []), + {ok, proplists:get_value(options, CompileInfo, [])}; + {error, beam_lib, Reason} -> + ?WARN("Couldn't read debug info from ~p for reason: ~p", [Target, Reason]), + {error, Reason} + end. + +erl_compiler_opts_set() -> + EnvSet = case os:getenv("ERL_COMPILER_OPTIONS") of + false -> false; + _ -> true + end, + %% return false if changed env opts would have been caught in opts_changed/2 + EnvSet andalso not erlang:function_exported(compile, env_compiler_options, 0). + +erlcinfo_file(Dir) -> + filename:join(rebar_dir:local_cache_dir(Dir), ?ERLCINFO_FILE). + +%% Get dependency graph of given Erls files and their dependencies (header files, +%% parse transforms, behaviours etc.) located in their directories or given +%% InclDirs. Note that last modification times stored in vertices already respect +%% dependencies induced by given graph G. +init_erlcinfo(InclDirs, Erls, Dir, OutDir) -> + G = digraph:new([acyclic]), + try restore_erlcinfo(G, InclDirs, Dir) + catch + _:_ -> + ?WARN("Failed to restore ~ts file. Discarding it.~n", [erlcinfo_file(Dir)]), + file:delete(erlcinfo_file(Dir)) + end, + Dirs = source_and_include_dirs(InclDirs, Erls), + %% A source file may have been renamed or deleted. Remove it from the graph + %% and remove any beam file for that source if it exists. + Modified = maybe_rm_beams_and_edges(G, OutDir, Erls), + Modified1 = lists:foldl(update_erlcinfo_fun(G, Dirs), Modified, Erls), + if Modified1 -> store_erlcinfo(G, InclDirs, Dir); not Modified1 -> ok end, + G. + +maybe_rm_beams_and_edges(G, Dir, Files) -> + Vertices = digraph:vertices(G), + case lists:filter(fun(File) -> + case filename:extension(File) =:= ".erl" of + true -> + maybe_rm_beam_and_edge(G, Dir, File); + false -> + false + end + end, lists:sort(Vertices) -- lists:sort(Files)) of + [] -> + false; + _ -> + true + end. + +source_and_include_dirs(InclDirs, Erls) -> + SourceDirs = lists:map(fun filename:dirname/1, Erls), + lists:usort(InclDirs ++ SourceDirs). + +update_erlcinfo(G, Dirs, Source) -> + case digraph:vertex(G, Source) of + {_, LastUpdated} -> + case filelib:last_modified(Source) of + 0 -> + %% The file doesn't exist anymore, + %% erase it from the graph. + %% All the edges will be erased automatically. + digraph:del_vertex(G, Source), + modified; + LastModified when LastUpdated < LastModified -> + modify_erlcinfo(G, Source, LastModified, filename:dirname(Source), Dirs); + _ -> + Modified = lists:foldl( + update_erlcinfo_fun(G, Dirs), + false, digraph:out_neighbours(G, Source)), + MaxModified = update_max_modified_deps(G, Source), + case Modified orelse MaxModified > LastUpdated of + true -> modified; + false -> unmodified + end + end; + false -> + modify_erlcinfo(G, Source, filelib:last_modified(Source), filename:dirname(Source), Dirs) + end. + +update_erlcinfo_fun(G, Dirs) -> + fun(Erl, Modified) -> + case update_erlcinfo(G, Dirs, Erl) of + modified -> true; + unmodified -> Modified + end + end. + +update_max_modified_deps(G, Source) -> + MaxModified = lists:max(lists:map( + fun(File) -> {_, MaxModified} = digraph:vertex(G, File), MaxModified end, + [Source|digraph:out_neighbours(G, Source)])), + digraph:add_vertex(G, Source, MaxModified), + MaxModified. + +modify_erlcinfo(G, Source, LastModified, Dir, Dirs) -> + {ok, Fd} = file:open(Source, [read]), + Incls = parse_attrs(Fd, [], Dir), + AbsIncls = expand_file_names(Incls, Dirs), + ok = file:close(Fd), + digraph:add_vertex(G, Source, LastModified), + digraph:del_edges(G, digraph:out_edges(G, Source)), + lists:foreach( + fun(Incl) -> + update_erlcinfo(G, Dirs, Incl), + digraph:add_edge(G, Source, Incl) + end, AbsIncls), + modified. + +restore_erlcinfo(G, InclDirs, Dir) -> + case file:read_file(erlcinfo_file(Dir)) of + {ok, Data} -> + % Since externally passed InclDirs can influence erlcinfo graph (see + % modify_erlcinfo), we have to check here that they didn't change. + #erlcinfo{vsn=?ERLCINFO_VSN, info={Vs, Es, InclDirs}} = + binary_to_term(Data), + lists:foreach( + fun({V, LastUpdated}) -> + digraph:add_vertex(G, V, LastUpdated) + end, Vs), + lists:foreach( + fun({_, V1, V2, _}) -> + digraph:add_edge(G, V1, V2) + end, Es); + {error, _} -> + ok + end. + +store_erlcinfo(G, InclDirs, Dir) -> + Vs = lists:map(fun(V) -> digraph:vertex(G, V) end, digraph:vertices(G)), + Es = lists:map(fun(E) -> digraph:edge(G, E) end, digraph:edges(G)), + File = erlcinfo_file(Dir), + ok = filelib:ensure_dir(File), + Data = term_to_binary(#erlcinfo{info={Vs, Es, InclDirs}}, [{compressed, 2}]), + file:write_file(File, Data). + +%% NOTE: If, for example, one of the entries in Files, refers to +%% gen_server.erl, that entry will be dropped. It is dropped because +%% such an entry usually refers to the beam file, and we don't pass a +%% list of OTP src dirs for finding gen_server.erl's full path. Also, +%% if gen_server.erl was modified, it's not rebar's task to compile a +%% new version of the beam file. Therefore, it's reasonable to drop +%% such entries. Also see process_attr(behaviour, Form, Includes). +-spec expand_file_names([file:filename()], + [file:filename()]) -> [file:filename()]. +expand_file_names(Files, Dirs) -> + %% We check if Files exist by itself or within the directories + %% listed in Dirs. + %% Return the list of files matched. + lists:flatmap( + fun(Incl) -> + case filelib:is_regular(Incl) of + true -> + [Incl]; + false -> + lists:flatmap( + fun(Dir) -> + FullPath = filename:join(Dir, Incl), + case filelib:is_regular(FullPath) of + true -> + [FullPath]; + false -> + [] + end + end, Dirs) + end + end, Files). + +-spec internal_erl_compile(rebar_dict(), file:filename(), file:filename(), + file:filename(), list(), rebar_dict()) -> + ok | {ok, any()} | {error, any(), any()}. +internal_erl_compile(Opts, Dir, Module, OutDir, ErlOpts, RebarOpts) -> + Target = target_base(OutDir, Module) ++ ".beam", + ok = filelib:ensure_dir(Target), + PrivIncludes = [{i, filename:join(Dir, Src)} + || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])], + AllOpts = [{outdir, filename:dirname(Target)}] ++ ErlOpts ++ PrivIncludes ++ + [{i, filename:join(Dir, "include")}, {i, Dir}, return], + case compile:file(Module, AllOpts) of + {ok, _Mod} -> + ok; + {ok, _Mod, Ws} -> + FormattedWs = format_error_sources(Ws, Opts), + rebar_base_compiler:ok_tuple(Module, FormattedWs); + {error, Es, Ws} -> + error_tuple(Module, Es, Ws, AllOpts, Opts) + end. + +error_tuple(Module, Es, Ws, AllOpts, Opts) -> + FormattedEs = format_error_sources(Es, Opts), + FormattedWs = format_error_sources(Ws, Opts), + rebar_base_compiler:error_tuple(Module, FormattedEs, FormattedWs, AllOpts). + +format_error_sources(Es, Opts) -> + [{rebar_base_compiler:format_error_source(Src, Opts), Desc} + || {Src, Desc} <- Es]. + +target_base(OutDir, Source) -> + filename:join(OutDir, filename:basename(Source, ".erl")). + +-spec compile_mib(rebar_app_info:t()) -> + fun((file:filename(), file:filename(), rebar_dict()) -> 'ok'). +compile_mib(AppInfo) -> + fun(Source, Target, Opts) -> + Dir = filename:dirname(Target), + Mib = filename:rootname(Target), + HrlFilename = Mib ++ ".hrl", + + AppInclude = filename:join([rebar_app_info:dir(AppInfo), "include"]), + + ok = filelib:ensure_dir(Target), + ok = filelib:ensure_dir(filename:join([AppInclude, "dummy.hrl"])), + + AllOpts = [{outdir, Dir} + ,{i, [Dir]}] ++ + rebar_opts:get(Opts, mib_opts, []), + + case snmpc:compile(Source, AllOpts) of + {ok, _} -> + MibToHrlOpts = + case proplists:get_value(verbosity, AllOpts, undefined) of + undefined -> + #options{specific = [], + cwd = rebar_dir:get_cwd()}; + Verbosity -> + #options{specific = [{verbosity, Verbosity}], + cwd = rebar_dir:get_cwd()} + end, + ok = snmpc:mib_to_hrl(Mib, Mib, MibToHrlOpts), + rebar_file_utils:mv(HrlFilename, AppInclude), + ok; + {error, compilation_failed} -> + ?FAIL + end + end. + +-spec compile_xrl(file:filename(), file:filename(), + rebar_dict()) -> 'ok'. +compile_xrl(Source, Target, Opts) -> + AllOpts = [{scannerfile, Target} | rebar_opts:get(Opts, xrl_opts, [])], + compile_xrl_yrl(Opts, Source, Target, AllOpts, leex). + +-spec compile_yrl(file:filename(), file:filename(), + rebar_dict()) -> 'ok'. +compile_yrl(Source, Target, Opts) -> + AllOpts = [{parserfile, Target} | rebar_opts:get(Opts, yrl_opts, [])], + compile_xrl_yrl(Opts, Source, Target, AllOpts, yecc). + +-spec compile_xrl_yrl(rebar_dict(), file:filename(), + file:filename(), list(), module()) -> 'ok'. +compile_xrl_yrl(_Opts, Source, Target, AllOpts, Mod) -> + %% FIX ME: should be the outdir or something + Dir = filename:dirname(filename:dirname(Target)), + AllOpts1 = [{includefile, filename:join(Dir, I)} || {includefile, I} <- AllOpts, + filename:pathtype(I) =:= relative], + case needs_compile(Source, Target) of + true -> + case Mod:file(Source, AllOpts1 ++ [{return, true}]) of + {ok, _} -> + ok; + {ok, _Mod, Ws} -> + rebar_base_compiler:ok_tuple(Source, Ws); + {error, Es, Ws} -> + rebar_base_compiler:error_tuple(Source, + Es, Ws, AllOpts1) + end; + false -> + skipped + end. + +needs_compile(Source, Target) -> + filelib:last_modified(Source) > filelib:last_modified(Target). + + + +-spec dirs(file:filename()) -> [file:filename()]. +dirs(Dir) -> + [F || F <- filelib:wildcard(filename:join([Dir, "*"])), filelib:is_dir(F)]. + +-spec delete_dir(file:filename(), [string()]) -> 'ok' | {'error', atom()}. +delete_dir(Dir, []) -> + file:del_dir(Dir); +delete_dir(Dir, Subdirs) -> + lists:foreach(fun(D) -> delete_dir(D, dirs(D)) end, Subdirs), + file:del_dir(Dir). + +parse_attrs(Fd, Includes, Dir) -> + case io:parse_erl_form(Fd, "") of + {ok, Form, _Line} -> + case erl_syntax:type(Form) of + attribute -> + NewIncludes = process_attr(Form, Includes, Dir), + parse_attrs(Fd, NewIncludes, Dir); + _ -> + parse_attrs(Fd, Includes, Dir) + end; + {eof, _} -> + Includes; + _Err -> + parse_attrs(Fd, Includes, Dir) + end. + +process_attr(Form, Includes, Dir) -> + AttrName = erl_syntax:atom_value(erl_syntax:attribute_name(Form)), + process_attr(AttrName, Form, Includes, Dir). + +process_attr(import, Form, Includes, _Dir) -> + case erl_syntax_lib:analyze_import_attribute(Form) of + {Mod, _Funs} -> + [module_to_erl(Mod)|Includes]; + Mod -> + [module_to_erl(Mod)|Includes] + end; +process_attr(file, Form, Includes, _Dir) -> + {File, _} = erl_syntax_lib:analyze_file_attribute(Form), + [File|Includes]; +process_attr(include, Form, Includes, _Dir) -> + [FileNode] = erl_syntax:attribute_arguments(Form), + File = erl_syntax:string_value(FileNode), + [File|Includes]; +process_attr(include_lib, Form, Includes, Dir) -> + [FileNode] = erl_syntax:attribute_arguments(Form), + RawFile = erl_syntax:string_value(FileNode), + maybe_expand_include_lib_path(RawFile, Dir) ++ Includes; +process_attr(behavior, Form, Includes, _Dir) -> + process_attr(behaviour, Form, Includes, _Dir); +process_attr(behaviour, Form, Includes, _Dir) -> + [FileNode] = erl_syntax:attribute_arguments(Form), + File = module_to_erl(erl_syntax:atom_value(FileNode)), + [File|Includes]; +process_attr(compile, Form, Includes, _Dir) -> + [Arg] = erl_syntax:attribute_arguments(Form), + case erl_syntax:concrete(Arg) of + {parse_transform, Mod} -> + [module_to_erl(Mod)|Includes]; + {core_transform, Mod} -> + [module_to_erl(Mod)|Includes]; + L when is_list(L) -> + lists:foldl( + fun({parse_transform, Mod}, Acc) -> + [module_to_erl(Mod)|Acc]; + ({core_transform, Mod}, Acc) -> + [module_to_erl(Mod)|Acc]; + (_, Acc) -> + Acc + end, Includes, L); + _ -> + Includes + end; +process_attr(_, _Form, Includes, _Dir) -> + Includes. + +module_to_erl(Mod) -> + atom_to_list(Mod) ++ ".erl". + +%% Given a path like "stdlib/include/erl_compile.hrl", return +%% "OTP_INSTALL_DIR/lib/erlang/lib/stdlib-x.y.z/include/erl_compile.hrl". +%% Usually a simple [Lib, SubDir, File1] = filename:split(File) should +%% work, but to not crash when an unusual include_lib path is used, +%% utilize more elaborate logic. +maybe_expand_include_lib_path(File, Dir) -> + File1 = filename:basename(File), + case filename:split(filename:dirname(File)) of + [_] -> + warn_and_find_path(File, Dir); + [Lib | SubDir] -> + case code:lib_dir(list_to_atom(Lib), list_to_atom(filename:join(SubDir))) of + {error, bad_name} -> + warn_and_find_path(File, Dir); + AppDir -> + [filename:join(AppDir, File1)] + end + end. + +%% The use of -include_lib was probably incorrect by the user but lets try to make it work. +%% We search in the outdir and outdir/../include to see if the header exists. +warn_and_find_path(File, Dir) -> + SrcHeader = filename:join(Dir, File), + case filelib:is_regular(SrcHeader) of + true -> + [SrcHeader]; + false -> + IncludeDir = filename:join(rebar_utils:droplast(filename:split(Dir))++["include"]), + IncludeHeader = filename:join(IncludeDir, File), + case filelib:is_regular(IncludeHeader) of + true -> + [filename:join(IncludeDir, File)]; + false -> + [] + end + end. + +%% +%% Ensure all files in a list are present and abort if one is missing +%% +-spec check_files([file:filename()]) -> [file:filename()]. +check_files(FileList) -> + [check_file(F) || F <- FileList]. + +check_file(File) -> + case filelib:is_regular(File) of + false -> ?ABORT("File ~p is missing, aborting\n", [File]); + true -> File + end. + +outdir(RebarOpts) -> + ErlOpts = rebar_opts:erl_opts(RebarOpts), + proplists:get_value(outdir, ErlOpts, ?DEFAULT_OUTDIR). + +include_abs_dirs(ErlOpts, BaseDir) -> + ErlOptIncludes = proplists:get_all_values(i, ErlOpts), + InclDirs = lists:map(fun(Incl) -> filename:absname(Incl) end, ErlOptIncludes), + [filename:join([BaseDir, "include"])|InclDirs]. + +dir_recursive(Opts, Dir, CompileOpts) when is_list(CompileOpts) -> + case proplists:get_value(recursive,CompileOpts) of + undefined -> rebar_dir:recursive(Opts, Dir); + Recursive -> Recursive + end. + +valid_erl_first_conf(FileList) -> + Strs = filter_file_list(FileList), + case rebar_utils:is_list_of_strings(Strs) of + true -> true; + false -> ?ABORT("An invalid file list (~p) was provided as part of your erl_first_files directive", + [FileList]) + end. + +filter_file_list(FileList) -> + Atoms = lists:filter( fun(X) -> is_atom(X) end, FileList), + case Atoms of + [] -> + FileList; + _ -> + atoms_in_erl_first_files_warning(Atoms), + lists:filter( fun(X) -> not(is_atom(X)) end, FileList) + end. + +atoms_in_erl_first_files_warning(Atoms) -> + W = "You have provided atoms as file entries in erl_first_files; " + "erl_first_files only expects lists of filenames as strings. " + "The following modules (~p) may not work as expected and it is advised " + "that you change these entires to string format " + "(e.g., \"src/module.erl\") ", + ?WARN(W, [Atoms]). + +warn_deprecated() -> + case get({deprecate_warn, ?MODULE}) of + undefined -> + ?WARN("Calling deprecated ~p compiler module. This module has been " + "replaced by rebar_compiler and rebar_compiler_erl, but will " + "remain available.", [?MODULE]), + put({deprecate_warn, ?MODULE}, true), + ok; + _ -> + ok + end. -- cgit v1.1 From 1df574a6f55506d606020d64f42c33f8bbf16a7b Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Wed, 10 Oct 2018 11:53:27 -0600 Subject: throw builder error when returned from project build (#1909) --- src/rebar_prv_compile.erl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index 54bdee2..a509704 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -208,8 +208,12 @@ build_app(AppInfo, State) -> %% load plugins since thats where project builders would be PluginDepsPaths = rebar_state:code_paths(State, all_plugin_deps), code:add_pathsa(PluginDepsPaths), - Module:build(AppInfo), - rebar_utils:remove_from_code_path(PluginDepsPaths); + case Module:build(AppInfo) of + ok -> + rebar_utils:remove_from_code_path(PluginDepsPaths); + {error, Reason} -> + throw({error, {Module, Reason}}) + end; _ -> throw(?PRV_ERROR({unknown_project_type, rebar_app_info:name(AppInfo), Type})) end -- cgit v1.1 From 9d788d893620f868b9c0ee00ddec8ae4d5d8fea7 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 5 Oct 2018 07:57:20 -0400 Subject: Abstracted path management Move path management out of rebar_utils manual code path function handling (which we leave there for backwards compat), and centralize them to allow easier coordination of paths between plugins and deps. On top of path handling, do a check of loaded modules to only purge and reload those that actually need it done in order to prevent all kinds of weird interaction and accidental purge kills. It also allows the possible cohabitation of both at once, with a "in case of conflict pick X" as a policy Changing path handling in providers also highlighted a bunch of bugs in some tests and appears to fix some in other providers, specifically around plugins. --- src/rebar_api.erl | 17 ++++ src/rebar_hooks.erl | 8 +- src/rebar_packages.erl | 2 +- src/rebar_paths.erl | 177 ++++++++++++++++++++++++++++++++++++++++++ src/rebar_plugins.erl | 4 +- src/rebar_prv_common_test.erl | 12 ++- src/rebar_prv_compile.erl | 36 ++++----- src/rebar_prv_dialyzer.erl | 5 +- src/rebar_prv_edoc.erl | 4 +- src/rebar_prv_eunit.erl | 8 +- src/rebar_prv_xref.erl | 5 +- 11 files changed, 233 insertions(+), 45 deletions(-) create mode 100644 src/rebar_paths.erl (limited to 'src') diff --git a/src/rebar_api.erl b/src/rebar_api.erl index 4dabe8a..00eb054 100644 --- a/src/rebar_api.erl +++ b/src/rebar_api.erl @@ -9,6 +9,8 @@ expand_env_variable/3, get_arch/0, wordsize/0, + set_paths/2, + unset_paths/2, add_deps_to_path/1, restore_code_path/1, processing_base_dir/1, @@ -67,6 +69,21 @@ get_arch() -> wordsize() -> rebar_utils:wordsize(). +%% @doc Set code paths. Takes arguments of the form +%% `[plugins, deps]' or `[deps, plugins]' and ensures the +%% project's app and dependencies are set in the right order +%% for the next bit of execution +-spec set_paths(rebar_paths:targets(), rebar_state:t()) -> ok. +set_paths(List, State) -> + rebar_paths:set_paths(List, State). + +%% @doc Unsets code paths. Takes arguments of the form +%% `[plugins, deps]' or `[deps, plugins]' and ensures the +%% paths are no longer active. +-spec unset_paths(rebar_paths:targets(), rebar_state:t()) -> ok. +unset_paths(List, State) -> + rebar_paths:unset_paths(List, State). + %% @doc Add deps to the code path -spec add_deps_to_path(rebar_state:t()) -> ok. add_deps_to_path(State) -> diff --git a/src/rebar_hooks.erl b/src/rebar_hooks.erl index ec6fe31..f2ef8a3 100644 --- a/src/rebar_hooks.erl +++ b/src/rebar_hooks.erl @@ -42,8 +42,9 @@ run_provider_hooks_(Dir, Type, Command, Providers, TypeHooks, State) -> [] -> State; HookProviders -> - PluginDepsPaths = lists:usort(rebar_state:code_paths(State, all_plugin_deps)), - code:add_pathsa(PluginDepsPaths), + %PluginDepsPaths = lists:usort(rebar_state:code_paths(State, all_plugin_deps)), + %code:add_pathsa(PluginDepsPaths), + rebar_paths:set_paths([plugins, deps], State), Providers1 = rebar_state:providers(State), State1 = rebar_state:providers(rebar_state:dir(State, Dir), Providers++Providers1), case rebar_core:do(HookProviders, State1) of @@ -51,7 +52,8 @@ run_provider_hooks_(Dir, Type, Command, Providers, TypeHooks, State) -> ?DEBUG(format_error({bad_provider, Type, Command, ProviderName}), []), throw(?PRV_ERROR({bad_provider, Type, Command, ProviderName})); {ok, State2} -> - rebar_utils:remove_from_code_path(PluginDepsPaths), + %rebar_utils:remove_from_code_path(PluginDepsPaths), + rebar_paths:set_paths([deps, plugins], State2), State2 end end. diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 7676213..096948c 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -82,7 +82,7 @@ get_package(Dep, Vsn, Hash, Repos, Table, State) -> not_found end. -new_package_table() -> +new_package_table() -> ?PACKAGE_TABLE = ets:new(?PACKAGE_TABLE, [named_table, public, ordered_set, {keypos, 2}]), ets:insert(?PACKAGE_TABLE, {?PACKAGE_INDEX_VERSION, package_index_version}). diff --git a/src/rebar_paths.erl b/src/rebar_paths.erl new file mode 100644 index 0000000..bb43897 --- /dev/null +++ b/src/rebar_paths.erl @@ -0,0 +1,177 @@ +-module(rebar_paths). +-include("rebar.hrl"). + +-type target() :: deps | plugins. +-type targets() :: [target(), ...]. +-export_type([target/0, targets/0]). +-export([set_paths/2, unset_paths/2]). + +-spec set_paths(targets(), rebar_state:t()) -> ok. +set_paths(UserTargets, State) -> + Targets = normalize_targets(UserTargets), + GroupPaths = path_groups(Targets, State), + Paths = lists:append([P || {_, P} <- GroupPaths]), + [code:del_path(P) || P <- Paths], + code:add_pathsa(lists:reverse(Paths)), + % set path breaks with escripts + %true = code:set_path(lists:append([P || {_, P} <- GroupPaths])), + AppGroups = app_groups(Targets, State), + purge_and_load(AppGroups, code:all_loaded(), sets:new()), + ok. + +-spec unset_paths(targets(), rebar_state:t()) -> ok. +unset_paths(UserTargets, State) -> + Targets = normalize_targets(UserTargets), + GroupPaths = path_groups(Targets, State), + Paths = lists:append([P || {_, P} <- GroupPaths]), + [code:del_path(P) || P <- Paths], + purge(Paths, code:all_loaded()), + ok. + + +%% The paths are to be set in the reverse order; i.e. the default +%% path is always last when possible (minimize cases where a build +%% tool version clashes with an app's), and put the highest priorities +%% first. +-spec normalize_targets(targets()) -> targets(). +normalize_targets(List) -> + %% Plan for the eventuality of getting values piped in + %% from future versions of rebar3, possibly from plugins and so on, + %% which means we'd risk failing kind of violently. We only support + %% deps and plugins + TmpList = lists:foldl( + fun(deps, [deps | _] = Acc) -> Acc; + (plugins, [plugins | _] = Acc) -> Acc; + (deps, Acc) -> [deps | Acc -- [deps]]; + (plugins, Acc) -> [plugins | Acc -- [plugins]]; + (_, Acc) -> Acc + end, + [], + List + ), + lists:reverse(TmpList). + +purge_and_load([], _, _) -> + ok; +purge_and_load([{_Group, Apps}|Rest], ModPaths, Seen) -> + %% We have: a list of all applications in the current priority group, + %% a list of all loaded modules with their active path, and a list of + %% seen applications. + %% + %% We do the following: + %% 1. identify the apps that have not been solved yet + %% 2. find the paths for all apps in the current group + %% 3. unload and reload apps that may have changed paths in order + %% to get updated module lists and specs + %% (we ignore started apps and apps that have not run for this) + %% 4. create a list of modules to check from that app list + %% 5. check the modules to match their currently loaded paths with + %% the path set from the apps in the current group; modules + %% that differ must be purged; others can stay + + %% 1) + AppNames = [AppName || App <- Apps, + AppName <- [rebar_app_info:name(App)], + not sets:is_element(AppName, Seen)], + GoodApps = [App || AppName <- AppNames, + App <- Apps, + rebar_app_info:name(App) =:= AppName], + %% 2) + %% TODO: add extra dirs (and test), and possibly the stdlib + GoodAppPaths = [rebar_app_info:ebin_dir(App) || App <- GoodApps], + %% ++ [code:lib_dir()], + %% 3) + [begin + AtomApp = binary_to_atom(AppName, utf8), + %% blind load/unload won't interrupt an already-running app, + %% preventing odd errors, maybe! + case application:unload(AtomApp) of + ok -> application:load(AtomApp); + _ -> ok + end + end || AppName <- AppNames, + %% Shouldn't unload ourselves; rebar runs without ever + %% being started and unloading breaks logging! + AppName =/= <<"rebar">>], + + %% 4) + CandidateMods = lists:append( + %% Start by asking the currently loaded app (if loaded) + %% since it would be the primary source of conflicting modules + [case application:get_key(AppName, modules) of + {ok, Mods} -> + Mods; + undefined -> + %% if not found, parse the app file on disk, in case + %% the app's modules are used without it being loaded + case rebar_app_info:app_details(App) of + [] -> []; + Details -> proplists:get_value(modules, Details, []) + end + end || App <- GoodApps, + AppName <- [binary_to_atom(rebar_app_info:name(App), utf8)]] + ), + + %% 5) + Mods = misloaded_modules(CandidateMods, GoodAppPaths, ModPaths), + [purge_mod(Mod) || Mod <- Mods], + purge_and_load(Rest, ModPaths, + sets:union(Seen, sets:from_list(AppNames))). + + +purge(Paths, ModPaths) -> + lists:map(fun purge_mod/1, lists:usort( + [Mod || {Mod, Path} <- ModPaths, + is_list(Path), % not 'preloaded' or mocked + any_prefix(Path, Paths)] + )). + +misloaded_modules(Mods, GoodAppPaths, ModPaths) -> + %% Identify paths that are invalid; i.e. app paths that cover an + %% app in the desired group, but are not in the desired group. + lists:usort( + [purge_mod(Mod) + || Mod <- Mods, + {_, Path} <- [lists:keyfind(Mod, 1, ModPaths)], + is_list(Path), % not 'preloaded' or mocked + not any_prefix(Path, GoodAppPaths)] + ). + +any_prefix(Path, Paths) -> + lists:any(fun(P) -> lists:prefix(P, Path) end, Paths). + +%% assume paths currently set are good; only unload a module so next call +%% uses the correctly set paths +purge_mod(Mod) -> + case erlang:check_process_code(self(), Mod) of + false -> + code:purge(Mod), + code:delete(Mod); + _ -> + %% cannot purge safely without killing ourselves + code:soft_purge(Mod) andalso + code:delete(Mod) + end. + +path_groups(Targets, State) -> + [{Target, get_paths(Target, State)} || Target <- Targets]. + +app_groups(Targets, State) -> + [{Target, get_apps(Target, State)} || Target <- Targets]. + +get_paths(deps, State) -> + rebar_state:code_paths(State, all_deps); +get_paths(plugins, State) -> + rebar_state:code_paths(State, all_plugin_deps). + +get_apps(deps, State) -> + %% The code paths for deps also include the top level apps + %% and the extras, which we don't have here; we have to + %% add the apps by hand + rebar_state:all_deps(State) ++ + case rebar_state:project_apps(State) of + undefined -> []; + List -> List + end; +get_apps(plugins, State) -> + rebar_state:all_plugin_deps(State). diff --git a/src/rebar_plugins.erl b/src/rebar_plugins.erl index d58675d..a9c7e00 100644 --- a/src/rebar_plugins.erl +++ b/src/rebar_plugins.erl @@ -122,12 +122,10 @@ handle_plugin(Profile, Plugin, State, Upgrade) -> %% Add newly built deps and plugin to code path State3 = rebar_state:update_all_plugin_deps(State2, Apps), NewCodePaths = [rebar_app_info:ebin_dir(A) || A <- ToBuild], - AllPluginEbins = filelib:wildcard(filename:join([rebar_dir:plugins_dir(State), "*", "ebin"])), - CodePaths = PreBuiltPaths++(AllPluginEbins--ToBuild), - code:add_pathsa(NewCodePaths++CodePaths), %% Store plugin code paths so we can remove them when compiling project apps State4 = rebar_state:update_code_paths(State3, all_plugin_deps, PreBuiltPaths++NewCodePaths), + rebar_paths:set_paths([plugins, deps], State4), {plugin_providers(Plugin), State4} catch diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index 9e71ee7..3d3bd8a 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -58,7 +58,7 @@ do(State) -> do(State, Tests) -> ?INFO("Running Common Test suites...", []), - rebar_utils:update_code(rebar_state:code_paths(State, all_deps), [soft_purge]), + rebar_paths:set_paths([deps, plugins], State), %% Run ct provider prehooks Providers = rebar_state:providers(State), @@ -73,14 +73,14 @@ do(State, Tests) -> ok -> %% Run ct provider post hooks for all project apps and top level project hooks rebar_hooks:run_project_and_app_hooks(Cwd, post, ?PROVIDER, Providers, State), - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), + rebar_paths:set_paths([plugins, deps], State), {ok, State}; Error -> - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), + rebar_paths:set_paths([plugins, deps], State), Error end; Error -> - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), + rebar_paths:set_paths([plugins, deps], State), Error end. @@ -250,11 +250,9 @@ select_tests(State, ProjectApps, CmdOpts, CfgOpts) -> end, SysConfigs), %% NB: load the applications (from user directories too) to support OTP < 17 %% to our best ability. - OldPath = code:get_path(), - code:add_pathsa(rebar_state:code_paths(State, all_deps)), + rebar_paths:set_paths([deps, plugins], State), [application:load(Application) || Config <- Configs, {Application, _} <- Config], rebar_utils:reread_config(Configs), - code:set_path(OldPath), Opts = merge_opts(CmdOpts,CfgOpts), discover_tests(State, ProjectApps, Opts). diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index a509704..405f73a 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -37,10 +37,7 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> IsDepsOnly = is_deps_only(State), - DepsPaths = rebar_state:code_paths(State, all_deps), - PluginDepsPaths = rebar_state:code_paths(State, all_plugin_deps), - rebar_utils:remove_from_code_path(PluginDepsPaths), - code:add_pathsa(DepsPaths), + rebar_paths:set_paths([deps, plugins], State), Providers = rebar_state:providers(State), Deps = rebar_state:deps_to_build(State), @@ -50,11 +47,10 @@ do(State) -> true -> State; false -> - handle_project_apps(DepsPaths, Providers, State) + handle_project_apps(Providers, State) end, - rebar_utils:cleanup_code_path(rebar_state:code_paths(State1, default) - ++ rebar_state:code_paths(State, all_plugin_deps)), + rebar_paths:set_paths([plugins, deps], State1), {ok, State1}. @@ -62,7 +58,7 @@ is_deps_only(State) -> {Args, _} = rebar_state:command_parsed_args(State), proplists:get_value(deps_only, Args, false). -handle_project_apps(DepsPaths, Providers, State) -> +handle_project_apps(Providers, State) -> Cwd = rebar_state:dir(State), ProjectApps = rebar_state:project_apps(State), {ok, ProjectApps1} = rebar_digraph:compile_order(ProjectApps), @@ -76,7 +72,7 @@ handle_project_apps(DepsPaths, Providers, State) -> %% projects with structures like /apps/foo,/apps/bar,/test build_extra_dirs(State, ProjectApps2), - State3 = update_code_paths(State2, ProjectApps2, DepsPaths), + State3 = update_code_paths(State2, ProjectApps2), rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State2), case rebar_state:has_all_artifacts(State3) of @@ -176,11 +172,10 @@ compile(State, Providers, AppInfo) -> %% The rebar_otp_app compilation step is safe regarding the %% overall path management, so we can just load all plugins back %% in memory. - PluginDepsPaths = rebar_state:code_paths(State, all_plugin_deps), - code:add_pathsa(PluginDepsPaths), + rebar_paths:set_paths([plugins, deps], State), AppFileCompileResult = rebar_otp_app:compile(State, AppInfo4), %% Clean up after ourselves, leave things as they were. - rebar_utils:remove_from_code_path(PluginDepsPaths), + rebar_paths:set_paths([deps, plugins], State), case AppFileCompileResult of {ok, AppInfo5} -> @@ -206,21 +201,22 @@ build_app(AppInfo, State) -> case lists:keyfind(Type, 1, ProjectBuilders) of {_, Module} -> %% load plugins since thats where project builders would be - PluginDepsPaths = rebar_state:code_paths(State, all_plugin_deps), - code:add_pathsa(PluginDepsPaths), - case Module:build(AppInfo) of - ok -> - rebar_utils:remove_from_code_path(PluginDepsPaths); - {error, Reason} -> - throw({error, {Module, Reason}}) + rebar_paths:set_paths([plugins, deps], State), + Res = Module:build(AppInfo), + rebar_paths:set_paths([deps, plugins], State), + case Res of + ok -> ok; + {error, Reason} -> throw({error, {Module, Reason}}) end; _ -> throw(?PRV_ERROR({unknown_project_type, rebar_app_info:name(AppInfo), Type})) end end. -update_code_paths(State, ProjectApps, DepsPaths) -> + +update_code_paths(State, ProjectApps) -> ProjAppsPaths = paths_for_apps(ProjectApps), ExtrasPaths = paths_for_extras(State, ProjectApps), + DepsPaths = rebar_state:code_paths(State, all_deps), rebar_state:code_paths(State, all_deps, DepsPaths ++ ProjAppsPaths ++ ExtrasPaths). paths_for_apps(Apps) -> paths_for_apps(Apps, []). diff --git a/src/rebar_prv_dialyzer.erl b/src/rebar_prv_dialyzer.erl index 99a7698..585051c 100644 --- a/src/rebar_prv_dialyzer.erl +++ b/src/rebar_prv_dialyzer.erl @@ -85,7 +85,8 @@ short_desc() -> do(State) -> maybe_fix_env(), ?INFO("Dialyzer starting, this may take a while...", []), - code:add_pathsa(rebar_state:code_paths(State, all_deps)), + rebar_paths:unset_paths([plugins], State), % no plugins in analysis + rebar_paths:set_paths([deps], State), Plt = get_plt(State), try @@ -104,7 +105,7 @@ do(State) -> throw:{output_file_error, _, _} = Error -> ?PRV_ERROR(Error) after - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)) + rebar_paths:set_paths([plugins,deps], State) end. %% This is used to workaround dialyzer quirk discussed here diff --git a/src/rebar_prv_edoc.erl b/src/rebar_prv_edoc.erl index 9517335..c78296a 100644 --- a/src/rebar_prv_edoc.erl +++ b/src/rebar_prv_edoc.erl @@ -32,7 +32,7 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()} | {error, {module(), any()}}. do(State) -> - code:add_pathsa(rebar_state:code_paths(State, all_deps)), + rebar_paths:set_paths([deps, plugins], State), ProjectApps = rebar_state:project_apps(State), Providers = rebar_state:providers(State), EdocOpts = rebar_state:get(State, edoc_opts, []), @@ -64,7 +64,7 @@ do(State) -> {app_failed, AppName} end, rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State), - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), + rebar_paths:set_paths([plugins, deps], State), case Res of {app_failed, App} -> ?PRV_ERROR({app_failed, App}); diff --git a/src/rebar_prv_eunit.erl b/src/rebar_prv_eunit.erl index 4b71416..f120926 100644 --- a/src/rebar_prv_eunit.erl +++ b/src/rebar_prv_eunit.erl @@ -54,7 +54,7 @@ do(State, Tests) -> ?INFO("Performing EUnit tests...", []), setup_name(State), - rebar_utils:update_code(rebar_state:code_paths(State, all_deps), [soft_purge]), + rebar_paths:set_paths([deps, plugins], State), %% Run eunit provider prehooks Providers = rebar_state:providers(State), @@ -67,14 +67,14 @@ do(State, Tests) -> {ok, State1} -> %% Run eunit provider posthooks rebar_hooks:run_project_and_app_hooks(Cwd, post, ?PROVIDER, Providers, State1), - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), + rebar_paths:set_paths([plugins, deps], State), {ok, State1}; Error -> - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), + rebar_paths:set_paths([plugins, deps], State), Error end; Error -> - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), + rebar_paths:set_paths([plugins, deps], State), Error end. diff --git a/src/rebar_prv_xref.erl b/src/rebar_prv_xref.erl index 2405ebb..a6b1f73 100644 --- a/src/rebar_prv_xref.erl +++ b/src/rebar_prv_xref.erl @@ -36,8 +36,7 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - OldPath = code:get_path(), - code:add_pathsa(rebar_state:code_paths(State, all_deps)), + rebar_paths:set_paths([deps, plugins], State), XrefChecks = prepare(State), XrefIgnores = rebar_state:get(State, xref_ignores, []), %% Run xref checks @@ -48,7 +47,7 @@ do(State) -> QueryChecks = rebar_state:get(State, xref_queries, []), QueryResults = lists:foldl(fun check_query/2, [], QueryChecks), stopped = xref:stop(xref), - rebar_utils:cleanup_code_path(OldPath), + rebar_paths:set_paths([plugins, deps], State), case XrefResults =:= [] andalso QueryResults =:= [] of true -> {ok, State}; -- cgit v1.1 From 311ee6b1371c3eea3611dc5d7945b1b5667c75bd Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sat, 6 Oct 2018 11:38:33 -0400 Subject: Fix a bug in compiler path handling Also handle some formatting --- src/rebar_compiler.erl | 7 +++---- src/rebar_paths.erl | 13 ++++++------- src/rebar_prv_compile.erl | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-) (limited to 'src') diff --git a/src/rebar_compiler.erl b/src/rebar_compiler.erl index c2c514a..6e94cb2 100644 --- a/src/rebar_compiler.erl +++ b/src/rebar_compiler.erl @@ -34,11 +34,10 @@ -define(RE_PREFIX, "^(?!\\._)"). compile_all(Compilers, AppInfo) -> - OutDir = rebar_utils:to_list(rebar_app_info:out_dir(AppInfo)), - + EbinDir = rebar_utils:to_list(rebar_app_info:ebin_dir(AppInfo)), %% Make sure that outdir is on the path - ok = rebar_file_utils:ensure_dir(OutDir), - true = code:add_patha(filename:absname(OutDir)), + ok = rebar_file_utils:ensure_dir(EbinDir), + true = code:add_patha(filename:absname(EbinDir)), %% necessary for erlang:function_exported/3 to work as expected %% called here for clarity as it's required by both opts_changed/2 diff --git a/src/rebar_paths.erl b/src/rebar_paths.erl index bb43897..71f0016 100644 --- a/src/rebar_paths.erl +++ b/src/rebar_paths.erl @@ -130,11 +130,10 @@ misloaded_modules(Mods, GoodAppPaths, ModPaths) -> %% Identify paths that are invalid; i.e. app paths that cover an %% app in the desired group, but are not in the desired group. lists:usort( - [purge_mod(Mod) - || Mod <- Mods, - {_, Path} <- [lists:keyfind(Mod, 1, ModPaths)], - is_list(Path), % not 'preloaded' or mocked - not any_prefix(Path, GoodAppPaths)] + [Mod || Mod <- Mods, + {_, Path} <- [lists:keyfind(Mod, 1, ModPaths)], + is_list(Path), % not 'preloaded' or mocked + not any_prefix(Path, GoodAppPaths)] ). any_prefix(Path, Paths) -> @@ -168,10 +167,10 @@ get_apps(deps, State) -> %% The code paths for deps also include the top level apps %% and the extras, which we don't have here; we have to %% add the apps by hand - rebar_state:all_deps(State) ++ case rebar_state:project_apps(State) of undefined -> []; List -> List - end; + end ++ + rebar_state:all_deps(State); get_apps(plugins, State) -> rebar_state:all_plugin_deps(State). diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index 405f73a..4b36636 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -115,7 +115,7 @@ copy_and_build_project_apps(State, Providers, Apps) -> rebar_app_info:dir(AppInfo), rebar_app_info:out_dir(AppInfo)) || AppInfo <- Apps], - code:add_pathsa([rebar_app_info:out_dir(AppInfo) || AppInfo <- Apps]), + code:add_pathsa([rebar_app_info:ebin_dir(AppInfo) || AppInfo <- Apps]), [compile(State, Providers, AppInfo) || AppInfo <- Apps]. -- cgit v1.1 From af5cecd8eec9692f43d04ad53c8f28734012b873 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 8 Oct 2018 12:41:30 -0400 Subject: Clean path code, add tests, add clash detection Some finishing touch to that code --- src/rebar_paths.erl | 47 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/rebar_paths.erl b/src/rebar_paths.erl index 71f0016..23fc755 100644 --- a/src/rebar_paths.erl +++ b/src/rebar_paths.erl @@ -5,6 +5,11 @@ -type targets() :: [target(), ...]. -export_type([target/0, targets/0]). -export([set_paths/2, unset_paths/2]). +-export([clashing_apps/2]). + +-ifdef(TEST). +-export([misloaded_modules/3]). +-endif. -spec set_paths(targets(), rebar_state:t()) -> ok. set_paths(UserTargets, State) -> @@ -13,8 +18,8 @@ set_paths(UserTargets, State) -> Paths = lists:append([P || {_, P} <- GroupPaths]), [code:del_path(P) || P <- Paths], code:add_pathsa(lists:reverse(Paths)), - % set path breaks with escripts - %true = code:set_path(lists:append([P || {_, P} <- GroupPaths])), + % set path breaks with escripts; we gotta do it by hand + % true = code:set_path(lists:append([P || {_, P} <- GroupPaths])), AppGroups = app_groups(Targets, State), purge_and_load(AppGroups, code:all_loaded(), sets:new()), ok. @@ -28,6 +33,16 @@ unset_paths(UserTargets, State) -> purge(Paths, code:all_loaded()), ok. +clashing_apps(Targets, State) -> + AppGroups = app_groups(Targets, State), + AppNames = [{G, sets:from_list( + [rebar_app_info:name(App) || App <- Apps] + )} || {G, Apps} <- AppGroups], + clashing_app_names(sets:new(), AppNames, []). + +%%%%%%%%%%%%%%% +%%% PRIVATE %%% +%%%%%%%%%%%%%%% %% The paths are to be set in the reverse order; i.e. the default %% path is always last when possible (minimize cases where a build @@ -77,9 +92,9 @@ purge_and_load([{_Group, Apps}|Rest], ModPaths, Seen) -> App <- Apps, rebar_app_info:name(App) =:= AppName], %% 2) - %% TODO: add extra dirs (and test), and possibly the stdlib + %% (no need for extra_src_dirs since those get put into ebin; + %% also no need for OTP libs; we want to allow overtaking them) GoodAppPaths = [rebar_app_info:ebin_dir(App) || App <- GoodApps], - %% ++ [code:lib_dir()], %% 3) [begin AtomApp = binary_to_atom(AppName, utf8), @@ -118,12 +133,12 @@ purge_and_load([{_Group, Apps}|Rest], ModPaths, Seen) -> purge_and_load(Rest, ModPaths, sets:union(Seen, sets:from_list(AppNames))). - purge(Paths, ModPaths) -> + SortedPaths = lists:sort(Paths), lists:map(fun purge_mod/1, lists:usort( [Mod || {Mod, Path} <- ModPaths, is_list(Path), % not 'preloaded' or mocked - any_prefix(Path, Paths)] + any_prefix(Path, SortedPaths)] )). misloaded_modules(Mods, GoodAppPaths, ModPaths) -> @@ -152,6 +167,26 @@ purge_mod(Mod) -> code:delete(Mod) end. + +%% This is a tricky O(n²) check since we want to +%% know whether an app clashes with any of the top priority groups. +%% +%% For example, let's say we have `[deps, plugins]', then we want +%% to find the plugins that clash with deps: +%% +%% `[{deps, [ClashingPlugins]}, {plugins, []}]' +%% +%% In case we'd ever have alternative or additional types, we can +%% find all clashes from other 'groups'. +clashing_app_names(_, [], Acc) -> + lists:reverse(Acc); +clashing_app_names(PrevNames, [{G,AppNames} | Rest], Acc) -> + CurrentNames = sets:subtract(AppNames, PrevNames), + NextNames = sets:subtract(sets:union([A || {_, A} <- Rest]), PrevNames), + Clashes = sets:intersection(CurrentNames, NextNames), + NewAcc = [{G, sets:to_list(Clashes)} | Acc], + clashing_app_names(sets:union(PrevNames, CurrentNames), Rest, NewAcc). + path_groups(Targets, State) -> [{Target, get_paths(Target, State)} || Target <- Targets]. -- cgit v1.1 From dada4e36e6d9a5c4b41bbe1f68389520e7c59ace Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 11 Oct 2018 08:38:37 -0400 Subject: Optimize path handling - Only set paths that need to be put as a priority - Clean up paths before leaving API mode The first point accounted for some performance cost, but the latter one explains the 40% overhead in test runs: since rebar3 calls rebar3 a lot with a bunch of fake apps, and that the new mechanism for path handling by default does not _remove_ paths, it just _orders_ them, we would end up in a situation where as the tests ran, more and more fake paths would get added to the VM. By the time the run was over, all path handling would take longer since more paths needed filtering every time. By resetting paths at the end of an API run, we prevent a given 'project' from polluting another one's runtime and performance once the API successfully returns. --- src/rebar3.erl | 15 +++++++++++- src/rebar_hooks.erl | 7 ++---- src/rebar_paths.erl | 59 +++++++++++++++++++++++------------------------ src/rebar_plugins.erl | 2 +- src/rebar_prv_compile.erl | 12 +++++----- src/rebar_prv_xref.erl | 3 +-- 6 files changed, 53 insertions(+), 45 deletions(-) (limited to 'src') diff --git a/src/rebar3.erl b/src/rebar3.erl index c931a85..059d530 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -175,7 +175,20 @@ run_aux(State, RawArgs) -> State10 = rebar_state:code_paths(State9, default, code:get_path()), - rebar_core:init_command(rebar_state:command_args(State10, Args), Task). + case rebar_core:init_command(rebar_state:command_args(State10, Args), Task) of + {ok, State11} -> + case rebar_state:get(State11, caller, command_line) of + api -> + rebar_paths:unset_paths([deps, plugins], State11), + {ok, State11}; + _ -> + {ok, State11} + end; + Other -> + Other + end. + + %% @doc set up base configuration having to do with verbosity, where %% to find config files, and so on, and return an internal rebar3 state term. diff --git a/src/rebar_hooks.erl b/src/rebar_hooks.erl index f2ef8a3..358458e 100644 --- a/src/rebar_hooks.erl +++ b/src/rebar_hooks.erl @@ -42,9 +42,7 @@ run_provider_hooks_(Dir, Type, Command, Providers, TypeHooks, State) -> [] -> State; HookProviders -> - %PluginDepsPaths = lists:usort(rebar_state:code_paths(State, all_plugin_deps)), - %code:add_pathsa(PluginDepsPaths), - rebar_paths:set_paths([plugins, deps], State), + rebar_paths:set_paths([plugins], State), Providers1 = rebar_state:providers(State), State1 = rebar_state:providers(rebar_state:dir(State, Dir), Providers++Providers1), case rebar_core:do(HookProviders, State1) of @@ -52,8 +50,7 @@ run_provider_hooks_(Dir, Type, Command, Providers, TypeHooks, State) -> ?DEBUG(format_error({bad_provider, Type, Command, ProviderName}), []), throw(?PRV_ERROR({bad_provider, Type, Command, ProviderName})); {ok, State2} -> - %rebar_utils:remove_from_code_path(PluginDepsPaths), - rebar_paths:set_paths([deps, plugins], State2), + rebar_paths:set_paths([deps], State2), State2 end end. diff --git a/src/rebar_paths.erl b/src/rebar_paths.erl index 23fc755..900443d 100644 --- a/src/rebar_paths.erl +++ b/src/rebar_paths.erl @@ -1,3 +1,5 @@ +%% BEFORE THIS FIX: rebar3 ct 266.78s user 144.06s system 144% cpu 4:33.70 total +%% CURRENT TIME: rebar3 ct 419.30s user 301.00s system 152% cpu 7:51.98 total -module(rebar_paths). -include("rebar.hrl"). @@ -8,20 +10,17 @@ -export([clashing_apps/2]). -ifdef(TEST). --export([misloaded_modules/3]). +-export([misloaded_modules/2]). -endif. -spec set_paths(targets(), rebar_state:t()) -> ok. set_paths(UserTargets, State) -> Targets = normalize_targets(UserTargets), GroupPaths = path_groups(Targets, State), - Paths = lists:append([P || {_, P} <- GroupPaths]), - [code:del_path(P) || P <- Paths], - code:add_pathsa(lists:reverse(Paths)), - % set path breaks with escripts; we gotta do it by hand - % true = code:set_path(lists:append([P || {_, P} <- GroupPaths])), + Paths = lists:append(lists:reverse([P || {_, P} <- GroupPaths])), + code:add_pathsa(Paths), AppGroups = app_groups(Targets, State), - purge_and_load(AppGroups, code:all_loaded(), sets:new()), + purge_and_load(AppGroups, sets:new()), ok. -spec unset_paths(targets(), rebar_state:t()) -> ok. @@ -33,6 +32,7 @@ unset_paths(UserTargets, State) -> purge(Paths, code:all_loaded()), ok. +-spec clashing_apps(targets(), rebar_state:t()) -> [{target(), [binary()]}]. clashing_apps(Targets, State) -> AppGroups = app_groups(Targets, State), AppNames = [{G, sets:from_list( @@ -66,9 +66,9 @@ normalize_targets(List) -> ), lists:reverse(TmpList). -purge_and_load([], _, _) -> +purge_and_load([], _) -> ok; -purge_and_load([{_Group, Apps}|Rest], ModPaths, Seen) -> +purge_and_load([{_Group, Apps}|Rest], Seen) -> %% We have: a list of all applications in the current priority group, %% a list of all loaded modules with their active path, and a list of %% seen applications. @@ -79,7 +79,12 @@ purge_and_load([{_Group, Apps}|Rest], ModPaths, Seen) -> %% 3. unload and reload apps that may have changed paths in order %% to get updated module lists and specs %% (we ignore started apps and apps that have not run for this) - %% 4. create a list of modules to check from that app list + %% This part turns out to be the bottleneck of this module, so + %% to speed it up, using clash detection proves useful: + %% only reload apps that clashed since others are unlikely to + %% conflict in significant ways + %% 4. create a list of modules to check from that app list—only loaded + %% modules make sense to check. %% 5. check the modules to match their currently loaded paths with %% the path set from the apps in the current group; modules %% that differ must be purged; others can stay @@ -126,27 +131,29 @@ purge_and_load([{_Group, Apps}|Rest], ModPaths, Seen) -> end || App <- GoodApps, AppName <- [binary_to_atom(rebar_app_info:name(App), utf8)]] ), + ModPaths = [{Mod,Path} || Mod <- CandidateMods, + erlang:function_exported(Mod, module_info, 0), + {file, Path} <- [code:is_loaded(Mod)]], %% 5) - Mods = misloaded_modules(CandidateMods, GoodAppPaths, ModPaths), + Mods = misloaded_modules(GoodAppPaths, ModPaths), [purge_mod(Mod) || Mod <- Mods], - purge_and_load(Rest, ModPaths, - sets:union(Seen, sets:from_list(AppNames))). + + purge_and_load(Rest, sets:union(Seen, sets:from_list(AppNames))). purge(Paths, ModPaths) -> SortedPaths = lists:sort(Paths), - lists:map(fun purge_mod/1, lists:usort( - [Mod || {Mod, Path} <- ModPaths, - is_list(Path), % not 'preloaded' or mocked - any_prefix(Path, SortedPaths)] - )). + lists:map(fun purge_mod/1, + [Mod || {Mod, Path} <- ModPaths, + is_list(Path), % not 'preloaded' or mocked + any_prefix(Path, SortedPaths)] + ). -misloaded_modules(Mods, GoodAppPaths, ModPaths) -> +misloaded_modules(GoodAppPaths, ModPaths) -> %% Identify paths that are invalid; i.e. app paths that cover an %% app in the desired group, but are not in the desired group. lists:usort( - [Mod || Mod <- Mods, - {_, Path} <- [lists:keyfind(Mod, 1, ModPaths)], + [Mod || {Mod, Path} <- ModPaths, is_list(Path), % not 'preloaded' or mocked not any_prefix(Path, GoodAppPaths)] ). @@ -157,15 +164,7 @@ any_prefix(Path, Paths) -> %% assume paths currently set are good; only unload a module so next call %% uses the correctly set paths purge_mod(Mod) -> - case erlang:check_process_code(self(), Mod) of - false -> - code:purge(Mod), - code:delete(Mod); - _ -> - %% cannot purge safely without killing ourselves - code:soft_purge(Mod) andalso - code:delete(Mod) - end. + code:soft_purge(Mod) andalso code:delete(Mod). %% This is a tricky O(n²) check since we want to diff --git a/src/rebar_plugins.erl b/src/rebar_plugins.erl index a9c7e00..2a78c6e 100644 --- a/src/rebar_plugins.erl +++ b/src/rebar_plugins.erl @@ -125,7 +125,7 @@ handle_plugin(Profile, Plugin, State, Upgrade) -> %% Store plugin code paths so we can remove them when compiling project apps State4 = rebar_state:update_code_paths(State3, all_plugin_deps, PreBuiltPaths++NewCodePaths), - rebar_paths:set_paths([plugins, deps], State4), + rebar_paths:set_paths([plugins], State4), {plugin_providers(Plugin), State4} catch diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index 4b36636..ee96d9f 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -37,7 +37,7 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> IsDepsOnly = is_deps_only(State), - rebar_paths:set_paths([deps, plugins], State), + rebar_paths:set_paths([deps], State), Providers = rebar_state:providers(State), Deps = rebar_state:deps_to_build(State), @@ -50,7 +50,7 @@ do(State) -> handle_project_apps(Providers, State) end, - rebar_paths:set_paths([plugins, deps], State1), + rebar_paths:set_paths([plugins], State1), {ok, State1}. @@ -172,10 +172,10 @@ compile(State, Providers, AppInfo) -> %% The rebar_otp_app compilation step is safe regarding the %% overall path management, so we can just load all plugins back %% in memory. - rebar_paths:set_paths([plugins, deps], State), + rebar_paths:set_paths([plugins], State), AppFileCompileResult = rebar_otp_app:compile(State, AppInfo4), - %% Clean up after ourselves, leave things as they were. - rebar_paths:set_paths([deps, plugins], State), + %% Clean up after ourselves, leave things as they were with deps first + rebar_paths:set_paths([deps], State), case AppFileCompileResult of {ok, AppInfo5} -> @@ -203,7 +203,7 @@ build_app(AppInfo, State) -> %% load plugins since thats where project builders would be rebar_paths:set_paths([plugins, deps], State), Res = Module:build(AppInfo), - rebar_paths:set_paths([deps, plugins], State), + rebar_paths:set_paths([deps], State), case Res of ok -> ok; {error, Reason} -> throw({error, {Module, Reason}}) diff --git a/src/rebar_prv_xref.erl b/src/rebar_prv_xref.erl index a6b1f73..12063d5 100644 --- a/src/rebar_prv_xref.erl +++ b/src/rebar_prv_xref.erl @@ -36,7 +36,7 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - rebar_paths:set_paths([deps, plugins], State), + rebar_paths:set_paths([deps], State), XrefChecks = prepare(State), XrefIgnores = rebar_state:get(State, xref_ignores, []), %% Run xref checks @@ -47,7 +47,6 @@ do(State) -> QueryChecks = rebar_state:get(State, xref_queries, []), QueryResults = lists:foldl(fun check_query/2, [], QueryChecks), stopped = xref:stop(xref), - rebar_paths:set_paths([plugins, deps], State), case XrefResults =:= [] andalso QueryResults =:= [] of true -> {ok, State}; -- cgit v1.1 From a80d1988963698717548ba269d7e5f915fca4e27 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sun, 14 Oct 2018 09:37:50 -0600 Subject: fix finding transitive deps with prerelease versions (#1914) --- src/rebar_packages.erl | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 7676213..6faa5e1 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -55,11 +55,13 @@ get_all_names(State) -> _='_'}, [], ['$1']}])). --spec get_package_versions(unicode:unicode_binary(), unicode:unicode_binary(), +-spec get_package_versions(unicode:unicode_binary(), ec_semver:semver(), + unicode:unicode_binary(), ets:tid(), rebar_state:t()) -> [vsn()]. -get_package_versions(Dep, Repo, Table, State) -> +get_package_versions(Dep, {_, AlphaInfo}, Repo, Table, State) -> ?MODULE:verify_table(State), - AllowPreRelease = rebar_state:get(State, deps_allow_prerelease, false), + AllowPreRelease = rebar_state:get(State, deps_allow_prerelease, false) + orelse AlphaInfo =/= {[],[]}, ets:select(Table, [{#package{key={Dep, {'$1', '$2'}, Repo}, _='_'}, [{'==', '$2', {{[],[]}}} || not AllowPreRelease], [{{'$1', '$2'}}]}]). @@ -180,7 +182,7 @@ find_highest_matching(Dep, Constraint, Repo, Table, State) -> end. find_highest_matching_(Dep, Constraint, #{name := Repo}, Table, State) -> - try get_package_versions(Dep, Repo, Table, State) of + try get_package_versions(Dep, Constraint, Repo, Table, State) of [Vsn] -> handle_single_vsn(Vsn, Constraint); Vsns -> @@ -292,7 +294,7 @@ resolve_version(Dep, DepVsn, Hash, HexRegistry, State) when is_binary(Hash) -> end; resolve_version(Dep, undefined, Hash, HexRegistry, State) -> Fun = fun(Repo) -> - case highest_matching(Dep, "0", Repo, HexRegistry, State) of + case highest_matching(Dep, {0,{[],[]}}, Repo, HexRegistry, State) of none -> not_found; {ok, Vsn} -> @@ -360,9 +362,9 @@ resolve_version_(Dep, DepVsn, Repo, HexRegistry, State) -> end. rm_ws(<<" ", R/binary>>) -> - rm_ws(R); + ec_semver:parse(rm_ws(R)); rm_ws(R) -> - R. + ec_semver:parse(R). valid_vsn(Vsn) -> %% Regepx from https://github.com/sindresorhus/semver-regex/blob/master/index.js @@ -375,7 +377,7 @@ highest_matching(Dep, Vsn, Repo, HexRegistry, State) -> find_highest_matching_(Dep, Vsn, #{name => Repo}, HexRegistry, State). cmp(Dep, Vsn, Repo, HexRegistry, State, CmpFun) -> - case get_package_versions(Dep, Repo, HexRegistry, State) of + case get_package_versions(Dep, Vsn, Repo, HexRegistry, State) of [] -> none; Vsns -> @@ -398,7 +400,7 @@ cmp_(BestMatch, MinVsn, [Vsn | R], CmpFun) -> %% We need to treat this differently since we want a version that is LOWER but %% the higest possible one. cmpl(Dep, Vsn, Repo, HexRegistry, State, CmpFun) -> - case get_package_versions(Dep, Repo, HexRegistry, State) of + case get_package_versions(Dep, Vsn, Repo, HexRegistry, State) of [] -> none; Vsns -> -- cgit v1.1 From fb6de6e0e5dc0da1c4e64c166bcb69327420cb60 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sun, 14 Oct 2018 12:43:28 -0400 Subject: Drop outdated comments --- src/rebar_paths.erl | 2 -- 1 file changed, 2 deletions(-) (limited to 'src') diff --git a/src/rebar_paths.erl b/src/rebar_paths.erl index 900443d..82c0218 100644 --- a/src/rebar_paths.erl +++ b/src/rebar_paths.erl @@ -1,5 +1,3 @@ -%% BEFORE THIS FIX: rebar3 ct 266.78s user 144.06s system 144% cpu 4:33.70 total -%% CURRENT TIME: rebar3 ct 419.30s user 301.00s system 152% cpu 7:51.98 total -module(rebar_paths). -include("rebar.hrl"). -- cgit v1.1 From 86519cf743204eab1d922ca8133fbf00c66f9ee8 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sun, 14 Oct 2018 10:50:16 -0600 Subject: fix resolving versions from vcs (#1915) app_info was turning any vsn into a binary which the vcs_vsn function interprets as being an actual version and not something like <<"git">>. original_vsn shouldn't be converted as it is then not longer "original". --- src/rebar_app_info.erl | 10 +++++----- src/rebar_pkg_resource.erl | 2 +- src/rebar_state.erl | 6 +++++- src/rebar_utils.erl | 11 ++++++++++- 4 files changed, 21 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index b1c6243..9dfe278 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -124,14 +124,14 @@ new(AppName) -> {ok, t()}. new(AppName, Vsn) -> {ok, #app_info_t{name=rebar_utils:to_binary(AppName), - original_vsn=rebar_utils:to_binary(Vsn)}}. + original_vsn=Vsn}}. %% @doc build a complete version of the app info with all fields set. -spec new(atom() | binary() | string(), binary() | string(), file:name()) -> {ok, t()}. new(AppName, Vsn, Dir) -> {ok, #app_info_t{name=rebar_utils:to_binary(AppName), - original_vsn=rebar_utils:to_binary(Vsn), + original_vsn=Vsn, dir=rebar_utils:to_list(Dir), out_dir=rebar_utils:to_list(Dir), ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin")}}. @@ -141,7 +141,7 @@ new(AppName, Vsn, Dir) -> {ok, t()}. new(AppName, Vsn, Dir, Deps) -> {ok, #app_info_t{name=rebar_utils:to_binary(AppName), - original_vsn=rebar_utils:to_binary(Vsn), + original_vsn=Vsn, dir=rebar_utils:to_list(Dir), out_dir=rebar_utils:to_list(Dir), ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin"), @@ -153,7 +153,7 @@ new(AppName, Vsn, Dir, Deps) -> new(Parent, AppName, Vsn, Dir, Deps) -> {ok, #app_info_t{name=rebar_utils:to_binary(AppName), parent=Parent, - original_vsn=rebar_utils:to_binary(Vsn), + original_vsn=Vsn, dir=rebar_utils:to_list(Dir), out_dir=rebar_utils:to_list(Dir), ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin"), @@ -390,7 +390,7 @@ original_vsn(#app_info_t{original_vsn=Vsn}) -> %% asking for a semver) -spec original_vsn(t(), binary() | string()) -> t(). original_vsn(AppInfo=#app_info_t{}, Vsn) -> - AppInfo#app_info_t{original_vsn=rebar_utils:to_binary(Vsn)}. + AppInfo#app_info_t{original_vsn=Vsn}. %% @doc returns the list of applications the app depends on. -spec applications(t()) -> list(). diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 97eabb6..823b7fc 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -60,7 +60,7 @@ lock(AppInfo, _) -> Res :: boolean(). needs_update(AppInfo, _) -> {pkg, _Name, Vsn, _Hash, _} = rebar_app_info:source(AppInfo), - case rebar_app_info:original_vsn(AppInfo) =:= rebar_utils:to_binary(Vsn) of + case rebar_utils:to_binary(rebar_app_info:original_vsn(AppInfo)) =:= rebar_utils:to_binary(Vsn) of true -> false; false -> diff --git a/src/rebar_state.erl b/src/rebar_state.erl index c8a9400..31d3a08 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -43,7 +43,7 @@ project_builders/1, add_project_builder/3, - create_resources/2, + create_resources/2, set_resources/2, resources/1, resources/2, add_resource/2, providers/1, providers/2, add_provider/2, allow_provider_overrides/1, allow_provider_overrides/2 @@ -368,6 +368,10 @@ namespace(State=#state_t{}, Namespace) -> resources(#state_t{resources=Resources}) -> Resources. +-spec set_resources(t(), [{rebar_resource_v2:type(), module()}]) -> t(). +set_resources(State, Resources) -> + State#state_t{resources=Resources}. + -spec resources(t(), [{rebar_resource_v2:type(), module()}]) -> t(). resources(State, NewResources) -> lists:foldl(fun(Resource, StateAcc) -> diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 419802a..1769b79 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -715,6 +715,15 @@ escript_foldl(Fun, Acc, File) -> Error end. +%% TODO: this is just for rebar3_hex and maybe other plugins +%% but eventually it should be dropped +vcs_vsn(OriginalVsn, Dir, Resources) when is_list(Dir) , + is_list(Resources) -> + ?WARN("Using deprecated rebar_utils:vcs_vsn/3. Please upgrade your plugins.", []), + FakeState = rebar_state:new(), + {ok, AppInfo} = rebar_app_info:new(fake, OriginalVsn, Dir), + vcs_vsn(AppInfo, OriginalVsn, + rebar_state:set_resources(FakeState, Resources)); vcs_vsn(AppInfo, Vcs, State) -> case vcs_vsn_cmd(AppInfo, Vcs, State) of {plain, VsnString} -> @@ -728,7 +737,7 @@ vcs_vsn(AppInfo, Vcs, State) -> end. %% Temp work around for repos like relx that use "semver" -vcs_vsn_cmd(_AppInfo, Vsn, _) when is_binary(Vsn) -> +vcs_vsn_cmd(_, Vsn, _) when is_binary(Vsn) -> {plain, Vsn}; vcs_vsn_cmd(AppInfo, VCS, State) when VCS =:= semver ; VCS =:= "semver" -> vcs_vsn_cmd(AppInfo, git, State); -- cgit v1.1 From c9fbb4a44feac75a325c5831293901d5cb12075a Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 18 Oct 2018 15:38:48 -0400 Subject: Fix shell hook expansion on windows Dollar Variable expansion (`$VAR`) was inadvertently disabled for windows variables, although %VARIABLES% already worked. This reduced the portability of hooks in general. Additionally, tests would fail on windows due to bad quoting of paths: the path C:/a/b/c would fail when passed to the command `cmd /q /c C:/a/b/c` because it would interpret /a /b and /c as 3 options. Using quotes makes the tests pass. --- src/rebar_utils.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 1769b79..11add61 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -156,7 +156,7 @@ sh_send(Command0, String, Options0) -> Options = [expand_sh_flag(V) || V <- proplists:compact(Options0 ++ DefaultOptions)], - Command = lists:flatten(patch_on_windows(Command0, proplists:get_value(env, Options, []))), + Command = lists:flatten(patch_on_windows(Command0, proplists:get_value(env, Options0, []))), PortSettings = proplists:get_all_values(port_settings, Options) ++ [exit_status, {line, 16384}, use_stdio, stderr_to_stdout, hide], Port = open_port({spawn, Command}, PortSettings), @@ -187,7 +187,7 @@ sh(Command0, Options0) -> ErrorHandler = proplists:get_value(error_handler, Options), OutputHandler = proplists:get_value(output_handler, Options), - Command = lists:flatten(patch_on_windows(Command0, proplists:get_value(env, Options, []))), + Command = lists:flatten(patch_on_windows(Command0, proplists:get_value(env, Options0, []))), PortSettings = proplists:get_all_values(port_settings, Options) ++ [exit_status, {line, 16384}, use_stdio, stderr_to_stdout, hide, eof], ?DEBUG("Port Cmd: ~ts\nPort Opts: ~p\n", [Command, PortSettings]), -- cgit v1.1 From bf039fa4c4d7ea9dc0d9a842f2ab2aa93a9c61d3 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sat, 20 Oct 2018 09:54:51 -0400 Subject: Fallback when logging isn't initialized In some cases, such as when the global rebar.config file contains typoes and invalid terms, the rebar3 executable fails when trying to log the error since it hasn't been set yet, such as in #1792 This patch fixes that by going for a fallback mechanism. --- src/rebar_log.erl | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/rebar_log.erl b/src/rebar_log.erl index 9150346..7fc2312 100644 --- a/src/rebar_log.erl +++ b/src/rebar_log.erl @@ -93,11 +93,18 @@ get_level() -> end. log(Level = error, Str, Args) -> - {ok, LogState} = application:get_env(rebar, log), - ec_cmd_log:Level(LogState, lists:flatten(cf:format("~!^~ts~n", [Str])), Args); + case application:get_env(rebar, log) of + {ok, LogState} -> + NewStr = lists:flatten(cf:format("~!^~ts~n", [Str])), + ec_cmd_log:Level( LogState, NewStr, Args); + undefined -> % fallback + io:format(standard_error, Str++"~n", Args) + end; log(Level, Str, Args) -> - {ok, LogState} = application:get_env(rebar, log), - ec_cmd_log:Level(LogState, Str++"~n", Args). + case application:get_env(rebar, log) of + {ok, LogState} -> ec_cmd_log:Level(LogState, Str++"~n", Args); + undefined -> io:format(Str++"~n", Args) + end. crashdump(Str, Args) -> crashdump("rebar3.crashdump", Str, Args). -- cgit v1.1 From a279020d1b860313719a3ba07972004b8235146d Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sat, 20 Oct 2018 16:14:03 -0400 Subject: check if git/hg is installed This PR is a simpler and mergeable version of #1853 by @shamis, since the provider changed format enough to make merging difficult. Compared to #1853, this also puts the responsibility on each resource to check rather than adding a new optional callback. The process dictionary is use as a warning/check cache. --- src/rebar_git_resource.erl | 19 +++++++++++++++++++ src/rebar_hg_resource.erl | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) (limited to 'src') diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl index 29c9ad7..03aa6d8 100644 --- a/src/rebar_git_resource.erl +++ b/src/rebar_git_resource.erl @@ -21,6 +21,7 @@ init(Type, _State) -> {ok, Resource}. lock(AppInfo, _) -> + check_type_support(), lock_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). lock_(AppDir, {git, Url, _}) -> @@ -43,6 +44,7 @@ lock_(AppDir, {git, Url}) -> %% Return true if either the git url or tag/branch/ref is not the same as the currently %% checked out git repo for the dep needs_update(AppInfo, _) -> + check_type_support(), needs_update_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). needs_update_(Dir, {git, Url, {tag, Tag}}) -> @@ -111,6 +113,7 @@ parse_git_url(not_scp, Url) -> end. download(TmpDir, AppInfo, State, _) -> + check_type_support(), case download_(TmpDir, rebar_app_info:source(AppInfo), State) of {ok, _} -> ok; @@ -307,3 +310,19 @@ parse_tags(Dir) -> end end end. + +check_type_support() -> + case get({is_supported, ?MODULE}) of + true -> + ok; + _ -> + case rebar_utils:sh("git --version", [{return_on_error, true}, + {use_stdout, false}]) of + {error, _} -> + ?ABORT("git not installed", []); + _ -> + put({is_supported, ?MODULE}, true), + ok + end + end. + diff --git a/src/rebar_hg_resource.erl b/src/rebar_hg_resource.erl index 21d4b9d..8139d04 100644 --- a/src/rebar_hg_resource.erl +++ b/src/rebar_hg_resource.erl @@ -18,6 +18,7 @@ init(Type, _State) -> {ok, Resource}. lock(AppInfo, _) -> + check_type_support(), lock_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). lock_(AppDir, {hg, Url, _}) -> @@ -61,6 +62,7 @@ needs_update_(Dir, {hg, Url, Ref}) -> not ((LocalRef =:= TargetRef) andalso compare_url(Dir, Url)). download(TmpDir, AppInfo, State, _) -> + check_type_support(), case download_(TmpDir, rebar_app_info:source(AppInfo), State) of {ok, _} -> ok; @@ -110,6 +112,7 @@ download_(Dir, {hg, Url, Rev}, _State) -> [{cd, filename:dirname(Dir)}]). make_vsn(AppInfo, _) -> + check_type_support(), make_vsn_(rebar_app_info:dir(AppInfo)). make_vsn_(Dir) -> @@ -183,3 +186,19 @@ parse_hg_url("http://" ++ HostPath) -> parse_hg_url("https://" ++ HostPath) -> [Host | Path] = rebar_string:lexemes(HostPath, "/"), {Host, filename:rootname(filename:join(Path), ".hg")}. + +check_type_support() -> + case get({is_supported, ?MODULE}) of + true -> + ok; + false -> + case rebar_utils:sh("hg --version", [{return_on_error, true}, + {use_stdout, false}]) of + {error, _} -> + ?ABORT("hg not installed", []); + _ -> + put({is_supported, ?MODULE}, true), + ok + end + end. + -- cgit v1.1 From 9b03dacf2b7829b584d26a999f80c315ae8ce897 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 22 Oct 2018 19:46:09 -0400 Subject: Allow Breakpoints during task runs This is mostly useful for tests, where a test suite of any kind can be interrupted halfway through so that the user can probe the running system to see what is happening. This is done as follows: 1. the user must call `r3:break()` in a test suite 2. the user runs the task as `r3:async_do(ct)` 3. the test holds up and the user can do whatever 4. the user calls `r3:resume()` and the test proceeds as normal A safeguard is added so that breakpoints are only triggered in the shell in async mode Sample session: $ rebar3 shell ... 1> rebar_agent:async_do(ct). ok ... Running Common Test suites... %%% rebar_alias_SUITE: . === BREAK === 2> % 2> r3:resume(). ok 3> ..... %%% rebar_as_SUITE: ........... %%% rebar_compile_SUITE: ...... ... --- src/r3.erl | 43 ++++++++++++++++++++++++++++++++++++++++++- src/rebar_agent.erl | 38 +++++++++++++++++++++++++++++++++++++- 2 files changed, 79 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/r3.erl b/src/r3.erl index bbf9eea..a79cc3a 100644 --- a/src/r3.erl +++ b/src/r3.erl @@ -1,8 +1,9 @@ %%% @doc external alias for `rebar_agent' for more convenient %%% calls from a shell. -module(r3). --export([do/1, do/2]). +-export([do/1, do/2, async_do/1, async_do/2, break/0, resume/0]). -export(['$handle_undefined_function'/2]). +-include("rebar.hrl"). %% @doc alias for `rebar_agent:do/1' -spec do(atom()) -> ok | {error, term()}. @@ -12,6 +13,46 @@ do(Command) -> rebar_agent:do(Command). -spec do(atom(), atom()) -> ok | {error, term()}. do(Namespace, Command) -> rebar_agent:do(Namespace, Command). +%% @async_doc alias for `rebar_agent:async_do/1' +-spec async_do(atom()) -> ok | {error, term()}. +async_do(Command) -> rebar_agent:async_do(Command). + +%% @async_doc alias for `rebar_agent:async_do/2' +-spec async_do(atom(), atom()) -> ok | {error, term()}. +async_do(Namespace, Command) -> rebar_agent:async_do(Namespace, Command). + +break() -> + case whereis(rebar_agent) of % is the shell running + undefined -> + ok; + Pid -> + {dictionary, Dict} = process_info(Pid, dictionary), + case lists:keyfind(cmd_type, 1, Dict) of + {cmd_type, async} -> + Self = self(), + Ref = make_ref(), + spawn_link(fun() -> + register(r3_breakpoint_handler, self()), + receive + resume -> + Self ! Ref + end + end), + io:format(user, "~n=== BREAK ===~n", []), + receive + Ref -> ok + end; + _ -> + ?DEBUG("ignoring breakpoint since command is not run " + "in async mode", []), + ok + end + end. + +resume() -> + r3_breakpoint_handler ! resume, + ok. + %% @private defer to rebar_agent '$handle_undefined_function'(Cmd, Args) -> rebar_agent:'$handle_undefined_function'(Cmd, Args). diff --git a/src/rebar_agent.erl b/src/rebar_agent.erl index 445ae54..b4734f1 100644 --- a/src/rebar_agent.erl +++ b/src/rebar_agent.erl @@ -1,7 +1,7 @@ %%% @doc Runs a process that holds a rebar3 state and can be used %%% to statefully maintain loaded project state into a running VM. -module(rebar_agent). --export([start_link/1, do/1, do/2]). +-export([start_link/1, do/1, do/2, async_do/1, async_do/2]). -export(['$handle_undefined_function'/2]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, @@ -35,6 +35,18 @@ do(Namespace, Command) when is_atom(Namespace), is_atom(Command) -> do(Namespace, Args) when is_atom(Namespace), is_list(Args) -> gen_server:call(?MODULE, {cmd, Namespace, do, Args}, infinity). +-spec async_do(atom()) -> ok | {error, term()}. +async_do(Command) when is_atom(Command) -> + gen_server:cast(?MODULE, {cmd, Command}); +async_do(Args) when is_list(Args) -> + gen_server:cast(?MODULE, {cmd, default, do, Args}). + +-spec async_do(atom(), atom()) -> ok. +async_do(Namespace, Command) when is_atom(Namespace), is_atom(Command) -> + gen_server:cast(?MODULE, {cmd, Namespace, Command}); +async_do(Namespace, Args) when is_atom(Namespace), is_list(Args) -> + gen_server:cast(?MODULE, {cmd, Namespace, do, Args}). + '$handle_undefined_function'(Cmd, [Namespace, Args]) -> gen_server:call(?MODULE, {cmd, Namespace, Cmd, Args}, infinity); '$handle_undefined_function'(Cmd, [Args]) -> @@ -54,20 +66,44 @@ init(State) -> %% @private handle_call({cmd, Command}, _From, State=#state{state=RState, cwd=Cwd}) -> MidState = maybe_show_warning(State), + put(cmd_type, sync), {Res, NewRState} = run(default, Command, "", RState, Cwd), + put(cmd_type, undefined), {reply, Res, MidState#state{state=NewRState}, hibernate}; handle_call({cmd, Namespace, Command}, _From, State = #state{state=RState, cwd=Cwd}) -> MidState = maybe_show_warning(State), + put(cmd_type, sync), {Res, NewRState} = run(Namespace, Command, "", RState, Cwd), + put(cmd_type, undefined), {reply, Res, MidState#state{state=NewRState}, hibernate}; handle_call({cmd, Namespace, Command, Args}, _From, State = #state{state=RState, cwd=Cwd}) -> MidState = maybe_show_warning(State), + put(cmd_type, sync), {Res, NewRState} = run(Namespace, Command, Args, RState, Cwd), + put(cmd_type, undefined), {reply, Res, MidState#state{state=NewRState}, hibernate}; handle_call(_Call, _From, State) -> {noreply, State}. %% @private +handle_cast({cmd, Command}, State=#state{state=RState, cwd=Cwd}) -> + MidState = maybe_show_warning(State), + put(cmd_type, async), + {_, NewRState} = run(default, Command, "", RState, Cwd), + put(cmd_type, undefined), + {noreply, MidState#state{state=NewRState}, hibernate}; +handle_cast({cmd, Namespace, Command}, State = #state{state=RState, cwd=Cwd}) -> + MidState = maybe_show_warning(State), + put(cmd_type, async), + {_, NewRState} = run(Namespace, Command, "", RState, Cwd), + put(cmd_type, undefined), + {noreply, MidState#state{state=NewRState}, hibernate}; +handle_cast({cmd, Namespace, Command, Args}, State = #state{state=RState, cwd=Cwd}) -> + MidState = maybe_show_warning(State), + put(cmd_type, async), + {_, NewRState} = run(Namespace, Command, Args, RState, Cwd), + put(cmd_type, undefined), + {noreply, MidState#state{state=NewRState}, hibernate}; handle_cast(_Cast, State) -> {noreply, State}. -- cgit v1.1 From 0f6f1630176da37af5971a6fa51f57c0733f949d Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Mon, 29 Oct 2018 11:38:00 -0600 Subject: set app's dir before setting app_info deps (#1928) --- src/rebar_app_discover.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index 74681c7..21dea29 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -280,7 +280,8 @@ find_apps(LibDirs, SrcDirs, Validate, State) -> -spec find_app(file:filename_all(), valid | invalid | all) -> {true, rebar_app_info:t()} | false. find_app(AppDir, Validate) -> {Config, SrcDirs} = find_config_src(AppDir, ["src"]), - AppInfo = rebar_app_info:update_opts(rebar_app_info:new(), dict:new(), Config), + AppInfo = rebar_app_info:update_opts(rebar_app_info:dir(rebar_app_info:new(), AppDir), + dict:new(), Config), find_app_(AppInfo, AppDir, SrcDirs, Validate). %% @doc check that a given app in a directory is there, and whether it's -- cgit v1.1 From 60dc0504bf46e9ece179da9b20c54e0e2a3e11c6 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 8 Nov 2018 14:09:07 -0700 Subject: fix compilation of global plugins (#1935) --- src/rebar3.erl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/rebar3.erl b/src/rebar3.erl index 059d530..a490a15 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -402,18 +402,21 @@ state_from_global_config(Config, GlobalConfigFile) -> Resources = application:get_env(rebar, resources, []), GlobalConfigThrowAway = rebar_state:create_resources(Resources, GlobalConfigThrowAway0), - GlobalState = case rebar_state:get(GlobalConfigThrowAway, plugins, []) of + Compilers = application:get_env(rebar, compilers, []), + GlobalConfigThrowAway1 = rebar_state:compilers(GlobalConfigThrowAway, Compilers), + + GlobalState = case rebar_state:get(GlobalConfigThrowAway1, plugins, []) of [] -> - GlobalConfigThrowAway; + GlobalConfigThrowAway1; GlobalPluginsToInstall -> rebar_plugins:handle_plugins(global, GlobalPluginsToInstall, - GlobalConfigThrowAway) + GlobalConfigThrowAway1) end, GlobalPlugins = rebar_state:providers(GlobalState), GlobalConfig2 = rebar_state:set(GlobalConfig, plugins, []), GlobalConfig3 = rebar_state:set(GlobalConfig2, {plugins, global}, - rebar_state:get(GlobalConfigThrowAway, plugins, [])), + rebar_state:get(GlobalConfigThrowAway1, plugins, [])), rebar_state:providers(rebar_state:new(GlobalConfig3, Config), GlobalPlugins). -spec test_state(rebar_state:t()) -> [{'extra_src_dirs',[string()]} | {'erl_opts',[any()]}]. -- cgit v1.1 From 8d31d559c18b12d3d06655de2ca9c7f76c67c948 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 9 Nov 2018 12:22:12 -0700 Subject: check last modified time on erl files for xrl and yrl files before compiling --- src/rebar_compiler.erl | 13 +++++++++++-- src/rebar_compiler_erl.erl | 4 ++-- src/rebar_compiler_mib.erl | 4 ++-- src/rebar_compiler_xrl.erl | 8 +++++--- src/rebar_compiler_yrl.erl | 8 +++++--- 5 files changed, 25 insertions(+), 12 deletions(-) (limited to 'src') diff --git a/src/rebar_compiler.erl b/src/rebar_compiler.erl index 6e94cb2..256794a 100644 --- a/src/rebar_compiler.erl +++ b/src/rebar_compiler.erl @@ -3,6 +3,7 @@ -export([compile_all/2, clean/2, + needs_compile/3, ok_tuple/2, error_tuple/4, maybe_report/1, @@ -18,7 +19,7 @@ include_dirs => [file:dirname()], src_ext => extension(), out_mappings => out_mappings()}. --callback needed_files(digraph:graph(), [file:filename()], rebar_app_info:t()) -> [file:filename()]. +-callback needed_files(digraph:graph(), [file:filename()], list(), rebar_app_info:t()) -> [file:filename()]. -callback dependencies(file:filename(), file:dirname(), [file:dirname()]) -> [file:filename()]. -callback compile(file:filename(), out_mappings(), rebar_dict(), list()) -> ok | {ok, [string()]} | {ok, [string()], [string()]}. @@ -66,7 +67,8 @@ run(CompilerMod, AppInfo) -> OutDir = rebar_app_info:out_dir(AppInfo), AbsSrcDirs = [filename:join(BaseDir, SrcDir) || SrcDir <- SrcDirs], G = init_dag(CompilerMod, AbsInclDirs, AbsSrcDirs, FoundFiles, OutDir, EbinDir), - {{FirstFiles, FirstFileOpts}, {RestFiles, Opts}} = CompilerMod:needed_files(G, FoundFiles, AppInfo), + {{FirstFiles, FirstFileOpts}, {RestFiles, Opts}} = CompilerMod:needed_files(G, FoundFiles, + Mappings, AppInfo), true = digraph:delete(G), compile_each(FirstFiles, FirstFileOpts, BaseOpts, Mappings, CompilerMod), @@ -111,6 +113,13 @@ clean_(CompilerMod, AppInfo) -> CompilerMod:clean(FoundFiles, AppInfo), rebar_file_utils:rm_rf(dag_file(CompilerMod, EbinDir)). +-spec needs_compile(filename:all(), extension(), [{extension(), file:dirname()}]) -> boolean(). +needs_compile(Source, OutExt, Mappings) -> + Ext = filename:extension(Source), + BaseName = filename:basename(Source, Ext), + {_, OutDir} = lists:keyfind(OutExt, 1, Mappings), + Target = filename:join(OutDir, BaseName++OutExt), + filelib:last_modified(Source) > filelib:last_modified(Target). run_on_extra_src_dirs(CompilerMod, AppInfo, Fun) -> ExtraDirs = rebar_dir:extra_src_dirs(rebar_app_info:opts(AppInfo), []), diff --git a/src/rebar_compiler_erl.erl b/src/rebar_compiler_erl.erl index d9bc69b..f0ae6f4 100644 --- a/src/rebar_compiler_erl.erl +++ b/src/rebar_compiler_erl.erl @@ -3,7 +3,7 @@ -behaviour(rebar_compiler). -export([context/1, - needed_files/3, + needed_files/4, dependencies/3, compile/4, clean/2]). @@ -31,7 +31,7 @@ context(AppInfo) -> out_mappings => Mappings}. -needed_files(Graph, FoundFiles, AppInfo) -> +needed_files(Graph, FoundFiles, _, AppInfo) -> OutDir = rebar_app_info:out_dir(AppInfo), Dir = rebar_app_info:dir(AppInfo), EbinDir = rebar_app_info:ebin_dir(AppInfo), diff --git a/src/rebar_compiler_mib.erl b/src/rebar_compiler_mib.erl index 32516bf..c731c27 100644 --- a/src/rebar_compiler_mib.erl +++ b/src/rebar_compiler_mib.erl @@ -3,7 +3,7 @@ -behaviour(rebar_compiler). -export([context/1, - needed_files/3, + needed_files/4, dependencies/3, compile/4, clean/2]). @@ -21,7 +21,7 @@ context(AppInfo) -> src_ext => ".mib", out_mappings => Mappings}. -needed_files(_, FoundFiles, AppInfo) -> +needed_files(_, FoundFiles, _, AppInfo) -> FirstFiles = [], %% Remove first files from found files diff --git a/src/rebar_compiler_xrl.erl b/src/rebar_compiler_xrl.erl index 5c023f0..23ed1eb 100644 --- a/src/rebar_compiler_xrl.erl +++ b/src/rebar_compiler_xrl.erl @@ -3,7 +3,7 @@ -behaviour(rebar_compiler). -export([context/1, - needed_files/3, + needed_files/4, dependencies/3, compile/4, clean/2]). @@ -16,11 +16,13 @@ context(AppInfo) -> src_ext => ".xrl", out_mappings => Mappings}. -needed_files(_, FoundFiles, AppInfo) -> +needed_files(_, FoundFiles, Mappings, AppInfo) -> FirstFiles = [], %% Remove first files from found files - RestFiles = [Source || Source <- FoundFiles, not lists:member(Source, FirstFiles)], + RestFiles = [Source || Source <- FoundFiles, + not lists:member(Source, FirstFiles), + rebar_compiler:needs_compile(Source, ".erl", Mappings)], Opts = rebar_opts:get(rebar_app_info:opts(AppInfo), xrl_opts, []), diff --git a/src/rebar_compiler_yrl.erl b/src/rebar_compiler_yrl.erl index 41d93b1..9c1767e 100644 --- a/src/rebar_compiler_yrl.erl +++ b/src/rebar_compiler_yrl.erl @@ -3,7 +3,7 @@ -behaviour(rebar_compiler). -export([context/1, - needed_files/3, + needed_files/4, dependencies/3, compile/4, clean/2]). @@ -16,11 +16,13 @@ context(AppInfo) -> src_ext => ".yrl", out_mappings => Mappings}. -needed_files(_, FoundFiles, AppInfo) -> +needed_files(_, FoundFiles, Mappings, AppInfo) -> FirstFiles = [], %% Remove first files from found files - RestFiles = [Source || Source <- FoundFiles, not lists:member(Source, FirstFiles)], + RestFiles = [Source || Source <- FoundFiles, + not lists:member(Source, FirstFiles), + rebar_compiler:needs_compile(Source, ".erl", Mappings)], Opts = rebar_opts:get(rebar_app_info:opts(AppInfo), yrl_opts, []), {{FirstFiles, Opts}, {RestFiles, Opts}}. -- cgit v1.1 From 7dd8545e52c0730dbf7d11306522ac830f337615 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andra=CC=81s=20Boroska?= Date: Sun, 11 Nov 2018 19:03:11 +0100 Subject: Fix plugins list display to include project plugins --- src/rebar_prv_plugins.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_plugins.erl b/src/rebar_prv_plugins.erl index d66b645..fba5da0 100644 --- a/src/rebar_prv_plugins.erl +++ b/src/rebar_prv_plugins.erl @@ -42,10 +42,11 @@ do(State) -> RebarOpts = rebar_state:opts(State), SrcDirs = rebar_dir:src_dirs(RebarOpts, ["src"]), Plugins = rebar_state:get(State, plugins, []), + ProjectPlugins = rebar_state:get(State, project_plugins, []), PluginsDirs = filelib:wildcard(filename:join(rebar_dir:plugins_dir(State), "*")), CheckoutsDirs = filelib:wildcard(filename:join(rebar_dir:checkouts_dir(State), "*")), Apps = rebar_app_discover:find_apps(CheckoutsDirs++PluginsDirs, SrcDirs, all, State), - display_plugins("Local plugins", Apps, Plugins), + display_plugins("Local plugins", Apps, Plugins ++ ProjectPlugins), {ok, State}. -spec format_error(any()) -> iolist(). -- cgit v1.1 From 5944b14f570ee3b42debe71f4628c7ef7824984c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A1s=20Boroska?= Date: Fri, 16 Nov 2018 16:03:59 +0100 Subject: Fix package upgrade (issue #1945) (#1946) --- src/rebar_app_utils.erl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index 605944e..19ed4dd 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -259,8 +259,7 @@ update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) -> maybe_warn_retired(PkgName, PkgVsn1, Hash, Retired), PkgVsn2 = list_to_binary(lists:flatten(ec_semver:format(PkgVsn1))), AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName, PkgVsn2, Hash1, RepoConfig}), - AppInfo2 = rebar_app_info:update_opts_deps(AppInfo1, Deps), - rebar_app_info:original_vsn(AppInfo2, PkgVsn2); + rebar_app_info:update_opts_deps(AppInfo1, Deps); not_found -> throw(?PRV_ERROR({missing_package, PkgName, PkgVsn})); {error, {invalid_vsn, InvalidVsn}} -> -- cgit v1.1 From 5facb035fe67a0e2683c6460833ac1ec8baa8c82 Mon Sep 17 00:00:00 2001 From: tothlac Date: Sun, 11 Nov 2018 17:12:41 +0100 Subject: Support alias format {Namespace, Cmd} and {Namespace, Cmd, Args} (#1940) --- src/rebar_prv_alias.erl | 15 ++++++++++++++- src/rebar_prv_do.erl | 24 +++++++++++++++++++++--- 2 files changed, 35 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_alias.erl b/src/rebar_prv_alias.erl index 736417b..ce56f29 100644 --- a/src/rebar_prv_alias.erl +++ b/src/rebar_prv_alias.erl @@ -73,8 +73,12 @@ desc(Cmds) -> "Equivalent to running: rebar3 do " ++ rebar_string:join(lists:map(fun to_desc/1, Cmds), ","). -to_desc({Cmd, Args}) -> +to_desc({Cmd, Args}) when is_list(Args) -> atom_to_list(Cmd) ++ " " ++ Args; +to_desc({Namespace, Cmd}) -> + atom_to_list(Namespace) ++ " " ++ atom_to_list(Cmd); +to_desc({Namespace, Cmd, Args}) -> + atom_to_list(Namespace) ++ " " ++ atom_to_list(Cmd) ++ " " ++ Args; to_desc(Cmd) -> atom_to_list(Cmd). @@ -98,6 +102,12 @@ make_args(Cmds) -> lists:map(fun make_tuple/1, lists:map(fun make_arg/1, Cmds))). +make_arg({Namespace, Command, Args}) when is_atom(Namespace), is_atom(Command) -> + {make_atom(Namespace), + make_atom(Command), + make_list([make_string(A) || A <- split_args(Args)])}; +make_arg({Namespace, Command}) when is_atom(Namespace), is_atom(Command) -> + {make_atom(Namespace), make_atom(Command)}; make_arg({Cmd, Args}) -> {make_string(Cmd), make_list([make_string(A) || A <- split_args(Args)])}; make_arg(Cmd) -> @@ -117,6 +127,9 @@ make_string(Atom) when is_atom(Atom) -> make_string(String) when is_list(String) -> {string, 1, String}. +make_atom(Atom) when is_atom(Atom) -> + {atom, 1, Atom}. + %% In case someone used the long option format, the option needs to get %% separated from its value. split_args(Args) -> diff --git a/src/rebar_prv_do.erl b/src/rebar_prv_do.erl index f850135..5f7aa12 100644 --- a/src/rebar_prv_do.erl +++ b/src/rebar_prv_do.erl @@ -44,13 +44,31 @@ do(State) -> do_tasks(Tasks, State) end. +-spec do_tasks(list(Task), State) -> Res when + Task :: {string(), string()} | + {string(), atom()} | + {atom(), atom(), string()}, + State :: rebar_state:t(), + Res :: {ok, rebar_state:t()} | + {error, term()}. do_tasks([], State) -> {ok, State}; -do_tasks([{TaskStr, Args}|Tail], State) -> +do_tasks([{TaskStr, Args} | Tail], State) when is_list(Args) -> Task = list_to_atom(TaskStr), State1 = rebar_state:set(State, task, Task), State2 = rebar_state:command_args(State1, Args), Namespace = rebar_state:namespace(State2), + do_task(TaskStr, Args, Tail, State, Namespace); +do_tasks([{Namespace, Task} | Tail], State) -> + do_task(atom_to_list(Task), [], Tail, State, Namespace); +do_tasks([{Namespace, Task, Args} | Tail], State) + when is_atom(Namespace), is_atom(Task) -> + do_task(atom_to_list(Task), Args, Tail, State, Namespace). + +do_task(TaskStr, Args, Tail, State, Namespace) -> + Task = list_to_atom(TaskStr), + State1 = rebar_state:set(State, task, Task), + State2 = rebar_state:command_args(State1, Args), case Namespace of default -> %% The first task we hit might be a namespace! @@ -65,7 +83,8 @@ do_tasks([{TaskStr, Args}|Tail], State) -> _ -> %% We're already in a non-default namespace, check the %% task directly. - case rebar_core:process_command(State2, Task) of + State3 = rebar_state:namespace(State2, Namespace), + case rebar_core:process_command(State3, Task) of {ok, FinalState} when Tail =:= [] -> {ok, FinalState}; {ok, _} -> @@ -75,7 +94,6 @@ do_tasks([{TaskStr, Args}|Tail], State) -> end end. - -spec format_error(any()) -> iolist(). format_error(Reason) -> io_lib:format("~p", [Reason]). -- cgit v1.1 From beb8878a6c73d37e05a167b974d965ede005f2f8 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sun, 18 Nov 2018 07:36:54 -0700 Subject: fix needed_files type spec --- src/rebar_compiler.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_compiler.erl b/src/rebar_compiler.erl index 256794a..092f898 100644 --- a/src/rebar_compiler.erl +++ b/src/rebar_compiler.erl @@ -19,7 +19,8 @@ include_dirs => [file:dirname()], src_ext => extension(), out_mappings => out_mappings()}. --callback needed_files(digraph:graph(), [file:filename()], list(), rebar_app_info:t()) -> [file:filename()]. +-callback needed_files(digraph:graph(), [file:filename()], out_mappings(), + rebar_app_info:t()) -> [file:filename()]. -callback dependencies(file:filename(), file:dirname(), [file:dirname()]) -> [file:filename()]. -callback compile(file:filename(), out_mappings(), rebar_dict(), list()) -> ok | {ok, [string()]} | {ok, [string()], [string()]}. -- cgit v1.1 From 74f1de612c1e30b57bf36a4ca24862bcb2c111d9 Mon Sep 17 00:00:00 2001 From: Stavros Aronis Date: Thu, 22 Nov 2018 20:37:37 +0100 Subject: Add hint for `rebar3 new help` on `rebar3 help new` --- src/rebar_prv_new.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_new.erl b/src/rebar_prv_new.erl index f632362..c6a1e9b 100644 --- a/src/rebar_prv_new.erl +++ b/src/rebar_prv_new.erl @@ -82,7 +82,9 @@ info() -> "Create rebar3 project based on template and vars.~n" "~n" "Valid command line options:~n" - "