From 963c49f5eb9ab5b34e1843fb43305743720917ac Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sun, 6 Aug 2017 07:26:21 -0400 Subject: Unicode support in all the places This is done through 3 main change groups: - replacing `~s` by `~ts` in format strings, so that strings that contain unicode are properly printed rather than crashing - adding the `unicode` argument to all function of the `re` module to ensure transformations on strings containing unicode data are valid instead of crashing (see issue #1302) - replacing `ec_cnv:to_binary/1` and `ec_cnv:to_list/1` with matching functions in `rebar_utils`. The last point has been done, rather than modifying and updating erlware commons, because binary and list conversions can be a contentious subject. For example, if what is being handled is actually bytes from a given binary stream, then forcing a byte-oriented interpretation of the data can corrupt it. As such, it does not appear safe to modify erlware commons' conversion functions since it may not be safe for all its users. Instead, rebar3 reimplements a subset of them (only converting atoms and chardata, ignoring numbers) with the explicit purpose of handling unicode string data. Tests were left as unchanged as possible. This may impact the ability to run rebar3's own suites in a unicode path, but respects a principle of least change for such a large patch. --- src/rebar_hooks.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/rebar_hooks.erl') diff --git a/src/rebar_hooks.erl b/src/rebar_hooks.erl index d6a0e2b..48aa928 100644 --- a/src/rebar_hooks.erl +++ b/src/rebar_hooks.erl @@ -57,9 +57,9 @@ run_provider_hooks_(Dir, Type, Command, Providers, TypeHooks, State) -> end. format_error({bad_provider, Type, Command, {Name, Namespace}}) -> - io_lib:format("Unable to run ~s hooks for '~p', command '~p' in namespace '~p' not found.", [Type, Command, Namespace, Name]); + io_lib:format("Unable to run ~ts hooks for '~p', command '~p' in namespace '~p' not found.", [Type, Command, Namespace, Name]); format_error({bad_provider, Type, Command, Name}) -> - io_lib:format("Unable to run ~s hooks for '~p', command '~p' not found.", [Type, Command, Name]). + io_lib:format("Unable to run ~ts hooks for '~p', command '~p' not found.", [Type, Command, Name]). %% @doc The following environment variables are exported when running %% a hook (absolute paths): @@ -143,7 +143,7 @@ join_dirs(BaseDir, Dirs) -> string:join([ filename:join(BaseDir, Dir) || Dir <- Dirs ], ":"). re_version(Path) -> - case re:run(Path, "^.*-(?[^/-]*)$", [{capture, [1], list}]) of + case re:run(Path, "^.*-(?[^/-]*)$", [{capture,[1],list}, unicode]) of nomatch -> ""; {match, [Ver]} -> Ver end. -- cgit v1.1 From 2d5cd9c00cfa4e58066b48beee4057fdd52cc7be Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 1 Nov 2017 19:38:03 -0400 Subject: OTP-21 readiness, Full Unicode support This replaces all deprecated function usage by alternative ones based on a version switch enacted at compile time, preventing all warnings. This will likely introduce some possible runtime errors in using a Rebar3 compiled on OTP-20 or OTP-21 back in versions 19 and earlier, but we can't really work around that. A bunch of dependencies have been updated to support OTP-21 without warnings as well. --- src/rebar_hooks.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/rebar_hooks.erl') diff --git a/src/rebar_hooks.erl b/src/rebar_hooks.erl index 48aa928..8893f2a 100644 --- a/src/rebar_hooks.erl +++ b/src/rebar_hooks.erl @@ -140,7 +140,7 @@ create_env(State, Opts) -> ]. join_dirs(BaseDir, Dirs) -> - string:join([ filename:join(BaseDir, Dir) || Dir <- Dirs ], ":"). + rebar_string:join([filename:join(BaseDir, Dir) || Dir <- Dirs], ":"). re_version(Path) -> case re:run(Path, "^.*-(?[^/-]*)$", [{capture,[1],list}, unicode]) of -- cgit v1.1 From dd9680fe42b42774190c2e95e56d052080f2e939 Mon Sep 17 00:00:00 2001 From: Carl-Johan Kjellander Date: Fri, 23 Feb 2018 15:24:55 +0100 Subject: move and export create_env for use by providers --- src/rebar_hooks.erl | 58 +---------------------------------------------------- 1 file changed, 1 insertion(+), 57 deletions(-) (limited to 'src/rebar_hooks.erl') diff --git a/src/rebar_hooks.erl b/src/rebar_hooks.erl index 8893f2a..ec6fe31 100644 --- a/src/rebar_hooks.erl +++ b/src/rebar_hooks.erl @@ -61,29 +61,6 @@ format_error({bad_provider, Type, Command, {Name, Namespace}}) -> format_error({bad_provider, Type, Command, Name}) -> io_lib:format("Unable to run ~ts hooks for '~p', command '~p' not found.", [Type, Command, Name]). -%% @doc The following environment variables are exported when running -%% a hook (absolute paths): -%% -%% REBAR_DEPS_DIR = rebar_dir:deps_dir/1 -%% REBAR_BUILD_DIR = rebar_dir:base_dir/1 -%% REBAR_ROOT_DIR = rebar_dir:root_dir/1 -%% REBAR_CHECKOUTS_DIR = rebar_dir:checkouts_dir/1 -%% REBAR_PLUGINS_DIR = rebar_dir:plugins_dir/1 -%% REBAR_GLOBAL_CONFIG_DIR = rebar_dir:global_config_dir/1 -%% REBAR_GLOBAL_CACHE_DIR = rebar_dir:global_cache_dir/1 -%% REBAR_TEMPLATE_DIR = rebar_dir:template_dir/1 -%% REBAR_APP_DIRS = rebar_dir:lib_dirs/1 -%% REBAR_SRC_DIRS = rebar_dir:src_dirs/1 -%% -%% autoconf compatible variables -%% (see: http://www.gnu.org/software/autoconf/manual/autoconf.html#Erlang-Libraries): -%% ERLANG_ERTS_VER = erlang:system_info(version) -%% ERLANG_ROOT_DIR = code:root_dir/0 -%% ERLANG_LIB_DIR_erl_interface = code:lib_dir(erl_interface) -%% ERLANG_LIB_VER_erl_interface = version part of path returned by code:lib_dir(erl_interface) -%% ERL = ERLANG_ROOT_DIR/bin/erl -%% ERLC = ERLANG_ROOT_DIR/bin/erl -%% run_hooks(Dir, pre, Command, Opts, State) -> run_hooks(Dir, pre_hooks, Command, Opts, State); run_hooks(Dir, post, Command, Opts, State) -> @@ -94,7 +71,7 @@ run_hooks(Dir, Type, Command, Opts, State) -> ?DEBUG("run_hooks(~p, ~p, ~p) -> no hooks defined\n", [Dir, Type, Command]), ok; Hooks -> - Env = create_env(State, Opts), + Env = rebar_env:create_env(State, Opts), lists:foreach(fun({_, C, _}=Hook) when C =:= Command -> apply_hook(Dir, Env, Hook); ({C, _}=Hook) when C =:= Command -> @@ -114,36 +91,3 @@ apply_hook(Dir, Env, {Arch, Command, Hook}) -> apply_hook(Dir, Env, {Command, Hook}) -> Msg = lists:flatten(io_lib:format("Hook for ~p failed!~n", [Command])), rebar_utils:sh(Hook, [use_stdout, {cd, Dir}, {env, Env}, {abort_on_error, Msg}]). - -create_env(State, Opts) -> - BaseDir = rebar_dir:base_dir(State), - [ - {"REBAR_DEPS_DIR", filename:absname(rebar_dir:deps_dir(State))}, - {"REBAR_BUILD_DIR", filename:absname(rebar_dir:base_dir(State))}, - {"REBAR_ROOT_DIR", filename:absname(rebar_dir:root_dir(State))}, - {"REBAR_CHECKOUTS_DIR", filename:absname(rebar_dir:checkouts_dir(State))}, - {"REBAR_PLUGINS_DIR", filename:absname(rebar_dir:plugins_dir(State))}, - {"REBAR_GLOBAL_CONFIG_DIR", filename:absname(rebar_dir:global_config_dir(State))}, - {"REBAR_GLOBAL_CACHE_DIR", filename:absname(rebar_dir:global_cache_dir(Opts))}, - {"REBAR_TEMPLATE_DIR", filename:absname(rebar_dir:template_dir(State))}, - {"REBAR_APP_DIRS", join_dirs(BaseDir, rebar_dir:lib_dirs(State))}, - {"REBAR_SRC_DIRS", join_dirs(BaseDir, rebar_dir:all_src_dirs(Opts))}, - {"ERLANG_ERTS_VER", erlang:system_info(version)}, - {"ERLANG_ROOT_DIR", code:root_dir()}, - {"ERLANG_LIB_DIR_erl_interface", code:lib_dir(erl_interface)}, - {"ERLANG_LIB_VER_erl_interface", re_version(code:lib_dir(erl_interface))}, - {"ERL", filename:join([code:root_dir(), "bin", "erl"])}, - {"ERLC", filename:join([code:root_dir(), "bin", "erlc"])}, - {"ERLANG_ARCH" , rebar_api:wordsize()}, - {"ERLANG_TARGET", rebar_api:get_arch()} - - ]. - -join_dirs(BaseDir, Dirs) -> - rebar_string:join([filename:join(BaseDir, Dir) || Dir <- Dirs], ":"). - -re_version(Path) -> - case re:run(Path, "^.*-(?[^/-]*)$", [{capture,[1],list}, unicode]) of - nomatch -> ""; - {match, [Ver]} -> Ver - end. -- cgit v1.1 From 9d788d893620f868b9c0ee00ddec8ae4d5d8fea7 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 5 Oct 2018 07:57:20 -0400 Subject: Abstracted path management Move path management out of rebar_utils manual code path function handling (which we leave there for backwards compat), and centralize them to allow easier coordination of paths between plugins and deps. On top of path handling, do a check of loaded modules to only purge and reload those that actually need it done in order to prevent all kinds of weird interaction and accidental purge kills. It also allows the possible cohabitation of both at once, with a "in case of conflict pick X" as a policy Changing path handling in providers also highlighted a bunch of bugs in some tests and appears to fix some in other providers, specifically around plugins. --- src/rebar_hooks.erl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src/rebar_hooks.erl') diff --git a/src/rebar_hooks.erl b/src/rebar_hooks.erl index ec6fe31..f2ef8a3 100644 --- a/src/rebar_hooks.erl +++ b/src/rebar_hooks.erl @@ -42,8 +42,9 @@ run_provider_hooks_(Dir, Type, Command, Providers, TypeHooks, State) -> [] -> State; HookProviders -> - PluginDepsPaths = lists:usort(rebar_state:code_paths(State, all_plugin_deps)), - code:add_pathsa(PluginDepsPaths), + %PluginDepsPaths = lists:usort(rebar_state:code_paths(State, all_plugin_deps)), + %code:add_pathsa(PluginDepsPaths), + rebar_paths:set_paths([plugins, deps], State), Providers1 = rebar_state:providers(State), State1 = rebar_state:providers(rebar_state:dir(State, Dir), Providers++Providers1), case rebar_core:do(HookProviders, State1) of @@ -51,7 +52,8 @@ run_provider_hooks_(Dir, Type, Command, Providers, TypeHooks, State) -> ?DEBUG(format_error({bad_provider, Type, Command, ProviderName}), []), throw(?PRV_ERROR({bad_provider, Type, Command, ProviderName})); {ok, State2} -> - rebar_utils:remove_from_code_path(PluginDepsPaths), + %rebar_utils:remove_from_code_path(PluginDepsPaths), + rebar_paths:set_paths([deps, plugins], State2), State2 end end. -- cgit v1.1 From dada4e36e6d9a5c4b41bbe1f68389520e7c59ace Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 11 Oct 2018 08:38:37 -0400 Subject: Optimize path handling - Only set paths that need to be put as a priority - Clean up paths before leaving API mode The first point accounted for some performance cost, but the latter one explains the 40% overhead in test runs: since rebar3 calls rebar3 a lot with a bunch of fake apps, and that the new mechanism for path handling by default does not _remove_ paths, it just _orders_ them, we would end up in a situation where as the tests ran, more and more fake paths would get added to the VM. By the time the run was over, all path handling would take longer since more paths needed filtering every time. By resetting paths at the end of an API run, we prevent a given 'project' from polluting another one's runtime and performance once the API successfully returns. --- src/rebar_hooks.erl | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'src/rebar_hooks.erl') diff --git a/src/rebar_hooks.erl b/src/rebar_hooks.erl index f2ef8a3..358458e 100644 --- a/src/rebar_hooks.erl +++ b/src/rebar_hooks.erl @@ -42,9 +42,7 @@ run_provider_hooks_(Dir, Type, Command, Providers, TypeHooks, State) -> [] -> State; HookProviders -> - %PluginDepsPaths = lists:usort(rebar_state:code_paths(State, all_plugin_deps)), - %code:add_pathsa(PluginDepsPaths), - rebar_paths:set_paths([plugins, deps], State), + rebar_paths:set_paths([plugins], State), Providers1 = rebar_state:providers(State), State1 = rebar_state:providers(rebar_state:dir(State, Dir), Providers++Providers1), case rebar_core:do(HookProviders, State1) of @@ -52,8 +50,7 @@ run_provider_hooks_(Dir, Type, Command, Providers, TypeHooks, State) -> ?DEBUG(format_error({bad_provider, Type, Command, ProviderName}), []), throw(?PRV_ERROR({bad_provider, Type, Command, ProviderName})); {ok, State2} -> - %rebar_utils:remove_from_code_path(PluginDepsPaths), - rebar_paths:set_paths([deps, plugins], State2), + rebar_paths:set_paths([deps], State2), State2 end end. -- cgit v1.1