From 0303567d95f0769f3d76df35d890458d9741cf4d Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 10 Sep 2018 10:20:57 -0400 Subject: Reload logger config in shell This requires some fancy dynamic work since the logger is started as part of the kernel and we lost the sys.config from users when working from there. We start conservatively by making it an optional thing, turning it on only where we know it to be safe. The changes are applied _after_ having loaded the rest of configs so if an inoffensive error happens, the shell works (with a bad error message) rather than plain exploding. --- src/rebar_utils.erl | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) (limited to 'src/rebar_utils.erl') diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 2ded481..7e57d01 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -72,7 +72,7 @@ info_useless/2, list_dir/1, user_agent/0, - reread_config/1, + reread_config/1, reread_config/2, get_proxy_auth/0, is_list_of_strings/1]). @@ -436,6 +436,18 @@ user_agent() -> ?FMT("Rebar/~ts (OTP/~ts)", [Vsn, otp_release()]). reread_config(ConfigList) -> + %% Default to not re-configuring the logger for now; + %% this can leak logs in CT redirection when setting up hooks + %% for example. If we want to turn it on by default, we may + %% want to disable it in CT at the same time or figure out a + %% way to silence it. + %% The same pattern may apply to other tasks, so let's enable + %% case-by-case. + reread_config(ConfigList, []). + +reread_config(ConfigList, Opts) -> + UpdateLoggerConfig = erlang:function_exported(logger, module_info, 0) andalso + proplists:get_value(update_logger, Opts, false), %% NB: we attempt to mimic -config here, which survives app reload, %% hence {persistent, true}. SetEnv = case version_tuple(?MODULE:otp_release()) of @@ -445,15 +457,52 @@ reread_config(ConfigList) -> fun (App, Key, Val) -> application:set_env(App, Key, Val, [{persistent, true}]) end end, try + Res = [SetEnv(Application, Key, Val) || Config <- ConfigList, {Application, Items} <- Config, - {Key, Val} <- Items] + {Key, Val} <- Items], + case UpdateLoggerConfig of + true -> reread_logger_config(); + false -> ok + end, + Res catch _:_ -> ?ERROR("The configuration file submitted could not be read " "and will be ignored.", []) end. +%% @private since the kernel app is already booted, re-reading its config +%% requires doing some magic to dynamically patch running handlers to +%% deal with the current value. +reread_logger_config() -> + KernelCfg = application:get_all_env(kernel), + LogCfg = proplists:get_value(logger, KernelCfg), + case LogCfg of + undefined -> + ok; + _ -> + %% Extract and apply settings related to primary configuration + %% -- primary config is used for settings shared across handlers + LogLvlPrimary = proplists:get_value(logger_info, KernelCfg, all), + {FilterDefault, Filters} = + case lists:keyfind(filters, 1, KernelCfg) of + false -> {log, []}; + {filters, FoundDef, FoundFilter} -> {FoundDef, FoundFilter} + end, + Primary = #{level => LogLvlPrimary, + filter_default => FilterDefault, + filters => Filters}, + %% Load the correct handlers based on their individual config. + [case Id of + default -> logger:update_handler_config(Id, Cfg); + _ -> logger:add_handler(Id, Mod, Cfg) + end || {handler, Id, Mod, Cfg} <- LogCfg], + logger:set_primary_config(Primary), + ok + end. + + %% @doc Given env. variable `FOO' we want to expand all references to %% it in `InStr'. References can have two forms: `$FOO' and `${FOO}' %% The end of form `$FOO' is delimited with whitespace or EOL -- cgit v1.1 From 56b7d88975aa8da6446857cfd92de0825024bf63 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 13 Sep 2018 19:36:00 -0600 Subject: support for hex v2, multiple repository fetching, private organizations (#1884) * update to hex_core for hex-v2 repo support (#1865) * update to hex_core for hex-v2 repo support This patch adds only single repo hex-v2 support through hex_core. Packages no longer filtered out by buildtool metadata and the package index is updated per-package instead of fetched as one large ets dump. * tell travis to also build hex_core branch * support list of repos for hex packages (#1866) * support list of repos for hex packages repos are defined under the hex key in rebar configs. They can be defined at the top level of a project or globally, but not in profiles and the repos configured in dependencies are also ignored. Searching for packages involves first checking for a match in the local repo index cache, in the order repos are defined. If not found each repo is checked through the hex api for any known versions of the package and the first repo with a version that fits the constraint is used. * add {repos, replace, []} for overriding the global & default repos * add hex auth handling for repos (#1874) auth token are kept in a hex.config file that is modified by the rebar3 hex plugin. Repo names that have a : separating a parent and child are considered organizations. The parent repo's auth will be included with the child. So an organization named hexpm:rebar3_test will include any hexpm auth tokens found in the rebar3_test organization's configuration. * move packages to top level of of hexpm cache dir (#1876) * move packages to top level of of hexpm cache dir * append organization name to parent's repo_url when parsing repos * only eval config scripts and apply overrides once per app (#1879) * only eval config scripts and apply overrides once per app * move new resource behaviour to rebar_resource_v2 and keep v1 * cleanup use of rebar_resource module and unused functions * cleanup error messages and unused code * when discovering apps support mix packages as unbuilt apps (#1882) * use hex_core tarball unpacking support in pkg resource (#1883) * use hex_core tarball unpacking support in pkg resource * ignore etag if package doesn't exist and delete if checksum fails * add back tests for bad package checksums * improve bad registry checksum error message --- src/rebar_utils.erl | 157 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 127 insertions(+), 30 deletions(-) (limited to 'src/rebar_utils.erl') diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 7e57d01..995d212 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -74,13 +74,15 @@ user_agent/0, reread_config/1, reread_config/2, get_proxy_auth/0, - is_list_of_strings/1]). + is_list_of_strings/1, + ssl_opts/1]). %% for internal use only -export([otp_release/0]). -include("rebar.hrl"). +-include_lib("public_key/include/OTP-PUB-KEY.hrl"). -define(ONE_LEVEL_INDENT, " "). -define(APP_NAME_INDEX, 2). @@ -710,12 +712,12 @@ escript_foldl(Fun, Acc, File) -> Error end. -vcs_vsn(Vcs, Dir, Resources) -> - case vcs_vsn_cmd(Vcs, Dir, Resources) of +vcs_vsn(AppInfo, Vcs, State) -> + case vcs_vsn_cmd(AppInfo, Vcs, State) of {plain, VsnString} -> VsnString; {cmd, CmdString} -> - vcs_vsn_invoke(CmdString, Dir); + vcs_vsn_invoke(CmdString, rebar_app_info:dir(AppInfo)); unknown -> ?ABORT("vcs_vsn: Unknown vsn format: ~p", [Vcs]); {error, Reason} -> @@ -723,23 +725,18 @@ vcs_vsn(Vcs, Dir, Resources) -> end. %% Temp work around for repos like relx that use "semver" -vcs_vsn_cmd(Vsn, _, _) when is_binary(Vsn) -> +vcs_vsn_cmd(_AppInfo, Vsn, _) when is_binary(Vsn) -> {plain, Vsn}; -vcs_vsn_cmd(VCS, Dir, Resources) when VCS =:= semver ; VCS =:= "semver" -> - vcs_vsn_cmd(git, Dir, Resources); -vcs_vsn_cmd({cmd, _Cmd}=Custom, _, _) -> +vcs_vsn_cmd(AppInfo, VCS, State) when VCS =:= semver ; VCS =:= "semver" -> + vcs_vsn_cmd(AppInfo, git, State); +vcs_vsn_cmd(_AppInfo, {cmd, _Cmd}=Custom, _) -> Custom; -vcs_vsn_cmd(VCS, Dir, Resources) when is_atom(VCS) -> - case find_resource_module(VCS, Resources) of - {ok, Module} -> - Module:make_vsn(Dir); - {error, _} -> - unknown - end; -vcs_vsn_cmd(VCS, Dir, Resources) when is_list(VCS) -> +vcs_vsn_cmd(AppInfo, VCS, State) when is_atom(VCS) -> + rebar_resource_v2:make_vsn(AppInfo, VCS, State); +vcs_vsn_cmd(AppInfo, VCS, State) when is_list(VCS) -> try list_to_existing_atom(VCS) of AVCS -> - case vcs_vsn_cmd(AVCS, Dir, Resources) of + case vcs_vsn_cmd(AppInfo, AVCS, State) of unknown -> {plain, VCS}; Other -> Other end @@ -754,19 +751,6 @@ vcs_vsn_invoke(Cmd, Dir) -> {ok, VsnString} = rebar_utils:sh(Cmd, [{cd, Dir}, {use_stdout, false}]), rebar_string:trim(VsnString, trailing, "\n"). -find_resource_module(Type, Resources) -> - case lists:keyfind(Type, 1, Resources) of - false -> - case code:which(Type) of - non_existing -> - {error, unknown}; - _ -> - {ok, Type} - end; - {Type, Module} -> - {ok, Module} - end. - %% @doc ident to the level specified -spec indent(non_neg_integer()) -> iolist(). indent(Amount) when erlang:is_integer(Amount) -> @@ -977,3 +961,116 @@ is_list_of_strings(List) when is_list(hd(List)) -> true; is_list_of_strings(List) when is_list(List) -> true. + +%%------------------------------------------------------------------------------ +%% @doc +%% Return the SSL options adequate for the project based on +%% its configuration, including for validation of certs. +%% @end +%%------------------------------------------------------------------------------ +-spec ssl_opts(Url) -> Res when + Url :: string(), + Res :: proplists:proplist(). +ssl_opts(Url) -> + case get_ssl_config() of + ssl_verify_enabled -> + ssl_opts(ssl_verify_enabled, Url); + ssl_verify_disabled -> + [{verify, verify_none}] + end. + +%%------------------------------------------------------------------------------ +%% @doc +%% Return the SSL options adequate for the project based on +%% its configuration, including for validation of certs. +%% @end +%%------------------------------------------------------------------------------ +-spec ssl_opts(Enabled, Url) -> Res when + Enabled :: atom(), + Url :: string(), + Res :: proplists:proplist(). +ssl_opts(ssl_verify_enabled, Url) -> + case check_ssl_version() of + true -> + {ok, {_, _, Hostname, _, _, _}} = + http_uri:parse(rebar_utils:to_list(Url)), + VerifyFun = {fun ssl_verify_hostname:verify_fun/3, + [{check_hostname, Hostname}]}, + CACerts = certifi:cacerts(), + [{verify, verify_peer}, {depth, 2}, {cacerts, CACerts}, + {partial_chain, fun partial_chain/1}, {verify_fun, VerifyFun}]; + false -> + ?WARN("Insecure HTTPS request (peer verification disabled), " + "please update to OTP 17.4 or later", []), + [{verify, verify_none}] + end. + +-spec partial_chain(Certs) -> Res when + Certs :: list(any()), + Res :: unknown_ca | {trusted_ca, any()}. +partial_chain(Certs) -> + Certs1 = [{Cert, public_key:pkix_decode_cert(Cert, otp)} || Cert <- Certs], + CACerts = certifi:cacerts(), + CACerts1 = [public_key:pkix_decode_cert(Cert, otp) || Cert <- CACerts], + case ec_lists:find(fun({_, Cert}) -> + check_cert(CACerts1, Cert) + end, Certs1) of + {ok, Trusted} -> + {trusted_ca, element(1, Trusted)}; + _ -> + unknown_ca + end. + +-spec extract_public_key_info(Cert) -> Res when + Cert :: #'OTPCertificate'{tbsCertificate::#'OTPTBSCertificate'{}}, + Res :: any(). +extract_public_key_info(Cert) -> + ((Cert#'OTPCertificate'.tbsCertificate)#'OTPTBSCertificate'.subjectPublicKeyInfo). + +-spec check_cert(CACerts, Cert) -> Res when + CACerts :: list(any()), + Cert :: any(), + Res :: boolean(). +check_cert(CACerts, Cert) -> + lists:any(fun(CACert) -> + extract_public_key_info(CACert) == extract_public_key_info(Cert) + end, CACerts). + +-spec check_ssl_version() -> + boolean(). +check_ssl_version() -> + case application:get_key(ssl, vsn) of + {ok, Vsn} -> + parse_vsn(Vsn) >= {5, 3, 6}; + _ -> + false + end. + +-spec get_ssl_config() -> + ssl_verify_disabled | ssl_verify_enabled. +get_ssl_config() -> + GlobalConfigFile = rebar_dir:global_config(), + Config = rebar_config:consult_file(GlobalConfigFile), + case proplists:get_value(ssl_verify, Config, []) of + false -> + ssl_verify_disabled; + _ -> + ssl_verify_enabled + end. + +-spec parse_vsn(Vsn) -> Res when + Vsn :: string(), + Res :: {integer(), integer(), integer()}. +parse_vsn(Vsn) -> + version_pad(rebar_string:lexemes(Vsn, ".-")). + +-spec version_pad(list(nonempty_string())) -> Res when + Res :: {integer(), integer(), integer()}. +version_pad([Major]) -> + {list_to_integer(Major), 0, 0}; +version_pad([Major, Minor]) -> + {list_to_integer(Major), list_to_integer(Minor), 0}; +version_pad([Major, Minor, Patch]) -> + {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}; +version_pad([Major, Minor, Patch | _]) -> + {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}. -- cgit v1.1 From dec484643c233fda9c17d38c1854ba7f3f37547b Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 5 Oct 2018 08:29:07 -0600 Subject: compiler behaviour (#1893) * add compile type for dynamic project compilation * new rebar_compiler abstraction for running multiple compilers rebar_compiler is a new behaviour that a plugin can implement to be called on any ues of the compile provider to compile source files and keep track of their dependencies. * fix check that modules in .app modules list are from src_dirs * use project_type to find module for building projects * allow plugins to add project builders and compilers --- src/rebar_utils.erl | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 82 insertions(+), 5 deletions(-) (limited to 'src/rebar_utils.erl') diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 995d212..419802a 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -37,8 +37,9 @@ escript_foldl/3, find_files/2, find_files/3, + find_files_in_dirs/3, + find_source/3, beam_to_mod/1, - beam_to_mod/2, erl_to_mod/1, beams/1, find_executable/1, @@ -206,6 +207,12 @@ sh(Command0, Options0) -> find_files(Dir, Regex) -> find_files(Dir, Regex, true). +find_files_in_dirs([], _Regex, _Recursive) -> + []; +find_files_in_dirs([Dir | T], Regex, Recursive) -> + find_files(Dir, Regex, Recursive) ++ find_files_in_dirs(T, Regex, Recursive). + + find_files(Dir, Regex, Recursive) -> filelib:fold_files(Dir, Regex, Recursive, fun(F, Acc) -> [F | Acc] end, []). @@ -671,10 +678,6 @@ sh_loop(Port, Fun, Acc) -> end end. -beam_to_mod(Dir, Filename) -> - [Dir | Rest] = filename:split(Filename), - list_to_atom(filename:basename(rebar_string:join(Rest, "."), ".beam")). - beam_to_mod(Filename) -> list_to_atom(filename:basename(Filename, ".beam")). @@ -1074,3 +1077,77 @@ version_pad([Major, Minor, Patch]) -> {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}; version_pad([Major, Minor, Patch | _]) -> {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}. + + +-ifdef(filelib_find_source). +find_source(Filename, Dir, Rules) -> + filelib:find_source(Filename, Dir, Rules). +-else. +%% Looks for a file relative to a given directory + +-type find_file_rule() :: {ObjDirSuffix::string(), SrcDirSuffix::string()}. + +%% Looks for a source file relative to the object file name and directory + +-type find_source_rule() :: {ObjExtension::string(), SrcExtension::string(), + [find_file_rule()]}. + +keep_suffix_search_rules(Rules) -> + [T || {_,_,_}=T <- Rules]. + +-spec find_source(file:filename(), file:filename(), [find_source_rule()]) -> + {ok, file:filename()} | {error, not_found}. +find_source(Filename, Dir, Rules) -> + try_suffix_rules(keep_suffix_search_rules(Rules), Filename, Dir). + +try_suffix_rules(Rules, Filename, Dir) -> + Ext = filename:extension(Filename), + try_suffix_rules(Rules, filename:rootname(Filename, Ext), Dir, Ext). + +try_suffix_rules([{Ext,Src,Rules}|Rest], Root, Dir, Ext) + when is_list(Src), is_list(Rules) -> + case try_dir_rules(add_local_search(Rules), Root ++ Src, Dir) of + {ok, File} -> {ok, File}; + _Other -> + try_suffix_rules(Rest, Root, Dir, Ext) + end; +try_suffix_rules([_|Rest], Root, Dir, Ext) -> + try_suffix_rules(Rest, Root, Dir, Ext); +try_suffix_rules([], _Root, _Dir, _Ext) -> + {error, not_found}. + +%% ensuring we check the directory of the object file before any other directory +add_local_search(Rules) -> + Local = {"",""}, + [Local] ++ lists:filter(fun (X) -> X =/= Local end, Rules). + +try_dir_rules([{From, To}|Rest], Filename, Dir) + when is_list(From), is_list(To) -> + case try_dir_rule(Dir, Filename, From, To) of + {ok, File} -> {ok, File}; + error -> try_dir_rules(Rest, Filename, Dir) + end; +try_dir_rules([], _Filename, _Dir) -> + {error, not_found}. + +try_dir_rule(Dir, Filename, From, To) -> + case lists:suffix(From, Dir) of + true -> + NewDir = lists:sublist(Dir, 1, length(Dir)-length(From))++To, + Src = filename:join(NewDir, Filename), + case filelib:is_regular(Src) of + true -> {ok, Src}; + false -> find_regular_file(filelib:wildcard(Src)) + end; + false -> + error + end. + +find_regular_file([]) -> + error; +find_regular_file([File|Files]) -> + case filelib:is_regular(File) of + true -> {ok, File}; + false -> find_regular_file(Files) + end. +-endif. -- cgit v1.1 From 86519cf743204eab1d922ca8133fbf00c66f9ee8 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sun, 14 Oct 2018 10:50:16 -0600 Subject: fix resolving versions from vcs (#1915) app_info was turning any vsn into a binary which the vcs_vsn function interprets as being an actual version and not something like <<"git">>. original_vsn shouldn't be converted as it is then not longer "original". --- src/rebar_utils.erl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'src/rebar_utils.erl') diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 419802a..1769b79 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -715,6 +715,15 @@ escript_foldl(Fun, Acc, File) -> Error end. +%% TODO: this is just for rebar3_hex and maybe other plugins +%% but eventually it should be dropped +vcs_vsn(OriginalVsn, Dir, Resources) when is_list(Dir) , + is_list(Resources) -> + ?WARN("Using deprecated rebar_utils:vcs_vsn/3. Please upgrade your plugins.", []), + FakeState = rebar_state:new(), + {ok, AppInfo} = rebar_app_info:new(fake, OriginalVsn, Dir), + vcs_vsn(AppInfo, OriginalVsn, + rebar_state:set_resources(FakeState, Resources)); vcs_vsn(AppInfo, Vcs, State) -> case vcs_vsn_cmd(AppInfo, Vcs, State) of {plain, VsnString} -> @@ -728,7 +737,7 @@ vcs_vsn(AppInfo, Vcs, State) -> end. %% Temp work around for repos like relx that use "semver" -vcs_vsn_cmd(_AppInfo, Vsn, _) when is_binary(Vsn) -> +vcs_vsn_cmd(_, Vsn, _) when is_binary(Vsn) -> {plain, Vsn}; vcs_vsn_cmd(AppInfo, VCS, State) when VCS =:= semver ; VCS =:= "semver" -> vcs_vsn_cmd(AppInfo, git, State); -- cgit v1.1