From 1d10f7ae8c7b73c427609bfbfbde4c81a48359c0 Mon Sep 17 00:00:00 2001 From: Shamis Shukoor Date: Fri, 3 Aug 2018 09:23:20 +0200 Subject: fixed typo erl_first_files --- src/rebar_erlc_compiler.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index 920c3b4..a398ddd 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -802,7 +802,7 @@ valid_erl_first_conf(FileList) -> Strs = filter_file_list(FileList), case rebar_utils:is_list_of_strings(Strs) of true -> true; - false -> ?ABORT("An invalid file list (~p) was provided as part of your erl_files_first directive", + false -> ?ABORT("An invalid file list (~p) was provided as part of your erl_first_files directive", [FileList]) end. -- cgit v1.1 From e157b6c23fcf86e76a87edf38283d5684df1307f Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 2 Aug 2018 19:43:57 -0400 Subject: Friendlier output on include_lib errors Turns output like this: ===> Compiling apps/c/src/c.erl failed apps/c/src/c.erl:3: can't find include lib "parse_trans/include/codegen.hrl" Into: ===> Compiling apps/c/src/c.erl failed apps/c/src/c.erl:3: can't find include lib "parse_trans/include/codegen.hrl"; Make sure parse_trans is in your app file's 'applications' list Which is likely going to help newcomers encountering issues. --- src/rebar_base_compiler.erl | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rebar_base_compiler.erl b/src/rebar_base_compiler.erl index 3f273f1..00b0533 100644 --- a/src/rebar_base_compiler.erl +++ b/src/rebar_base_compiler.erl @@ -264,14 +264,24 @@ report(Messages) -> -spec format_errors(_, Extra, [err_or_warn()]) -> [string()] when Extra :: string(). format_errors(_MainSource, Extra, Errors) -> - [begin - [format_error(Source, Extra, Desc) || Desc <- Descs] - end + [[format_error(Source, Extra, Desc) || Desc <- Descs] || {Source, Descs} <- Errors]. %% @private format compiler errors into proper outputtable strings -spec format_error(file:filename(), Extra, err_or_warn()) -> string() when Extra :: string(). +format_error(Source, Extra, {Line, Mod=epp, Desc={include,lib,File}}) -> + %% Special case for include file errors, overtaking the default one + BaseDesc = Mod:format_error(Desc), + Friendly = case filename:split(File) of + [Lib, "include", _] -> + io_lib:format("; Make sure ~s is in your app " + "file's 'applications' list", [Lib]); + _ -> + "" + end, + FriendlyDesc = BaseDesc ++ Friendly, + ?FMT("~ts:~w: ~ts~ts~n", [Source, Line, Extra, FriendlyDesc]); format_error(Source, Extra, {{Line, Column}, Mod, Desc}) -> ErrorDesc = Mod:format_error(Desc), ?FMT("~ts:~w:~w: ~ts~ts~n", [Source, Line, Column, Extra, ErrorDesc]); -- cgit v1.1 From 8f36dfdc65a8dc6c4fb07c346d753e142b2735bd Mon Sep 17 00:00:00 2001 From: Gerald Xv Date: Wed, 5 Sep 2018 04:55:17 +0000 Subject: add report in the effects_code_generation, as it is the short form for both report_errors and report_warnings --- src/rebar_erlc_compiler.erl | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index a398ddd..57e5398 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -363,6 +363,7 @@ effects_code_generation(Option) -> report_errors -> false; return_errors-> false; return_warnings-> false; + report -> false; warnings_as_errors -> false; binary -> false; verbose -> false; -- cgit v1.1 From 0303567d95f0769f3d76df35d890458d9741cf4d Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 10 Sep 2018 10:20:57 -0400 Subject: Reload logger config in shell This requires some fancy dynamic work since the logger is started as part of the kernel and we lost the sys.config from users when working from there. We start conservatively by making it an optional thing, turning it on only where we know it to be safe. The changes are applied _after_ having loaded the rest of configs so if an inoffensive error happens, the shell works (with a bad error message) rather than plain exploding. --- src/rebar_prv_shell.erl | 2 +- src/rebar_utils.erl | 53 +++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 52 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_shell.erl b/src/rebar_prv_shell.erl index af8d99f..5b8d789 100644 --- a/src/rebar_prv_shell.erl +++ b/src/rebar_prv_shell.erl @@ -385,7 +385,7 @@ reread_config(AppsToStart, State) -> lists:member(App, Running), lists:member(App, AppsToStart), not lists:member(App, BlackList)], - _ = rebar_utils:reread_config(ConfigList), + _ = rebar_utils:reread_config(ConfigList, [update_logger]), ok end. diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 2ded481..7e57d01 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -72,7 +72,7 @@ info_useless/2, list_dir/1, user_agent/0, - reread_config/1, + reread_config/1, reread_config/2, get_proxy_auth/0, is_list_of_strings/1]). @@ -436,6 +436,18 @@ user_agent() -> ?FMT("Rebar/~ts (OTP/~ts)", [Vsn, otp_release()]). reread_config(ConfigList) -> + %% Default to not re-configuring the logger for now; + %% this can leak logs in CT redirection when setting up hooks + %% for example. If we want to turn it on by default, we may + %% want to disable it in CT at the same time or figure out a + %% way to silence it. + %% The same pattern may apply to other tasks, so let's enable + %% case-by-case. + reread_config(ConfigList, []). + +reread_config(ConfigList, Opts) -> + UpdateLoggerConfig = erlang:function_exported(logger, module_info, 0) andalso + proplists:get_value(update_logger, Opts, false), %% NB: we attempt to mimic -config here, which survives app reload, %% hence {persistent, true}. SetEnv = case version_tuple(?MODULE:otp_release()) of @@ -445,15 +457,52 @@ reread_config(ConfigList) -> fun (App, Key, Val) -> application:set_env(App, Key, Val, [{persistent, true}]) end end, try + Res = [SetEnv(Application, Key, Val) || Config <- ConfigList, {Application, Items} <- Config, - {Key, Val} <- Items] + {Key, Val} <- Items], + case UpdateLoggerConfig of + true -> reread_logger_config(); + false -> ok + end, + Res catch _:_ -> ?ERROR("The configuration file submitted could not be read " "and will be ignored.", []) end. +%% @private since the kernel app is already booted, re-reading its config +%% requires doing some magic to dynamically patch running handlers to +%% deal with the current value. +reread_logger_config() -> + KernelCfg = application:get_all_env(kernel), + LogCfg = proplists:get_value(logger, KernelCfg), + case LogCfg of + undefined -> + ok; + _ -> + %% Extract and apply settings related to primary configuration + %% -- primary config is used for settings shared across handlers + LogLvlPrimary = proplists:get_value(logger_info, KernelCfg, all), + {FilterDefault, Filters} = + case lists:keyfind(filters, 1, KernelCfg) of + false -> {log, []}; + {filters, FoundDef, FoundFilter} -> {FoundDef, FoundFilter} + end, + Primary = #{level => LogLvlPrimary, + filter_default => FilterDefault, + filters => Filters}, + %% Load the correct handlers based on their individual config. + [case Id of + default -> logger:update_handler_config(Id, Cfg); + _ -> logger:add_handler(Id, Mod, Cfg) + end || {handler, Id, Mod, Cfg} <- LogCfg], + logger:set_primary_config(Primary), + ok + end. + + %% @doc Given env. variable `FOO' we want to expand all references to %% it in `InStr'. References can have two forms: `$FOO' and `${FOO}' %% The end of form `$FOO' is delimited with whitespace or EOL -- cgit v1.1 From 4ec1d288c0b90ce92dffc687bb6287fcf5a2776d Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 13 Sep 2018 18:15:46 -0400 Subject: Bump to 3.6.2 --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index c96f65c..cc6b11e 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.6.2"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 5e678754d7951a1700e2fcca4e1eb91ecc862e1a Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 13 Sep 2018 18:17:36 -0400 Subject: Back to git-based versioning --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index cc6b11e..c96f65c 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.6.2"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 56b7d88975aa8da6446857cfd92de0825024bf63 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 13 Sep 2018 19:36:00 -0600 Subject: support for hex v2, multiple repository fetching, private organizations (#1884) * update to hex_core for hex-v2 repo support (#1865) * update to hex_core for hex-v2 repo support This patch adds only single repo hex-v2 support through hex_core. Packages no longer filtered out by buildtool metadata and the package index is updated per-package instead of fetched as one large ets dump. * tell travis to also build hex_core branch * support list of repos for hex packages (#1866) * support list of repos for hex packages repos are defined under the hex key in rebar configs. They can be defined at the top level of a project or globally, but not in profiles and the repos configured in dependencies are also ignored. Searching for packages involves first checking for a match in the local repo index cache, in the order repos are defined. If not found each repo is checked through the hex api for any known versions of the package and the first repo with a version that fits the constraint is used. * add {repos, replace, []} for overriding the global & default repos * add hex auth handling for repos (#1874) auth token are kept in a hex.config file that is modified by the rebar3 hex plugin. Repo names that have a : separating a parent and child are considered organizations. The parent repo's auth will be included with the child. So an organization named hexpm:rebar3_test will include any hexpm auth tokens found in the rebar3_test organization's configuration. * move packages to top level of of hexpm cache dir (#1876) * move packages to top level of of hexpm cache dir * append organization name to parent's repo_url when parsing repos * only eval config scripts and apply overrides once per app (#1879) * only eval config scripts and apply overrides once per app * move new resource behaviour to rebar_resource_v2 and keep v1 * cleanup use of rebar_resource module and unused functions * cleanup error messages and unused code * when discovering apps support mix packages as unbuilt apps (#1882) * use hex_core tarball unpacking support in pkg resource (#1883) * use hex_core tarball unpacking support in pkg resource * ignore etag if package doesn't exist and delete if checksum fails * add back tests for bad package checksums * improve bad registry checksum error message --- src/rebar.app.src | 2 + src/rebar.hrl | 31 ++- src/rebar3.erl | 19 +- src/rebar_api.erl | 2 +- src/rebar_app_discover.erl | 49 ++-- src/rebar_app_info.erl | 70 +++--- src/rebar_app_utils.erl | 109 ++++----- src/rebar_config.erl | 2 +- src/rebar_fetch.erl | 108 ++++----- src/rebar_git_resource.erl | 69 ++++-- src/rebar_hex_repos.erl | 142 ++++++++++++ src/rebar_hg_resource.erl | 67 ++++-- src/rebar_otp_app.erl | 16 +- src/rebar_packages.erl | 480 ++++++++++++++++++++++++++++------------ src/rebar_pkg_resource.erl | 437 +++++++++++++----------------------- src/rebar_prv_deps.erl | 8 +- src/rebar_prv_deps_tree.erl | 6 +- src/rebar_prv_install_deps.erl | 70 +++--- src/rebar_prv_local_upgrade.erl | 28 ++- src/rebar_prv_lock.erl | 9 +- src/rebar_prv_packages.erl | 106 +++++---- src/rebar_prv_repos.erl | 47 ++++ src/rebar_prv_shell.erl | 2 + src/rebar_prv_update.erl | 238 ++------------------ src/rebar_prv_upgrade.erl | 35 ++- src/rebar_resource.erl | 44 +++- src/rebar_resource_v2.erl | 147 ++++++++++++ src/rebar_state.erl | 76 ++++--- src/rebar_string.erl | 5 +- src/rebar_utils.erl | 157 ++++++++++--- 30 files changed, 1520 insertions(+), 1061 deletions(-) create mode 100644 src/rebar_hex_repos.erl create mode 100644 src/rebar_prv_repos.erl create mode 100644 src/rebar_resource_v2.erl (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index c96f65c..530a79e 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -30,6 +30,7 @@ relx, cf, inets, + hex_core, eunit_formatters]}, {env, [ %% Default log level @@ -67,6 +68,7 @@ rebar_prv_release, rebar_prv_relup, rebar_prv_report, + rebar_prv_repos, rebar_prv_shell, rebar_prv_state, rebar_prv_tar, diff --git a/src/rebar.hrl b/src/rebar.hrl index f461c70..572cbe8 100644 --- a/src/rebar.hrl +++ b/src/rebar.hrl @@ -25,14 +25,35 @@ -define(CONFIG_VERSION, "1.1.0"). -define(DEFAULT_CDN, "https://repo.hex.pm/"). -define(REMOTE_PACKAGE_DIR, "tarballs"). --define(REMOTE_REGISTRY_FILE, "registry.ets.gz"). -define(LOCK_FILE, "rebar.lock"). -define(DEFAULT_COMPILER_SOURCE_FORMAT, relative). +-define(PACKAGE_INDEX_VERSION, 4). +-define(PACKAGE_TABLE, package_index_v4). +-define(INDEX_FILE, "packages-v4.idx"). +-define(HEX_AUTH_FILE, "hex.config"). +-define(PUBLIC_HEX_REPO, <<"hexpm">>). --define(PACKAGE_INDEX_VERSION, 3). --define(PACKAGE_TABLE, package_index). --define(INDEX_FILE, "packages.idx"). --define(REGISTRY_FILE, "registry"). +%% ignore this function in all modules +%% not every module that exports it and relies on it being called implements provider +-ignore_xref([{format_error, 1}]). + +%% the package record is used in a select match spec which upsets dialyzer +%% this is the suggested workaround from Tobias +%% http://erlang.org/pipermail/erlang-questions/2009-February/041445.html +-type ms_field() :: '$1' | '_'. + +%% TODO: change package and requirement keys to be required (:=) after dropping support for OTP-18 +-record(package, {key :: {unicode:unicode_binary() | ms_field(), unicode:unicode_binary() | ms_field(), + unicode:unicode_binary() | ms_field()}, + checksum :: binary() | ms_field(), + retired :: boolean() | ms_field(), + dependencies :: [#{package => unicode:unicode_binary(), + requirement => unicode:unicode_binary()}] | ms_field()}). + +-record(resource, {type :: atom(), + module :: module(), + state :: term(), + implementation :: rebar_resource | rebar_resource_v2}). -ifdef(namespaced_types). -type rebar_dict() :: dict:dict(). diff --git a/src/rebar3.erl b/src/rebar3.erl index ec8e953..e87cb19 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -103,7 +103,7 @@ run(RawArgs) -> case erlang:system_info(version) of "6.1" -> ?WARN("Due to a filelib bug in Erlang 17.1 it is recommended" - "you update to a newer release.", []); + "you update to a newer release.", []); _ -> ok end, @@ -139,8 +139,14 @@ run_aux(State, RawArgs) -> rebar_state:set(State1, rebar_packages_cdn, CDN) end, + %% TODO: this means use of REBAR_PROFILE=profile will replace the repos with + %% the repos defined in the profile. But it will not work with `as profile`. + %% Maybe it shouldn't work with either to be consistent? + Resources = application:get_env(rebar, resources, []), + State2_ = rebar_state:create_resources(Resources, State2), + %% bootstrap test profile - State3 = rebar_state:add_to_profile(State2, test, test_state(State1)), + State3 = rebar_state:add_to_profile(State2_, test, test_state(State1)), %% Process each command, resetting any state between each one BaseDir = rebar_state:get(State, base_dir, ?DEFAULT_BASE_DIR), @@ -375,7 +381,11 @@ state_from_global_config(Config, GlobalConfigFile) -> %% We don't want to worry about global plugin install state effecting later %% usage. So we throw away the global profile state used for plugin install. - GlobalConfigThrowAway = rebar_state:current_profiles(GlobalConfig, [global]), + GlobalConfigThrowAway0 = rebar_state:current_profiles(GlobalConfig, [global]), + + Resources = application:get_env(rebar, resources, []), + GlobalConfigThrowAway = rebar_state:create_resources(Resources, GlobalConfigThrowAway0), + GlobalState = case rebar_state:get(GlobalConfigThrowAway, plugins, []) of [] -> GlobalConfigThrowAway; @@ -386,7 +396,8 @@ state_from_global_config(Config, GlobalConfigFile) -> end, GlobalPlugins = rebar_state:providers(GlobalState), GlobalConfig2 = rebar_state:set(GlobalConfig, plugins, []), - GlobalConfig3 = rebar_state:set(GlobalConfig2, {plugins, global}, rebar_state:get(GlobalConfigThrowAway, plugins, [])), + GlobalConfig3 = rebar_state:set(GlobalConfig2, {plugins, global}, + rebar_state:get(GlobalConfigThrowAway, plugins, [])), rebar_state:providers(rebar_state:new(GlobalConfig3, Config), GlobalPlugins). -spec test_state(rebar_state:t()) -> [{'extra_src_dirs',[string()]} | {'erl_opts',[any()]}]. diff --git a/src/rebar_api.erl b/src/rebar_api.erl index 9d9071e..4dabe8a 100644 --- a/src/rebar_api.erl +++ b/src/rebar_api.erl @@ -88,4 +88,4 @@ processing_base_dir(State) -> %% its configuration, including for validation of certs. -spec ssl_opts(string()) -> [term()]. ssl_opts(Url) -> - rebar_pkg_resource:ssl_opts(Url). + rebar_utils:ssl_opts(Url). diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index 382b36b..e82403c 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -9,8 +9,7 @@ find_apps/2, find_apps/3, find_app/2, - find_app/3, - find_app/4]). + find_app/3]). -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). @@ -95,7 +94,7 @@ format_error({missing_module, Module}) -> merge_deps(AppInfo, State) -> %% These steps make sure that hooks and artifacts are run in the context of %% the application they are defined at. If an umbrella structure is used and - %% they are deifned at the top level they will instead run in the context of + %% they are defined at the top level they will instead run in the context of %% the State and at the top level, not as part of an application. CurrentProfiles = rebar_state:current_profiles(State), Default = reset_hooks(rebar_state:default(State), CurrentProfiles), @@ -205,7 +204,7 @@ reset_hooks(Opts, CurrentProfiles) -> -spec all_app_dirs([file:name()]) -> [{file:name(), [file:name()]}]. all_app_dirs(LibDirs) -> lists:flatmap(fun(LibDir) -> - SrcDirs = find_config_src(LibDir, ["src"]), + {_, SrcDirs} = find_config_src(LibDir, ["src"]), app_dirs(LibDir, SrcDirs) end, LibDirs). @@ -278,8 +277,9 @@ find_apps(LibDirs, SrcDirs, Validate) -> %% app info record. -spec find_app(file:filename_all(), valid | invalid | all) -> {true, rebar_app_info:t()} | false. find_app(AppDir, Validate) -> - SrcDirs = find_config_src(AppDir, ["src"]), - find_app(rebar_app_info:new(), AppDir, SrcDirs, Validate). + {Config, SrcDirs} = find_config_src(AppDir, ["src"]), + AppInfo = rebar_app_info:update_opts(rebar_app_info:new(), dict:new(), Config), + find_app_(AppInfo, AppDir, SrcDirs, Validate). %% @doc check that a given app in a directory is there, and whether it's %% valid or not based on the second argument. Returns the related @@ -291,7 +291,7 @@ find_app(AppInfo, AppDir, Validate) -> %% of src/ AppOpts = rebar_app_info:opts(AppInfo), SrcDirs = rebar_dir:src_dirs(AppOpts, ["src"]), - find_app(AppInfo, AppDir, SrcDirs, Validate). + find_app_(AppInfo, AppDir, SrcDirs, Validate). %% @doc check that a given app in a directory is there, and whether it's %% valid or not based on the second argument. The third argument includes @@ -301,6 +301,14 @@ find_app(AppInfo, AppDir, Validate) -> [file:filename_all()], valid | invalid | all) -> {true, rebar_app_info:t()} | false. find_app(AppInfo, AppDir, SrcDirs, Validate) -> + Config = rebar_config:consult(AppDir), + AppInfo1 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), Config), + find_app_(AppInfo1, AppDir, SrcDirs, Validate). + +-spec find_app_(rebar_app_info:t(), file:filename_all(), + [file:filename_all()], valid | invalid | all) -> + {true, rebar_app_info:t()} | false. +find_app_(AppInfo, AppDir, SrcDirs, Validate) -> AppFile = filelib:wildcard(filename:join([AppDir, "ebin", "*.app"])), AppSrcFile = lists:append( [filelib:wildcard(filename:join([AppDir, SrcDir, "*.app.src"])) @@ -331,17 +339,14 @@ create_app_info(AppInfo, AppDir, AppFile) -> AppInfo2 = rebar_app_info:applications( rebar_app_info:app_details(AppInfo1, AppDetails), IncludedApplications++Applications), - C = rebar_config:consult(AppDir), - AppInfo3 = rebar_app_info:update_opts(AppInfo2, - rebar_app_info:opts(AppInfo2), C), - Valid = case rebar_app_utils:validate_application_info(AppInfo3) =:= true - andalso rebar_app_info:has_all_artifacts(AppInfo3) =:= true of + Valid = case rebar_app_utils:validate_application_info(AppInfo2) =:= true + andalso rebar_app_info:has_all_artifacts(AppInfo2) =:= true of true -> true; _ -> false end, - rebar_app_info:dir(rebar_app_info:valid(AppInfo3, Valid), AppDir). + rebar_app_info:dir(rebar_app_info:valid(AppInfo2, Valid), AppDir). %% @doc Read in and parse the .app file if it is availabe. Do the same for %% the .app.src file if it exists. @@ -403,12 +408,20 @@ try_handle_app_file(_AppInfo, Other, _AppDir, _AppSrcFile, _, _Validate) -> AppFile :: file:filename(), AppDir :: file:filename(), AppSrcFile :: file:filename(). -try_handle_app_src_file(_AppInfo, _, _AppDir, [], _Validate) -> - false; +try_handle_app_src_file(AppInfo, _, _AppDir, [], _Validate) -> + %% if .app and .app.src are not found check for a mix config file + %% it is assumed a plugin will build the application, including + %% a .app after this step + case filelib:is_file(filename:join(rebar_app_info:dir(AppInfo), "mix.exs")) of + true -> + {true, AppInfo}; + false -> + false + end; try_handle_app_src_file(_AppInfo, _, _AppDir, _AppSrcFile, valid) -> false; try_handle_app_src_file(AppInfo, _, AppDir, [File], Validate) when Validate =:= invalid - ; Validate =:= all -> + ; Validate =:= all -> AppInfo1 = rebar_app_info:app_file(AppInfo, undefined), AppInfo2 = create_app_info(AppInfo1, AppDir, File), case filename:extension(File) of @@ -437,8 +450,8 @@ to_atom(Bin) -> find_config_src(AppDir, Default) -> case rebar_config:consult(AppDir) of [] -> - Default; + {[], Default}; Terms -> %% TODO: handle profiles I guess, but we don't have that info - proplists:get_value(src_dirs, Terms, Default) + {Terms, proplists:get_value(src_dirs, Terms, Default)} end. diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index 88d6335..56ae4c0 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -7,6 +7,7 @@ new/4, new/5, update_opts/3, + update_opts_deps/2, discover/1, name/1, name/2, @@ -43,8 +44,6 @@ get/2, get/3, set/3, - resource_type/1, - resource_type/2, source/1, source/2, is_lock/1, @@ -53,6 +52,8 @@ is_checkout/2, valid/1, valid/2, + is_available/1, + is_available/2, verify_otp_vsn/1, has_all_artifacts/1, @@ -72,7 +73,7 @@ app_file_src :: file:filename_all() | undefined, app_file_src_script:: file:filename_all() | undefined, app_file :: file:filename_all() | undefined, - original_vsn :: binary() | string() | undefined, + original_vsn :: binary() | undefined, parent=root :: binary() | root, app_details=[] :: list(), applications=[] :: list(), @@ -83,11 +84,11 @@ dep_level=0 :: integer(), dir :: file:name(), out_dir :: file:name(), - resource_type :: pkg | src | undefined, source :: string() | tuple() | checkout | undefined, is_lock=false :: boolean(), is_checkout=false :: boolean(), - valid :: boolean() | undefined}). + valid :: boolean() | undefined, + is_available=false :: boolean()}). %%============================================================================ %% types @@ -114,14 +115,14 @@ new(AppName) -> {ok, t()}. new(AppName, Vsn) -> {ok, #app_info_t{name=rebar_utils:to_binary(AppName), - original_vsn=Vsn}}. + original_vsn=rebar_utils:to_binary(Vsn)}}. %% @doc build a complete version of the app info with all fields set. -spec new(atom() | binary() | string(), binary() | string(), file:name()) -> {ok, t()}. new(AppName, Vsn, Dir) -> {ok, #app_info_t{name=rebar_utils:to_binary(AppName), - original_vsn=Vsn, + original_vsn=rebar_utils:to_binary(Vsn), dir=rebar_utils:to_list(Dir), out_dir=rebar_utils:to_list(Dir)}}. @@ -130,7 +131,7 @@ new(AppName, Vsn, Dir) -> {ok, t()}. new(AppName, Vsn, Dir, Deps) -> {ok, #app_info_t{name=rebar_utils:to_binary(AppName), - original_vsn=Vsn, + original_vsn=rebar_utils:to_binary(Vsn), dir=rebar_utils:to_list(Dir), out_dir=rebar_utils:to_list(Dir), deps=Deps}}. @@ -141,7 +142,7 @@ new(AppName, Vsn, Dir, Deps) -> new(Parent, AppName, Vsn, Dir, Deps) -> {ok, #app_info_t{name=rebar_utils:to_binary(AppName), parent=Parent, - original_vsn=Vsn, + original_vsn=rebar_utils:to_binary(Vsn), dir=rebar_utils:to_list(Dir), out_dir=rebar_utils:to_list(Dir), deps=Deps}}. @@ -150,10 +151,12 @@ new(Parent, AppName, Vsn, Dir, Deps) -> %% file for the app -spec update_opts(t(), rebar_dict(), [any()]) -> t(). update_opts(AppInfo, Opts, Config) -> - LockDeps = case resource_type(AppInfo) of - pkg -> - Deps = deps(AppInfo), - [{{locks, default}, Deps}, {{deps, default}, Deps}]; + LockDeps = case source(AppInfo) of + Tuple when is_tuple(Tuple) andalso element(1, Tuple) =:= pkg -> + %% Deps are set separate for packages + %% instead of making it seem we have no deps + %% don't set anything here. + []; _ -> deps_from_config(dir(AppInfo), Config) end, @@ -165,8 +168,18 @@ update_opts(AppInfo, Opts, Config) -> NewOpts = rebar_opts:merge_opts(LocalOpts, Opts), - AppInfo#app_info_t{opts=NewOpts - ,default=NewOpts}. + AppInfo#app_info_t{opts=NewOpts, + default=NewOpts}. + +%% @doc update the opts based on new deps, usually from an app's hex registry metadata +-spec update_opts_deps(t(), [any()]) -> t(). +update_opts_deps(AppInfo=#app_info_t{opts=Opts}, Deps) -> + LocalOpts = dict:from_list([{{locks, default}, Deps}, {{deps, default}, Deps}]), + NewOpts = rebar_opts:merge_opts(LocalOpts, Opts), + AppInfo#app_info_t{opts=NewOpts, + default=NewOpts, + deps=Deps}. + %% @private extract the deps for an app in `Dir' based on its config file data -spec deps_from_config(file:filename(), [any()]) -> [{tuple(), any()}, ...]. @@ -350,15 +363,15 @@ parent(AppInfo=#app_info_t{}, Parent) -> %% @doc returns the original version of the app (unevaluated if %% asking for a semver) --spec original_vsn(t()) -> string(). +-spec original_vsn(t()) -> binary(). original_vsn(#app_info_t{original_vsn=Vsn}) -> Vsn. %% @doc stores the original version of the app (unevaluated if %% asking for a semver) --spec original_vsn(t(), string()) -> t(). +-spec original_vsn(t(), binary() | string()) -> t(). original_vsn(AppInfo=#app_info_t{}, Vsn) -> - AppInfo#app_info_t{original_vsn=Vsn}. + AppInfo#app_info_t{original_vsn=rebar_utils:to_binary(Vsn)}. %% @doc returns the list of applications the app depends on. -spec applications(t()) -> list(). @@ -438,16 +451,6 @@ ebin_dir(#app_info_t{out_dir=OutDir}) -> priv_dir(#app_info_t{out_dir=OutDir}) -> rebar_utils:to_list(filename:join(OutDir, "priv")). -%% @doc returns whether the app is source app or a package app. --spec resource_type(t()) -> pkg | src. -resource_type(#app_info_t{resource_type=ResourceType}) -> - ResourceType. - -%% @doc sets whether the app is source app or a package app. --spec resource_type(t(), pkg | src) -> t(). -resource_type(AppInfo=#app_info_t{}, Type) -> - AppInfo#app_info_t{resource_type=Type}. - %% @doc finds the source specification for the app -spec source(t()) -> string() | tuple(). source(#app_info_t{source=Source}) -> @@ -478,6 +481,17 @@ is_checkout(#app_info_t{is_checkout=IsCheckout}) -> is_checkout(AppInfo=#app_info_t{}, IsCheckout) -> AppInfo#app_info_t{is_checkout=IsCheckout}. +%% @doc returns whether the app source exists in the deps dir +-spec is_available(t()) -> boolean(). +is_available(#app_info_t{is_available=IsAvailable}) -> + IsAvailable. + +%% @doc sets whether the app's source is available +%% only set if the app's source is found in the expected dep directory +-spec is_available(t(), boolean()) -> t(). +is_available(AppInfo=#app_info_t{}, IsAvailable) -> + AppInfo#app_info_t{is_available=IsAvailable}. + %% @doc returns whether the app is valid (built) or not -spec valid(t()) -> boolean(). valid(AppInfo=#app_info_t{valid=undefined}) -> diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index 1d7ef5b..35e908c 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -217,26 +217,23 @@ parse_dep(_, Dep, _, _, _) -> dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> CheckoutsDir = rebar_utils:to_list(rebar_dir:checkouts_dir(State, Name)), AppInfo = case rebar_app_info:discover(CheckoutsDir) of - {ok, App} -> - rebar_app_info:source(rebar_app_info:is_checkout(App, true), checkout); - not_found -> - Dir = rebar_utils:to_list(filename:join(DepsDir, Name)), - {ok, AppInfo0} = - case rebar_app_info:discover(Dir) of - {ok, App} -> - {ok, rebar_app_info:parent(App, Parent)}; - not_found -> - rebar_app_info:new(Parent, Name, Vsn, Dir, []) - end, - rebar_app_info:source(AppInfo0, Source) - end, - C = rebar_config:consult(rebar_app_info:dir(AppInfo)), - AppInfo1 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), C), - Overrides = rebar_state:get(State, overrides, []), - AppInfo2 = rebar_app_info:set(AppInfo1, overrides, rebar_app_info:get(AppInfo, overrides, [])++Overrides), - AppInfo3 = rebar_app_info:apply_overrides(rebar_app_info:get(AppInfo2, overrides, []), AppInfo2), - AppInfo4 = rebar_app_info:apply_profiles(AppInfo3, [default, prod]), - AppInfo5 = rebar_app_info:profiles(AppInfo4, [default]), + {ok, App} -> + rebar_app_info:source(rebar_app_info:is_checkout(App, true), checkout); + not_found -> + Dir = rebar_utils:to_list(filename:join(DepsDir, Name)), + {ok, AppInfo0} = + case rebar_app_info:discover(Dir) of + {ok, App} -> + {ok, rebar_app_info:is_available(rebar_app_info:parent(App, Parent), + true)}; + not_found -> + rebar_app_info:new(Parent, Name, Vsn, Dir, []) + end, + rebar_app_info:source(AppInfo0, Source) + end, + Overrides = rebar_app_info:get(AppInfo, overrides, []) ++ rebar_state:get(State, overrides, []), + AppInfo2 = rebar_app_info:set(AppInfo, overrides, Overrides), + AppInfo5 = rebar_app_info:profiles(AppInfo2, [default]), rebar_app_info:is_lock(AppInfo5, IsLock). %% @doc Takes a given application app_info record along with the project. @@ -250,52 +247,36 @@ expand_deps_sources(Dep, State) -> %% around version if required. -spec update_source(rebar_app_info:t(), Source, rebar_state:t()) -> rebar_app_info:t() when - Source :: tuple() | atom() | binary(). % TODO: meta to source() + Source :: rebar_resource_v2:source(). update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) -> - {PkgName1, PkgVsn1} = case PkgVsn of - undefined -> - get_package(PkgName, "0", State); - <<"~>", Vsn/binary>> -> - [Vsn1] = [X || X <- binary:split(Vsn, [<<" ">>], [global]), X =/= <<>>], - get_package(PkgName, Vsn1, State); - _ -> - {PkgName, PkgVsn} - end, - %% store the expected hash for the dependency - Hash1 = case Hash of - undefined -> % unknown, define the hash since we know the dep - fetch_checksum(PkgName1, PkgVsn1, Hash, State); - _ -> % keep as is - Hash - end, - AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName1, PkgVsn1, Hash1}), - Deps = rebar_packages:deps(PkgName1 - ,PkgVsn1 - ,State), - AppInfo2 = rebar_app_info:resource_type(rebar_app_info:deps(AppInfo1, Deps), pkg), - rebar_app_info:original_vsn(AppInfo2, PkgVsn1); + case rebar_packages:resolve_version(PkgName, PkgVsn, Hash, + ?PACKAGE_TABLE, State) of + {ok, Package, RepoConfig} -> + #package{key={_, PkgVsn1, _}, + checksum=Hash1, + dependencies=Deps} = Package, + AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName, PkgVsn1, Hash1, RepoConfig}), + AppInfo2 = rebar_app_info:update_opts_deps(AppInfo1, Deps), + rebar_app_info:original_vsn(AppInfo2, PkgVsn1); + not_found -> + throw(?PRV_ERROR({missing_package, PkgName, PkgVsn})); + {error, {invalid_vsn, InvalidVsn}} -> + throw(?PRV_ERROR({invalid_vsn, PkgName, InvalidVsn})) + end; update_source(AppInfo, Source, _State) -> rebar_app_info:source(AppInfo, Source). -%% @doc grab the checksum for a given package --spec fetch_checksum(atom(), string(), iodata() | undefined, rebar_state:t()) -> - iodata() | no_return(). -fetch_checksum(PkgName, PkgVsn, Hash, State) -> - try - rebar_packages:registry_checksum({pkg, PkgName, PkgVsn, Hash}, State) - catch - _:_ -> - ?INFO("Package ~ts-~ts not found. Fetching registry updates and trying again...", [PkgName, PkgVsn]), - {ok, _} = rebar_prv_update:do(State), - rebar_packages:registry_checksum({pkg, PkgName, PkgVsn, Hash}, State) - end. - %% @doc convert a given exception's payload into an io description. -spec format_error(any()) -> iolist(). -format_error({missing_package, Package}) -> - io_lib:format("Package not found in registry: ~ts", [Package]); +format_error({missing_package, Name, undefined}) -> + io_lib:format("Package not found in any repo: ~ts.", [rebar_utils:to_binary(Name)]); +format_error({missing_package, Name, Vsn}) -> + io_lib:format("Package not found in any repo: ~ts-~ts.", [rebar_utils:to_binary(Name), + rebar_utils:to_binary(Vsn)]); format_error({parse_dep, Dep}) -> io_lib:format("Failed parsing dep ~p", [Dep]); +format_error({invalid_vsn, Dep, InvalidVsn}) -> + io_lib:format("Dep ~ts has invalid version ~ts", [Dep, InvalidVsn]); format_error(Error) -> io_lib:format("~p", [Error]). @@ -303,18 +284,6 @@ format_error(Error) -> %% Internal functions %% =================================================================== -%% @private find the correct version of a package based on the version -%% and name passed in. --spec get_package(binary(), binary() | string(), rebar_state:t()) -> - term() | no_return(). -get_package(Dep, Vsn, State) -> - case rebar_packages:find_highest_matching(Dep, Vsn, ?PACKAGE_TABLE, State) of - {ok, HighestDepVsn} -> - {Dep, HighestDepVsn}; - none -> - throw(?PRV_ERROR({missing_package, rebar_utils:to_binary(Dep)})) - end. - %% @private checks that all the beam files have been properly %% created. -spec has_all_beams(file:filename_all(), [module()]) -> diff --git a/src/rebar_config.erl b/src/rebar_config.erl index 797dddc..2651ca1 100644 --- a/src/rebar_config.erl +++ b/src/rebar_config.erl @@ -74,7 +74,7 @@ consult_lock_file(File) -> read_attrs(beta, Locks, []); [{Vsn, Locks}|Attrs] when is_list(Locks) -> % versioned lock file %% Because this is the first version of rebar3 to introduce a lock - %% file, all versionned lock files with a different versions have + %% file, all versioned lock files with a different version have %% to be newer. case Vsn of ?CONFIG_VERSION -> diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index d2c7706..9c76e0e 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -7,104 +7,74 @@ %% ------------------------------------------------------------------- -module(rebar_fetch). --export([lock_source/3, - download_source/3, - needs_update/3]). +-export([lock_source/2, + download_source/2, + needs_update/2]). -export([format_error/1]). -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). --spec lock_source(file:filename_all(), rebar_resource:resource(), rebar_state:t()) -> - rebar_resource:resource() | {error, string()}. -lock_source(AppDir, Source, State) -> - Resources = rebar_state:resources(State), - Module = get_resource_type(Source, Resources), - Module:lock(AppDir, Source). +-spec lock_source(rebar_app_info:t(), rebar_state:t()) + -> rebar_resource_v2:source() | {error, string()}. +lock_source(AppInfo, State) -> + rebar_resource_v2:lock(AppInfo, State). --spec download_source(file:filename_all(), rebar_resource:resource(), rebar_state:t()) -> - true | {error, any()}. -download_source(AppDir, Source, State) -> - try download_source_(AppDir, Source, State) of - true -> - true; - Error -> - throw(?PRV_ERROR(Error)) +-spec download_source(rebar_app_info:t(), rebar_state:t()) + -> rebar_app_info:t() | {error, any()}. +download_source(AppInfo, State) -> + AppDir = rebar_app_info:dir(AppInfo), + try download_source_(AppInfo, State) of + ok -> + %% freshly downloaded, update the app info opts to reflect the new config + Config = rebar_config:consult(AppDir), + AppInfo1 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), Config), + case rebar_app_discover:find_app(AppInfo1, AppDir, all) of + {true, AppInfo2} -> + rebar_app_info:is_available(AppInfo2, true); + false -> + throw(?PRV_ERROR({dep_app_not_found, rebar_app_info:name(AppInfo1)})) + end; + {error, Reason} -> + throw(?PRV_ERROR(Reason)) catch + throw:{no_resource, Type, Location} -> + throw(?PRV_ERROR({no_resource, Location, Type})); ?WITH_STACKTRACE(C,T,S) ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, S]), - throw(?PRV_ERROR({fetch_fail, Source})) + throw(?PRV_ERROR({fetch_fail, rebar_app_info:source(AppInfo)})) end. -download_source_(AppDir, Source, State) -> - Resources = rebar_state:resources(State), - Module = get_resource_type(Source, Resources), +download_source_(AppInfo, State) -> + AppDir = rebar_app_info:dir(AppInfo), TmpDir = ec_file:insecure_mkdtemp(), AppDir1 = rebar_utils:to_list(AppDir), - case Module:download(TmpDir, Source, State) of - {ok, _} -> + case rebar_resource_v2:download(TmpDir, AppInfo, State) of + ok -> ec_file:mkdir_p(AppDir1), code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), ok = rebar_file_utils:rm_rf(filename:absname(AppDir1)), ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]), - ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)), - true; + rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)); Error -> Error end. --spec needs_update(file:filename_all(), rebar_resource:resource(), rebar_state:t()) -> boolean() | {error, string()}. -needs_update(AppDir, Source, State) -> - Resources = rebar_state:resources(State), - Module = get_resource_type(Source, Resources), +-spec needs_update(rebar_app_info:t(), rebar_state:t()) + -> boolean() | {error, string()}. +needs_update(AppInfo, State) -> try - Module:needs_update(AppDir, Source) + rebar_resource_v2:needs_update(AppInfo, State) catch _:_ -> true end. -format_error({bad_download, CachePath}) -> - io_lib:format("Download of package does not match md5sum from server: ~ts", [CachePath]); -format_error({unexpected_hash, CachePath, Expected, Found}) -> - io_lib:format("The checksum for package at ~ts (~ts) does not match the " - "checksum previously locked (~ts). Either unlock or " - "upgrade the package, or make sure you fetched it from " - "the same index from which it was initially fetched.", - [CachePath, Found, Expected]); -format_error({failed_extract, CachePath}) -> - io_lib:format("Failed to extract package: ~ts", [CachePath]); -format_error({bad_etag, Source}) -> - io_lib:format("MD5 Checksum comparison failed for: ~ts", [Source]); format_error({fetch_fail, Name, Vsn}) -> io_lib:format("Failed to fetch and copy dep: ~ts-~ts", [Name, Vsn]); format_error({fetch_fail, Source}) -> io_lib:format("Failed to fetch and copy dep: ~p", [Source]); -format_error({bad_checksum, File}) -> - io_lib:format("Checksum mismatch against tarball in ~ts", [File]); -format_error({bad_registry_checksum, File}) -> - io_lib:format("Checksum mismatch against registry in ~ts", [File]). - -get_resource_type({Type, Location}, Resources) -> - find_resource_module(Type, Location, Resources); -get_resource_type({Type, Location, _}, Resources) -> - find_resource_module(Type, Location, Resources); -get_resource_type({Type, _, _, Location}, Resources) -> - find_resource_module(Type, Location, Resources); -get_resource_type(_, _) -> - rebar_pkg_resource. - -find_resource_module(Type, Location, Resources) -> - case lists:keyfind(Type, 1, Resources) of - false -> - case code:which(Type) of - non_existing -> - {error, io_lib:format("Cannot handle dependency ~ts.~n" - " No module for resource type ~p", [Location, Type])}; - _ -> - Type - end; - {Type, Module} -> - Module - end. +format_error({dep_app_not_found, AppName}) -> + io_lib:format("Dependency failure: source for ~ts does not contain a " + "recognizable project and can not be built", [AppName]). diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl index 3aa875f..29c9ad7 100644 --- a/src/rebar_git_resource.erl +++ b/src/rebar_git_resource.erl @@ -2,21 +2,30 @@ %% ex: ts=4 sw=4 et -module(rebar_git_resource). --behaviour(rebar_resource). +-behaviour(rebar_resource_v2). --export([lock/2 - ,download/3 - ,needs_update/2 - ,make_vsn/1]). +-export([init/2, + lock/2, + download/4, + needs_update/2, + make_vsn/2]). -include("rebar.hrl"). %% Regex used for parsing scp style remote url -define(SCP_PATTERN, "\\A(?[^@]+)@(?[^:]+):(?.+)\\z"). -lock(AppDir, {git, Url, _}) -> - lock(AppDir, {git, Url}); -lock(AppDir, {git, Url}) -> +-spec init(atom(), rebar_state:t()) -> {ok, rebar_resource_v2:resource()}. +init(Type, _State) -> + Resource = rebar_resource_v2:new(Type, ?MODULE, #{}), + {ok, Resource}. + +lock(AppInfo, _) -> + lock_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). + +lock_(AppDir, {git, Url, _}) -> + lock_(AppDir, {git, Url}); +lock_(AppDir, {git, Url}) -> AbortMsg = lists:flatten(io_lib:format("Locking of git dependency failed in ~ts", [AppDir])), Dir = rebar_utils:escape_double_quotes(AppDir), {ok, VsnString} = @@ -33,14 +42,17 @@ lock(AppDir, {git, Url}) -> %% Return true if either the git url or tag/branch/ref is not the same as the currently %% checked out git repo for the dep -needs_update(Dir, {git, Url, {tag, Tag}}) -> +needs_update(AppInfo, _) -> + needs_update_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). + +needs_update_(Dir, {git, Url, {tag, Tag}}) -> {ok, Current} = rebar_utils:sh(?FMT("git describe --tags --exact-match", []), [{cd, Dir}]), Current1 = rebar_string:trim(rebar_string:trim(Current, both, "\n"), both, "\r"), ?DEBUG("Comparing git tag ~ts with ~ts", [Tag, Current1]), not ((Current1 =:= Tag) andalso compare_url(Dir, Url)); -needs_update(Dir, {git, Url, {branch, Branch}}) -> +needs_update_(Dir, {git, Url, {branch, Branch}}) -> %% Fetch remote so we can check if the branch has changed SafeBranch = rebar_utils:escape_chars(Branch), {ok, _} = rebar_utils:sh(?FMT("git fetch origin ~ts", [SafeBranch]), @@ -50,9 +62,9 @@ needs_update(Dir, {git, Url, {branch, Branch}}) -> [{cd, Dir}]), ?DEBUG("Checking git branch ~ts for updates", [Branch]), not ((Current =:= []) andalso compare_url(Dir, Url)); -needs_update(Dir, {git, Url, "master"}) -> - needs_update(Dir, {git, Url, {branch, "master"}}); -needs_update(Dir, {git, _, Ref}) -> +needs_update_(Dir, {git, Url, "master"}) -> + needs_update_(Dir, {git, Url, {branch, "master"}}); +needs_update_(Dir, {git, _, Ref}) -> {ok, Current} = rebar_utils:sh(?FMT("git rev-parse --short=7 -q HEAD", []), [{cd, Dir}]), Current1 = rebar_string:trim(rebar_string:trim(Current, both, "\n"), @@ -98,25 +110,35 @@ parse_git_url(not_scp, Url) -> {error, Reason} end. -download(Dir, {git, Url}, State) -> +download(TmpDir, AppInfo, State, _) -> + case download_(TmpDir, rebar_app_info:source(AppInfo), State) of + {ok, _} -> + ok; + {error, Reason} -> + {error, Reason}; + Error -> + {error, Error} + end. + +download_(Dir, {git, Url}, State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), - download(Dir, {git, Url, {branch, "master"}}, State); -download(Dir, {git, Url, ""}, State) -> + download_(Dir, {git, Url, {branch, "master"}}, State); +download_(Dir, {git, Url, ""}, State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), - download(Dir, {git, Url, {branch, "master"}}, State); -download(Dir, {git, Url, {branch, Branch}}, _State) -> + download_(Dir, {git, Url, {branch, "master"}}, State); +download_(Dir, {git, Url, {branch, Branch}}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), git_clone(branch, git_vsn(), Url, Dir, Branch); -download(Dir, {git, Url, {tag, Tag}}, _State) -> +download_(Dir, {git, Url, {tag, Tag}}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), git_clone(tag, git_vsn(), Url, Dir, Tag); -download(Dir, {git, Url, {ref, Ref}}, _State) -> +download_(Dir, {git, Url, {ref, Ref}}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), git_clone(ref, git_vsn(), Url, Dir, Ref); -download(Dir, {git, Url, Rev}, _State) -> +download_(Dir, {git, Url, Rev}, _State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), @@ -195,7 +217,10 @@ git_vsn_fetch() -> undefined end. -make_vsn(Dir) -> +make_vsn(AppInfo, _) -> + make_vsn_(rebar_app_info:dir(AppInfo)). + +make_vsn_(Dir) -> case collect_default_refcount(Dir) of Vsn={plain, _} -> Vsn; diff --git a/src/rebar_hex_repos.erl b/src/rebar_hex_repos.erl new file mode 100644 index 0000000..ebee191 --- /dev/null +++ b/src/rebar_hex_repos.erl @@ -0,0 +1,142 @@ +-module(rebar_hex_repos). + +-export([from_state/2, + get_repo_config/2, + auth_config/1, + update_auth_config/2, + format_error/1]). + +-ifdef(TEST). +%% exported for test purposes +-export([repos/1, merge_repos/1]). +-endif. + +-include("rebar.hrl"). +-include_lib("providers/include/providers.hrl"). + +-export_type([repo/0]). + +-type repo() :: #{name => unicode:unicode_binary(), + api_url => binary(), + api_key => binary(), + repo_url => binary(), + repo_public_key => binary(), + repo_verify => binary()}. + +from_state(BaseConfig, State) -> + HexConfig = rebar_state:get(State, hex, []), + Repos = repos(HexConfig), + %% auth is stored in a separate config file since the plugin generates and modifies it + Auth = ?MODULE:auth_config(State), + %% add base config entries that are specific to use by rebar3 and not overridable + Repos1 = merge_with_base_and_auth(Repos, BaseConfig, Auth), + %% merge organizations parent repo options into each oraganization repo + update_organizations(Repos1). + +-spec get_repo_config(unicode:unicode_binary(), rebar_state:t() | [repo()]) + -> {ok, repo()} | error. +get_repo_config(RepoName, Repos) when is_list(Repos) -> + case ec_lists:find(fun(#{name := N}) -> N =:= RepoName end, Repos) of + error -> + throw(?PRV_ERROR({repo_not_found, RepoName})); + {ok, RepoConfig} -> + {ok, RepoConfig} + end; +get_repo_config(RepoName, State) -> + Resources = rebar_state:resources(State), + #{repos := Repos} = rebar_resource_v2:find_resource_state(pkg, Resources), + get_repo_config(RepoName, Repos). + +merge_with_base_and_auth(Repos, BaseConfig, Auth) -> + [maps:merge(maps:get(maps:get(name, Repo), Auth, #{}), + maps:merge(Repo, BaseConfig)) || Repo <- Repos]. + +%% A user's list of repos are merged by name while keeping the order +%% intact. The order is based on the first use of a repo by name in the +%% list. The default repo is appended to the user's list. +repos(HexConfig) -> + HexDefaultConfig = default_repo(), + case [R || R <- HexConfig, element(1, R) =:= repos] of + [] -> + [HexDefaultConfig]; + %% we only care if the first element is a replace entry + [{repos, replace, Repos} | _]-> + merge_repos(Repos); + Repos -> + RepoList = repo_list(Repos), + merge_repos(RepoList ++ [HexDefaultConfig]) + end. + +-spec merge_repos([repo()]) -> [repo()]. +merge_repos(Repos) -> + lists:foldl(fun(R=#{name := Name}, ReposAcc) -> + %% private organizations include the parent repo before a : + case rebar_string:split(Name, <<":">>) of + [Repo, Org] -> + update_repo_list(R#{name => Name, + organization => Org, + parent => Repo}, ReposAcc); + _ -> + update_repo_list(R, ReposAcc) + end + end, [], Repos). + +update_organizations(Repos) -> + lists:map(fun(Repo=#{organization := Organization, + parent := ParentName}) -> + {ok, Parent} = get_repo_config(ParentName, Repos), + ParentRepoUrl = rebar_utils:to_list(maps:get(repo_url, Parent)), + {ok, RepoUrl} = + rebar_utils:url_append_path(ParentRepoUrl, + filename:join("repos", rebar_utils:to_list(Organization))), + %% still let the organization config override this constructed repo url + maps:merge(Parent#{repo_url => rebar_utils:to_binary(RepoUrl)}, Repo); + (Repo) -> + Repo + end, Repos). + +update_repo_list(R=#{name := N}, [H=#{name := HN} | Rest]) when N =:= HN -> + [maps:merge(R, H) | Rest]; +update_repo_list(R, [H | Rest]) -> + [H | update_repo_list(R, Rest)]; +update_repo_list(R, []) -> + [R]. + +default_repo() -> + HexDefaultConfig = hex_core:default_config(), + HexDefaultConfig#{name => ?PUBLIC_HEX_REPO}. + +repo_list([]) -> + []; +repo_list([{repos, Repos} | T]) -> + Repos ++ repo_list(T); +repo_list([{repos, replace, Repos} | T]) -> + Repos ++ repo_list(T). + +format_error({repo_not_found, RepoName}) -> + io_lib:format("The repo ~ts was not found in the configuration.", [RepoName]). + +%% auth functions + +%% authentication is in a separate config file because the hex plugin updates it + +-spec auth_config_file(rebar_state:t()) -> file:filename_all(). +auth_config_file(State) -> + filename:join(rebar_dir:global_config_dir(State), ?HEX_AUTH_FILE). + +-spec auth_config(rebar_state:t()) -> map(). +auth_config(State) -> + case file:consult(auth_config_file(State)) of + {ok, [Config]} -> + Config; + _ -> + #{} + end. + +-spec update_auth_config(map(), rebar_state:t()) -> ok. +update_auth_config(Updates, State) -> + Config = auth_config(State), + AuthConfigFile = auth_config_file(State), + ok = filelib:ensure_dir(AuthConfigFile), + NewConfig = iolist_to_binary([io_lib:print(maps:merge(Config, Updates)) | ".\n"]), + ok = file:write_file(AuthConfigFile, NewConfig). diff --git a/src/rebar_hg_resource.erl b/src/rebar_hg_resource.erl index abcca88..21d4b9d 100644 --- a/src/rebar_hg_resource.erl +++ b/src/rebar_hg_resource.erl @@ -2,39 +2,51 @@ %% ex: ts=4 sw=4 et -module(rebar_hg_resource). --behaviour(rebar_resource). +-behaviour(rebar_resource_v2). --export([lock/2 - ,download/3 - ,needs_update/2 - ,make_vsn/1]). +-export([init/2, + lock/2, + download/4, + needs_update/2, + make_vsn/2]). -include("rebar.hrl"). -lock(AppDir, {hg, Url, _}) -> - lock(AppDir, {hg, Url}); -lock(AppDir, {hg, Url}) -> +-spec init(atom(), rebar_state:t()) -> {ok, rebar_resource_v2:resource()}. +init(Type, _State) -> + Resource = rebar_resource_v2:new(Type, ?MODULE, #{}), + {ok, Resource}. + +lock(AppInfo, _) -> + lock_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). + +lock_(AppDir, {hg, Url, _}) -> + lock_(AppDir, {hg, Url}); +lock_(AppDir, {hg, Url}) -> Ref = get_ref(AppDir), {hg, Url, {ref, Ref}}. %% Return `true' if either the hg url or tag/branch/ref is not the same as %% the currently checked out repo for the dep -needs_update(Dir, {hg, Url, {tag, Tag}}) -> +needs_update(AppInfo, _) -> + needs_update_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). + +needs_update_(Dir, {hg, Url, {tag, Tag}}) -> Ref = get_ref(Dir), {ClosestTag, Distance} = get_tag_distance(Dir, Ref), ?DEBUG("Comparing hg tag ~ts with ref ~ts (closest tag is ~ts at distance ~ts)", [Tag, Ref, ClosestTag, Distance]), not ((Distance =:= "0") andalso (Tag =:= ClosestTag) andalso compare_url(Dir, Url)); -needs_update(Dir, {hg, Url, {branch, Branch}}) -> +needs_update_(Dir, {hg, Url, {branch, Branch}}) -> Ref = get_ref(Dir), BRef = get_branch_ref(Dir, Branch), not ((Ref =:= BRef) andalso compare_url(Dir, Url)); -needs_update(Dir, {hg, Url, "default"}) -> +needs_update_(Dir, {hg, Url, "default"}) -> Ref = get_ref(Dir), BRef = get_branch_ref(Dir, "default"), not ((Ref =:= BRef) andalso compare_url(Dir, Url)); -needs_update(Dir, {hg, Url, Ref}) -> +needs_update_(Dir, {hg, Url, Ref}) -> LocalRef = get_ref(Dir), TargetRef = case Ref of {ref, Ref1} -> @@ -48,13 +60,23 @@ needs_update(Dir, {hg, Url, Ref}) -> ?DEBUG("Comparing hg ref ~ts with ~ts", [Ref1, LocalRef]), not ((LocalRef =:= TargetRef) andalso compare_url(Dir, Url)). -download(Dir, {hg, Url}, State) -> +download(TmpDir, AppInfo, State, _) -> + case download_(TmpDir, rebar_app_info:source(AppInfo), State) of + {ok, _} -> + ok; + {error, Reason} -> + {error, Reason}; + Error -> + {error, Error} + end. + +download_(Dir, {hg, Url}, State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), - download(Dir, {hg, Url, {branch, "default"}}, State); -download(Dir, {hg, Url, ""}, State) -> + download_(Dir, {hg, Url, {branch, "default"}}, State); +download_(Dir, {hg, Url, ""}, State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), - download(Dir, {hg, Url, {branch, "default"}}, State); -download(Dir, {hg, Url, {branch, Branch}}, _State) -> + download_(Dir, {hg, Url, {branch, "default"}}, State); +download_(Dir, {hg, Url, {branch, Branch}}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), rebar_utils:sh(?FMT("hg clone -q -b ~ts ~ts ~ts", @@ -62,7 +84,7 @@ download(Dir, {hg, Url, {branch, Branch}}, _State) -> rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]); -download(Dir, {hg, Url, {tag, Tag}}, _State) -> +download_(Dir, {hg, Url, {tag, Tag}}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), rebar_utils:sh(?FMT("hg clone -q -u ~ts ~ts ~ts", @@ -70,7 +92,7 @@ download(Dir, {hg, Url, {tag, Tag}}, _State) -> rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]); -download(Dir, {hg, Url, {ref, Ref}}, _State) -> +download_(Dir, {hg, Url, {ref, Ref}}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), rebar_utils:sh(?FMT("hg clone -q -r ~ts ~ts ~ts", @@ -78,7 +100,7 @@ download(Dir, {hg, Url, {ref, Ref}}, _State) -> rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]); -download(Dir, {hg, Url, Rev}, _State) -> +download_(Dir, {hg, Url, Rev}, _State) -> ok = filelib:ensure_dir(Dir), maybe_warn_local_url(Url), rebar_utils:sh(?FMT("hg clone -q -r ~ts ~ts ~ts", @@ -87,7 +109,10 @@ download(Dir, {hg, Url, Rev}, _State) -> rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]). -make_vsn(Dir) -> +make_vsn(AppInfo, _) -> + make_vsn_(rebar_app_info:dir(AppInfo)). + +make_vsn_(Dir) -> BaseHg = "hg -R \"" ++ rebar_utils:escape_double_quotes(Dir) ++ "\" ", Ref = get_ref(Dir), Cmd = BaseHg ++ "log --template \"{latesttag}+build.{latesttagdistance}.rev.{node|short}\"" diff --git a/src/rebar_otp_app.erl b/src/rebar_otp_app.erl index f5bb9cf..1d854da 100644 --- a/src/rebar_otp_app.erl +++ b/src/rebar_otp_app.erl @@ -59,8 +59,8 @@ compile(State, App) -> format_error({missing_app_file, Filename}) -> io_lib:format("App file is missing: ~ts", [Filename]); -format_error({file_read, File, Reason}) -> - io_lib:format("Failed to read required file ~ts for processing: ~ts", [File, file:format_error(Reason)]); +format_error({file_read, AppName, File, Reason}) -> + io_lib:format("Failed to read required ~ts file for processing the application '~ts': ~ts", [File, AppName, file:format_error(Reason)]); format_error({invalid_name, File, AppName}) -> io_lib:format("Invalid ~ts: name of application (~p) must match filename.", [File, AppName]). @@ -79,7 +79,7 @@ validate_app(State, App) -> Error end; {error, Reason} -> - ?PRV_ERROR({file_read, AppFile, Reason}) + ?PRV_ERROR({file_read, rebar_app_info:name(App), ".app", Reason}) end. validate_app_modules(State, App, AppData) -> @@ -110,7 +110,7 @@ preprocess(State, AppInfo, AppSrcFile) -> A1 = apply_app_vars(AppVars, AppData), %% AppSrcFile may contain instructions for generating a vsn number - Vsn = app_vsn(AppData, AppSrcFile, State), + Vsn = app_vsn(AppInfo, AppData, AppSrcFile, State), A2 = lists:keystore(vsn, 1, A1, {vsn, Vsn}), %% systools:make_relup/4 fails with {missing_param, registered} @@ -131,7 +131,7 @@ preprocess(State, AppInfo, AppSrcFile) -> AppFile; {error, Reason} -> - throw(?PRV_ERROR({file_read, AppSrcFile, Reason})) + throw(?PRV_ERROR({file_read, rebar_app_info:name(AppInfo), ".app.src", Reason})) end. load_app_vars(State) -> @@ -226,10 +226,8 @@ consult_app_file(Filename) -> end end. -app_vsn(AppData, AppFile, State) -> - AppDir = filename:dirname(filename:dirname(AppFile)), - Resources = rebar_state:resources(State), - rebar_utils:vcs_vsn(get_value(vsn, AppData, AppFile), AppDir, Resources). +app_vsn(AppInfo, AppData, AppFile, State) -> + rebar_utils:vcs_vsn(AppInfo, get_value(vsn, AppData, AppFile), State). get_value(Key, AppInfo, AppFile) -> case proplists:get_value(Key, AppInfo) of diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 8cebeca..8a3ffea 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -1,18 +1,18 @@ -module(rebar_packages). --export([packages/1 - ,close_packages/0 - ,load_and_verify_version/1 - ,deps/3 +-export([get/2 + ,get_all_names/1 ,registry_dir/1 - ,package_dir/1 - ,registry_checksum/2 - ,find_highest_matching/6 - ,find_highest_matching/4 - ,find_highest_matching_/6 - ,find_all/3 + ,package_dir/2 + ,find_highest_matching/5 ,verify_table/1 - ,format_error/1]). + ,format_error/1 + ,update_package/3 + ,resolve_version/5]). + +-ifdef(TEST). +-export([new_package_table/0, find_highest_matching_/5, cmp_/4, cmpl_/4, valid_vsn/1]). +-endif. -export_type([package/0]). @@ -23,119 +23,133 @@ -type vsn() :: binary(). -type package() :: pkg_name() | {pkg_name(), vsn()}. --spec packages(rebar_state:t()) -> ets:tid(). -packages(State) -> - catch ets:delete(?PACKAGE_TABLE), - case load_and_verify_version(State) of - true -> - ok; - false -> - ?DEBUG("Error loading package index.", []), - handle_bad_index(State) +format_error({missing_package, Name, Vsn}) -> + io_lib:format("Package not found in any repo: ~ts-~ts.", [rebar_utils:to_binary(Name), + rebar_utils:to_binary(Vsn)]); +format_error({missing_package, Pkg}) -> + io_lib:format("Package not found in any repo: ~p.", [Pkg]). + +-spec get(rebar_hex_repos:repo(), binary()) -> {ok, map()} | {error, term()}. +get(Config, Name) -> + try hex_api_package:get(Config, Name) of + {ok, {200, _Headers, PkgInfo}} -> + {ok, PkgInfo}; + {ok, {404, _, _}} -> + {error, not_found}; + Error -> + ?DEBUG("Hex api request failed: ~p", [Error]), + {error, unknown} + catch + error:{badmatch, {error, {failed_connect, _}}} -> + {error, failed_to_connect}; + _:Exception -> + ?DEBUG("hex_api_package:get failed: ~p", [Exception]), + {error, unknown} end. -handle_bad_index(State) -> - ?ERROR("Bad packages index. Trying to fix by updating the registry.", []), - {ok, State1} = rebar_prv_update:do(State), - case load_and_verify_version(State1) of - true -> - ok; - false -> - %% Still unable to load after an update, create an empty registry - ets:new(?PACKAGE_TABLE, [named_table, public]) + +-spec get_all_names(rebar_state:t()) -> [binary()]. +get_all_names(State) -> + verify_table(State), + lists:usort(ets:select(?PACKAGE_TABLE, [{#package{key={'$1', '_', '_'}, + _='_'}, + [], ['$1']}])). + +-spec get_package_versions(unicode:unicode_binary(), unicode:unicode_binary(), ets:tid(), rebar_state:t()) + -> [vsn()]. +get_package_versions(Dep, Repo, Table, State) -> + ?MODULE:verify_table(State), + ets:select(Table, [{#package{key={Dep,'$1', Repo}, + _='_'}, + [], ['$1']}]). + + +get_package(Dep, Vsn, Hash, Repo, Table, State) -> + get_package(Dep, Vsn, Hash, false, [Repo], Table, State). + +-spec get_package(unicode:unicode_binary(), unicode:unicode_binary(), + binary() | undefined | '_', boolean() | '_', + unicode:unicode_binary() | '_' | list(), ets:tab(), rebar_state:t()) + -> {ok, #package{}} | not_found. +get_package(Dep, Vsn, undefined, Retired, Repo, Table, State) -> + get_package(Dep, Vsn, '_', Retired, Repo, Table, State); +get_package(Dep, Vsn, Hash, Retired, Repos, Table, State) -> + ?MODULE:verify_table(State), + case ets:select(Table, [{#package{key={Dep, Vsn, Repo}, + checksum=Hash, + retired=Retired, + _='_'}, [], ['$_']} || Repo <- Repos]) of + %% have to allow multiple matches in the list for cases that Repo is `_` + [Package | _] -> + {ok, Package}; + _ -> + not_found end. -close_packages() -> - catch ets:delete(?PACKAGE_TABLE). +new_package_table() -> + ?PACKAGE_TABLE = ets:new(?PACKAGE_TABLE, [named_table, public, ordered_set, {keypos, 2}]), + ets:insert(?PACKAGE_TABLE, {?PACKAGE_INDEX_VERSION, package_index_version}). load_and_verify_version(State) -> {ok, RegistryDir} = registry_dir(State), case ets:file2tab(filename:join(RegistryDir, ?INDEX_FILE)) of {ok, _} -> - case ets:lookup_element(?PACKAGE_TABLE, package_index_version, 2) of + case ets:lookup_element(?PACKAGE_TABLE, package_index_version, 1) of ?PACKAGE_INDEX_VERSION -> true; - _ -> + V -> + %% no reason to confuse the user since we just start fresh and they + %% shouldn't notice, so log as a debug message only + ?DEBUG("Package index version mismatch. Current version ~p, this rebar3 expecting ~p", + [V, ?PACKAGE_INDEX_VERSION]), (catch ets:delete(?PACKAGE_TABLE)), - rebar_prv_update:hex_to_index(State) + new_package_table() end; - _ -> - rebar_prv_update:hex_to_index(State) + _ -> + new_package_table() end. -deps(Name, Vsn, State) -> - try - deps_(Name, Vsn, State) - catch - _:_ -> - handle_missing_package({Name, Vsn}, State, fun(State1) -> deps_(Name, Vsn, State1) end) - end. - -deps_(Name, Vsn, State) -> - ?MODULE:verify_table(State), - ets:lookup_element(?PACKAGE_TABLE, {rebar_utils:to_binary(Name), rebar_utils:to_binary(Vsn)}, 2). - -handle_missing_package(Dep, State, Fun) -> - case Dep of - {Name, Vsn} -> - ?INFO("Package ~ts-~ts not found. Fetching registry updates and trying again...", [Name, Vsn]); - _ -> - ?INFO("Package ~p not found. Fetching registry updates and trying again...", [Dep]) - end, +handle_missing_package(PkgKey, Repo, State, Fun) -> + Name = + case PkgKey of + {N, Vsn, _Repo} -> + ?DEBUG("Package ~ts-~ts not found. Fetching registry updates for " + "package and trying again...", [N, Vsn]), + N; + _ -> + ?DEBUG("Package ~p not found. Fetching registry updates for " + "package and trying again...", [PkgKey]), + PkgKey + end, - {ok, State1} = rebar_prv_update:do(State), - try - Fun(State1) + update_package(Name, Repo, State), + try + Fun(State) catch _:_ -> %% Even after an update the package is still missing, time to error out - throw(?PRV_ERROR({missing_package, Dep})) + throw(?PRV_ERROR({missing_package, PkgKey})) end. registry_dir(State) -> CacheDir = rebar_dir:global_cache_dir(rebar_state:opts(State)), - case rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN) of - ?DEFAULT_CDN -> - RegistryDir = filename:join([CacheDir, "hex", "default"]), - case filelib:ensure_dir(filename:join(RegistryDir, "placeholder")) of - ok -> ok; - {error, Posix} when Posix == eaccess; Posix == enoent -> - ?ABORT("Could not write to ~p. Please ensure the path is writeable.", - [RegistryDir]) - end, - {ok, RegistryDir}; - CDN -> - case rebar_utils:url_append_path(CDN, ?REMOTE_PACKAGE_DIR) of - {ok, Parsed} -> - {ok, {_, _, Host, _, Path, _}} = http_uri:parse(Parsed), - CDNHostPath = lists:reverse(rebar_string:lexemes(Host, ".")), - CDNPath = tl(filename:split(Path)), - RegistryDir = filename:join([CacheDir, "hex"] ++ CDNHostPath ++ CDNPath), - ok = filelib:ensure_dir(filename:join(RegistryDir, "placeholder")), - {ok, RegistryDir}; - _ -> - {uri_parse_error, CDN} - end - end. + RegistryDir = filename:join([CacheDir, "hex"]), + case filelib:ensure_dir(filename:join(RegistryDir, "placeholder")) of + ok -> ok; + {error, Posix} when Posix == eaccess; Posix == enoent -> + ?ABORT("Could not write to ~p. Please ensure the path is writeable.", + [RegistryDir]) + end, + {ok, RegistryDir}. -package_dir(State) -> - case registry_dir(State) of - {ok, RegistryDir} -> - PackageDir = filename:join([RegistryDir, "packages"]), - ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")), - {ok, PackageDir}; - Error -> - Error - end. +-spec package_dir(rebar_hex_repos:repo(), rebar_state:t()) -> {ok, file:filename_all()}. +package_dir(Repo, State) -> + {ok, RegistryDir} = registry_dir(State), + RepoName = maps:get(name, Repo), + PackageDir = filename:join([RegistryDir, rebar_utils:to_list(RepoName), "packages"]), + ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")), + {ok, PackageDir}. -registry_checksum({pkg, Name, Vsn, _Hash}, State) -> - try - ?MODULE:verify_table(State), - ets:lookup_element(?PACKAGE_TABLE, {Name, Vsn}, 3) - catch - _:_ -> - throw(?PRV_ERROR({missing_package, rebar_utils:to_binary(Name), rebar_utils:to_binary(Vsn)})) - end. %% Hex supports use of ~> to specify the version required for a dependency. %% Since rebar3 requires exact versions to choose from we find the highest @@ -152,31 +166,28 @@ registry_checksum({pkg, Name, Vsn, _Hash}, State) -> %% `~> 2.1.3-dev` | `>= 2.1.3-dev and < 2.2.0` %% `~> 2.0` | `>= 2.0.0 and < 3.0.0` %% `~> 2.1` | `>= 2.1.0 and < 3.0.0` -find_highest_matching(Dep, Constraint, Table, State) -> - find_highest_matching(undefined, undefined, Dep, Constraint, Table, State). - -find_highest_matching(Pkg, PkgVsn, Dep, Constraint, Table, State) -> - try find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State) of +find_highest_matching(Dep, Constraint, Repo, Table, State) -> + try find_highest_matching_(Dep, Constraint, Repo, Table, State) of none -> - handle_missing_package(Dep, State, + handle_missing_package(Dep, Repo, State, fun(State1) -> - find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State1) + find_highest_matching_(Dep, Constraint, Repo, Table, State1) end); Result -> Result catch _:_ -> - handle_missing_package(Dep, State, + handle_missing_package(Dep, Repo, State, fun(State1) -> - find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State1) + find_highest_matching_(Dep, Constraint, Repo, Table, State1) end) end. -find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State) -> - try find_all(Dep, Table, State) of - {ok, [Vsn]} -> - handle_single_vsn(Pkg, PkgVsn, Dep, Vsn, Constraint); - {ok, Vsns} -> +find_highest_matching_(Dep, Constraint, #{name := Repo}, Table, State) -> + try get_package_versions(Dep, Repo, Table, State) of + [Vsn] -> + handle_single_vsn(Vsn, Constraint); + Vsns -> case handle_vsns(Constraint, Vsns) of none -> none; @@ -188,18 +199,6 @@ find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State) -> none end. -find_all(Dep, Table, State) -> - ?MODULE:verify_table(State), - try ets:lookup_element(Table, Dep, 2) of - [Vsns] when is_list(Vsns)-> - {ok, Vsns}; - Vsns -> - {ok, Vsns} - catch - error:badarg -> - none - end. - handle_vsns(Constraint, Vsns) -> lists:foldl(fun(Version, Highest) -> case ec_semver:pes(Version, Constraint) andalso @@ -211,26 +210,227 @@ handle_vsns(Constraint, Vsns) -> end end, none, Vsns). -handle_single_vsn(Pkg, PkgVsn, Dep, Vsn, Constraint) -> +handle_single_vsn(Vsn, Constraint) -> case ec_semver:pes(Vsn, Constraint) of true -> {ok, Vsn}; false -> - case {Pkg, PkgVsn} of - {undefined, undefined} -> - ?DEBUG("Only existing version of ~ts is ~ts which does not match constraint ~~> ~ts. " - "Using anyway, but it is not guaranteed to work.", [Dep, Vsn, Constraint]); - _ -> - ?DEBUG("[~ts:~ts] Only existing version of ~ts is ~ts which does not match constraint ~~> ~ts. " - "Using anyway, but it is not guaranteed to work.", [Pkg, PkgVsn, Dep, Vsn, Constraint]) - end, - {ok, Vsn} + none end. -format_error({missing_package, Name, Vsn}) -> - io_lib:format("Package not found in registry: ~ts-~ts.", [rebar_utils:to_binary(Name), rebar_utils:to_binary(Vsn)]); -format_error({missing_package, Dep}) -> - io_lib:format("Package not found in registry: ~p.", [Dep]). - verify_table(State) -> ets:info(?PACKAGE_TABLE, named_table) =:= true orelse load_and_verify_version(State). + +parse_deps(Deps) -> + [{maps:get(app, D, Name), {pkg, Name, Constraint, undefined}} + || D=#{package := Name, + requirement := Constraint} <- Deps]. + +parse_checksum(<>) -> + list_to_binary( + rebar_string:uppercase( + lists:flatten(io_lib:format("~64.16.0b", [Checksum])))); +parse_checksum(Checksum) -> + Checksum. + +update_package(Name, RepoConfig=#{name := Repo}, State) -> + ?MODULE:verify_table(State), + try hex_repo:get_package(RepoConfig#{repo_key => maps:get(read_key, RepoConfig, <<>>)}, Name) of + {ok, {200, _Headers, #{releases := Releases}}} -> + _ = insert_releases(Name, Releases, Repo, ?PACKAGE_TABLE), + {ok, RegistryDir} = rebar_packages:registry_dir(State), + PackageIndex = filename:join(RegistryDir, ?INDEX_FILE), + ok = ets:tab2file(?PACKAGE_TABLE, PackageIndex); + {ok, {403, _Headers, <<>>}} -> + not_found; + {ok, {404, _Headers, _}} -> + not_found; + Error -> + ?DEBUG("Hex get_package request failed: ~p", [Error]), + %% TODO: add better log message. hex_core should export a format_error + ?WARN("Failed to update package from repo ~ts", [Repo]), + fail + catch + _:Exception -> + ?DEBUG("hex_repo:get_package failed for package ~p: ~p", [Name, Exception]), + fail + end. + +insert_releases(Name, Releases, Repo, Table) -> + [true = ets:insert(Table, + #package{key={Name, Version, Repo}, + checksum=parse_checksum(Checksum), + retired=maps:get(retired, Release, false), + dependencies=parse_deps(Dependencies)}) + || Release=#{checksum := Checksum, + version := Version, + dependencies := Dependencies} <- Releases]. + +-spec resolve_version(unicode:unicode_binary(), unicode:unicode_binary() | undefined, + binary() | undefined, + ets:tab(), rebar_state:t()) + -> {error, {invalid_vsn, unicode:unicode_binary()}} | + not_found | + {ok, #package{}, map()}. +%% if checksum is defined search for any matching repo matching pkg-vsn and checksum +resolve_version(Dep, DepVsn, Hash, HexRegistry, State) when is_binary(Hash) -> + Resources = rebar_state:resources(State), + #{repos := RepoConfigs} = rebar_resource_v2:find_resource_state(pkg, Resources), + RepoNames = [RepoName || #{name := RepoName} <- RepoConfigs], + + %% allow retired packages when we have a checksum + case get_package(Dep, DepVsn, Hash, '_', RepoNames, HexRegistry, State) of + {ok, Package=#package{key={_, _, RepoName}}} -> + {ok, RepoConfig} = rebar_hex_repos:get_repo_config(RepoName, RepoConfigs), + {ok, Package, RepoConfig}; + _ -> + Fun = fun(Repo) -> + case resolve_version_(Dep, DepVsn, Repo, HexRegistry, State) of + none -> + not_found; + {ok, Vsn} -> + get_package(Dep, Vsn, Hash, Repo, HexRegistry, State) + end + end, + handle_missing_no_exception(Fun, Dep, State) + end; +resolve_version(Dep, undefined, Hash, HexRegistry, State) -> + Fun = fun(Repo) -> + case highest_matching(Dep, "0", Repo, HexRegistry, State) of + none -> + not_found; + {ok, Vsn} -> + get_package(Dep, Vsn, Hash, Repo, HexRegistry, State) + end + end, + handle_missing_no_exception(Fun, Dep, State); +resolve_version(Dep, DepVsn, Hash, HexRegistry, State) -> + case valid_vsn(DepVsn) of + false -> + {error, {invalid_vsn, DepVsn}}; + _ -> + Fun = fun(Repo) -> + case resolve_version_(Dep, DepVsn, Repo, HexRegistry, State) of + none -> + not_found; + {ok, Vsn} -> + get_package(Dep, Vsn, Hash, Repo, HexRegistry, State) + end + end, + handle_missing_no_exception(Fun, Dep, State) + end. + +check_all_repos(Fun, RepoConfigs) -> + ec_lists:search(fun(#{name := R}) -> + Fun(R) + end, RepoConfigs). + +handle_missing_no_exception(Fun, Dep, State) -> + Resources = rebar_state:resources(State), + #{repos := RepoConfigs} = rebar_resource_v2:find_resource_state(pkg, Resources), + + %% first check all repos in order for a local match + %% if none is found then we step through checking after updating the repo registry + case check_all_repos(Fun, RepoConfigs) of + not_found -> + ec_lists:search(fun(Config=#{name := R}) -> + case ?MODULE:update_package(Dep, Config, State) of + ok -> + Fun(R); + _ -> + not_found + end + end, RepoConfigs); + Result -> + Result + end. + +resolve_version_(Dep, DepVsn, Repo, HexRegistry, State) -> + case DepVsn of + <<"~>", Vsn/binary>> -> + highest_matching(Dep, rm_ws(Vsn), Repo, HexRegistry, State); + <<">=", Vsn/binary>> -> + cmp(Dep, rm_ws(Vsn), Repo, HexRegistry, State, fun ec_semver:gte/2); + <<">", Vsn/binary>> -> + cmp(Dep, rm_ws(Vsn), Repo, HexRegistry, State, fun ec_semver:gt/2); + <<"<=", Vsn/binary>> -> + cmpl(Dep, rm_ws(Vsn), Repo, HexRegistry, State, fun ec_semver:lte/2); + <<"<", Vsn/binary>> -> + cmpl(Dep, rm_ws(Vsn), Repo, HexRegistry, State, fun ec_semver:lt/2); + <<"==", Vsn/binary>> -> + {ok, Vsn}; + Vsn -> + {ok, Vsn} + end. + +rm_ws(<<" ", R/binary>>) -> + rm_ws(R); +rm_ws(R) -> + R. + +valid_vsn(Vsn) -> + %% Regepx from https://github.com/sindresorhus/semver-regex/blob/master/index.js + SemVerRegExp = "v?(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)(\\.(0|[1-9][0-9]*))?" + "(-[0-9a-z-]+(\\.[0-9a-z-]+)*)?(\\+[0-9a-z-]+(\\.[0-9a-z-]+)*)?", + SupportedVersions = "^(>=?|<=?|~>|==)?\\s*" ++ SemVerRegExp ++ "$", + re:run(Vsn, SupportedVersions, [unicode]) =/= nomatch. + +highest_matching(Dep, Vsn, Repo, HexRegistry, State) -> + find_highest_matching_(Dep, Vsn, #{name => Repo}, HexRegistry, State). + +cmp(Dep, Vsn, Repo, HexRegistry, State, CmpFun) -> + case get_package_versions(Dep, Repo, HexRegistry, State) of + [] -> + none; + Vsns -> + cmp_(undefined, Vsn, Vsns, CmpFun) + end. + +cmp_(undefined, MinVsn, [], _CmpFun) -> + {ok, MinVsn}; +cmp_(HighestDepVsn, _MinVsn, [], _CmpFun) -> + {ok, HighestDepVsn}; + +cmp_(BestMatch, MinVsn, [Vsn | R], CmpFun) -> + case CmpFun(Vsn, MinVsn) of + true -> + cmp_(Vsn, Vsn, R, CmpFun); + false -> + cmp_(BestMatch, MinVsn, R, CmpFun) + end. + +%% We need to treat this differently since we want a version that is LOWER but +%% the higest possible one. +cmpl(Dep, Vsn, Repo, HexRegistry, State, CmpFun) -> + case get_package_versions(Dep, Repo, HexRegistry, State) of + [] -> + none; + Vsns -> + cmpl_(undefined, Vsn, Vsns, CmpFun) + end. + +cmpl_(undefined, MaxVsn, [], _CmpFun) -> + {ok, MaxVsn}; +cmpl_(HighestDepVsn, _MaxVsn, [], _CmpFun) -> + {ok, HighestDepVsn}; + +cmpl_(undefined, MaxVsn, [Vsn | R], CmpFun) -> + case CmpFun(Vsn, MaxVsn) of + true -> + cmpl_(Vsn, MaxVsn, R, CmpFun); + false -> + cmpl_(undefined, MaxVsn, R, CmpFun) + end; + +cmpl_(BestMatch, MaxVsn, [Vsn | R], CmpFun) -> + case CmpFun(Vsn, MaxVsn) of + true -> + case ec_semver:gte(Vsn, BestMatch) of + true -> + cmpl_(Vsn, MaxVsn, R, CmpFun); + false -> + cmpl_(BestMatch, MaxVsn, R, CmpFun) + end; + false -> + cmpl_(BestMatch, MaxVsn, R, CmpFun) + end. diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 2cf167e..97eabb6 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -2,42 +2,51 @@ %% ex: ts=4 sw=4 et -module(rebar_pkg_resource). --behaviour(rebar_resource). +-behaviour(rebar_resource_v2). --export([lock/2 - ,download/3 - ,download/4 - ,needs_update/2 - ,make_vsn/1]). +-export([init/2, + lock/2, + download/4, + download/5, + needs_update/2, + make_vsn/2, + format_error/1]). --export([request/2 - ,etag/1 - ,ssl_opts/1]). - -%% Exported for ct +-ifdef(TEST). +%% exported for test purposes -export([store_etag_in_cache/2]). +-endif. -include("rebar.hrl"). --include_lib("public_key/include/OTP-PUB-KEY.hrl"). - --type cached_result() :: {'bad_checksum',string()} | - {'bad_registry_checksum',string()} | - {'failed_extract',string()} | - {'ok','true'} | - {'unexpected_hash',string(),_,binary()}. +-include_lib("providers/include/providers.hrl"). --type download_result() :: {bad_download, binary() | string()} | - {fetch_fail, _, _} | cached_result(). +-type package() :: {pkg, binary(), binary(), binary(), rebar_hex_repos:repo()}. %%============================================================================== %% Public API %%============================================================================== --spec lock(AppDir, Source) -> Res when - AppDir :: file:name(), - Source :: tuple(), - Res :: {atom(), string(), any()}. -lock(_AppDir, Source) -> - Source. + +-spec init(atom(), rebar_state:t()) -> {ok, rebar_resource_v2:resource()}. +init(Type, State) -> + {ok, Vsn} = application:get_key(rebar, vsn), + BaseConfig = #{http_adapter => hex_http_httpc, + http_user_agent_fragment => + <<"(rebar3/", (list_to_binary(Vsn))/binary, ") (httpc)">>, + http_adapter_config => #{profile => rebar}}, + Repos = rebar_hex_repos:from_state(BaseConfig, State), + Resource = rebar_resource_v2:new(Type, ?MODULE, #{repos => Repos, + base_config => BaseConfig}), + {ok, Resource}. + + + +-spec lock(AppInfo, ResourceState) -> Res when + AppInfo :: rebar_app_info:t(), + ResourceState :: rebar_resource_v2:resource_state(), + Res :: {atom(), string(), any(), binary()}. +lock(AppInfo, _) -> + {pkg, Name, Vsn, Hash, _RepoConfig} = rebar_app_info:source(AppInfo), + {pkg, Name, Vsn, Hash}. %%------------------------------------------------------------------------------ %% @doc @@ -45,13 +54,13 @@ lock(_AppDir, Source) -> %% version. %% @end %%------------------------------------------------------------------------------ --spec needs_update(Dir, Pkg) -> Res when - Dir :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, +-spec needs_update(AppInfo, ResourceState) -> Res when + AppInfo :: rebar_app_info:t(), + ResourceState :: rebar_resource_v2:resource_state(), Res :: boolean(). -needs_update(Dir, {pkg, _Name, Vsn, _Hash}) -> - [AppInfo] = rebar_app_discover:find_apps([Dir], all), - case rebar_app_info:original_vsn(AppInfo) =:= rebar_utils:to_list(Vsn) of +needs_update(AppInfo, _) -> + {pkg, _Name, Vsn, _Hash, _} = rebar_app_info:source(AppInfo), + case rebar_app_info:original_vsn(AppInfo) =:= rebar_utils:to_binary(Vsn) of true -> false; false -> @@ -63,13 +72,19 @@ needs_update(Dir, {pkg, _Name, Vsn, _Hash}) -> %% Download the given pkg. %% @end %%------------------------------------------------------------------------------ --spec download(TmpDir, Pkg, State) -> Res when +-spec download(TmpDir, AppInfo, State, ResourceState) -> Res when TmpDir :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + AppInfo :: rebar_app_info:t(), + ResourceState :: rebar_resource_v2:resource_state(), State :: rebar_state:t(), - Res :: {'error',_} | {'ok',_} | {'tarball',binary() | string()}. -download(TmpDir, Pkg, State) -> - download(TmpDir, Pkg, State, true). + Res :: ok | {error,_}. +download(TmpDir, AppInfo, State, ResourceState) -> + case download(TmpDir, rebar_app_info:source(AppInfo), State, ResourceState, true) of + ok -> + ok; + Error -> + {error, Error} + end. %%------------------------------------------------------------------------------ %% @doc @@ -78,26 +93,28 @@ download(TmpDir, Pkg, State) -> %% is different. %% @end %%------------------------------------------------------------------------------ --spec download(TmpDir, Pkg, State, UpdateETag) -> Res when +-spec download(TmpDir, Pkg, State, ResourceState, UpdateETag) -> Res when TmpDir :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + Pkg :: package(), State :: rebar_state:t(), + ResourceState:: rebar_resource_v2:resource_state(), UpdateETag :: boolean(), - Res :: download_result(). -download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State, UpdateETag) -> - CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), - {ok, PackageDir} = rebar_packages:package_dir(State), + Res :: ok | {error,_} | {unexpected_hash, string(), integer(), integer()} | + {fetch_fail, binary(), binary()}. +download(TmpDir, Pkg={pkg, Name, Vsn, _Hash, Repo}, State, _ResourceState, UpdateETag) -> + {ok, PackageDir} = rebar_packages:package_dir(Repo, State), Package = binary_to_list(<>), ETagFile = binary_to_list(<>), CachePath = filename:join(PackageDir, Package), ETagPath = filename:join(PackageDir, ETagFile), - case rebar_utils:url_append_path(CDN, filename:join(?REMOTE_PACKAGE_DIR, - Package)) of - {ok, Url} -> - cached_download(TmpDir, CachePath, Pkg, Url, etag(ETagPath), State, - ETagPath, UpdateETag); - _ -> - {fetch_fail, Name, Vsn} + case cached_download(TmpDir, CachePath, Pkg, etag(CachePath, ETagPath), ETagPath, UpdateETag) of + {bad_registry_checksum, Expected, Found} -> + %% checksum comparison failed. in case this is from a modified cached package + %% overwrite the etag if it exists so it is not relied on again + store_etag_in_cache(ETagPath, <<>>), + ?PRV_ERROR({bad_registry_checksum, Name, Vsn, Expected, Found}); + Result -> + Result end. %%------------------------------------------------------------------------------ @@ -106,12 +123,19 @@ download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State, UpdateETag) -> %% Returns {error, string()} as this operation is not supported for pkg sources. %% @end %%------------------------------------------------------------------------------ --spec make_vsn(Vsn) -> Res when - Vsn :: any(), - Res :: {'error',[1..255,...]}. -make_vsn(_) -> +-spec make_vsn(AppInfo, ResourceState) -> Res when + AppInfo :: rebar_app_info:t(), + ResourceState :: rebar_resource_v2:resource_state(), + Res :: {'error', string()}. +make_vsn(_, _) -> {error, "Replacing version of type pkg not supported."}. +format_error({bad_registry_checksum, Name, Vsn, Expected, Found}) -> + io_lib:format("The checksum for package at ~ts-~ts (~ts) does not match the " + "checksum expected from the registry (~ts). " + "Run `rebar3 do unlock ~ts, update` and then try again.", + [Name, Vsn, Found, Expected, Name]). + %%------------------------------------------------------------------------------ %% @doc %% Download the pkg belonging to the given address. If the etag of the pkg @@ -120,29 +144,24 @@ make_vsn(_) -> %% {ok, Contents, NewEtag}, otherwise if some error occured return error. %% @end %%------------------------------------------------------------------------------ --spec request(Url, ETag) -> Res when - Url :: string(), - ETag :: false | string(), - Res :: 'error' | {ok, cached} | {ok, any(), string()}. -request(Url, ETag) -> - HttpOptions = [{ssl, ssl_opts(Url)}, - {relaxed, true} | rebar_utils:get_proxy_auth()], - case httpc:request(get, {Url, [{"if-none-match", "\"" ++ ETag ++ "\""} - || ETag =/= false] ++ - [{"User-Agent", rebar_utils:user_agent()}]}, - HttpOptions, [{body_format, binary}], rebar) of - {ok, {{_Version, 200, _Reason}, Headers, Body}} -> - ?DEBUG("Successfully downloaded ~ts", [Url]), - {"etag", ETag1} = lists:keyfind("etag", 1, Headers), - {ok, Body, rebar_string:trim(ETag1, both, [$"])}; - {ok, {{_Version, 304, _Reason}, _Headers, _Body}} -> - ?DEBUG("Cached copy of ~ts still valid", [Url]), +-spec request(rebar_hex_repos:repo(), binary(), binary(), false | binary()) + -> {ok, cached} | {ok, binary(), binary()} | error. +request(Config, Name, Version, ETag) -> + Config1 = Config#{http_etag => ETag}, + try hex_repo:get_tarball(Config1, Name, Version) of + {ok, {200, #{<<"etag">> := ETag1}, Tarball}} -> + {ok, Tarball, ETag1}; + {ok, {304, _Headers, _}} -> {ok, cached}; - {ok, {{_Version, Code, _Reason}, _Headers, _Body}} -> - ?DEBUG("Request to ~p failed: status code ~p", [Url, Code]), + {ok, {Code, _Headers, _Body}} -> + ?DEBUG("Request for package ~s-~s failed: status code ~p", [Name, Version, Code]), error; {error, Reason} -> - ?DEBUG("Request to ~p failed: ~p", [Url, Reason]), + ?DEBUG("Request for package ~s-~s failed: ~p", [Name, Version, Reason]), + error + catch + _:Exception -> + ?DEBUG("hex_repo:get_tarball failed: ~p", [Exception]), error end. @@ -153,32 +172,23 @@ request(Url, ETag) -> %% returned from the hexpm server. The name is package-vsn.etag. %% @end %%------------------------------------------------------------------------------ --spec etag(Path) -> Res when - Path :: file:name(), - Res :: false | string(). -etag(Path) -> - case file:read_file(Path) of +-spec etag(PackagePath, ETagPath) -> Res when + PackagePath :: file:name(), + ETagPath :: file:name(), + Res :: binary(). +etag(PackagePath, ETagPath) -> + case file:read_file(ETagPath) of {ok, Bin} -> - binary_to_list(Bin); + %% just in case a user deleted a cached package but not its etag + %% verify the package is also there, and if not, ignore the etag + case filelib:is_file(PackagePath) of + true -> + Bin; + false -> + <<>> + end; {error, _} -> - false - end. - -%%------------------------------------------------------------------------------ -%% @doc -%% Return the SSL options adequate for the project based on -%% its configuration, including for validation of certs. -%% @end -%%------------------------------------------------------------------------------ --spec ssl_opts(Url) -> Res when - Url :: string(), - Res :: proplists:proplist(). -ssl_opts(Url) -> - case get_ssl_config() of - ssl_verify_enabled -> - ssl_opts(ssl_verify_enabled, Url); - ssl_verify_disabled -> - [{verify, verify_none}] + <<>> end. %%------------------------------------------------------------------------------ @@ -188,7 +198,7 @@ ssl_opts(Url) -> %%------------------------------------------------------------------------------ -spec store_etag_in_cache(File, ETag) -> Res when File :: file:name(), - ETag :: string(), + ETag :: binary(), Res :: ok. store_etag_in_cache(Path, ETag) -> _ = file:write_file(Path, ETag). @@ -196,223 +206,74 @@ store_etag_in_cache(Path, ETag) -> %%%============================================================================= %%% Private functions %%%============================================================================= --spec cached_download(TmpDir, CachePath, Pkg, Url, ETag, State, ETagPath, - UpdateETag) -> Res when +-spec cached_download(TmpDir, CachePath, Pkg, ETag, ETagPath, UpdateETag) -> Res when TmpDir :: file:name(), CachePath :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, - Url :: string(), - ETag :: false | string(), - State :: rebar_state:t(), + Pkg :: package(), + ETag :: binary(), ETagPath :: file:name(), UpdateETag :: boolean(), - Res :: download_result(). -cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, Url, ETag, - State, ETagPath, UpdateETag) -> - case request(Url, ETag) of + Res :: ok | {unexpected_hash, integer(), integer()} | {fetch_fail, binary(), binary()}. +cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash, RepoConfig}, ETag, + ETagPath, UpdateETag) -> + case request(RepoConfig, Name, Vsn, ETag) of {ok, cached} -> ?INFO("Version cached at ~ts is up to date, reusing it", [CachePath]), - serve_from_cache(TmpDir, CachePath, Pkg, State); + serve_from_cache(TmpDir, CachePath, Pkg); {ok, Body, NewETag} -> ?INFO("Downloaded package, caching at ~ts", [CachePath]), maybe_store_etag_in_cache(UpdateETag, ETagPath, NewETag), - serve_from_download(TmpDir, CachePath, Pkg, NewETag, Body, State, - ETagPath); - error when ETag =/= false -> + serve_from_download(TmpDir, CachePath, Pkg, Body); + error when ETag =/= <<>> -> store_etag_in_cache(ETagPath, ETag), ?INFO("Download error, using cached file at ~ts", [CachePath]), - serve_from_cache(TmpDir, CachePath, Pkg, State); + serve_from_cache(TmpDir, CachePath, Pkg); error -> {fetch_fail, Name, Vsn} end. --spec serve_from_cache(TmpDir, CachePath, Pkg, State) -> Res when +-spec serve_from_cache(TmpDir, CachePath, Pkg) -> Res when TmpDir :: file:name(), CachePath :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, - State :: rebar_state:t(), - Res :: cached_result(). -serve_from_cache(TmpDir, CachePath, Pkg, State) -> - {Files, Contents, Version, Meta} = extract(TmpDir, CachePath), - case checksums(Pkg, Files, Contents, Version, Meta, State) of - {Chk, Chk, Chk, Chk} -> - ok = erl_tar:extract({binary, Contents}, [{cwd, TmpDir}, compressed]), - {ok, true}; - {_Hash, Chk, Chk, Chk} -> - ?DEBUG("Expected hash ~p does not match checksums ~p", [_Hash, Chk]), - {unexpected_hash, CachePath, _Hash, Chk}; - {Chk, _Bin, Chk, Chk} -> - ?DEBUG("Checksums: registry: ~p, pkg: ~p", [Chk, _Bin]), - {failed_extract, CachePath}; - {Chk, Chk, _Reg, Chk} -> - ?DEBUG("Checksums: registry: ~p, pkg: ~p", [_Reg, Chk]), - {bad_registry_checksum, CachePath}; - {_Hash, _Bin, _Reg, _Tar} -> - ?DEBUG("Checksums: expected: ~p, registry: ~p, pkg: ~p, meta: ~p", - [_Hash, _Reg, _Bin, _Tar]), - {bad_checksum, CachePath} + Pkg :: package(), + Res :: ok | {error,_} | {bad_registry_checksum, integer(), integer()}. +serve_from_cache(TmpDir, CachePath, Pkg) -> + {ok, Binary} = file:read_file(CachePath), + serve_from_memory(TmpDir, Binary, Pkg). + +-spec serve_from_memory(TmpDir, Tarball, Package) -> Res when + TmpDir :: file:name(), + Tarball :: binary(), + Package :: package(), + Res :: ok | {error,_} | {bad_registry_checksum, integer(), integer()}. +serve_from_memory(TmpDir, Binary, {pkg, _Name, _Vsn, Hash, _RepoConfig}) -> + RegistryChecksum = list_to_integer(binary_to_list(Hash), 16), + case hex_tarball:unpack(Binary, TmpDir) of + {ok, #{checksum := <>}} when RegistryChecksum =/= Checksum -> + ?DEBUG("Expected hash ~64.16.0B does not match checksum of fetched package ~64.16.0B", + [RegistryChecksum, Checksum]), + {bad_registry_checksum, RegistryChecksum, Checksum}; + {ok, #{checksum := <>}} -> + ok; + {error, Reason} -> + {error, {hex_tarball, Reason}} end. --spec serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State, - ETagPath) -> Res when +-spec serve_from_download(TmpDir, CachePath, Package, Binary) -> Res when TmpDir :: file:name(), CachePath :: file:name(), - Package :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, - ETag :: string(), + Package :: package(), Binary :: binary(), - State :: rebar_state:t(), - ETagPath :: file:name(), - Res :: download_result(). -serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State, ETagPath) -> + Res :: ok | {error,_}. +serve_from_download(TmpDir, CachePath, Package, Binary) -> ?DEBUG("Writing ~p to cache at ~ts", [Package, CachePath]), file:write_file(CachePath, Binary), - case etag(ETagPath) of - ETag -> - serve_from_cache(TmpDir, CachePath, Package, State); - FileETag -> - ?DEBUG("Downloaded file ~ts ETag ~ts doesn't match returned ETag ~ts", - [CachePath, ETag, FileETag]), - {bad_download, CachePath} - end. - --spec extract(TmpDir, CachePath) -> Res when - TmpDir :: file:name(), - CachePath :: file:name(), - Res :: {Files, Contents, Version, Meta}, - Files :: list({file:name(), binary()}), - Contents :: binary(), - Version :: binary(), - Meta :: binary(). -extract(TmpDir, CachePath) -> - ec_file:mkdir_p(TmpDir), - {ok, Files} = erl_tar:extract(CachePath, [memory]), - {"contents.tar.gz", Contents} = lists:keyfind("contents.tar.gz", 1, Files), - {"VERSION", Version} = lists:keyfind("VERSION", 1, Files), - {"metadata.config", Meta} = lists:keyfind("metadata.config", 1, Files), - {Files, Contents, Version, Meta}. - --spec checksums(Pkg, Files, Contents, Version, Meta, State) -> Res when - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, - Files :: list({file:name(), binary()}), - Contents :: binary(), - Version :: binary(), - Meta :: binary(), - State :: rebar_state:t(), - Res :: {Hash, BinChecksum, RegistryChecksum, TarChecksum}, - Hash :: binary(), - BinChecksum :: binary(), - RegistryChecksum :: any(), - TarChecksum :: binary(). -checksums(Pkg={pkg, _Name, _Vsn, Hash}, Files, Contents, Version, Meta, State) -> - Blob = <>, - <> = crypto:hash(sha256, Blob), - BinChecksum = list_to_binary( - rebar_string:uppercase( - lists:flatten(io_lib:format("~64.16.0b", [X])))), - RegistryChecksum = rebar_packages:registry_checksum(Pkg, State), - {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files), - {Hash, BinChecksum, RegistryChecksum, TarChecksum}. - -%%------------------------------------------------------------------------------ -%% @doc -%% Return the SSL options adequate for the project based on -%% its configuration, including for validation of certs. -%% @end -%%------------------------------------------------------------------------------ --spec ssl_opts(Enabled, Url) -> Res when - Enabled :: atom(), - Url :: string(), - Res :: proplists:proplist(). -ssl_opts(ssl_verify_enabled, Url) -> - case check_ssl_version() of - true -> - {ok, {_, _, Hostname, _, _, _}} = - http_uri:parse(rebar_utils:to_list(Url)), - VerifyFun = {fun ssl_verify_hostname:verify_fun/3, - [{check_hostname, Hostname}]}, - CACerts = certifi:cacerts(), - [{verify, verify_peer}, {depth, 2}, {cacerts, CACerts}, - {partial_chain, fun partial_chain/1}, {verify_fun, VerifyFun}]; - false -> - ?WARN("Insecure HTTPS request (peer verification disabled), " - "please update to OTP 17.4 or later", []), - [{verify, verify_none}] - end. - --spec partial_chain(Certs) -> Res when - Certs :: list(any()), - Res :: unknown_ca | {trusted_ca, any()}. -partial_chain(Certs) -> - Certs1 = [{Cert, public_key:pkix_decode_cert(Cert, otp)} || Cert <- Certs], - CACerts = certifi:cacerts(), - CACerts1 = [public_key:pkix_decode_cert(Cert, otp) || Cert <- CACerts], - case ec_lists:find(fun({_, Cert}) -> - check_cert(CACerts1, Cert) - end, Certs1) of - {ok, Trusted} -> - {trusted_ca, element(1, Trusted)}; - _ -> - unknown_ca - end. - --spec extract_public_key_info(Cert) -> Res when - Cert :: #'OTPCertificate'{tbsCertificate::#'OTPTBSCertificate'{}}, - Res :: any(). -extract_public_key_info(Cert) -> - ((Cert#'OTPCertificate'.tbsCertificate)#'OTPTBSCertificate'.subjectPublicKeyInfo). - --spec check_cert(CACerts, Cert) -> Res when - CACerts :: list(any()), - Cert :: any(), - Res :: boolean(). -check_cert(CACerts, Cert) -> - lists:any(fun(CACert) -> - extract_public_key_info(CACert) == extract_public_key_info(Cert) - end, CACerts). - --spec check_ssl_version() -> - boolean(). -check_ssl_version() -> - case application:get_key(ssl, vsn) of - {ok, Vsn} -> - parse_vsn(Vsn) >= {5, 3, 6}; - _ -> - false - end. - --spec get_ssl_config() -> - ssl_verify_disabled | ssl_verify_enabled. -get_ssl_config() -> - GlobalConfigFile = rebar_dir:global_config(), - Config = rebar_config:consult_file(GlobalConfigFile), - case proplists:get_value(ssl_verify, Config, []) of - false -> - ssl_verify_disabled; - _ -> - ssl_verify_enabled - end. - --spec parse_vsn(Vsn) -> Res when - Vsn :: string(), - Res :: {integer(), integer(), integer()}. -parse_vsn(Vsn) -> - version_pad(rebar_string:lexemes(Vsn, ".-")). - --spec version_pad(list(nonempty_string())) -> Res when - Res :: {integer(), integer(), integer()}. -version_pad([Major]) -> - {list_to_integer(Major), 0, 0}; -version_pad([Major, Minor]) -> - {list_to_integer(Major), list_to_integer(Minor), 0}; -version_pad([Major, Minor, Patch]) -> - {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}; -version_pad([Major, Minor, Patch | _]) -> - {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}. + serve_from_memory(TmpDir, Binary, Package). -spec maybe_store_etag_in_cache(UpdateETag, Path, ETag) -> Res when UpdateETag :: boolean(), Path :: file:name(), - ETag :: string(), + ETag :: binary(), Res :: ok. maybe_store_etag_in_cache(false = _UpdateETag, _Path, _ETag) -> ok; diff --git a/src/rebar_prv_deps.erl b/src/rebar_prv_deps.erl index a88b014..577a859 100644 --- a/src/rebar_prv_deps.erl +++ b/src/rebar_prv_deps.erl @@ -97,10 +97,11 @@ display_dep(_State, {Name, _Vsn, Source}) when is_tuple(Source) -> display_dep(_State, {Name, _Vsn, Source, _Opts}) when is_tuple(Source) -> ?CONSOLE("~ts* (~ts source)", [rebar_utils:to_binary(Name), type(Source)]); %% Locked -display_dep(State, {Name, Source={pkg, _, Vsn}, Level}) when is_integer(Level) -> +display_dep(State, {Name, _Source={pkg, _, Vsn}, Level}) when is_integer(Level) -> DepsDir = rebar_dir:deps_dir(State), AppDir = filename:join([DepsDir, rebar_utils:to_binary(Name)]), - NeedsUpdate = case rebar_fetch:needs_update(AppDir, Source, State) of + {ok, AppInfo} = rebar_app_info:discover(AppDir), + NeedsUpdate = case rebar_fetch:needs_update(AppInfo, State) of true -> "*"; false -> "" end, @@ -108,7 +109,8 @@ display_dep(State, {Name, Source={pkg, _, Vsn}, Level}) when is_integer(Level) - display_dep(State, {Name, Source, Level}) when is_tuple(Source), is_integer(Level) -> DepsDir = rebar_dir:deps_dir(State), AppDir = filename:join([DepsDir, rebar_utils:to_binary(Name)]), - NeedsUpdate = case rebar_fetch:needs_update(AppDir, Source, State) of + {ok, AppInfo} = rebar_app_info:discover(AppDir), + NeedsUpdate = case rebar_fetch:needs_update(AppInfo, State) of true -> "*"; false -> "" end, diff --git a/src/rebar_prv_deps_tree.erl b/src/rebar_prv_deps_tree.erl index 07c7972..d7b49c5 100644 --- a/src/rebar_prv_deps_tree.erl +++ b/src/rebar_prv_deps_tree.erl @@ -39,18 +39,16 @@ format_error(Reason) -> %% Internal functions print_deps_tree(SrcDeps, Verbose, State) -> - Resources = rebar_state:resources(State), D = lists:foldl(fun(App, Dict) -> Name = rebar_app_info:name(App), Vsn = rebar_app_info:original_vsn(App), - AppDir = rebar_app_info:dir(App), - Vsn1 = rebar_utils:vcs_vsn(Vsn, AppDir, Resources), + Vsn1 = rebar_utils:vcs_vsn(App, Vsn, State), Source = rebar_app_info:source(App), Parent = rebar_app_info:parent(App), dict:append_list(Parent, [{Name, Vsn1, Source}], Dict) end, dict:new(), SrcDeps), ProjectAppNames = [{rebar_app_info:name(App) - ,rebar_utils:vcs_vsn(rebar_app_info:original_vsn(App), rebar_app_info:dir(App), Resources) + ,rebar_utils:vcs_vsn(App, rebar_app_info:original_vsn(App), State) ,project} || App <- rebar_state:project_apps(State)], case dict:find(root, D) of {ok, Children} -> diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index b735ed0..bad5af4 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -277,10 +277,8 @@ update_unseen_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Loc -spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer()) -> {rebar_app_info:t(), [rebar_app_info:t()], rebar_state:t()}. handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) -> Name = rebar_app_info:name(AppInfo), - C = rebar_config:consult(rebar_app_info:dir(AppInfo)), - AppInfo0 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), C), - AppInfo1 = rebar_app_info:apply_overrides(rebar_app_info:get(AppInfo, overrides, []), AppInfo0), + AppInfo1 = rebar_app_info:apply_overrides(rebar_app_info:get(AppInfo, overrides, []), AppInfo), AppInfo2 = rebar_app_info:apply_profiles(AppInfo1, [default, prod]), Plugins = rebar_app_info:get(AppInfo2, plugins, []), @@ -297,34 +295,33 @@ handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) -> AppInfo4 = rebar_app_info:deps(AppInfo3, rebar_state:deps_names(Deps)), %% Keep all overrides from the global config and this dep when parsing its deps - Overrides = rebar_app_info:get(AppInfo0, overrides, []), + Overrides = rebar_app_info:get(AppInfo, overrides, []), Deps1 = rebar_app_utils:parse_deps(Name, DepsDir, Deps, rebar_state:set(State, overrides, Overrides) ,Locks, Level+1), {AppInfo4, Deps1, State1}. -spec maybe_fetch(rebar_app_info:t(), atom(), boolean(), - sets:set(binary()), rebar_state:t()) -> {boolean(), rebar_app_info:t()}. + sets:set(binary()), rebar_state:t()) -> {ok, rebar_app_info:t()}. maybe_fetch(AppInfo, Profile, Upgrade, Seen, State) -> AppDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)), %% Don't fetch dep if it exists in the _checkouts dir case rebar_app_info:is_checkout(AppInfo) of true -> - {false, AppInfo}; + {ok, AppInfo}; false -> - case rebar_app_discover:find_app(AppInfo, AppDir, all) of + case rebar_app_info:is_available(AppInfo) of false -> - true = fetch_app(AppInfo, AppDir, State), - maybe_symlink_default(State, Profile, AppDir, AppInfo), - {true, rebar_app_info:valid(update_app_info(AppDir, AppInfo), false)}; - {true, AppInfo1} -> - case sets:is_element(rebar_app_info:name(AppInfo1), Seen) of + AppInfo1 = fetch_app(AppInfo, State), + maybe_symlink_default(State, Profile, AppDir, AppInfo1), + {ok, rebar_app_info:is_available(rebar_app_info:valid(AppInfo1, false), true)}; + true -> + case sets:is_element(rebar_app_info:name(AppInfo), Seen) of true -> - {false, AppInfo1}; + {ok, AppInfo}; false -> - maybe_symlink_default(State, Profile, AppDir, AppInfo1), - MaybeUpgrade = maybe_upgrade(AppInfo, AppDir, Upgrade, State), - AppInfo2 = update_app_info(AppDir, AppInfo1), - {MaybeUpgrade, AppInfo2} + maybe_symlink_default(State, Profile, AppDir, AppInfo), + AppInfo1 = maybe_upgrade(AppInfo, AppDir, Upgrade, State), + {ok, AppInfo1} end end end. @@ -372,52 +369,37 @@ make_relative_to_root(State, Path) when is_list(Path) -> Root = rebar_dir:root_dir(State), rebar_dir:make_relative_path(Path, Root). -fetch_app(AppInfo, AppDir, State) -> +fetch_app(AppInfo, State) -> ?INFO("Fetching ~ts (~p)", [rebar_app_info:name(AppInfo), - format_source(rebar_app_info:source(AppInfo))]), - Source = rebar_app_info:source(AppInfo), - true = rebar_fetch:download_source(AppDir, Source, State). - -format_source({pkg, Name, Vsn, _Hash}) -> {pkg, Name, Vsn}; -format_source(Source) -> Source. - -%% This is called after the dep has been downloaded and unpacked, if it hadn't been already. -%% So this is the first time for newly downloaded apps that its .app/.app.src data can -%% be read in an parsed. -update_app_info(AppDir, AppInfo) -> - case rebar_app_discover:find_app(AppInfo, AppDir, all) of - {true, AppInfo1} -> - AppInfo1; - false -> - throw(?PRV_ERROR({dep_app_not_found, AppDir, rebar_app_info:name(AppInfo)})) - end. + rebar_resource_v2:format_source(rebar_app_info:source(AppInfo))]), + rebar_fetch:download_source(AppInfo, State). -maybe_upgrade(AppInfo, AppDir, Upgrade, State) -> - Source = rebar_app_info:source(AppInfo), +maybe_upgrade(AppInfo, _AppDir, Upgrade, State) -> case Upgrade orelse rebar_app_info:is_lock(AppInfo) of true -> - case rebar_fetch:needs_update(AppDir, Source, State) of + case rebar_fetch:needs_update(AppInfo, State) of true -> - ?INFO("Upgrading ~ts (~p)", [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)]), - true = rebar_fetch:download_source(AppDir, Source, State); + ?INFO("Upgrading ~ts (~p)", [rebar_app_info:name(AppInfo), + rebar_resource_v2:format_source(rebar_app_info:source(AppInfo))]), + rebar_fetch:download_source(AppInfo, State); false -> case Upgrade of true -> ?INFO("No upgrade needed for ~ts", [rebar_app_info:name(AppInfo)]), - false; + AppInfo; false -> - false + AppInfo end end; false -> - false + AppInfo end. warn_skip_deps(AppInfo, State) -> Msg = "Skipping ~ts (from ~p) as an app of the same name " "has already been fetched", Args = [rebar_app_info:name(AppInfo), - rebar_app_info:source(AppInfo)], + rebar_resource_v2:format_source(rebar_app_info:source(AppInfo))], case rebar_state:get(State, deps_error_on_conflict, false) of false -> case rebar_state:get(State, deps_warning_on_conflict, true) of diff --git a/src/rebar_prv_local_upgrade.erl b/src/rebar_prv_local_upgrade.erl index 3b3c9cb..1931d65 100644 --- a/src/rebar_prv_local_upgrade.erl +++ b/src/rebar_prv_local_upgrade.erl @@ -77,7 +77,7 @@ get_md5(Rebar3Path) -> maybe_fetch_rebar3(Rebar3Md5) -> TmpDir = ec_file:insecure_mkdtemp(), TmpFile = filename:join(TmpDir, "rebar3"), - case rebar_pkg_resource:request("https://s3.amazonaws.com/rebar3/rebar3", Rebar3Md5) of + case request("https://s3.amazonaws.com/rebar3/rebar3", Rebar3Md5) of {ok, Binary, ETag} -> file:write_file(TmpFile, Binary), case etag(TmpFile) of @@ -101,3 +101,29 @@ etag(Path) -> {error, _} -> false end. + +-spec request(Url, ETag) -> Res when + Url :: string(), + ETag :: false | string(), + Res :: 'error' | {ok, cached} | {ok, any(), string()}. +request(Url, ETag) -> + HttpOptions = [{ssl, rebar_utils:ssl_opts(Url)}, + {relaxed, true} | rebar_utils:get_proxy_auth()], + case httpc:request(get, {Url, [{"if-none-match", "\"" ++ ETag ++ "\""} + || ETag =/= false] ++ + [{"User-Agent", rebar_utils:user_agent()}]}, + HttpOptions, [{body_format, binary}], rebar) of + {ok, {{_Version, 200, _Reason}, Headers, Body}} -> + ?DEBUG("Successfully downloaded ~ts", [Url]), + {"etag", ETag1} = lists:keyfind("etag", 1, Headers), + {ok, Body, rebar_string:trim(ETag1, both, [$"])}; + {ok, {{_Version, 304, _Reason}, _Headers, _Body}} -> + ?DEBUG("Cached copy of ~ts still valid", [Url]), + {ok, cached}; + {ok, {{_Version, Code, _Reason}, _Headers, _Body}} -> + ?DEBUG("Request to ~p failed: status code ~p", [Url, Code]), + error; + {error, Reason} -> + ?DEBUG("Request to ~p failed: ~p", [Url, Reason]), + error + end. diff --git a/src/rebar_prv_lock.erl b/src/rebar_prv_lock.erl index cbe8dfe..570c03f 100644 --- a/src/rebar_prv_lock.erl +++ b/src/rebar_prv_lock.erl @@ -54,12 +54,9 @@ format_error(Reason) -> build_locks(State) -> AllDeps = rebar_state:lock(State), [begin - Dir = rebar_app_info:dir(Dep), - Source = rebar_app_info:source(Dep), - %% If source is tuple it is a source dep %% e.g. {git, "git://github.com/ninenines/cowboy.git", "master"} - {rebar_app_info:name(Dep) - ,rebar_fetch:lock_source(Dir, Source, State) - ,rebar_app_info:dep_level(Dep)} + {rebar_app_info:name(Dep), + rebar_fetch:lock_source(Dep, State), + rebar_app_info:dep_level(Dep)} end || Dep <- AllDeps, not(rebar_app_info:is_checkout(Dep))]. diff --git a/src/rebar_prv_packages.erl b/src/rebar_prv_packages.erl index 6e8e683..3e54cdc 100644 --- a/src/rebar_prv_packages.erl +++ b/src/rebar_prv_packages.erl @@ -15,53 +15,75 @@ -spec init(rebar_state:t()) -> {ok, rebar_state:t()}. init(State) -> - State1 = rebar_state:add_provider(State, providers:create([{name, ?PROVIDER}, - {module, ?MODULE}, - {bare, true}, - {deps, ?DEPS}, - {example, "rebar3 pkgs"}, - {short_desc, "List available packages."}, - {desc, info("List available packages")}, - {opts, []}])), + State1 = rebar_state:add_provider(State, + providers:create([{name, ?PROVIDER}, + {module, ?MODULE}, + {bare, true}, + {deps, ?DEPS}, + {example, "rebar3 pkgs elli"}, + {short_desc, "List information for a package."}, + {desc, info("List information for a package")}, + {opts, [{package, undefined, undefined, string, + "Package to fetch information for."}]}])), {ok, State1}. -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - rebar_packages:packages(State), - case rebar_state:command_args(State) of - [Name] -> - print_packages(get_packages(rebar_utils:to_binary(Name))); - _ -> - print_packages(sort_packages()) - end, - {ok, State}. + {Args, _} = rebar_state:command_parsed_args(State), + case proplists:get_value(package, Args, undefined) of + undefined -> + ?PRV_ERROR(no_package_arg); + Name -> + Resources = rebar_state:resources(State), + #{repos := Repos} = rebar_resource_v2:find_resource_state(pkg, Resources), + Results = get_package(rebar_utils:to_binary(Name), Repos), + case lists:all(fun({_, {error, not_found}}) -> true; (_) -> false end, Results) of + true -> + ?PRV_ERROR({not_found, Name}); + false -> + [print_packages(Result) || Result <- Results], + {ok, State} + end + end. --spec format_error(any()) -> iolist(). -format_error(load_registry_fail) -> - "Failed to load package regsitry. Try running 'rebar3 update' to fix". +-spec get_package(binary(), [map()]) -> [{binary(), {ok, map()} | {error, term()}}]. +get_package(Name, Repos) -> + lists:foldl(fun(RepoConfig, Acc) -> + [{maps:get(name, RepoConfig), rebar_packages:get(RepoConfig, Name)} | Acc] + end, [], Repos). -print_packages(Pkgs) -> - orddict:map(fun(Name, Vsns) -> - SortedVsns = lists:sort(fun(A, B) -> - ec_semver:lte(ec_semver:parse(A) - ,ec_semver:parse(B)) - end, Vsns), - VsnStr = join(SortedVsns, <<", ">>), - ?CONSOLE("~ts:~n Versions: ~ts~n", [Name, VsnStr]) - end, Pkgs). -sort_packages() -> - ets:foldl(fun({package_index_version, _}, Acc) -> - Acc; - ({Pkg, Vsns}, Acc) -> - orddict:store(Pkg, Vsns, Acc); - (_, Acc) -> - Acc - end, orddict:new(), ?PACKAGE_TABLE). +-spec format_error(any()) -> iolist(). +format_error(no_package_arg) -> + "Missing package argument to `rebar3 pkgs` command."; +format_error({not_found, Name}) -> + io_lib:format("Package ~ts not found in any repo.", [Name]); +format_error(unknown) -> + "Something went wrong with fetching package metadata.". -get_packages(Name) -> - ets:lookup(?PACKAGE_TABLE, Name). +print_packages({RepoName, {error, not_found}}) -> + ?CONSOLE("~ts: Package not found in this repo.~n", [RepoName]); +print_packages({RepoName, {error, _}}) -> + ?CONSOLE("~ts: Error fetching from this repo.~n", [RepoName]); +print_packages({RepoName, {ok, #{<<"name">> := Name, + <<"meta">> := Meta, + <<"releases">> := Releases}}}) -> + Description = maps:get(<<"description">>, Meta, ""), + Licenses = join(maps:get(<<"licenses">>, Meta, []), <<", ">>), + Links = join_map(maps:get(<<"links">>, Meta, []), <<"\n ">>), + Maintainers = join(maps:get(<<"maintainers">>, Meta, []), <<", ">>), + Versions = [V || #{<<"version">> := V} <- Releases], + VsnStr = join(Versions, <<", ">>), + ?CONSOLE("~ts:~n" + " Name: ~ts~n" + " Description: ~ts~n" + " Licenses: ~ts~n" + " Maintainers: ~ts~n" + " Links:~n ~ts~n" + " Versions: ~ts~n", [RepoName, Name, Description, Licenses, Maintainers, Links, VsnStr]); +print_packages(_) -> + ok. -spec join([binary()], binary()) -> binary(). join([Bin], _Sep) -> @@ -69,6 +91,14 @@ join([Bin], _Sep) -> join([Bin | T], Sep) -> <>. +-spec join_map(map(), binary()) -> binary(). +join_map(Map, Sep) -> + join_tuple_list(maps:to_list(Map), Sep). + +join_tuple_list([{K, V}], _Sep) -> + <>; +join_tuple_list([{K, V} | T], Sep) -> + <>. info(Description) -> io_lib:format("~ts.~n", [Description]). diff --git a/src/rebar_prv_repos.erl b/src/rebar_prv_repos.erl new file mode 100644 index 0000000..0515910 --- /dev/null +++ b/src/rebar_prv_repos.erl @@ -0,0 +1,47 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et + +-module(rebar_prv_repos). + +-behaviour(provider). + +-export([init/1, + do/1, + format_error/1]). + +-include("rebar.hrl"). + +-define(PROVIDER, repos). +-define(DEPS, []). + +%% =================================================================== +%% Public API +%% =================================================================== + +-spec init(rebar_state:t()) -> {ok, rebar_state:t()}. +init(State) -> + Provider = providers:create( + [{name, ?PROVIDER}, + {module, ?MODULE}, + {bare, false}, + {deps, ?DEPS}, + {example, "rebar3 repos"}, + {short_desc, "Print current package repository configuration"}, + {desc, "Display repository configuration for debugging purpose"}, + {opts, []}]), + State1 = rebar_state:add_provider(State, Provider), + {ok, State1}. + +-spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. +do(State) -> + Resources = rebar_state:resources(State), + #{repos := Repos} = rebar_resource_v2:find_resource_state(pkg, Resources), + + ?CONSOLE("Repos:", []), + %%TODO: do some formatting + ?CONSOLE("~p", [Repos]), + {ok, State}. + +-spec format_error(any()) -> iolist(). +format_error(Reason) -> + io_lib:format("~p", [Reason]). diff --git a/src/rebar_prv_shell.erl b/src/rebar_prv_shell.erl index 5b8d789..760f0d8 100644 --- a/src/rebar_prv_shell.erl +++ b/src/rebar_prv_shell.erl @@ -40,6 +40,8 @@ -define(PROVIDER, shell). -define(DEPS, [compile]). +-dialyzer({nowarn_function, rewrite_leaders/2}). + %% =================================================================== %% Public API %% =================================================================== diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index 1744631..4c820c5 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -9,12 +9,6 @@ do/1, format_error/1]). --export([hex_to_index/1]). - --ifdef(TEST). --export([cmp_/6, cmpl_/6, valid_vsn/1]). --endif. - -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). @@ -39,44 +33,13 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - try - case rebar_packages:registry_dir(State) of - {ok, RegistryDir} -> - filelib:ensure_dir(filename:join(RegistryDir, "dummy")), - HexFile = filename:join(RegistryDir, "registry"), - ?INFO("Updating package registry...", []), - TmpDir = ec_file:insecure_mkdtemp(), - TmpFile = filename:join(TmpDir, "packages.gz"), - - CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), - case rebar_utils:url_append_path(CDN, ?REMOTE_REGISTRY_FILE) of - {ok, Url} -> - HttpOptions = [{relaxed, true} | rebar_utils:get_proxy_auth()], - ?DEBUG("Fetching registry from ~p", [Url]), - case httpc:request(get, {Url, [{"User-Agent", rebar_utils:user_agent()}]}, - HttpOptions, [{stream, TmpFile}, {sync, true}], - rebar) of - {ok, saved_to_file} -> - {ok, Data} = file:read_file(TmpFile), - Unzipped = zlib:gunzip(Data), - ok = file:write_file(HexFile, Unzipped), - ?INFO("Writing registry to ~ts", [HexFile]), - hex_to_index(State), - {ok, State}; - _ -> - ?PRV_ERROR(package_index_download) - end; - _ -> - ?PRV_ERROR({package_parse_cdn, CDN}) - end; - {uri_parse_error, CDN} -> - ?PRV_ERROR({package_parse_cdn, CDN}) - end - catch - ?WITH_STACKTRACE(_E, C, S) - ?DEBUG("Error creating package index: ~p ~p", [C, S]), - throw(?PRV_ERROR(package_index_write)) - end. + Names = rebar_packages:get_all_names(State), + Resources = rebar_state:resources(State), + #{repos := RepoConfigs} = rebar_resource_v2:find_resource_state(pkg, Resources), + [[update_package(Name, RepoConfig, State) + || Name <- Names] + || RepoConfig <- RepoConfigs], + {ok, State}. -spec format_error(any()) -> iolist(). format_error({package_parse_cdn, Uri}) -> @@ -86,186 +49,11 @@ format_error(package_index_download) -> format_error(package_index_write) -> "Failed to write package index.". -is_supported(<<"make">>) -> true; -is_supported(<<"rebar">>) -> true; -is_supported(<<"rebar3">>) -> true; -is_supported(_) -> false. - -hex_to_index(State) -> - {ok, RegistryDir} = rebar_packages:registry_dir(State), - HexFile = filename:join(RegistryDir, "registry"), - try ets:file2tab(HexFile) of - {ok, Registry} -> - try - PackageIndex = filename:join(RegistryDir, "packages.idx"), - ?INFO("Generating package index...", []), - (catch ets:delete(?PACKAGE_TABLE)), - ets:new(?PACKAGE_TABLE, [named_table, public]), - ets:foldl(fun({{Pkg, PkgVsn}, [Deps, Checksum, BuildTools | _]}, _) when is_list(BuildTools) -> - case lists:any(fun is_supported/1, BuildTools) of - true -> - DepsList = update_deps_list(Pkg, PkgVsn, Deps, Registry, State), - HashedDeps = update_deps_hashes(DepsList), - ets:insert(?PACKAGE_TABLE, {{Pkg, PkgVsn}, HashedDeps, Checksum}); - false -> - true - end; - (_, _) -> - true - end, true, Registry), - - ets:foldl(fun({Pkg, [[]]}, _) when is_binary(Pkg) -> - true; - ({Pkg, [Vsns=[_Vsn | _Rest]]}, _) when is_binary(Pkg) -> - %% Verify the package is of the right build tool by checking if the first - %% version exists in the table from the foldl above - case [V || V <- Vsns, ets:member(?PACKAGE_TABLE, {Pkg, V})] of - [] -> - true; - Vsns1 -> - ets:insert(?PACKAGE_TABLE, {Pkg, Vsns1}) - end; - (_, _) -> - true - end, true, Registry), - ets:insert(?PACKAGE_TABLE, {package_index_version, ?PACKAGE_INDEX_VERSION}), - ?INFO("Writing index to ~ts", [PackageIndex]), - ets:tab2file(?PACKAGE_TABLE, PackageIndex), - true - after - catch ets:delete(Registry) - end; - {error, Reason} -> - ?DEBUG("Error loading package registry: ~p", [Reason]), - false - catch - _:_ -> - fail - end. - -update_deps_list(Pkg, PkgVsn, Deps, HexRegistry, State) -> - lists:foldl(fun([Dep, DepVsn, false, AppName | _], DepsListAcc) -> - Dep1 = {Pkg, PkgVsn, Dep, AppName}, - case {valid_vsn(DepVsn), DepVsn} of - %% Those are all not perfectly implemented! - %% and doubled since spaces seem not to be - %% enforced - {false, Vsn} -> - ?DEBUG("[~ts:~ts], Bad dependency version for ~ts: ~ts.", - [Pkg, PkgVsn, Dep, Vsn]), - DepsListAcc; - {_, <<"~>", Vsn/binary>>} -> - highest_matching(Dep1, rm_ws(Vsn), HexRegistry, - State, DepsListAcc); - {_, <<">=", Vsn/binary>>} -> - cmp(Dep1, rm_ws(Vsn), HexRegistry, State, - DepsListAcc, fun ec_semver:gte/2); - {_, <<">", Vsn/binary>>} -> - cmp(Dep1, rm_ws(Vsn), HexRegistry, State, - DepsListAcc, fun ec_semver:gt/2); - {_, <<"<=", Vsn/binary>>} -> - cmpl(Dep1, rm_ws(Vsn), HexRegistry, State, - DepsListAcc, fun ec_semver:lte/2); - {_, <<"<", Vsn/binary>>} -> - cmpl(Dep1, rm_ws(Vsn), HexRegistry, State, - DepsListAcc, fun ec_semver:lt/2); - {_, <<"==", Vsn/binary>>} -> - [{AppName, {pkg, Dep, Vsn, undefined}} | DepsListAcc]; - {_, Vsn} -> - [{AppName, {pkg, Dep, Vsn, undefined}} | DepsListAcc] - end; - ([_Dep, _DepVsn, true, _AppName | _], DepsListAcc) -> - DepsListAcc - end, [], Deps). - -update_deps_hashes(List) -> - [{Name, {pkg, PkgName, Vsn, lookup_hash(PkgName, Vsn, Hash)}} - || {Name, {pkg, PkgName, Vsn, Hash}} <- List]. - -lookup_hash(Name, Vsn, undefined) -> - try - ets:lookup_element(?PACKAGE_TABLE, {Name, Vsn}, 3) - catch - _:_ -> - undefined - end; -lookup_hash(_, _, Hash) -> - Hash. - - -rm_ws(<<" ", R/binary>>) -> - rm_ws(R); -rm_ws(R) -> - R. - -valid_vsn(Vsn) -> - %% Regepx from https://github.com/sindresorhus/semver-regex/blob/master/index.js - SemVerRegExp = "v?(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)(\\.(0|[1-9][0-9]*))?" - "(-[0-9a-z-]+(\\.[0-9a-z-]+)*)?(\\+[0-9a-z-]+(\\.[0-9a-z-]+)*)?", - SupportedVersions = "^(>=?|<=?|~>|==)?\\s*" ++ SemVerRegExp ++ "$", - re:run(Vsn, SupportedVersions, [unicode]) =/= nomatch. - -highest_matching({Pkg, PkgVsn, Dep, App}, Vsn, HexRegistry, State, DepsListAcc) -> - case rebar_packages:find_highest_matching_(Pkg, PkgVsn, Dep, Vsn, HexRegistry, State) of - {ok, HighestDepVsn} -> - [{App, {pkg, Dep, HighestDepVsn, undefined}} | DepsListAcc]; - none -> - ?DEBUG("[~ts:~ts] Missing registry entry for package ~ts. Try to fix with `rebar3 update`", - [Pkg, PkgVsn, Dep]), - DepsListAcc - end. - -cmp({_Pkg, _PkgVsn, Dep, _App} = Dep1, Vsn, HexRegistry, State, DepsListAcc, CmpFun) -> - {ok, Vsns} = rebar_packages:find_all(Dep, HexRegistry, State), - cmp_(undefined, Vsn, Vsns, DepsListAcc, Dep1, CmpFun). - - -cmp_(undefined, _MinVsn, [], DepsListAcc, {Pkg, PkgVsn, Dep, _App}, _CmpFun) -> - ?DEBUG("[~ts:~ts] Missing registry entry for package ~ts. Try to fix with `rebar3 update`", - [Pkg, PkgVsn, Dep]), - DepsListAcc; -cmp_(HighestDepVsn, _MinVsn, [], DepsListAcc, {_Pkg, _PkgVsn, Dep, App}, _CmpFun) -> - [{App, {pkg, Dep, HighestDepVsn, undefined}} | DepsListAcc]; - -cmp_(BestMatch, MinVsn, [Vsn | R], DepsListAcc, Dep, CmpFun) -> - case CmpFun(Vsn, MinVsn) of - true -> - cmp_(Vsn, Vsn, R, DepsListAcc, Dep, CmpFun); - false -> - cmp_(BestMatch, MinVsn, R, DepsListAcc, Dep, CmpFun) - end. - -%% We need to treat this differently since we want a version that is LOWER but -%% the higest possible one. -cmpl({_Pkg, _PkgVsn, Dep, _App} = Dep1, Vsn, HexRegistry, State, DepsListAcc, CmpFun) -> - {ok, Vsns} = rebar_packages:find_all(Dep, HexRegistry, State), - cmpl_(undefined, Vsn, Vsns, DepsListAcc, Dep1, CmpFun). - -cmpl_(undefined, _MaxVsn, [], DepsListAcc, {Pkg, PkgVsn, Dep, _App}, _CmpFun) -> - ?DEBUG("[~ts:~ts] Missing registry entry for package ~ts. Try to fix with `rebar3 update`", - [Pkg, PkgVsn, Dep]), - DepsListAcc; - -cmpl_(HighestDepVsn, _MaxVsn, [], DepsListAcc, {_Pkg, _PkgVsn, Dep, App}, _CmpFun) -> - [{App, {pkg, Dep, HighestDepVsn, undefined}} | DepsListAcc]; - -cmpl_(undefined, MaxVsn, [Vsn | R], DepsListAcc, Dep, CmpFun) -> - case CmpFun(Vsn, MaxVsn) of - true -> - cmpl_(Vsn, MaxVsn, R, DepsListAcc, Dep, CmpFun); - false -> - cmpl_(undefined, MaxVsn, R, DepsListAcc, Dep, CmpFun) - end; -cmpl_(BestMatch, MaxVsn, [Vsn | R], DepsListAcc, Dep, CmpFun) -> - case CmpFun(Vsn, MaxVsn) of - true -> - case ec_semver:gte(Vsn, BestMatch) of - true -> - cmpl_(Vsn, MaxVsn, R, DepsListAcc, Dep, CmpFun); - false -> - cmpl_(BestMatch, MaxVsn, R, DepsListAcc, Dep, CmpFun) - end; - false -> - cmpl_(BestMatch, MaxVsn, R, DepsListAcc, Dep, CmpFun) +update_package(Name, RepoConfig, State) -> + case rebar_packages:update_package(Name, RepoConfig, State) of + fail -> + ?WARN("Failed to fetch updates for package ~ts from repo ~ts", [Name, maps:get(name, RepoConfig)]); + _ -> + ok end. diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl index e4469cf..b1b1b16 100644 --- a/src/rebar_prv_upgrade.erl +++ b/src/rebar_prv_upgrade.erl @@ -82,17 +82,22 @@ do_(State) -> Deps = [Dep || Dep <- TopDeps ++ ProfileDeps, % TopDeps > ProfileDeps is_atom(Dep) orelse is_atom(element(1, Dep))], Names = parse_names(rebar_utils:to_binary(proplists:get_value(package, Args, <<"">>)), Locks), + DepsDict = deps_dict(rebar_state:all_deps(State)), AltDeps = find_non_default_deps(Deps, State), FilteredNames = cull_default_names_if_profiles(Names, Deps, State), case prepare_locks(FilteredNames, Deps, Locks, [], DepsDict, AltDeps) of {error, Reason} -> {error, Reason}; - {Locks0, _Unlocks0} -> + {Locks0, Unlocks0} -> Deps0 = top_level_deps(Deps, Locks), State1 = rebar_state:set(State, {deps, default}, Deps0), DepsDir = rebar_prv_install_deps:profile_dep_dir(State, default), D = rebar_app_utils:parse_deps(root, DepsDir, Deps0, State1, Locks0, 0), + + %% first update the package index for the packages to be upgraded + update_pkg_deps(Unlocks0, D, State1), + State2 = rebar_state:set(State1, {parsed_deps, default}, D), State3 = rebar_state:set(State2, {locks, default}, Locks0), State4 = rebar_state:set(State3, upgrade, true), @@ -121,6 +126,34 @@ format_error({transitive_dependency, Name}) -> format_error(Reason) -> io_lib:format("~p", [Reason]). +%% fetch updates for package deps that have been unlocked for upgrade +update_pkg_deps([], _, _) -> + ok; +update_pkg_deps([{Name, _, _} | Rest], AppInfos, State) -> + case rebar_app_utils:find(Name, AppInfos) of + {ok, AppInfo} -> + case element(1, rebar_app_info:source(AppInfo)) of + pkg -> + Resources = rebar_state:resources(State), + #{repos := RepoConfigs} = rebar_resource_v2:find_resource_state(pkg, Resources), + [update_package(Name, RepoConfig, State) || RepoConfig <- RepoConfigs]; + _ -> + skip + end; + _ -> + %% this should be impossible... + skip + end, + update_pkg_deps(Rest, AppInfos, State). + +update_package(Name, RepoConfig, State) -> + case rebar_packages:update_package(Name, RepoConfig, State) of + fail -> + ?WARN("Failed to fetch updates for package ~ts from repo ~ts", [Name, maps:get(name, RepoConfig)]); + _ -> + ok + end. + parse_names(Bin, Locks) -> case lists:usort(re:split(Bin, <<" *, *">>, [trim, unicode])) of %% Nothing submitted, use *all* apps diff --git a/src/rebar_resource.erl b/src/rebar_resource.erl index cdce7a8..a3a8edb 100644 --- a/src/rebar_resource.erl +++ b/src/rebar_resource.erl @@ -2,23 +2,53 @@ %% ex: ts=4 sw=4 et -module(rebar_resource). --export([]). +-export([new/3, + lock/2, + download/4, + needs_update/2, + make_vsn/2]). --export_type([resource/0 - ,type/0 - ,location/0 - ,ref/0]). +-export_type([source/0, + type/0, + location/0, + ref/0]). --type resource() :: {type(), location(), ref()}. +-include("rebar.hrl"). + +-type source() :: {type(), location(), ref()} | {type(), location(), ref(), binary()}. -type type() :: atom(). -type location() :: string(). -type ref() :: any(). -callback lock(file:filename_all(), tuple()) -> - rebar_resource:resource(). + source(). -callback download(file:filename_all(), tuple(), rebar_state:t()) -> {tarball, file:filename_all()} | {ok, any()} | {error, any()}. -callback needs_update(file:filename_all(), tuple()) -> boolean(). -callback make_vsn(file:filename_all()) -> {plain, string()} | {error, string()}. + +-spec new(type(), module(), term()) -> rebar_resource_v2:resource(). +new(Type, Module, State) -> + #resource{type=Type, + module=Module, + state=State, + implementation=?MODULE}. + +lock(Module, AppInfo) -> + Module:lock(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). + +download(Module, TmpDir, AppInfo, State) -> + case Module:download(TmpDir, rebar_app_info:source(AppInfo), State) of + {ok, _} -> + ok; + Error -> + Error + end. + +needs_update(Module, AppInfo) -> + Module:needs_update(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). + +make_vsn(Module, AppInfo) -> + Module:make_vsn(rebar_app_info:dir(AppInfo)). diff --git a/src/rebar_resource_v2.erl b/src/rebar_resource_v2.erl new file mode 100644 index 0000000..f032f6e --- /dev/null +++ b/src/rebar_resource_v2.erl @@ -0,0 +1,147 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +-module(rebar_resource_v2). + +-export([new/3, + find_resource_state/2, + format_source/1, + lock/2, + download/3, + needs_update/2, + make_vsn/3, + format_error/1]). + +-export_type([resource/0, + source/0, + type/0, + location/0, + ref/0, + resource_state/0]). + +-include("rebar.hrl"). +-include_lib("providers/include/providers.hrl"). + +-type resource() :: #resource{}. +-type source() :: {type(), location(), ref()} | {type(), location(), ref(), binary()}. +-type type() :: atom(). +-type location() :: string(). +-type ref() :: any(). +-type resource_state() :: term(). + +-callback init(type(), rebar_state:t()) -> {ok, resource()}. +-callback lock(rebar_app_info:t(), resource_state()) -> source(). +-callback download(file:filename_all(), rebar_app_info:t(), resource_state(), rebar_state:t()) -> + ok | {error, any()}. +-callback needs_update(rebar_app_info:t(), resource_state()) -> boolean(). +-callback make_vsn(rebar_app_info:t(), resource_state()) -> + {plain, string()} | {error, string()}. + +-spec new(type(), module(), term()) -> resource(). +new(Type, Module, State) -> + #resource{type=Type, + module=Module, + state=State, + implementation=?MODULE}. + +-spec find_resource(type(), [resource()]) -> {ok, resource()} | {error, not_found}. +find_resource(Type, Resources) -> + case ec_lists:find(fun(#resource{type=T}) -> T =:= Type end, Resources) of + error when is_atom(Type) -> + case code:which(Type) of + non_existing -> + {error, not_found}; + _ -> + {ok, rebar_resource:new(Type, Type, #{})} + end; + error -> + {error, not_found}; + {ok, Resource} -> + {ok, Resource} + end. + +find_resource_state(Type, Resources) -> + case lists:keyfind(Type, #resource.type, Resources) of + false -> + {error, not_found}; + #resource{state=State} -> + State + end. + +format_source({pkg, Name, Vsn, _Hash, _}) -> {pkg, Name, Vsn}; +format_source(Source) -> Source. + +lock(AppInfo, State) -> + resource_run(lock, rebar_app_info:source(AppInfo), [AppInfo], State). + +resource_run(Function, Source, Args, State) -> + Resources = rebar_state:resources(State), + case get_resource_type(Source, Resources) of + {ok, #resource{type=_, + module=Module, + state=ResourceState, + implementation=?MODULE}} -> + erlang:apply(Module, Function, Args++[ResourceState]); + {ok, #resource{type=_, + module=Module, + state=_, + implementation=rebar_resource}} -> + erlang:apply(rebar_resource, Function, [Module | Args]) + end. + +download(TmpDir, AppInfo, State) -> + resource_run(download, rebar_app_info:source(AppInfo), [TmpDir, AppInfo, State], State). + +needs_update(AppInfo, State) -> + resource_run(needs_update, rebar_app_info:source(AppInfo), [AppInfo], State). + +%% this is a special case since it is used for project apps as well, not just deps +make_vsn(AppInfo, VcsType, State) -> + Resources = rebar_state:resources(State), + case is_resource_type(VcsType, Resources) of + true -> + case find_resource(VcsType, Resources) of + {ok, #resource{type=_, + module=Module, + state=ResourceState, + implementation=?MODULE}} -> + Module:make_vsn(AppInfo, ResourceState); + {ok, #resource{type=_, + module=Module, + state=_, + implementation=rebar_resource}} -> + rebar_resource:make_vsn(Module, AppInfo) + end; + false -> + unknown + end. + +format_error({no_resource, Location, Type}) -> + io_lib:format("Cannot handle dependency ~ts.~n" + " No module found for resource type ~p.", [Location, Type]); +format_error({no_resource, Source}) -> + io_lib:format("Cannot handle dependency ~ts.~n" + " No module found for unknown resource type.", [Source]). + +is_resource_type(Type, Resources) -> + lists:any(fun(#resource{type=T}) -> T =:= Type end, Resources). + +-spec get_resource_type(term(), [resource()]) -> {ok, resource()}. +get_resource_type({Type, Location}, Resources) -> + get_resource(Type, Location, Resources); +get_resource_type({Type, Location, _}, Resources) -> + get_resource(Type, Location, Resources); +get_resource_type({Type, _, _, Location}, Resources) -> + get_resource(Type, Location, Resources); +get_resource_type(Location={Type, _, _, _, _}, Resources) -> + get_resource(Type, Location, Resources); +get_resource_type(Source, _) -> + throw(?PRV_ERROR({no_resource, Source})). + +-spec get_resource(type(), term(), [resource()]) -> {ok, resource()}. +get_resource(Type, Location, Resources) -> + case find_resource(Type, Resources) of + {error, not_found} -> + throw(?PRV_ERROR({no_resource, Location, Type})); + {ok, Resource} -> + {ok, Resource} + end. diff --git a/src/rebar_state.erl b/src/rebar_state.erl index 3586dd6..ec74eea 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -38,6 +38,7 @@ to_list/1, + create_resources/2, resources/1, resources/2, add_resource/2, providers/1, providers/2, add_provider/2, allow_provider_overrides/1, allow_provider_overrides/2 @@ -75,26 +76,24 @@ -spec new() -> t(). new() -> - BaseState = base_state(), + BaseState = base_state(dict:new()), BaseState#state_t{dir = rebar_dir:get_cwd()}. -spec new(list()) -> t(). new(Config) when is_list(Config) -> - BaseState = base_state(), Opts = base_opts(Config), - BaseState#state_t { dir = rebar_dir:get_cwd(), - default = Opts, - opts = Opts }. + BaseState = base_state(Opts), + BaseState#state_t{dir=rebar_dir:get_cwd(), + default=Opts}. -spec new(t() | atom(), list()) -> t(). new(Profile, Config) when is_atom(Profile) , is_list(Config) -> - BaseState = base_state(), Opts = base_opts(Config), - BaseState#state_t { dir = rebar_dir:get_cwd(), - current_profiles = [Profile], - default = Opts, - opts = Opts }; + BaseState = base_state(Opts), + BaseState#state_t{dir = rebar_dir:get_cwd(), + current_profiles = [Profile], + default = Opts}; new(ParentState=#state_t{}, Config) -> %% Load terms from rebar.config, if it exists Dir = rebar_dir:get_cwd(), @@ -129,20 +128,15 @@ deps_from_config(Dir, Config) -> [{{locks, default}, D}, {{deps, default}, Deps}] end. -base_state() -> - case application:get_env(rebar, resources) of - undefined -> - Resources = []; - {ok, Resources} -> - Resources - end, - #state_t{resources=Resources}. +base_state(Opts) -> + #state_t{opts=Opts}. base_opts(Config) -> Deps = proplists:get_value(deps, Config, []), Plugins = proplists:get_value(plugins, Config, []), ProjectPlugins = proplists:get_value(project_plugins, Config, []), - Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins}, {{project_plugins, default}, ProjectPlugins} | Config], + Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins}, + {{project_plugins, default}, ProjectPlugins} | Config], true = rebar_config:verify_config_format(Terms), dict:from_list(Terms). @@ -359,18 +353,50 @@ namespace(#state_t{namespace=Namespace}) -> namespace(State=#state_t{}, Namespace) -> State#state_t{namespace=Namespace}. --spec resources(t()) -> [{rebar_resource:type(), module()}]. +-spec resources(t()) -> [{rebar_resource_v2:type(), module()}]. resources(#state_t{resources=Resources}) -> Resources. --spec resources(t(), [{rebar_resource:type(), module()}]) -> t(). +-spec resources(t(), [{rebar_resource_v2:type(), module()}]) -> t(). resources(State, NewResources) -> - State#state_t{resources=NewResources}. - --spec add_resource(t(), {rebar_resource:type(), module()}) -> t(). -add_resource(State=#state_t{resources=Resources}, Resource) -> + lists:foldl(fun(Resource, StateAcc) -> + add_resource(StateAcc, Resource) + end, State, NewResources). + +-spec add_resource(t(), {rebar_resource_v2:type(), module()}) -> t(). +add_resource(State=#state_t{resources=Resources}, {ResourceType, ResourceModule}) -> + _ = code:ensure_loaded(ResourceModule), + Resource = case erlang:function_exported(ResourceModule, init, 2) of + true -> + case ResourceModule:init(ResourceType, State) of + {ok, R=#resource{}} -> + R; + _ -> + %% init didn't return a resource + %% must be an old resource + warn_old_resource(ResourceModule), + rebar_resource:new(ResourceType, + ResourceModule, + #{}) + end; + false -> + %% no init, must be initial implementation + warn_old_resource(ResourceModule), + rebar_resource:new(ResourceType, + ResourceModule, + #{}) + end, State#state_t{resources=[Resource | Resources]}. +warn_old_resource(ResourceModule) -> + ?WARN("Using custom resource ~s that implements a deprecated api. " + "It should be upgraded to rebar_resource_v2.", [ResourceModule]). + +create_resources(Resources, State) -> + lists:foldl(fun(R, StateAcc) -> + add_resource(StateAcc, R) + end, State, Resources). + providers(#state_t{providers=Providers}) -> Providers. diff --git a/src/rebar_string.erl b/src/rebar_string.erl index 47cb15c..d03b14e 100644 --- a/src/rebar_string.erl +++ b/src/rebar_string.erl @@ -1,7 +1,7 @@ %%% @doc Compatibility module for string functionality %%% for pre- and post-unicode support. -module(rebar_string). --export([join/2, lexemes/2, trim/3, uppercase/1, lowercase/1, chr/2]). +-export([join/2, split/2, lexemes/2, trim/3, uppercase/1, lowercase/1, chr/2]). -ifdef(unicode_str). @@ -15,6 +15,7 @@ join([], Sep) when is_list(Sep) -> join([H|T], Sep) -> H ++ lists:append([Sep ++ X || X <- T]). +split(Str, SearchPattern) -> string:split(Str, SearchPattern). lexemes(Str, SepList) -> string:lexemes(Str, SepList). trim(Str, Direction, Cluster=[_]) -> string:trim(Str, Direction, Cluster). uppercase(Str) -> string:uppercase(Str). @@ -27,6 +28,8 @@ chr([], _C, _I) -> 0. -else. join(Strings, Separator) -> string:join(Strings, Separator). +split(Str, SearchPattern) when is_list(Str) -> string:split(Str, SearchPattern); +split(Str, SearchPattern) when is_binary(Str) -> binary:split(Str, SearchPattern). lexemes(Str, SepList) -> string:tokens(Str, SepList). trim(Str, Direction, [Char]) -> Dir = case Direction of diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 7e57d01..995d212 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -74,13 +74,15 @@ user_agent/0, reread_config/1, reread_config/2, get_proxy_auth/0, - is_list_of_strings/1]). + is_list_of_strings/1, + ssl_opts/1]). %% for internal use only -export([otp_release/0]). -include("rebar.hrl"). +-include_lib("public_key/include/OTP-PUB-KEY.hrl"). -define(ONE_LEVEL_INDENT, " "). -define(APP_NAME_INDEX, 2). @@ -710,12 +712,12 @@ escript_foldl(Fun, Acc, File) -> Error end. -vcs_vsn(Vcs, Dir, Resources) -> - case vcs_vsn_cmd(Vcs, Dir, Resources) of +vcs_vsn(AppInfo, Vcs, State) -> + case vcs_vsn_cmd(AppInfo, Vcs, State) of {plain, VsnString} -> VsnString; {cmd, CmdString} -> - vcs_vsn_invoke(CmdString, Dir); + vcs_vsn_invoke(CmdString, rebar_app_info:dir(AppInfo)); unknown -> ?ABORT("vcs_vsn: Unknown vsn format: ~p", [Vcs]); {error, Reason} -> @@ -723,23 +725,18 @@ vcs_vsn(Vcs, Dir, Resources) -> end. %% Temp work around for repos like relx that use "semver" -vcs_vsn_cmd(Vsn, _, _) when is_binary(Vsn) -> +vcs_vsn_cmd(_AppInfo, Vsn, _) when is_binary(Vsn) -> {plain, Vsn}; -vcs_vsn_cmd(VCS, Dir, Resources) when VCS =:= semver ; VCS =:= "semver" -> - vcs_vsn_cmd(git, Dir, Resources); -vcs_vsn_cmd({cmd, _Cmd}=Custom, _, _) -> +vcs_vsn_cmd(AppInfo, VCS, State) when VCS =:= semver ; VCS =:= "semver" -> + vcs_vsn_cmd(AppInfo, git, State); +vcs_vsn_cmd(_AppInfo, {cmd, _Cmd}=Custom, _) -> Custom; -vcs_vsn_cmd(VCS, Dir, Resources) when is_atom(VCS) -> - case find_resource_module(VCS, Resources) of - {ok, Module} -> - Module:make_vsn(Dir); - {error, _} -> - unknown - end; -vcs_vsn_cmd(VCS, Dir, Resources) when is_list(VCS) -> +vcs_vsn_cmd(AppInfo, VCS, State) when is_atom(VCS) -> + rebar_resource_v2:make_vsn(AppInfo, VCS, State); +vcs_vsn_cmd(AppInfo, VCS, State) when is_list(VCS) -> try list_to_existing_atom(VCS) of AVCS -> - case vcs_vsn_cmd(AVCS, Dir, Resources) of + case vcs_vsn_cmd(AppInfo, AVCS, State) of unknown -> {plain, VCS}; Other -> Other end @@ -754,19 +751,6 @@ vcs_vsn_invoke(Cmd, Dir) -> {ok, VsnString} = rebar_utils:sh(Cmd, [{cd, Dir}, {use_stdout, false}]), rebar_string:trim(VsnString, trailing, "\n"). -find_resource_module(Type, Resources) -> - case lists:keyfind(Type, 1, Resources) of - false -> - case code:which(Type) of - non_existing -> - {error, unknown}; - _ -> - {ok, Type} - end; - {Type, Module} -> - {ok, Module} - end. - %% @doc ident to the level specified -spec indent(non_neg_integer()) -> iolist(). indent(Amount) when erlang:is_integer(Amount) -> @@ -977,3 +961,116 @@ is_list_of_strings(List) when is_list(hd(List)) -> true; is_list_of_strings(List) when is_list(List) -> true. + +%%------------------------------------------------------------------------------ +%% @doc +%% Return the SSL options adequate for the project based on +%% its configuration, including for validation of certs. +%% @end +%%------------------------------------------------------------------------------ +-spec ssl_opts(Url) -> Res when + Url :: string(), + Res :: proplists:proplist(). +ssl_opts(Url) -> + case get_ssl_config() of + ssl_verify_enabled -> + ssl_opts(ssl_verify_enabled, Url); + ssl_verify_disabled -> + [{verify, verify_none}] + end. + +%%------------------------------------------------------------------------------ +%% @doc +%% Return the SSL options adequate for the project based on +%% its configuration, including for validation of certs. +%% @end +%%------------------------------------------------------------------------------ +-spec ssl_opts(Enabled, Url) -> Res when + Enabled :: atom(), + Url :: string(), + Res :: proplists:proplist(). +ssl_opts(ssl_verify_enabled, Url) -> + case check_ssl_version() of + true -> + {ok, {_, _, Hostname, _, _, _}} = + http_uri:parse(rebar_utils:to_list(Url)), + VerifyFun = {fun ssl_verify_hostname:verify_fun/3, + [{check_hostname, Hostname}]}, + CACerts = certifi:cacerts(), + [{verify, verify_peer}, {depth, 2}, {cacerts, CACerts}, + {partial_chain, fun partial_chain/1}, {verify_fun, VerifyFun}]; + false -> + ?WARN("Insecure HTTPS request (peer verification disabled), " + "please update to OTP 17.4 or later", []), + [{verify, verify_none}] + end. + +-spec partial_chain(Certs) -> Res when + Certs :: list(any()), + Res :: unknown_ca | {trusted_ca, any()}. +partial_chain(Certs) -> + Certs1 = [{Cert, public_key:pkix_decode_cert(Cert, otp)} || Cert <- Certs], + CACerts = certifi:cacerts(), + CACerts1 = [public_key:pkix_decode_cert(Cert, otp) || Cert <- CACerts], + case ec_lists:find(fun({_, Cert}) -> + check_cert(CACerts1, Cert) + end, Certs1) of + {ok, Trusted} -> + {trusted_ca, element(1, Trusted)}; + _ -> + unknown_ca + end. + +-spec extract_public_key_info(Cert) -> Res when + Cert :: #'OTPCertificate'{tbsCertificate::#'OTPTBSCertificate'{}}, + Res :: any(). +extract_public_key_info(Cert) -> + ((Cert#'OTPCertificate'.tbsCertificate)#'OTPTBSCertificate'.subjectPublicKeyInfo). + +-spec check_cert(CACerts, Cert) -> Res when + CACerts :: list(any()), + Cert :: any(), + Res :: boolean(). +check_cert(CACerts, Cert) -> + lists:any(fun(CACert) -> + extract_public_key_info(CACert) == extract_public_key_info(Cert) + end, CACerts). + +-spec check_ssl_version() -> + boolean(). +check_ssl_version() -> + case application:get_key(ssl, vsn) of + {ok, Vsn} -> + parse_vsn(Vsn) >= {5, 3, 6}; + _ -> + false + end. + +-spec get_ssl_config() -> + ssl_verify_disabled | ssl_verify_enabled. +get_ssl_config() -> + GlobalConfigFile = rebar_dir:global_config(), + Config = rebar_config:consult_file(GlobalConfigFile), + case proplists:get_value(ssl_verify, Config, []) of + false -> + ssl_verify_disabled; + _ -> + ssl_verify_enabled + end. + +-spec parse_vsn(Vsn) -> Res when + Vsn :: string(), + Res :: {integer(), integer(), integer()}. +parse_vsn(Vsn) -> + version_pad(rebar_string:lexemes(Vsn, ".-")). + +-spec version_pad(list(nonempty_string())) -> Res when + Res :: {integer(), integer(), integer()}. +version_pad([Major]) -> + {list_to_integer(Major), 0, 0}; +version_pad([Major, Minor]) -> + {list_to_integer(Major), list_to_integer(Minor), 0}; +version_pad([Major, Minor, Patch]) -> + {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}; +version_pad([Major, Minor, Patch | _]) -> + {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}. -- cgit v1.1 From 5889a6e32bf09e5c50b49be330229bd157b5fae9 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 13 Sep 2018 20:13:59 -0600 Subject: Bump to 3.7.0-rc1 --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 530a79e..c122e04 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.7.0-rc1"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 221cbf5cac0204f3d2456552c08c38325ff592e2 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 13 Sep 2018 20:26:47 -0600 Subject: Back to git-based versioning --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index c122e04..530a79e 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.7.0-rc1"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From cd858a45e37f6563f22bf3aebee5911521f3c497 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Mon, 17 Sep 2018 09:02:15 -0600 Subject: reset default profile app plugins to empty if at the top level (#1885) top level default profile plugins are installed in the run_aux function. This commit removes the plugins from the plugin opts of an application that is also at the top level and skips the default profile when installing top level plugins in project_apps_install so they aren't handled twice. Additionally, before handling a plugin the state's list of known plugin apps is checked and the plugin is skipped if it has already been handled. --- src/rebar_app_discover.erl | 35 +++++++++++++++++++++-------------- src/rebar_plugins.erl | 33 ++++++++++++++++++++++++--------- 2 files changed, 45 insertions(+), 23 deletions(-) (limited to 'src') diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index e82403c..9b1346d 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -98,8 +98,7 @@ merge_deps(AppInfo, State) -> %% the State and at the top level, not as part of an application. CurrentProfiles = rebar_state:current_profiles(State), Default = reset_hooks(rebar_state:default(State), CurrentProfiles), - {C, State1} = project_app_config(AppInfo, State), - AppInfo0 = rebar_app_info:update_opts(AppInfo, Default, C), + {AppInfo0, State1} = project_app_config(AppInfo, Default, State), Name = rebar_app_info:name(AppInfo0), @@ -154,28 +153,36 @@ parse_profile_deps(Profile, Name, Deps, Opts, State) -> %% @doc Find the app-level config and return the state updated %% with the relevant app-level data. --spec project_app_config(rebar_app_info:t(), rebar_state:t()) -> +-spec project_app_config(rebar_app_info:t(), rebar_dict(), rebar_state:t()) -> {Config, rebar_state:t()} when Config :: [any()]. -project_app_config(AppInfo, State) -> +project_app_config(AppInfo, Default, State) -> C = rebar_config:consult(rebar_app_info:dir(AppInfo)), + AppInfo1 = rebar_app_info:update_opts(AppInfo, Default, C), + {AppInfo2, State1} = maybe_reset_hooks_plugins(AppInfo1, State), + {AppInfo2, State1}. + +-spec maybe_reset_hooks_plugins(AppInfo, State) -> {AppInfo, State} when + AppInfo :: rebar_app_info:t(), + State :: rebar_state:t(). +maybe_reset_hooks_plugins(AppInfo, State) -> Dir = rebar_app_info:dir(AppInfo), - Opts = maybe_reset_hooks(Dir, rebar_state:opts(State), State), - {C, rebar_state:opts(State, Opts)}. - -%% @private Check if the app is at the root of the project. -%% If it is, then drop the hooks from the config so they aren't run twice --spec maybe_reset_hooks(file:filename(), Opts, rebar_state:t()) -> Opts when - Opts :: rebar_dict(). -maybe_reset_hooks(Dir, Opts, State) -> case ec_file:real_dir_path(rebar_dir:root_dir(State)) of Dir -> CurrentProfiles = rebar_state:current_profiles(State), - reset_hooks(Opts, CurrentProfiles); + Opts = reset_hooks(rebar_state:opts(State), CurrentProfiles), + State1 = rebar_state:opts(State, Opts), + + %% set plugins to empty since this is an app at the top level + %% and top level plugins are installed in run_aux + AppInfo1 = rebar_app_info:set(rebar_app_info:set(AppInfo, {plugins,default}, []), plugins, []), + + {AppInfo1, State1}; _ -> - Opts + {AppInfo, State} end. + %% @doc make the hooks empty for a given set of options -spec reset_hooks(Opts, Profiles) -> Opts when diff --git a/src/rebar_plugins.erl b/src/rebar_plugins.erl index bc6a1e0..d58675d 100644 --- a/src/rebar_plugins.erl +++ b/src/rebar_plugins.erl @@ -39,13 +39,18 @@ project_apps_install(State) -> Profiles = rebar_state:current_profiles(State), ProjectApps = rebar_state:project_apps(State), lists:foldl(fun(Profile, StateAcc) -> - Plugins = rebar_state:get(State, {plugins, Profile}, []), - StateAcc1 = handle_plugins(Profile, Plugins, StateAcc), + StateAcc1 = case Profile of + default -> + %% default profile top level plugins + %% are installed in run_aux + StateAcc; + _ -> + Plugins = rebar_state:get(State, {plugins, Profile}, []), + handle_plugins(Profile, Plugins, StateAcc) + end, lists:foldl(fun(AppInfo, StateAcc2) -> - C = rebar_config:consult(rebar_app_info:dir(AppInfo)), - AppInfo0 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), C), - Plugins2 = rebar_app_info:get(AppInfo0, {plugins, Profile}, []), + Plugins2 = rebar_app_info:get(AppInfo, {plugins, Profile}, []), handle_plugins(Profile, Plugins2, StateAcc2) end, StateAcc1, ProjectApps) end, State, Profiles). @@ -62,12 +67,25 @@ install(State, AppInfo) -> State2 = lists:foldl(fun(Profile, StateAcc) -> Plugins = rebar_app_info:get(AppInfo, {plugins, Profile}, []), - handle_plugins(Profile, Plugins, StateAcc) + Plugins1 = filter_existing_plugins(Plugins, StateAcc), + handle_plugins(Profile, Plugins1, StateAcc) end, State1, Profiles), %% Reset the overrides after processing the dep rebar_state:set(State2, overrides, StateOverrides). +filter_existing_plugins(Plugins, State) -> + PluginNames = lists:zip(Plugins, rebar_state:deps_names(Plugins)), + AllPlugins = rebar_state:all_plugin_deps(State), + lists:filtermap(fun({Plugin, PluginName}) -> + case rebar_app_utils:find(PluginName, AllPlugins) of + {ok, _} -> + false; + _ -> + {true, Plugin} + end + end, PluginNames). + handle_plugins(Profile, Plugins, State) -> handle_plugins(Profile, Plugins, State, false). @@ -76,7 +94,6 @@ handle_plugins(Profile, Plugins, State, Upgrade) -> Locks = rebar_state:lock(State), DepsDir = rebar_state:get(State, deps_dir, ?DEFAULT_DEPS_DIR), State1 = rebar_state:set(State, deps_dir, ?DEFAULT_PLUGINS_DIR), - %% Install each plugin individually so if one fails to install it doesn't effect the others {_PluginProviders, State2} = lists:foldl(fun(Plugin, {PluginAcc, StateAcc}) -> @@ -122,8 +139,6 @@ handle_plugin(Profile, Plugin, State, Upgrade) -> build_plugin(AppInfo, Apps, State) -> Providers = rebar_state:providers(State), - %Providers1 = rebar_state:providers(rebar_app_info:state(AppInfo)), - %rebar_app_info:state_or_new(State, AppInfo) S = rebar_state:all_deps(State, Apps), S1 = rebar_state:set(S, deps_dir, ?DEFAULT_PLUGINS_DIR), rebar_prv_compile:compile(S1, Providers, AppInfo). -- cgit v1.1 From 43769ae87f7c60a9b6cb3bf08085a4a4206961b7 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Mon, 17 Sep 2018 09:02:30 -0600 Subject: only print skip warning if version is different (#1886) --- src/rebar_prv_install_deps.erl | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index bad5af4..068c4c8 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -259,9 +259,21 @@ update_seen_dep(AppInfo, _Profile, _Level, Deps, Apps, State, Upgrade, Seen, Loc %% If seen from lock file or user requested an upgrade %% don't print warning about skipping case lists:keymember(Name, 1, Locks) of - false when Upgrade -> ok; - false when not Upgrade -> warn_skip_deps(AppInfo, State); - true -> ok + false when Upgrade -> + ok; + false when not Upgrade -> + {ok, SeenApp} = rebar_app_utils:find(Name, Apps), + Source = rebar_app_info:source(AppInfo), + case rebar_app_info:source(SeenApp) of + Source -> + %% dep is the same version and checksum as the one we already saw. + %% meaning there is no conflict, so don't warn about it. + skip; + _ -> + warn_skip_deps(Name, Source, State) + end; + true -> + ok end, {Deps, Apps, State, Seen}. @@ -395,11 +407,11 @@ maybe_upgrade(AppInfo, _AppDir, Upgrade, State) -> AppInfo end. -warn_skip_deps(AppInfo, State) -> +warn_skip_deps(Name, Source, State) -> Msg = "Skipping ~ts (from ~p) as an app of the same name " "has already been fetched", - Args = [rebar_app_info:name(AppInfo), - rebar_resource_v2:format_source(rebar_app_info:source(AppInfo))], + Args = [Name, + rebar_resource_v2:format_source(Source)], case rebar_state:get(State, deps_error_on_conflict, false) of false -> case rebar_state:get(State, deps_warning_on_conflict, true) of -- cgit v1.1 From 8e0ef83de48968fed80114cc4518004ed59fa649 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Wed, 19 Sep 2018 08:24:38 -0600 Subject: add option to compiler provider to only build dependencies (#1888) --- src/rebar_prv_compile.erl | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index 0b4fa5f..edb2d82 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -30,22 +30,41 @@ init(State) -> {example, "rebar3 compile"}, {short_desc, "Compile apps .app.src and .erl files."}, {desc, "Compile apps .app.src and .erl files."}, - {opts, []}])), + {opts, [{deps_only, $d, "deps_only", undefined, + "Only compile dependencies, no project apps will be built."}]}])), {ok, State1}. -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> + IsDepsOnly = is_deps_only(State), DepsPaths = rebar_state:code_paths(State, all_deps), PluginDepsPaths = rebar_state:code_paths(State, all_plugin_deps), rebar_utils:remove_from_code_path(PluginDepsPaths), code:add_pathsa(DepsPaths), - ProjectApps = rebar_state:project_apps(State), Providers = rebar_state:providers(State), Deps = rebar_state:deps_to_build(State), - Cwd = rebar_state:dir(State), - copy_and_build_apps(State, Providers, Deps), + + State1 = case IsDepsOnly of + true -> + State; + false -> + handle_project_apps(DepsPaths, Providers, State) + end, + + rebar_utils:cleanup_code_path(rebar_state:code_paths(State1, default) + ++ rebar_state:code_paths(State, all_plugin_deps)), + + {ok, State1}. + +is_deps_only(State) -> + {Args, _} = rebar_state:command_parsed_args(State), + proplists:get_value(deps_only, Args, false). + +handle_project_apps(DepsPaths, Providers, State) -> + Cwd = rebar_state:dir(State), + ProjectApps = rebar_state:project_apps(State), {ok, ProjectApps1} = rebar_digraph:compile_order(ProjectApps), %% Run top level hooks *before* project apps compiled but *after* deps are @@ -66,10 +85,9 @@ do(State) -> true -> true end, - rebar_utils:cleanup_code_path(rebar_state:code_paths(State3, default) - ++ rebar_state:code_paths(State, all_plugin_deps)), - {ok, State3}. + State3. + -spec format_error(any()) -> iolist(). format_error({missing_artifact, File}) -> -- cgit v1.1 From 2dfba204e4dea5d1c3821fd26d22bd7201595f6c Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 21 Sep 2018 10:32:57 -0600 Subject: properly support top level app erl_opts from REBAR_CONFIG os var (#1889) When REBAR_CONFIG was set it would not effect the top level app's configuration because app_discover was rereading the top level rebar.config which ignored REBAR_CONFIG. Instead this patch has it use the existing configuration from REBAR_CONFIG. --- src/rebar_app_discover.erl | 74 ++++++++++++++++++++++++--------------------- src/rebar_app_info.erl | 14 +++++++-- src/rebar_erlc_compiler.erl | 1 + src/rebar_prv_plugins.erl | 4 +-- 4 files changed, 54 insertions(+), 39 deletions(-) (limited to 'src') diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index 9b1346d..224539b 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -7,7 +7,7 @@ find_unbuilt_apps/1, find_apps/1, find_apps/2, - find_apps/3, + find_apps/4, find_app/2, find_app/3]). @@ -23,7 +23,7 @@ do(State, LibDirs) -> Dirs = [filename:join(BaseDir, LibDir) || LibDir <- LibDirs], RebarOpts = rebar_state:opts(State), SrcDirs = rebar_dir:src_dirs(RebarOpts, ["src"]), - Apps = find_apps(Dirs, SrcDirs, all), + Apps = find_apps(Dirs, SrcDirs, all, State), ProjectDeps = rebar_state:deps_names(State), DepsDir = rebar_dir:deps_dir(State), CurrentProfiles = rebar_state:current_profiles(State), @@ -52,7 +52,7 @@ do(State, LibDirs) -> Name = rebar_app_info:name(AppInfo), case enable(State, AppInfo) of true -> - {AppInfo1, StateAcc1} = merge_deps(AppInfo, StateAcc), + {AppInfo1, StateAcc1} = merge_opts(AppInfo, StateAcc), OutDir = filename:join(DepsDir, Name), AppInfo2 = rebar_app_info:out_dir(AppInfo1, OutDir), ProjectDeps1 = lists:delete(Name, ProjectDeps), @@ -87,33 +87,34 @@ format_error({module_list, File}) -> format_error({missing_module, Module}) -> io_lib:format("Module defined in app file missing: ~p~n", [Module]). -%% @doc handles the merging and application of profiles and overrides -%% for a given application, within its own context. --spec merge_deps(rebar_app_info:t(), rebar_state:t()) -> +%% @doc merges configuration of a project app and the top level state +%% some configuration like erl_opts must be merged into a subapp's opts +%% while plugins and hooks need to be kept defined to only either the +%% top level state or an individual application. +-spec merge_opts(rebar_app_info:t(), rebar_state:t()) -> {rebar_app_info:t(), rebar_state:t()}. -merge_deps(AppInfo, State) -> +merge_opts(AppInfo, State) -> %% These steps make sure that hooks and artifacts are run in the context of %% the application they are defined at. If an umbrella structure is used and %% they are defined at the top level they will instead run in the context of %% the State and at the top level, not as part of an application. CurrentProfiles = rebar_state:current_profiles(State), - Default = reset_hooks(rebar_state:default(State), CurrentProfiles), - {AppInfo0, State1} = project_app_config(AppInfo, Default, State), + {AppInfo1, State1} = maybe_reset_hooks_plugins(AppInfo, State), - Name = rebar_app_info:name(AppInfo0), + Name = rebar_app_info:name(AppInfo1), %% We reset the opts here to default so no profiles are applied multiple times - AppInfo1 = rebar_app_info:apply_overrides(rebar_state:get(State1, overrides, []), AppInfo0), - AppInfo2 = rebar_app_info:apply_profiles(AppInfo1, CurrentProfiles), + AppInfo2 = rebar_app_info:apply_overrides(rebar_state:get(State1, overrides, []), AppInfo1), + AppInfo3 = rebar_app_info:apply_profiles(AppInfo2, CurrentProfiles), %% Will throw an exception if checks fail - rebar_app_info:verify_otp_vsn(AppInfo2), + rebar_app_info:verify_otp_vsn(AppInfo3), State2 = lists:foldl(fun(Profile, StateAcc) -> - handle_profile(Profile, Name, AppInfo2, StateAcc) + handle_profile(Profile, Name, AppInfo3, StateAcc) end, State1, lists:reverse(CurrentProfiles)), - {AppInfo2, State2}. + {AppInfo3, State2}. %% @doc Applies a given profile for an app, ensuring the deps %% match the context it will require. @@ -151,25 +152,15 @@ parse_profile_deps(Profile, Name, Deps, Opts, State) -> ,Locks ,1). -%% @doc Find the app-level config and return the state updated -%% with the relevant app-level data. --spec project_app_config(rebar_app_info:t(), rebar_dict(), rebar_state:t()) -> - {Config, rebar_state:t()} when - Config :: [any()]. -project_app_config(AppInfo, Default, State) -> - C = rebar_config:consult(rebar_app_info:dir(AppInfo)), - AppInfo1 = rebar_app_info:update_opts(AppInfo, Default, C), - {AppInfo2, State1} = maybe_reset_hooks_plugins(AppInfo1, State), - {AppInfo2, State1}. - +%% reset the State hooks if there is a top level application -spec maybe_reset_hooks_plugins(AppInfo, State) -> {AppInfo, State} when AppInfo :: rebar_app_info:t(), State :: rebar_state:t(). maybe_reset_hooks_plugins(AppInfo, State) -> Dir = rebar_app_info:dir(AppInfo), + CurrentProfiles = rebar_state:current_profiles(State), case ec_file:real_dir_path(rebar_dir:root_dir(State)) of Dir -> - CurrentProfiles = rebar_state:current_profiles(State), Opts = reset_hooks(rebar_state:opts(State), CurrentProfiles), State1 = rebar_state:opts(State, Opts), @@ -179,7 +170,11 @@ maybe_reset_hooks_plugins(AppInfo, State) -> {AppInfo1, State1}; _ -> - {AppInfo, State} + %% if not in the top root directory then we need to merge in the + %% default state opts to this subapp's opts + Default = reset_hooks(rebar_state:default(State), CurrentProfiles), + AppInfo1 = rebar_app_info:update_opts(AppInfo, Default), + {AppInfo1, State} end. @@ -211,8 +206,8 @@ reset_hooks(Opts, CurrentProfiles) -> -spec all_app_dirs([file:name()]) -> [{file:name(), [file:name()]}]. all_app_dirs(LibDirs) -> lists:flatmap(fun(LibDir) -> - {_, SrcDirs} = find_config_src(LibDir, ["src"]), - app_dirs(LibDir, SrcDirs) + {_, SrcDirs} = find_config_src(LibDir, ["src"]), + app_dirs(LibDir, SrcDirs) end, LibDirs). %% @private find the directories for all apps based on their source dirs @@ -270,11 +265,11 @@ find_apps(LibDirs, Validate) -> %% @doc for each directory passed, with the configured source directories, %% find all apps according to the validity rule passed in. %% Returns all the related app info records. --spec find_apps([file:filename_all()], [file:filename_all()], valid | invalid | all) -> [rebar_app_info:t()]. -find_apps(LibDirs, SrcDirs, Validate) -> +-spec find_apps([file:filename_all()], [file:filename_all()], valid | invalid | all, rebar_state:t()) -> [rebar_app_info:t()]. +find_apps(LibDirs, SrcDirs, Validate, State) -> rebar_utils:filtermap( fun({AppDir, AppSrcDirs}) -> - find_app(rebar_app_info:new(), AppDir, AppSrcDirs, Validate) + find_app(rebar_app_info:new(), AppDir, AppSrcDirs, Validate, State) end, all_app_dirs(LibDirs, SrcDirs) ). @@ -305,8 +300,19 @@ find_app(AppInfo, AppDir, Validate) -> %% the directories where source files can be located. Returns the related %% app info record. -spec find_app(rebar_app_info:t(), file:filename_all(), - [file:filename_all()], valid | invalid | all) -> + [file:filename_all()], valid | invalid | all, rebar_state:t()) -> {true, rebar_app_info:t()} | false. +find_app(AppInfo, AppDir, SrcDirs, Validate, State) -> + AppInfo1 = case ec_file:real_dir_path(rebar_dir:root_dir(State)) of + AppDir -> + Opts = rebar_state:opts(State), + rebar_app_info:default(rebar_app_info:opts(AppInfo, Opts), Opts); + _ -> + Config = rebar_config:consult(AppDir), + rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), Config) + end, + find_app_(AppInfo1, AppDir, SrcDirs, Validate). + find_app(AppInfo, AppDir, SrcDirs, Validate) -> Config = rebar_config:consult(AppDir), AppInfo1 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), Config), diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index 56ae4c0..eb95311 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -7,6 +7,7 @@ new/4, new/5, update_opts/3, + update_opts/2, update_opts_deps/2, discover/1, name/1, @@ -158,7 +159,7 @@ update_opts(AppInfo, Opts, Config) -> %% don't set anything here. []; _ -> - deps_from_config(dir(AppInfo), Config) + deps_from_config(dir(AppInfo), proplists:get_value(deps, Config, [])) end, Plugins = proplists:get_value(plugins, Config, []), @@ -171,6 +172,13 @@ update_opts(AppInfo, Opts, Config) -> AppInfo#app_info_t{opts=NewOpts, default=NewOpts}. +%% @doc update current app info opts by merging in a new dict of opts +-spec update_opts(t(), rebar_dict()) -> t(). +update_opts(AppInfo=#app_info_t{opts=LocalOpts}, Opts) -> + NewOpts = rebar_opts:merge_opts(LocalOpts, Opts), + AppInfo#app_info_t{opts=NewOpts, + default=NewOpts}. + %% @doc update the opts based on new deps, usually from an app's hex registry metadata -spec update_opts_deps(t(), [any()]) -> t(). update_opts_deps(AppInfo=#app_info_t{opts=Opts}, Deps) -> @@ -183,10 +191,10 @@ update_opts_deps(AppInfo=#app_info_t{opts=Opts}, Deps) -> %% @private extract the deps for an app in `Dir' based on its config file data -spec deps_from_config(file:filename(), [any()]) -> [{tuple(), any()}, ...]. -deps_from_config(Dir, Config) -> +deps_from_config(Dir, ConfigDeps) -> case rebar_config:consult_lock_file(filename:join(Dir, ?LOCK_FILE)) of [] -> - [{{deps, default}, proplists:get_value(deps, Config, [])}]; + [{{deps, default}, ConfigDeps}]; D -> %% We want the top level deps only from the lock file. %% This ensures deterministic overrides for configs. diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index 57e5398..c63ca5b 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -99,6 +99,7 @@ compile(AppInfo, CompileOpts) when element(1, AppInfo) == app_info_t -> {recursive, dir_recursive(RebarOpts, "src", CompileOpts)}], MibsOpts = [check_last_mod, {recursive, dir_recursive(RebarOpts, "mibs", CompileOpts)}], + rebar_base_compiler:run(RebarOpts, check_files([filename:join(Dir, File) || File <- rebar_opts:get(RebarOpts, xrl_first_files, [])]), diff --git a/src/rebar_prv_plugins.erl b/src/rebar_prv_plugins.erl index 4bea3b3..d66b645 100644 --- a/src/rebar_prv_plugins.erl +++ b/src/rebar_prv_plugins.erl @@ -36,7 +36,7 @@ do(State) -> GlobalPlugins = rebar_state:get(GlobalConfig, plugins, []), GlobalSrcDirs = rebar_state:get(GlobalConfig, src_dirs, ["src"]), GlobalPluginsDir = filename:join([rebar_dir:global_cache_dir(rebar_state:opts(State)), "plugins", "*"]), - GlobalApps = rebar_app_discover:find_apps([GlobalPluginsDir], GlobalSrcDirs, all), + GlobalApps = rebar_app_discover:find_apps([GlobalPluginsDir], GlobalSrcDirs, all, State), display_plugins("Global plugins", GlobalApps, GlobalPlugins), RebarOpts = rebar_state:opts(State), @@ -44,7 +44,7 @@ do(State) -> Plugins = rebar_state:get(State, plugins, []), PluginsDirs = filelib:wildcard(filename:join(rebar_dir:plugins_dir(State), "*")), CheckoutsDirs = filelib:wildcard(filename:join(rebar_dir:checkouts_dir(State), "*")), - Apps = rebar_app_discover:find_apps(CheckoutsDirs++PluginsDirs, SrcDirs, all), + Apps = rebar_app_discover:find_apps(CheckoutsDirs++PluginsDirs, SrcDirs, all, State), display_plugins("Local plugins", Apps, Plugins), {ok, State}. -- cgit v1.1 From 8bcb7da1dd294f13bc8d2040ec88f0b41b941e1b Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Mon, 1 Oct 2018 15:41:34 -0600 Subject: warn if the matched package is retired, skip prerelease (#1897) retired packages are now used the same as any other but a warning will be printed when it is resolved. prerelease versions are skipped unless explicitly given as the version in the constraint or lock file. --- src/rebar.hrl | 6 +++--- src/rebar_app_utils.erl | 41 +++++++++++++++++++++++++++++++++++------ src/rebar_packages.erl | 44 ++++++++++++++++++++------------------------ 3 files changed, 58 insertions(+), 33 deletions(-) (limited to 'src') diff --git a/src/rebar.hrl b/src/rebar.hrl index 572cbe8..f11302d 100644 --- a/src/rebar.hrl +++ b/src/rebar.hrl @@ -27,9 +27,9 @@ -define(REMOTE_PACKAGE_DIR, "tarballs"). -define(LOCK_FILE, "rebar.lock"). -define(DEFAULT_COMPILER_SOURCE_FORMAT, relative). --define(PACKAGE_INDEX_VERSION, 4). --define(PACKAGE_TABLE, package_index_v4). --define(INDEX_FILE, "packages-v4.idx"). +-define(PACKAGE_INDEX_VERSION, 5). +-define(PACKAGE_TABLE, package_index). +-define(INDEX_FILE, "packages.idx"). -define(HEX_AUTH_FILE, "hex.config"). -define(PUBLIC_HEX_REPO, <<"hexpm">>). diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index 35e908c..fb08ea9 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -254,10 +254,13 @@ update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) -> {ok, Package, RepoConfig} -> #package{key={_, PkgVsn1, _}, checksum=Hash1, - dependencies=Deps} = Package, - AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName, PkgVsn1, Hash1, RepoConfig}), + dependencies=Deps, + retired=Retired} = Package, + maybe_warn_retired(PkgName, PkgVsn1, Hash, Retired), + PkgVsn2 = list_to_binary(lists:flatten(ec_semver:format(PkgVsn1))), + AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName, PkgVsn2, Hash1, RepoConfig}), AppInfo2 = rebar_app_info:update_opts_deps(AppInfo1, Deps), - rebar_app_info:original_vsn(AppInfo2, PkgVsn1); + rebar_app_info:original_vsn(AppInfo2, PkgVsn2); not_found -> throw(?PRV_ERROR({missing_package, PkgName, PkgVsn})); {error, {invalid_vsn, InvalidVsn}} -> @@ -269,10 +272,10 @@ update_source(AppInfo, Source, _State) -> %% @doc convert a given exception's payload into an io description. -spec format_error(any()) -> iolist(). format_error({missing_package, Name, undefined}) -> - io_lib:format("Package not found in any repo: ~ts.", [rebar_utils:to_binary(Name)]); + io_lib:format("Package not found in any repo: ~ts", [rebar_utils:to_binary(Name)]); format_error({missing_package, Name, Vsn}) -> - io_lib:format("Package not found in any repo: ~ts-~ts.", [rebar_utils:to_binary(Name), - rebar_utils:to_binary(Vsn)]); + io_lib:format("Package not found in any repo: ~ts ~ts", [rebar_utils:to_binary(Name), + rebar_utils:to_binary(Vsn)]); format_error({parse_dep, Dep}) -> io_lib:format("Failed parsing dep ~p", [Dep]); format_error({invalid_vsn, Dep, InvalidVsn}) -> @@ -284,6 +287,32 @@ format_error(Error) -> %% Internal functions %% =================================================================== +maybe_warn_retired(_, _, _, false) -> + ok; +maybe_warn_retired(_, _, Hash, _) when is_binary(Hash) -> + %% don't warn if this is a lock + ok; +maybe_warn_retired(Name, Vsn, _, R=#{reason := Reason}) -> + Message = maps:get(message, R, ""), + ?WARN("Warning: package ~s-~s is retired: (~s) ~s", + [Name, ec_semver:format(Vsn), retire_reason(Reason), Message]); +maybe_warn_retired(_, _, _, _) -> + ok. + +%% TODO: move to hex_core +retire_reason('RETIRED_OTHER') -> + "other"; +retire_reason('RETIRED_INVALID') -> + "invalid"; +retire_reason('RETIRED_SECURITY') -> + "security"; +retire_reason('RETIRED_DEPRECATED') -> + "deprecated"; +retire_reason('RETIRED_RENAMED') -> + "renamed"; +retire_reason(_Other) -> + "other". + %% @private checks that all the beam files have been properly %% created. -spec has_all_beams(file:filename_all(), [module()]) -> diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 8a3ffea..7533a44 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -24,10 +24,10 @@ -type package() :: pkg_name() | {pkg_name(), vsn()}. format_error({missing_package, Name, Vsn}) -> - io_lib:format("Package not found in any repo: ~ts-~ts.", [rebar_utils:to_binary(Name), - rebar_utils:to_binary(Vsn)]); + io_lib:format("Package not found in any repo: ~ts ~ts", [rebar_utils:to_binary(Name), + rebar_utils:to_binary(Vsn)]); format_error({missing_package, Pkg}) -> - io_lib:format("Package not found in any repo: ~p.", [Pkg]). + io_lib:format("Package not found in any repo: ~p", [Pkg]). -spec get(rebar_hex_repos:repo(), binary()) -> {ok, map()} | {error, term()}. get(Config, Name) -> @@ -55,29 +55,25 @@ get_all_names(State) -> _='_'}, [], ['$1']}])). --spec get_package_versions(unicode:unicode_binary(), unicode:unicode_binary(), ets:tid(), rebar_state:t()) - -> [vsn()]. +-spec get_package_versions(unicode:unicode_binary(), unicode:unicode_binary(), + ets:tid(), rebar_state:t()) -> [vsn()]. get_package_versions(Dep, Repo, Table, State) -> ?MODULE:verify_table(State), - ets:select(Table, [{#package{key={Dep,'$1', Repo}, - _='_'}, - [], ['$1']}]). - - -get_package(Dep, Vsn, Hash, Repo, Table, State) -> - get_package(Dep, Vsn, Hash, false, [Repo], Table, State). + AllowPreRelease = rebar_state:get(State, deps_allow_prerelease, false), + ets:select(Table, [{#package{key={Dep,{'$1', '$2'}, Repo}, + _='_'}, + [{'==', '$2', {{[],[]}}} || not AllowPreRelease], [{{'$1', '$2'}}]}]). -spec get_package(unicode:unicode_binary(), unicode:unicode_binary(), - binary() | undefined | '_', boolean() | '_', - unicode:unicode_binary() | '_' | list(), ets:tab(), rebar_state:t()) + binary() | undefined | '_', + [unicode:unicode_binary()] | ['_'], ets:tab(), rebar_state:t()) -> {ok, #package{}} | not_found. -get_package(Dep, Vsn, undefined, Retired, Repo, Table, State) -> - get_package(Dep, Vsn, '_', Retired, Repo, Table, State); -get_package(Dep, Vsn, Hash, Retired, Repos, Table, State) -> +get_package(Dep, Vsn, undefined, Repos, Table, State) -> + get_package(Dep, Vsn, '_', Repos, Table, State); +get_package(Dep, Vsn, Hash, Repos, Table, State) -> ?MODULE:verify_table(State), - case ets:select(Table, [{#package{key={Dep, Vsn, Repo}, + case ets:select(Table, [{#package{key={Dep, ec_semver:parse(Vsn), Repo}, checksum=Hash, - retired=Retired, _='_'}, [], ['$_']} || Repo <- Repos]) of %% have to allow multiple matches in the list for cases that Repo is `_` [Package | _] -> @@ -258,7 +254,7 @@ update_package(Name, RepoConfig=#{name := Repo}, State) -> insert_releases(Name, Releases, Repo, Table) -> [true = ets:insert(Table, - #package{key={Name, Version, Repo}, + #package{key={Name, ec_semver:parse(Version), Repo}, checksum=parse_checksum(Checksum), retired=maps:get(retired, Release, false), dependencies=parse_deps(Dependencies)}) @@ -279,7 +275,7 @@ resolve_version(Dep, DepVsn, Hash, HexRegistry, State) when is_binary(Hash) -> RepoNames = [RepoName || #{name := RepoName} <- RepoConfigs], %% allow retired packages when we have a checksum - case get_package(Dep, DepVsn, Hash, '_', RepoNames, HexRegistry, State) of + case get_package(Dep, DepVsn, Hash, RepoNames, HexRegistry, State) of {ok, Package=#package{key={_, _, RepoName}}} -> {ok, RepoConfig} = rebar_hex_repos:get_repo_config(RepoName, RepoConfigs), {ok, Package, RepoConfig}; @@ -289,7 +285,7 @@ resolve_version(Dep, DepVsn, Hash, HexRegistry, State) when is_binary(Hash) -> none -> not_found; {ok, Vsn} -> - get_package(Dep, Vsn, Hash, Repo, HexRegistry, State) + get_package(Dep, Vsn, Hash, [Repo], HexRegistry, State) end end, handle_missing_no_exception(Fun, Dep, State) @@ -300,7 +296,7 @@ resolve_version(Dep, undefined, Hash, HexRegistry, State) -> none -> not_found; {ok, Vsn} -> - get_package(Dep, Vsn, Hash, Repo, HexRegistry, State) + get_package(Dep, Vsn, Hash, [Repo], HexRegistry, State) end end, handle_missing_no_exception(Fun, Dep, State); @@ -314,7 +310,7 @@ resolve_version(Dep, DepVsn, Hash, HexRegistry, State) -> none -> not_found; {ok, Vsn} -> - get_package(Dep, Vsn, Hash, Repo, HexRegistry, State) + get_package(Dep, Vsn, Hash, [Repo], HexRegistry, State) end end, handle_missing_no_exception(Fun, Dep, State) -- cgit v1.1 From 6ea0a600b4f560565ca963b69c86b9e9cb0ea0b8 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 5 Oct 2018 07:58:36 -0600 Subject: upgrade relx and set base_dir in overlay_vars (#1902) --- src/rebar_relx.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_relx.erl b/src/rebar_relx.erl index 4548761..431e1bc 100644 --- a/src/rebar_relx.erl +++ b/src/rebar_relx.erl @@ -40,7 +40,8 @@ do(Module, Command, Provider, State) -> ,{caller, api} ,{log_level, LogLevel} | output_dir(OutputDir, Options)] ++ ErlOpts, AllOptions); Config -> - Config1 = merge_overlays(Config), + Config1 = [{overlay_vars, [{base_dir, rebar_dir:base_dir(State)}]} + | merge_overlays(Config)], relx:main([{lib_dirs, LibDirs} ,{config, Config1} ,{caller, api} -- cgit v1.1 From dec484643c233fda9c17d38c1854ba7f3f37547b Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 5 Oct 2018 08:29:07 -0600 Subject: compiler behaviour (#1893) * add compile type for dynamic project compilation * new rebar_compiler abstraction for running multiple compilers rebar_compiler is a new behaviour that a plugin can implement to be called on any ues of the compile provider to compile source files and keep track of their dependencies. * fix check that modules in .app modules list are from src_dirs * use project_type to find module for building projects * allow plugins to add project builders and compilers --- src/rebar.app.src | 3 + src/rebar3.erl | 5 +- src/rebar_app_discover.erl | 2 +- src/rebar_app_info.erl | 43 ++- src/rebar_app_utils.erl | 5 +- src/rebar_base_compiler.erl | 33 +- src/rebar_compiler.erl | 304 ++++++++++++++++ src/rebar_compiler_erl.erl | 368 ++++++++++++++++++++ src/rebar_compiler_mib.erl | 70 ++++ src/rebar_compiler_xrl.erl | 50 +++ src/rebar_compiler_yrl.erl | 49 +++ src/rebar_dir.erl | 5 +- src/rebar_erlc_compiler.erl | 827 -------------------------------------------- src/rebar_file_utils.erl | 11 +- src/rebar_otp_app.erl | 38 +- src/rebar_packages.erl | 2 +- src/rebar_prv_clean.erl | 3 +- src/rebar_prv_compile.erl | 43 ++- src/rebar_state.erl | 37 ++ src/rebar_utils.erl | 87 ++++- 20 files changed, 1080 insertions(+), 905 deletions(-) create mode 100644 src/rebar_compiler.erl create mode 100644 src/rebar_compiler_erl.erl create mode 100644 src/rebar_compiler_mib.erl create mode 100644 src/rebar_compiler_xrl.erl create mode 100644 src/rebar_compiler_yrl.erl delete mode 100644 src/rebar_erlc_compiler.erl (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 530a79e..724f921 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -40,6 +40,9 @@ {pkg, rebar_pkg_resource}, {hg, rebar_hg_resource}]}, + {app_compilers, [{erl, rebar_erlc_compiler}]}, + {compilers, [rebar_compiler_xrl, rebar_compiler_yrl, rebar_compiler_mib, rebar_compiler_erl]}, + {providers, [rebar_prv_app_discovery, rebar_prv_as, rebar_prv_bare_compile, diff --git a/src/rebar3.erl b/src/rebar3.erl index e87cb19..c931a85 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -139,11 +139,14 @@ run_aux(State, RawArgs) -> rebar_state:set(State1, rebar_packages_cdn, CDN) end, + Compilers = application:get_env(rebar, compilers, []), + State0 = rebar_state:compilers(State2, Compilers), + %% TODO: this means use of REBAR_PROFILE=profile will replace the repos with %% the repos defined in the profile. But it will not work with `as profile`. %% Maybe it shouldn't work with either to be consistent? Resources = application:get_env(rebar, resources, []), - State2_ = rebar_state:create_resources(Resources, State2), + State2_ = rebar_state:create_resources(Resources, State0), %% bootstrap test profile State3 = rebar_state:add_to_profile(State2_, test, test_state(State1)), diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index 224539b..74681c7 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -427,7 +427,7 @@ try_handle_app_src_file(AppInfo, _, _AppDir, [], _Validate) -> %% a .app after this step case filelib:is_file(filename:join(rebar_app_info:dir(AppInfo), "mix.exs")) of true -> - {true, AppInfo}; + {true, rebar_app_info:project_type(AppInfo, mix)}; false -> false end; diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index eb95311..b1c6243 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -24,7 +24,6 @@ parent/2, original_vsn/1, original_vsn/2, - ebin_dir/1, priv_dir/1, applications/1, applications/2, @@ -38,6 +37,8 @@ dir/2, out_dir/1, out_dir/2, + ebin_dir/1, + ebin_dir/2, default/1, default/2, opts/1, @@ -47,6 +48,8 @@ set/3, source/1, source/2, + project_type/1, + project_type/2, is_lock/1, is_lock/2, is_checkout/1, @@ -68,7 +71,10 @@ -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). --export_type([t/0]). +-export_type([t/0, + project_type/0]). + +-type project_type() :: rebar3 | mix | undefined. -record(app_info_t, {name :: binary() | undefined, app_file_src :: file:filename_all() | undefined, @@ -85,10 +91,12 @@ dep_level=0 :: integer(), dir :: file:name(), out_dir :: file:name(), + ebin_dir :: file:name(), source :: string() | tuple() | checkout | undefined, is_lock=false :: boolean(), is_checkout=false :: boolean(), valid :: boolean() | undefined, + project_type :: project_type(), is_available=false :: boolean()}). %%============================================================================ @@ -125,7 +133,8 @@ new(AppName, Vsn, Dir) -> {ok, #app_info_t{name=rebar_utils:to_binary(AppName), original_vsn=rebar_utils:to_binary(Vsn), dir=rebar_utils:to_list(Dir), - out_dir=rebar_utils:to_list(Dir)}}. + out_dir=rebar_utils:to_list(Dir), + ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin")}}. %% @doc build a complete version of the app info with all fields set. -spec new(atom() | binary() | string(), binary() | string(), file:name(), list()) -> @@ -135,6 +144,7 @@ new(AppName, Vsn, Dir, Deps) -> original_vsn=rebar_utils:to_binary(Vsn), dir=rebar_utils:to_list(Dir), out_dir=rebar_utils:to_list(Dir), + ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin"), deps=Deps}}. %% @doc build a complete version of the app info with all fields set. @@ -146,6 +156,7 @@ new(Parent, AppName, Vsn, Dir, Deps) -> original_vsn=rebar_utils:to_binary(Vsn), dir=rebar_utils:to_list(Dir), out_dir=rebar_utils:to_list(Dir), + ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin"), deps=Deps}}. %% @doc update the opts based on the contents of a config @@ -447,12 +458,21 @@ out_dir(#app_info_t{out_dir=OutDir}) -> %% should go -spec out_dir(t(), file:name()) -> t(). out_dir(AppInfo=#app_info_t{}, OutDir) -> - AppInfo#app_info_t{out_dir=rebar_utils:to_list(OutDir)}. + AppInfo#app_info_t{out_dir=rebar_utils:to_list(OutDir), + ebin_dir=filename:join(rebar_utils:to_list(OutDir), "ebin")}. %% @doc gets the directory where ebin files for the app should go -spec ebin_dir(t()) -> file:name(). -ebin_dir(#app_info_t{out_dir=OutDir}) -> - rebar_utils:to_list(filename:join(OutDir, "ebin")). +ebin_dir(#app_info_t{ebin_dir=undefined, + out_dir=OutDir}) -> + filename:join(rebar_utils:to_list(OutDir), "ebin"); +ebin_dir(#app_info_t{ebin_dir=EbinDir}) -> + EbinDir. + +%% @doc sets the directory where beam files should go +-spec ebin_dir(t(), file:name()) -> t(). +ebin_dir(AppInfo, EbinDir) -> + AppInfo#app_info_t{ebin_dir=EbinDir}. %% @doc gets the directory where private files for the app should go -spec priv_dir(t()) -> file:name(). @@ -500,6 +520,17 @@ is_available(#app_info_t{is_available=IsAvailable}) -> is_available(AppInfo=#app_info_t{}, IsAvailable) -> AppInfo#app_info_t{is_available=IsAvailable}. +%% @doc +-spec project_type(t()) -> atom(). +project_type(#app_info_t{project_type=ProjectType}) -> + ProjectType. + +%% @doc +-spec project_type(t(), atom()) -> t(). +project_type(AppInfo=#app_info_t{}, ProjectType) -> + AppInfo#app_info_t{project_type=ProjectType}. + + %% @doc returns whether the app is valid (built) or not -spec valid(t()) -> boolean(). valid(AppInfo=#app_info_t{valid=undefined}) -> diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index fb08ea9..605944e 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -273,9 +273,8 @@ update_source(AppInfo, Source, _State) -> -spec format_error(any()) -> iolist(). format_error({missing_package, Name, undefined}) -> io_lib:format("Package not found in any repo: ~ts", [rebar_utils:to_binary(Name)]); -format_error({missing_package, Name, Vsn}) -> - io_lib:format("Package not found in any repo: ~ts ~ts", [rebar_utils:to_binary(Name), - rebar_utils:to_binary(Vsn)]); +format_error({missing_package, Name, Constraint}) -> + io_lib:format("Package not found in any repo: ~ts ~ts", [Name, Constraint]); format_error({parse_dep, Dep}) -> io_lib:format("Failed parsing dep ~p", [Dep]); format_error({invalid_vsn, Dep, InvalidVsn}) -> diff --git a/src/rebar_base_compiler.erl b/src/rebar_base_compiler.erl index 00b0533..2fb3a12 100644 --- a/src/rebar_base_compiler.erl +++ b/src/rebar_base_compiler.erl @@ -33,6 +33,8 @@ run/8, ok_tuple/2, error_tuple/4, + report/1, + maybe_report/1, format_error_source/2]). -type desc() :: term(). @@ -94,14 +96,14 @@ run(Config, FirstFiles, SourceDir, SourceExt, TargetDir, TargetExt, TargetDir :: file:filename(), SourceExt :: string(), TargetExt :: string(). -run(Config, FirstFiles, SourceDir, SourceExt, TargetDir, TargetExt, +run(Config, FirstFiles, SourceDirs, SourceExt, TargetDir, TargetExt, Compile3Fn, Opts) -> %% Convert simple extension to proper regex SourceExtRe = "^(?!\\._).*\\" ++ SourceExt ++ [$$], Recursive = proplists:get_value(recursive, Opts, true), %% Find all possible source files - FoundFiles = rebar_utils:find_files(SourceDir, SourceExtRe, Recursive), + FoundFiles = rebar_utils:find_files_in_dirs(SourceDirs, SourceExtRe, Recursive), %% Remove first files from found files RestFiles = [Source || Source <- FoundFiles, not lists:member(Source, FirstFiles)], @@ -111,7 +113,7 @@ run(Config, FirstFiles, SourceDir, SourceExt, TargetDir, TargetExt, run(Config, FirstFiles, RestFiles, fun(S, C) -> - Target = target_file(S, SourceDir, SourceExt, + Target = target_file(S, SourceExt, TargetDir, TargetExt), simple_compile_wrapper(S, Target, Compile3Fn, C, CheckLastMod) end). @@ -160,32 +162,15 @@ simple_compile_wrapper(Source, Target, Compile3Fn, Config, true) -> %% @private take a basic source set of file fragments and a target location, %% create a file path and name for a compile artifact. --spec target_file(SourceFile, SourceDir, SourceExt, TargetDir, TargetExt) -> File when +-spec target_file(SourceFile, SourceExt, TargetDir, TargetExt) -> File when SourceFile :: file:filename(), - SourceDir :: file:filename(), TargetDir :: file:filename(), SourceExt :: string(), TargetExt :: string(), File :: file:filename(). -target_file(SourceFile, SourceDir, SourceExt, TargetDir, TargetExt) -> - BaseFile = remove_common_path(SourceFile, SourceDir), - filename:join([TargetDir, filename:basename(BaseFile, SourceExt) ++ TargetExt]). - -%% @private removes the common prefix between two file paths. -%% The remainder of the first file path passed will have its ending returned -%% when either path starts diverging. --spec remove_common_path(file:filename(), file:filename()) -> file:filename(). -remove_common_path(Fname, Path) -> - remove_common_path1(filename:split(Fname), filename:split(Path)). - -%% @private given two lists of file fragments, discard the identical -%% prefixed sections, and return the final bit of the first operand -%% as a filename. --spec remove_common_path1([string()], [string()]) -> file:filename(). -remove_common_path1([Part | RestFilename], [Part | RestPath]) -> - remove_common_path1(RestFilename, RestPath); -remove_common_path1(FilenameParts, _) -> - filename:join(FilenameParts). +target_file(SourceFile, SourceExt, TargetDir, TargetExt) -> + %% BaseFile = remove_common_path(SourceFile, SourceDir), + filename:join([TargetDir, filename:basename(SourceFile, SourceExt) ++ TargetExt]). %% @private runs the compile function `CompileFn' on every file %% passed internally, along with the related project configuration. diff --git a/src/rebar_compiler.erl b/src/rebar_compiler.erl new file mode 100644 index 0000000..c2c514a --- /dev/null +++ b/src/rebar_compiler.erl @@ -0,0 +1,304 @@ +-module(rebar_compiler). + +-export([compile_all/2, + clean/2, + + ok_tuple/2, + error_tuple/4, + maybe_report/1, + format_error_source/2, + report/1]). + +-include("rebar.hrl"). + +-type extension() :: string(). +-type out_mappings() :: [{extension(), file:filename()}]. + +-callback context(rebar_app_info:t()) -> #{src_dirs => [file:dirname()], + include_dirs => [file:dirname()], + src_ext => extension(), + out_mappings => out_mappings()}. +-callback needed_files(digraph:graph(), [file:filename()], rebar_app_info:t()) -> [file:filename()]. +-callback dependencies(file:filename(), file:dirname(), [file:dirname()]) -> [file:filename()]. +-callback compile(file:filename(), out_mappings(), rebar_dict(), list()) -> + ok | {ok, [string()]} | {ok, [string()], [string()]}. + +-define(DAG_VSN, 2). +-define(DAG_FILE, "source.dag"). +-type dag_v() :: {digraph:vertex(), term()} | 'false'. +-type dag_e() :: {digraph:vertex(), digraph:vertex()}. +-type dag() :: {list(dag_v()), list(dag_e()), list(string())}. +-record(dag, {vsn = ?DAG_VSN :: pos_integer(), + info = {[], [], []} :: dag()}). + +-define(RE_PREFIX, "^(?!\\._)"). + +compile_all(Compilers, AppInfo) -> + OutDir = rebar_utils:to_list(rebar_app_info:out_dir(AppInfo)), + + %% Make sure that outdir is on the path + ok = rebar_file_utils:ensure_dir(OutDir), + true = code:add_patha(filename:absname(OutDir)), + + %% necessary for erlang:function_exported/3 to work as expected + %% called here for clarity as it's required by both opts_changed/2 + %% and erl_compiler_opts_set/0 in needed_files + _ = code:ensure_loaded(compile), + + lists:foreach(fun(CompilerMod) -> + run(CompilerMod, AppInfo), + run_on_extra_src_dirs(CompilerMod, AppInfo, fun run/2) + end, Compilers), + ok. + +run(CompilerMod, AppInfo) -> + #{src_dirs := SrcDirs, + include_dirs := InclDirs, + src_ext := SrcExt, + out_mappings := Mappings} = CompilerMod:context(AppInfo), + + BaseDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)), + EbinDir = rebar_utils:to_list(rebar_app_info:ebin_dir(AppInfo)), + + BaseOpts = rebar_app_info:opts(AppInfo), + AbsInclDirs = [filename:join(BaseDir, InclDir) || InclDir <- InclDirs], + FoundFiles = find_source_files(BaseDir, SrcExt, SrcDirs, BaseOpts), + + OutDir = rebar_app_info:out_dir(AppInfo), + AbsSrcDirs = [filename:join(BaseDir, SrcDir) || SrcDir <- SrcDirs], + G = init_dag(CompilerMod, AbsInclDirs, AbsSrcDirs, FoundFiles, OutDir, EbinDir), + {{FirstFiles, FirstFileOpts}, {RestFiles, Opts}} = CompilerMod:needed_files(G, FoundFiles, AppInfo), + true = digraph:delete(G), + + compile_each(FirstFiles, FirstFileOpts, BaseOpts, Mappings, CompilerMod), + compile_each(RestFiles, Opts, BaseOpts, Mappings, CompilerMod). + +compile_each([], _Opts, _Config, _Outs, _CompilerMod) -> + ok; +compile_each([Source | Rest], Opts, Config, Outs, CompilerMod) -> + case CompilerMod:compile(Source, Outs, Config, Opts) of + ok -> + ?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]); + {ok, Warnings} -> + report(Warnings), + ?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]); + skipped -> + ?DEBUG("~tsSkipped ~ts", [rebar_utils:indent(1), filename:basename(Source)]); + Error -> + NewSource = format_error_source(Source, Config), + ?ERROR("Compiling ~ts failed", [NewSource]), + maybe_report(Error), + ?DEBUG("Compilation failed: ~p", [Error]), + ?FAIL + end, + compile_each(Rest, Opts, Config, Outs, CompilerMod). + +%% @doc remove compiled artifacts from an AppDir. +-spec clean([module()], rebar_app_info:t()) -> 'ok'. +clean(Compilers, AppInfo) -> + lists:foreach(fun(CompilerMod) -> + clean_(CompilerMod, AppInfo), + run_on_extra_src_dirs(CompilerMod, AppInfo, fun clean_/2) + end, Compilers). + +clean_(CompilerMod, AppInfo) -> + #{src_dirs := SrcDirs, + src_ext := SrcExt} = CompilerMod:context(AppInfo), + BaseDir = rebar_app_info:dir(AppInfo), + Opts = rebar_app_info:opts(AppInfo), + EbinDir = rebar_app_info:ebin_dir(AppInfo), + + FoundFiles = find_source_files(BaseDir, SrcExt, SrcDirs, Opts), + CompilerMod:clean(FoundFiles, AppInfo), + rebar_file_utils:rm_rf(dag_file(CompilerMod, EbinDir)). + + +run_on_extra_src_dirs(CompilerMod, AppInfo, Fun) -> + ExtraDirs = rebar_dir:extra_src_dirs(rebar_app_info:opts(AppInfo), []), + run_on_extra_src_dirs(ExtraDirs, CompilerMod, AppInfo, Fun). + +run_on_extra_src_dirs([], _CompilerMod, _AppInfo, _Fun) -> + ok; +run_on_extra_src_dirs([Dir | Rest], CompilerMod, AppInfo, Fun) -> + case filelib:is_dir(filename:join(rebar_app_info:dir(AppInfo), Dir)) of + true -> + EbinDir = filename:join(rebar_app_info:out_dir(AppInfo), Dir), + AppInfo1 = rebar_app_info:ebin_dir(AppInfo, EbinDir), + AppInfo2 = rebar_app_info:set(AppInfo1, src_dirs, [Dir]), + AppInfo3 = rebar_app_info:set(AppInfo2, extra_src_dirs, ["src"]), + Fun(CompilerMod, AppInfo3); + _ -> + ok + end, + run_on_extra_src_dirs(Rest, CompilerMod, AppInfo, Fun). + +%% These functions are here for the ultimate goal of getting rid of +%% rebar_base_compiler. This can't be done because of existing plugins. + +ok_tuple(Source, Ws) -> + rebar_base_compiler:ok_tuple(Source, Ws). + +error_tuple(Source, Es, Ws, Opts) -> + rebar_base_compiler:error_tuple(Source, Es, Ws, Opts). + +maybe_report(Reportable) -> + rebar_base_compiler:maybe_report(Reportable). + +format_error_source(Path, Opts) -> + rebar_base_compiler:format_error_source(Path, Opts). + +report(Messages) -> + rebar_base_compiler:report(Messages). + +%% private functions + +find_source_files(BaseDir, SrcExt, SrcDirs, Opts) -> + SourceExtRe = "^(?!\\._).*\\" ++ SrcExt ++ [$$], + lists:flatmap(fun(SrcDir) -> + Recursive = rebar_dir:recursive(Opts, SrcDir), + rebar_utils:find_files_in_dirs([filename:join(BaseDir, SrcDir)], SourceExtRe, Recursive) + end, SrcDirs). + +dag_file(CompilerMod, Dir) -> + filename:join([rebar_dir:local_cache_dir(Dir), CompilerMod, ?DAG_FILE]). + +%% private graph functions + +%% Get dependency graph of given Erls files and their dependencies (header files, +%% parse transforms, behaviours etc.) located in their directories or given +%% InclDirs. Note that last modification times stored in vertices already respect +%% dependencies induced by given graph G. +init_dag(Compiler, InclDirs, SrcDirs, Erls, Dir, EbinDir) -> + G = digraph:new([acyclic]), + try restore_dag(Compiler, G, InclDirs, Dir) + catch + _:_ -> + ?WARN("Failed to restore ~ts file. Discarding it.~n", [dag_file(Compiler, Dir)]), + file:delete(dag_file(Compiler, Dir)) + end, + Dirs = lists:usort(InclDirs ++ SrcDirs), + %% A source file may have been renamed or deleted. Remove it from the graph + %% and remove any beam file for that source if it exists. + Modified = maybe_rm_beams_and_edges(G, EbinDir, Erls), + Modified1 = lists:foldl(update_dag_fun(G, Compiler, Dirs), Modified, Erls), + if Modified1 -> store_dag(Compiler, G, InclDirs, Dir); not Modified1 -> ok end, + G. + +maybe_rm_beams_and_edges(G, Dir, Files) -> + Vertices = digraph:vertices(G), + case lists:filter(fun(File) -> + case filename:extension(File) =:= ".erl" of + true -> + maybe_rm_beam_and_edge(G, Dir, File); + false -> + false + end + end, lists:sort(Vertices) -- lists:sort(Files)) of + [] -> + false; + _ -> + true + end. + +maybe_rm_beam_and_edge(G, OutDir, Source) -> + %% This is NOT a double check it is the only check that the source file is actually gone + case filelib:is_regular(Source) of + true -> + %% Actually exists, don't delete + false; + false -> + Target = target_base(OutDir, Source) ++ ".beam", + ?DEBUG("Source ~ts is gone, deleting previous beam file if it exists ~ts", [Source, Target]), + file:delete(Target), + digraph:del_vertex(G, Source), + true + end. + + +target_base(OutDir, Source) -> + filename:join(OutDir, filename:basename(Source, ".erl")). + +restore_dag(Compiler, G, InclDirs, Dir) -> + case file:read_file(dag_file(Compiler, Dir)) of + {ok, Data} -> + % Since externally passed InclDirs can influence dependency graph (see + % modify_dag), we have to check here that they didn't change. + #dag{vsn=?DAG_VSN, info={Vs, Es, InclDirs}} = + binary_to_term(Data), + lists:foreach( + fun({V, LastUpdated}) -> + digraph:add_vertex(G, V, LastUpdated) + end, Vs), + lists:foreach( + fun({_, V1, V2, _}) -> + digraph:add_edge(G, V1, V2) + end, Es); + {error, _} -> + ok + end. + +store_dag(Compiler, G, InclDirs, Dir) -> + Vs = lists:map(fun(V) -> digraph:vertex(G, V) end, digraph:vertices(G)), + Es = lists:map(fun(E) -> digraph:edge(G, E) end, digraph:edges(G)), + File = dag_file(Compiler, Dir), + ok = filelib:ensure_dir(File), + Data = term_to_binary(#dag{info={Vs, Es, InclDirs}}, [{compressed, 2}]), + file:write_file(File, Data). + +update_dag(G, Compiler, Dirs, Source) -> + case digraph:vertex(G, Source) of + {_, LastUpdated} -> + case filelib:last_modified(Source) of + 0 -> + %% The file doesn't exist anymore, + %% erase it from the graph. + %% All the edges will be erased automatically. + digraph:del_vertex(G, Source), + modified; + LastModified when LastUpdated < LastModified -> + modify_dag(G, Compiler, Source, LastModified, filename:dirname(Source), Dirs); + _ -> + Modified = lists:foldl( + update_dag_fun(G, Compiler, Dirs), + false, digraph:out_neighbours(G, Source)), + MaxModified = update_max_modified_deps(G, Source), + case Modified orelse MaxModified > LastUpdated of + true -> modified; + false -> unmodified + end + end; + false -> + modify_dag(G, Compiler, Source, filelib:last_modified(Source), filename:dirname(Source), Dirs) + end. + +modify_dag(G, Compiler, Source, LastModified, SourceDir, Dirs) -> + AbsIncls = Compiler:dependencies(Source, SourceDir, Dirs), + digraph:add_vertex(G, Source, LastModified), + digraph:del_edges(G, digraph:out_edges(G, Source)), + lists:foreach( + fun(Incl) -> + update_dag(G, Compiler, Dirs, Incl), + digraph:add_edge(G, Source, Incl) + end, AbsIncls), + modified. + +update_dag_fun(G, Compiler, Dirs) -> + fun(Erl, Modified) -> + case update_dag(G, Compiler, Dirs, Erl) of + modified -> true; + unmodified -> Modified + end + end. + +update_max_modified_deps(G, Source) -> + MaxModified = + lists:foldl(fun(File, Acc) -> + case digraph:vertex(G, File) of + {_, MaxModified} when MaxModified > Acc -> + MaxModified; + _ -> + Acc + end + end, 0, [Source | digraph:out_neighbours(G, Source)]), + digraph:add_vertex(G, Source, MaxModified), + MaxModified. diff --git a/src/rebar_compiler_erl.erl b/src/rebar_compiler_erl.erl new file mode 100644 index 0000000..d9bc69b --- /dev/null +++ b/src/rebar_compiler_erl.erl @@ -0,0 +1,368 @@ +-module(rebar_compiler_erl). + +-behaviour(rebar_compiler). + +-export([context/1, + needed_files/3, + dependencies/3, + compile/4, + clean/2]). + +-include("rebar.hrl"). + +context(AppInfo) -> + EbinDir = rebar_app_info:ebin_dir(AppInfo), + Mappings = [{".beam", EbinDir}], + + OutDir = rebar_app_info:dir(AppInfo), + SrcDirs = rebar_dir:src_dirs(rebar_app_info:opts(AppInfo), ["src"]), + ExistingSrcDirs = lists:filter(fun(D) -> + ec_file:is_dir(filename:join(OutDir, D)) + end, SrcDirs), + + RebarOpts = rebar_app_info:opts(AppInfo), + ErlOpts = rebar_opts:erl_opts(RebarOpts), + ErlOptIncludes = proplists:get_all_values(i, ErlOpts), + InclDirs = lists:map(fun(Incl) -> filename:absname(Incl) end, ErlOptIncludes), + + #{src_dirs => ExistingSrcDirs, + include_dirs => [filename:join([OutDir, "include"]) | InclDirs], + src_ext => ".erl", + out_mappings => Mappings}. + + +needed_files(Graph, FoundFiles, AppInfo) -> + OutDir = rebar_app_info:out_dir(AppInfo), + Dir = rebar_app_info:dir(AppInfo), + EbinDir = rebar_app_info:ebin_dir(AppInfo), + RebarOpts = rebar_app_info:opts(AppInfo), + ErlOpts = rebar_opts:erl_opts(RebarOpts), + ?DEBUG("erlopts ~p", [ErlOpts]), + ?DEBUG("files to compile ~p", [FoundFiles]), + + %% Make sure that the ebin dir is on the path + ok = rebar_file_utils:ensure_dir(EbinDir), + true = code:add_patha(filename:absname(EbinDir)), + + {ParseTransforms, Rest} = split_source_files(FoundFiles, ErlOpts), + NeededErlFiles = case needed_files(Graph, ErlOpts, RebarOpts, OutDir, EbinDir, ParseTransforms) of + [] -> + needed_files(Graph, ErlOpts, RebarOpts, OutDir, EbinDir, Rest); + _ -> + %% at least one parse transform in the opts needs updating, so recompile all + FoundFiles + end, + {ErlFirstFiles, ErlOptsFirst} = erl_first_files(RebarOpts, ErlOpts, Dir, NeededErlFiles), + SubGraph = digraph_utils:subgraph(Graph, NeededErlFiles), + DepErlsOrdered = digraph_utils:topsort(SubGraph), + OtherErls = lists:reverse(DepErlsOrdered), + + PrivIncludes = [{i, filename:join(OutDir, Src)} + || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])], + AdditionalOpts = PrivIncludes ++ [{i, filename:join(OutDir, "include")}, {i, OutDir}, return], + + true = digraph:delete(SubGraph), + + {{ErlFirstFiles, ErlOptsFirst ++ AdditionalOpts}, + {[Erl || Erl <- OtherErls, + not lists:member(Erl, ErlFirstFiles)], ErlOpts ++ AdditionalOpts}}. + +dependencies(Source, SourceDir, Dirs) -> + {ok, Fd} = file:open(Source, [read]), + Incls = parse_attrs(Fd, [], SourceDir), + AbsIncls = expand_file_names(Incls, Dirs), + ok = file:close(Fd), + AbsIncls. + +compile(Source, [{_, OutDir}], Config, ErlOpts) -> + case compile:file(Source, [{outdir, OutDir} | ErlOpts]) of + {ok, _Mod} -> + ok; + {ok, _Mod, []} -> + ok; + {ok, _Mod, Ws} -> + FormattedWs = format_error_sources(Ws, Config), + rebar_compiler:ok_tuple(Source, FormattedWs); + {error, Es, Ws} -> + error_tuple(Source, Es, Ws, Config, ErlOpts); + error -> + error + end. + +clean(Files, AppInfo) -> + EbinDir = rebar_app_info:ebin_dir(AppInfo), + [begin + Source = filename:basename(File, ".erl"), + Target = target_base(EbinDir, Source) ++ ".beam", + file:delete(Target) + end || File <- Files]. + +%% + +error_tuple(Module, Es, Ws, AllOpts, Opts) -> + FormattedEs = format_error_sources(Es, AllOpts), + FormattedWs = format_error_sources(Ws, AllOpts), + rebar_compiler:error_tuple(Module, FormattedEs, FormattedWs, Opts). + +format_error_sources(Es, Opts) -> + [{rebar_compiler:format_error_source(Src, Opts), Desc} + || {Src, Desc} <- Es]. + +%% Get files which need to be compiled first, i.e. those specified in erl_first_files +%% and parse_transform options. Also produce specific erl_opts for these first +%% files, so that yet to be compiled parse transformations are excluded from it. +erl_first_files(Opts, ErlOpts, Dir, NeededErlFiles) -> + ErlFirstFilesConf = rebar_opts:get(Opts, erl_first_files, []), + valid_erl_first_conf(ErlFirstFilesConf), + NeededSrcDirs = lists:usort(lists:map(fun filename:dirname/1, NeededErlFiles)), + %% NOTE: order of files here is important! + ErlFirstFiles = + [filename:join(Dir, File) || File <- ErlFirstFilesConf, + lists:member(filename:join(Dir, File), NeededErlFiles)], + {ParseTransforms, ParseTransformsErls} = + lists:unzip(lists:flatmap( + fun(PT) -> + PTerls = [filename:join(D, module_to_erl(PT)) || D <- NeededSrcDirs], + [{PT, PTerl} || PTerl <- PTerls, lists:member(PTerl, NeededErlFiles)] + end, proplists:get_all_values(parse_transform, ErlOpts))), + ErlOptsFirst = lists:filter(fun({parse_transform, PT}) -> + not lists:member(PT, ParseTransforms); + (_) -> + true + end, ErlOpts), + {ErlFirstFiles ++ ParseTransformsErls, ErlOptsFirst}. + +split_source_files(SourceFiles, ErlOpts) -> + ParseTransforms = proplists:get_all_values(parse_transform, ErlOpts), + lists:partition(fun(Source) -> + lists:member(filename_to_atom(Source), ParseTransforms) + end, SourceFiles). + +filename_to_atom(F) -> list_to_atom(filename:rootname(filename:basename(F))). + +%% Get subset of SourceFiles which need to be recompiled, respecting +%% dependencies induced by given graph G. +needed_files(Graph, ErlOpts, RebarOpts, Dir, OutDir, SourceFiles) -> + lists:filter(fun(Source) -> + TargetBase = target_base(OutDir, Source), + Target = TargetBase ++ ".beam", + PrivIncludes = [{i, filename:join(Dir, Src)} + || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])], + AllOpts = [{outdir, filename:dirname(Target)} + ,{i, filename:join(Dir, "include")} + ,{i, Dir}] ++ PrivIncludes ++ ErlOpts, + digraph:vertex(Graph, Source) > {Source, filelib:last_modified(Target)} + orelse opts_changed(AllOpts, TargetBase) + orelse erl_compiler_opts_set() + end, SourceFiles). + +target_base(OutDir, Source) -> + filename:join(OutDir, filename:basename(Source, ".erl")). + +opts_changed(NewOpts, Target) -> + TotalOpts = case erlang:function_exported(compile, env_compiler_options, 0) of + true -> NewOpts ++ compile:env_compiler_options(); + false -> NewOpts + end, + case compile_info(Target) of + {ok, Opts} -> lists:any(fun effects_code_generation/1, lists:usort(TotalOpts) -- lists:usort(Opts)); + _ -> true + end. + +effects_code_generation(Option) -> + case Option of + beam -> false; + report_warnings -> false; + report_errors -> false; + return_errors-> false; + return_warnings-> false; + report -> false; + warnings_as_errors -> false; + binary -> false; + verbose -> false; + {cwd,_} -> false; + {outdir, _} -> false; + _ -> true + end. + +compile_info(Target) -> + case beam_lib:chunks(Target, [compile_info]) of + {ok, {_mod, Chunks}} -> + CompileInfo = proplists:get_value(compile_info, Chunks, []), + {ok, proplists:get_value(options, CompileInfo, [])}; + {error, beam_lib, Reason} -> + ?WARN("Couldn't read debug info from ~p for reason: ~p", [Target, Reason]), + {error, Reason} + end. + +erl_compiler_opts_set() -> + EnvSet = case os:getenv("ERL_COMPILER_OPTIONS") of + false -> false; + _ -> true + end, + %% return false if changed env opts would have been caught in opts_changed/2 + EnvSet andalso not erlang:function_exported(compile, env_compiler_options, 0). + +valid_erl_first_conf(FileList) -> + Strs = filter_file_list(FileList), + case rebar_utils:is_list_of_strings(Strs) of + true -> true; + false -> ?ABORT("An invalid file list (~p) was provided as part of your erl_first_files directive", + [FileList]) + end. + +filter_file_list(FileList) -> + Atoms = lists:filter( fun(X) -> is_atom(X) end, FileList), + case Atoms of + [] -> + FileList; + _ -> + atoms_in_erl_first_files_warning(Atoms), + lists:filter( fun(X) -> not(is_atom(X)) end, FileList) + end. + +atoms_in_erl_first_files_warning(Atoms) -> + W = "You have provided atoms as file entries in erl_first_files; " + "erl_first_files only expects lists of filenames as strings. " + "The following modules (~p) may not work as expected and it is advised " + "that you change these entires to string format " + "(e.g., \"src/module.erl\") ", + ?WARN(W, [Atoms]). + +module_to_erl(Mod) -> + atom_to_list(Mod) ++ ".erl". + +parse_attrs(Fd, Includes, Dir) -> + case io:parse_erl_form(Fd, "") of + {ok, Form, _Line} -> + case erl_syntax:type(Form) of + attribute -> + NewIncludes = process_attr(Form, Includes, Dir), + parse_attrs(Fd, NewIncludes, Dir); + _ -> + parse_attrs(Fd, Includes, Dir) + end; + {eof, _} -> + Includes; + _Err -> + parse_attrs(Fd, Includes, Dir) + end. + +process_attr(Form, Includes, Dir) -> + AttrName = erl_syntax:atom_value(erl_syntax:attribute_name(Form)), + process_attr(AttrName, Form, Includes, Dir). + +process_attr(import, Form, Includes, _Dir) -> + case erl_syntax_lib:analyze_import_attribute(Form) of + {Mod, _Funs} -> + [module_to_erl(Mod)|Includes]; + Mod -> + [module_to_erl(Mod)|Includes] + end; +process_attr(file, Form, Includes, _Dir) -> + {File, _} = erl_syntax_lib:analyze_file_attribute(Form), + [File|Includes]; +process_attr(include, Form, Includes, _Dir) -> + [FileNode] = erl_syntax:attribute_arguments(Form), + File = erl_syntax:string_value(FileNode), + [File|Includes]; +process_attr(include_lib, Form, Includes, Dir) -> + [FileNode] = erl_syntax:attribute_arguments(Form), + RawFile = erl_syntax:string_value(FileNode), + maybe_expand_include_lib_path(RawFile, Dir) ++ Includes; +process_attr(behavior, Form, Includes, _Dir) -> + process_attr(behaviour, Form, Includes, _Dir); +process_attr(behaviour, Form, Includes, _Dir) -> + [FileNode] = erl_syntax:attribute_arguments(Form), + File = module_to_erl(erl_syntax:atom_value(FileNode)), + [File|Includes]; +process_attr(compile, Form, Includes, _Dir) -> + [Arg] = erl_syntax:attribute_arguments(Form), + case erl_syntax:concrete(Arg) of + {parse_transform, Mod} -> + [module_to_erl(Mod)|Includes]; + {core_transform, Mod} -> + [module_to_erl(Mod)|Includes]; + L when is_list(L) -> + lists:foldl( + fun({parse_transform, Mod}, Acc) -> + [module_to_erl(Mod)|Acc]; + ({core_transform, Mod}, Acc) -> + [module_to_erl(Mod)|Acc]; + (_, Acc) -> + Acc + end, Includes, L); + _ -> + Includes + end; +process_attr(_, _Form, Includes, _Dir) -> + Includes. + +%% NOTE: If, for example, one of the entries in Files, refers to +%% gen_server.erl, that entry will be dropped. It is dropped because +%% such an entry usually refers to the beam file, and we don't pass a +%% list of OTP src dirs for finding gen_server.erl's full path. Also, +%% if gen_server.erl was modified, it's not rebar's task to compile a +%% new version of the beam file. Therefore, it's reasonable to drop +%% such entries. Also see process_attr(behaviour, Form, Includes). +-spec expand_file_names([file:filename()], + [file:filename()]) -> [file:filename()]. +expand_file_names(Files, Dirs) -> + %% We check if Files exist by itself or within the directories + %% listed in Dirs. + %% Return the list of files matched. + lists:flatmap( + fun(Incl) -> + case filelib:is_regular(Incl) of + true -> + [Incl]; + false -> + lists:flatmap( + fun(Dir) -> + FullPath = filename:join(Dir, Incl), + case filelib:is_regular(FullPath) of + true -> + [FullPath]; + false -> + [] + end + end, Dirs) + end + end, Files). + +%% Given a path like "stdlib/include/erl_compile.hrl", return +%% "OTP_INSTALL_DIR/lib/erlang/lib/stdlib-x.y.z/include/erl_compile.hrl". +%% Usually a simple [Lib, SubDir, File1] = filename:split(File) should +%% work, but to not crash when an unusual include_lib path is used, +%% utilize more elaborate logic. +maybe_expand_include_lib_path(File, Dir) -> + File1 = filename:basename(File), + case filename:split(filename:dirname(File)) of + [_] -> + warn_and_find_path(File, Dir); + [Lib | SubDir] -> + case code:lib_dir(list_to_atom(Lib), list_to_atom(filename:join(SubDir))) of + {error, bad_name} -> + warn_and_find_path(File, Dir); + AppDir -> + [filename:join(AppDir, File1)] + end + end. + +%% The use of -include_lib was probably incorrect by the user but lets try to make it work. +%% We search in the outdir and outdir/../include to see if the header exists. +warn_and_find_path(File, Dir) -> + SrcHeader = filename:join(Dir, File), + case filelib:is_regular(SrcHeader) of + true -> + [SrcHeader]; + false -> + IncludeDir = filename:join(rebar_utils:droplast(filename:split(Dir))++["include"]), + IncludeHeader = filename:join(IncludeDir, File), + case filelib:is_regular(IncludeHeader) of + true -> + [filename:join(IncludeDir, File)]; + false -> + [] + end + end. diff --git a/src/rebar_compiler_mib.erl b/src/rebar_compiler_mib.erl new file mode 100644 index 0000000..32516bf --- /dev/null +++ b/src/rebar_compiler_mib.erl @@ -0,0 +1,70 @@ +-module(rebar_compiler_mib). + +-behaviour(rebar_compiler). + +-export([context/1, + needed_files/3, + dependencies/3, + compile/4, + clean/2]). + +-include("rebar.hrl"). +-include_lib("stdlib/include/erl_compile.hrl"). + +context(AppInfo) -> + Dir = rebar_app_info:dir(AppInfo), + Mappings = [{".bin", filename:join([Dir, "priv", "mibs"])}, + {".hrl", filename:join(Dir, "include")}], + + #{src_dirs => ["mibs"], + include_dirs => [], + src_ext => ".mib", + out_mappings => Mappings}. + +needed_files(_, FoundFiles, AppInfo) -> + FirstFiles = [], + + %% Remove first files from found files + RestFiles = [Source || Source <- FoundFiles, not lists:member(Source, FirstFiles)], + + Opts = rebar_opts:get(rebar_app_info:opts(AppInfo), mib_opts, []), + {{FirstFiles, Opts}, {RestFiles, Opts}}. + +dependencies(_, _, _) -> + []. + +compile(Source, OutDirs, _, Opts) -> + {_, BinOut} = lists:keyfind(".bin", 1, OutDirs), + {_, HrlOut} = lists:keyfind(".hrl", 1, OutDirs), + + ok = rebar_file_utils:ensure_dir(BinOut), + ok = rebar_file_utils:ensure_dir(HrlOut), + Mib = filename:join(BinOut, filename:basename(Source, ".mib")), + HrlFilename = Mib ++ ".hrl", + + AllOpts = [{outdir, BinOut}, {i, [BinOut]}] ++ Opts, + + case snmpc:compile(Source, AllOpts) of + {ok, _} -> + MibToHrlOpts = + case proplists:get_value(verbosity, AllOpts, undefined) of + undefined -> + #options{specific = [], + cwd = rebar_dir:get_cwd()}; + Verbosity -> + #options{specific = [{verbosity, Verbosity}], + cwd = rebar_dir:get_cwd()} + end, + ok = snmpc:mib_to_hrl(Mib, Mib, MibToHrlOpts), + rebar_file_utils:mv(HrlFilename, HrlOut), + ok; + {error, compilation_failed} -> + ?FAIL + end. + +clean(MibFiles, AppInfo) -> + AppDir = rebar_app_info:dir(AppInfo), + MIBs = [filename:rootname(filename:basename(MIB)) || MIB <- MibFiles], + rebar_file_utils:delete_each( + [filename:join([AppDir, "include", MIB++".hrl"]) || MIB <- MIBs]), + ok = rebar_file_utils:rm_rf(filename:join([AppDir, "priv/mibs/*.bin"])). diff --git a/src/rebar_compiler_xrl.erl b/src/rebar_compiler_xrl.erl new file mode 100644 index 0000000..5c023f0 --- /dev/null +++ b/src/rebar_compiler_xrl.erl @@ -0,0 +1,50 @@ +-module(rebar_compiler_xrl). + +-behaviour(rebar_compiler). + +-export([context/1, + needed_files/3, + dependencies/3, + compile/4, + clean/2]). + +context(AppInfo) -> + Dir = rebar_app_info:dir(AppInfo), + Mappings = [{".erl", filename:join([Dir, "src"])}], + #{src_dirs => ["src"], + include_dirs => [], + src_ext => ".xrl", + out_mappings => Mappings}. + +needed_files(_, FoundFiles, AppInfo) -> + FirstFiles = [], + + %% Remove first files from found files + RestFiles = [Source || Source <- FoundFiles, not lists:member(Source, FirstFiles)], + + Opts = rebar_opts:get(rebar_app_info:opts(AppInfo), xrl_opts, []), + + {{FirstFiles, Opts}, {RestFiles, Opts}}. + +dependencies(_, _, _) -> + []. + +compile(Source, [{_, OutDir}], _, Opts) -> + BaseName = filename:basename(Source), + Target = filename:join([OutDir, BaseName]), + AllOpts = [{parserfile, Target} | Opts], + AllOpts1 = [{includefile, filename:join(OutDir, I)} || {includefile, I} <- AllOpts, + filename:pathtype(I) =:= relative], + case leex:file(Source, AllOpts1 ++ [{return, true}]) of + {ok, _} -> + ok; + {ok, _Mod, Ws} -> + rebar_compiler:ok_tuple(Source, Ws); + {error, Es, Ws} -> + rebar_compiler:error_tuple(Source, Es, Ws, AllOpts1) + end. + +clean(XrlFiles, _AppInfo) -> + rebar_file_utils:delete_each( + [rebar_utils:to_list(re:replace(F, "\\.xrl$", ".erl", [unicode])) + || F <- XrlFiles]). diff --git a/src/rebar_compiler_yrl.erl b/src/rebar_compiler_yrl.erl new file mode 100644 index 0000000..765ea36 --- /dev/null +++ b/src/rebar_compiler_yrl.erl @@ -0,0 +1,49 @@ +-module(rebar_compiler_yrl). + +-behaviour(rebar_compiler). + +-export([context/1, + needed_files/3, + dependencies/3, + compile/4, + clean/2]). + +context(AppInfo) -> + Dir = rebar_app_info:dir(AppInfo), + Mappings = [{".erl", filename:join([Dir, "src"])}], + #{src_dirs => ["src"], + include_dirs => [], + src_ext => ".yrl", + out_mappings => Mappings}. + +needed_files(_, FoundFiles, AppInfo) -> + FirstFiles = [], + + %% Remove first files from found files + RestFiles = [Source || Source <- FoundFiles, not lists:member(Source, FirstFiles)], + + Opts = rebar_opts:get(rebar_app_info:opts(AppInfo), yrl_opts, []), + {{FirstFiles, Opts}, {RestFiles, Opts}}. + +dependencies(_, _, _) -> + []. + +compile(Source, [{_, OutDir}], _, Opts) -> + BaseName = filename:basename(Source), + Target = filename:join([OutDir, BaseName]), + AllOpts = [{parserfile, Target} | Opts], + AllOpts1 = [{includefile, filename:join(OutDir, I)} || {includefile, I} <- AllOpts, + filename:pathtype(I) =:= relative], + case yeec:file(Source, AllOpts1 ++ [{return, true}]) of + {ok, _} -> + ok; + {ok, _Mod, Ws} -> + rebar_compiler:ok_tuple(Source, Ws); + {error, Es, Ws} -> + rebar_compiler:error_tuple(Source, Es, Ws, AllOpts1) + end. + +clean(YrlFiles, _AppInfo) -> + rebar_file_utils:delete_each( + [rebar_utils:to_list(re:replace(F, "\\.yrl$", ".erl", [unicode])) + || F <- YrlFiles]). diff --git a/src/rebar_dir.erl b/src/rebar_dir.erl index d7be423..17bc48e 100644 --- a/src/rebar_dir.erl +++ b/src/rebar_dir.erl @@ -301,9 +301,8 @@ all_src_dirs(Opts, SrcDefault, ExtraDefault) -> src_dir_opts(Opts, Dir) -> RawSrcDirs = raw_src_dirs(src_dirs, Opts, []), RawExtraSrcDirs = raw_src_dirs(extra_src_dirs, Opts, []), - AllOpts = [Opt || {D,Opt} <- RawSrcDirs++RawExtraSrcDirs, - D==Dir], - lists:ukeysort(1,proplists:unfold(lists:append(AllOpts))). + AllOpts = [Opt || {D, Opt} <- RawSrcDirs++RawExtraSrcDirs, D==Dir], + lists:ukeysort(1, proplists:unfold(lists:append(AllOpts))). %%% @doc %%% Return the value of the 'recursive' option for the given directory. diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl deleted file mode 100644 index c63ca5b..0000000 --- a/src/rebar_erlc_compiler.erl +++ /dev/null @@ -1,827 +0,0 @@ -%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- -%% ex: ts=4 sw=4 et -%% ------------------------------------------------------------------- -%% -%% rebar: Erlang Build Tools -%% -%% Copyright (c) 2009, 2010 Dave Smith (dizzyd@dizzyd.com) -%% -%% Permission is hereby granted, free of charge, to any person obtaining a copy -%% of this software and associated documentation files (the "Software"), to deal -%% in the Software without restriction, including without limitation the rights -%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -%% copies of the Software, and to permit persons to whom the Software is -%% furnished to do so, subject to the following conditions: -%% -%% The above copyright notice and this permission notice shall be included in -%% all copies or substantial portions of the Software. -%% -%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -%% THE SOFTWARE. -%% ------------------------------------------------------------------- --module(rebar_erlc_compiler). - --export([compile/1, compile/2, compile/3, - compile_dir/3, compile_dir/4, - compile_dirs/5, - clean/1]). - --include("rebar.hrl"). --include_lib("stdlib/include/erl_compile.hrl"). - --define(ERLCINFO_VSN, 2). --define(ERLCINFO_FILE, "erlcinfo"). --type erlc_info_v() :: {digraph:vertex(), term()} | 'false'. --type erlc_info_e() :: {digraph:vertex(), digraph:vertex()}. --type erlc_info() :: {list(erlc_info_v()), list(erlc_info_e()), list(string())}. --record(erlcinfo, { - vsn = ?ERLCINFO_VSN :: pos_integer(), - info = {[], [], []} :: erlc_info() -}). - --type compile_opts() :: [compile_opt()]. --type compile_opt() :: {recursive, boolean()}. - --define(DEFAULT_OUTDIR, "ebin"). --define(RE_PREFIX, "^(?!\\._)"). - -%% =================================================================== -%% Public API -%% =================================================================== - -%% Supported configuration variables: -%% -%% * erl_opts - Erlang list of options passed to compile:file/2 -%% It is also possible to specify platform specific -%% options by specifying a pair or a triplet where the -%% first string is a regex that is checked against the -%% string -%% -%% OtpRelease ++ "-" ++ SysArch ++ "-" ++ Words. -%% -%% where -%% -%% OtpRelease = erlang:system_info(otp_release). -%% SysArch = erlang:system_info(system_architecture). -%% Words = integer_to_list(8 * -%% erlang:system_info({wordsize, external})). -%% -%% E.g. to define HAVE_SENDFILE only on systems with -%% sendfile(), to define BACKLOG on Linux/FreeBSD as 128, -%% and to define 'old_inets' for R13 OTP release do: -%% -%% {erl_opts, [{platform_define, -%% "(linux|solaris|freebsd|darwin)", -%% 'HAVE_SENDFILE'}, -%% {platform_define, "(linux|freebsd)", -%% 'BACKLOG', 128}, -%% {platform_define, "R13", -%% 'old_inets'}]}. -%% - -%% @equiv compile(AppInfo, []) --spec compile(rebar_app_info:t()) -> ok. -compile(AppInfo) when element(1, AppInfo) == app_info_t -> - compile(AppInfo, []). - -%% @doc compile an individual application. --spec compile(rebar_app_info:t(), compile_opts()) -> ok. -compile(AppInfo, CompileOpts) when element(1, AppInfo) == app_info_t -> - Dir = rebar_utils:to_list(rebar_app_info:out_dir(AppInfo)), - RebarOpts = rebar_app_info:opts(AppInfo), - - SrcOpts = [check_last_mod, - {recursive, dir_recursive(RebarOpts, "src", CompileOpts)}], - MibsOpts = [check_last_mod, - {recursive, dir_recursive(RebarOpts, "mibs", CompileOpts)}], - - rebar_base_compiler:run(RebarOpts, - check_files([filename:join(Dir, File) - || File <- rebar_opts:get(RebarOpts, xrl_first_files, [])]), - filename:join(Dir, "src"), ".xrl", filename:join(Dir, "src"), ".erl", - fun compile_xrl/3, SrcOpts), - rebar_base_compiler:run(RebarOpts, - check_files([filename:join(Dir, File) - || File <- rebar_opts:get(RebarOpts, yrl_first_files, [])]), - filename:join(Dir, "src"), ".yrl", filename:join(Dir, "src"), ".erl", - fun compile_yrl/3, SrcOpts), - rebar_base_compiler:run(RebarOpts, - check_files([filename:join(Dir, File) - || File <- rebar_opts:get(RebarOpts, mib_first_files, [])]), - filename:join(Dir, "mibs"), ".mib", filename:join([Dir, "priv", "mibs"]), ".bin", - compile_mib(AppInfo), MibsOpts), - - SrcDirs = lists:map(fun(SrcDir) -> filename:join(Dir, SrcDir) end, - rebar_dir:src_dirs(RebarOpts, ["src"])), - OutDir = filename:join(Dir, outdir(RebarOpts)), - compile_dirs(RebarOpts, Dir, SrcDirs, OutDir, CompileOpts), - - ExtraDirs = rebar_dir:extra_src_dirs(RebarOpts), - F = fun(D) -> - case ec_file:is_dir(filename:join([Dir, D])) of - true -> compile_dirs(RebarOpts, Dir, [D], D, CompileOpts); - false -> ok - end - end, - lists:foreach(F, lists:map(fun(SrcDir) -> filename:join(Dir, SrcDir) end, ExtraDirs)). - -%% @hidden -%% these are kept for backwards compatibility but they're bad functions with -%% bad interfaces you probably shouldn't use -%% State/RebarOpts have to have src_dirs set and BaseDir must be the parent -%% directory of those src_dirs - --spec compile(rebar_dict() | rebar_state:t(), file:name(), file:name()) -> ok. -compile(State, BaseDir, OutDir) when element(1, State) == state_t -> - compile(rebar_state:opts(State), BaseDir, OutDir, [{recursive, false}]); -compile(RebarOpts, BaseDir, OutDir) -> - compile(RebarOpts, BaseDir, OutDir, [{recursive, false}]). - -%% @hidden - --spec compile(rebar_dict() | rebar_state:t(), file:name(), file:name(), compile_opts()) -> ok. -compile(State, BaseDir, OutDir, CompileOpts) when element(1, State) == state_t -> - compile(rebar_state:opts(State), BaseDir, OutDir, CompileOpts); -compile(RebarOpts, BaseDir, OutDir, CompileOpts) -> - SrcDirs = lists:map(fun(SrcDir) -> filename:join(BaseDir, SrcDir) end, - rebar_dir:src_dirs(RebarOpts, ["src"])), - compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, CompileOpts), - - ExtraDirs = rebar_dir:extra_src_dirs(RebarOpts), - F = fun(D) -> - case ec_file:is_dir(filename:join([BaseDir, D])) of - true -> compile_dirs(RebarOpts, BaseDir, [D], D, CompileOpts); - false -> ok - end - end, - lists:foreach(F, lists:map(fun(SrcDir) -> filename:join(BaseDir, SrcDir) end, ExtraDirs)). - -%% @equiv compile_dirs(Context, BaseDir, [Dir], Dir, [{recursive, false}]) --spec compile_dir(rebar_dict() | rebar_state:t(), file:name(), file:name()) -> ok. -compile_dir(State, BaseDir, Dir) when element(1, State) == state_t -> - compile_dir(rebar_state:opts(State), BaseDir, Dir, [{recursive, false}]); -compile_dir(RebarOpts, BaseDir, Dir) -> - compile_dir(RebarOpts, BaseDir, Dir, [{recursive, false}]). - -%% @equiv compile_dirs(Context, BaseDir, [Dir], Dir, Opts) --spec compile_dir(rebar_dict() | rebar_state:t(), file:name(), file:name(), compile_opts()) -> ok. -compile_dir(State, BaseDir, Dir, Opts) when element(1, State) == state_t -> - compile_dirs(rebar_state:opts(State), BaseDir, [Dir], Dir, Opts); -compile_dir(RebarOpts, BaseDir, Dir, Opts) -> - compile_dirs(RebarOpts, BaseDir, [Dir], Dir, Opts). - -%% @doc compile a list of directories with the given opts. --spec compile_dirs(rebar_dict() | rebar_state:t(), - file:filename(), - [file:filename()], - file:filename(), - compile_opts()) -> ok. -compile_dirs(State, BaseDir, Dirs, OutDir, CompileOpts) when element(1, State) == state_t -> - compile_dirs(rebar_state:opts(State), BaseDir, Dirs, OutDir, CompileOpts); -compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, CompileOpts) -> - ErlOpts = rebar_opts:erl_opts(RebarOpts), - ?DEBUG("erlopts ~p", [ErlOpts]), - AllErlFiles = gather_src(RebarOpts, BaseDir, SrcDirs, CompileOpts), - ?DEBUG("files to compile ~p", [AllErlFiles]), - - %% Make sure that outdir is on the path - ok = filelib:ensure_dir(filename:join(OutDir, "dummy.beam")), - true = code:add_patha(filename:absname(OutDir)), - - G = init_erlcinfo(include_abs_dirs(ErlOpts, BaseDir), AllErlFiles, BaseDir, OutDir), - - {ParseTransforms, Rest} = split_source_files(AllErlFiles, ErlOpts), - NeededErlFiles = case needed_files(G, ErlOpts, RebarOpts, BaseDir, OutDir, ParseTransforms) of - [] -> needed_files(G, ErlOpts, RebarOpts, BaseDir, OutDir, Rest); - %% at least one parse transform in the opts needs updating, so recompile all - _ -> AllErlFiles - end, - - {ErlFirstFiles, ErlOptsFirst} = erl_first_files(RebarOpts, ErlOpts, BaseDir, NeededErlFiles), - {DepErls, OtherErls} = lists:partition( - fun(Source) -> digraph:in_degree(G, Source) > 0 end, - [File || File <- NeededErlFiles, not lists:member(File, ErlFirstFiles)]), - SubGraph = digraph_utils:subgraph(G, DepErls), - DepErlsOrdered = digraph_utils:topsort(SubGraph), - FirstErls = ErlFirstFiles ++ lists:reverse(DepErlsOrdered), - try - rebar_base_compiler:run( - RebarOpts, FirstErls, OtherErls, - fun(S, C) -> - ErlOpts1 = case lists:member(S, ErlFirstFiles) of - true -> ErlOptsFirst; - false -> ErlOpts - end, - internal_erl_compile(C, BaseDir, S, OutDir, ErlOpts1, RebarOpts) - end) - after - true = digraph:delete(SubGraph), - true = digraph:delete(G) - end, - ok. - -%% @doc remove compiled artifacts from an AppDir. --spec clean(rebar_app_info:t()) -> 'ok'. -clean(AppInfo) -> - AppDir = rebar_app_info:out_dir(AppInfo), - - MibFiles = rebar_utils:find_files(filename:join([AppDir, "mibs"]), ?RE_PREFIX".*\\.mib\$"), - MIBs = [filename:rootname(filename:basename(MIB)) || MIB <- MibFiles], - rebar_file_utils:delete_each( - [filename:join([AppDir, "include",MIB++".hrl"]) || MIB <- MIBs]), - ok = rebar_file_utils:rm_rf(filename:join([AppDir, "priv/mibs/*.bin"])), - - YrlFiles = rebar_utils:find_files(filename:join([AppDir, "src"]), ?RE_PREFIX".*\\.[x|y]rl\$"), - rebar_file_utils:delete_each( - [rebar_utils:to_list(re:replace(F, "\\.[x|y]rl$", ".erl", [unicode])) - || F <- YrlFiles]), - - BinDirs = ["ebin"|rebar_dir:extra_src_dirs(rebar_app_info:opts(AppInfo))], - ok = clean_dirs(AppDir, BinDirs), - - %% Delete the build graph, if any - rebar_file_utils:rm_rf(erlcinfo_file(AppDir)). - -clean_dirs(_AppDir, []) -> ok; -clean_dirs(AppDir, [Dir|Rest]) -> - ok = rebar_file_utils:rm_rf(filename:join([AppDir, Dir, "*.beam"])), - %% Erlang compilation is recursive, so it's possible that we have a nested - %% directory structure in ebin with .beam files within. As such, we want - %% to scan whatever is left in the app's out_dir directory for sub-dirs which - %% satisfy our criteria. - BeamFiles = rebar_utils:find_files(filename:join([AppDir, Dir]), ?RE_PREFIX".*\\.beam\$"), - rebar_file_utils:delete_each(BeamFiles), - lists:foreach(fun(D) -> delete_dir(D, dirs(D)) end, dirs(filename:join([AppDir, Dir]))), - clean_dirs(AppDir, Rest). - - -%% =================================================================== -%% Internal functions -%% =================================================================== - -gather_src(Opts, BaseDir, Dirs, CompileOpts) -> - gather_src(Opts, filename:split(BaseDir), Dirs, [], CompileOpts). - -gather_src(_Opts, _BaseDirParts, [], Srcs, _CompileOpts) -> Srcs; -gather_src(Opts, BaseDirParts, [Dir|Rest], Srcs, CompileOpts) -> - DirParts = filename:split(Dir), - RelDir = case lists:prefix(BaseDirParts,DirParts) of - true -> - case lists:nthtail(length(BaseDirParts),DirParts) of - [] -> "."; - RestParts -> filename:join(RestParts) - end; - false -> Dir - end, - DirRecursive = dir_recursive(Opts, RelDir, CompileOpts), - gather_src(Opts, BaseDirParts, Rest, Srcs ++ rebar_utils:find_files(Dir, ?RE_PREFIX".*\\.erl\$", DirRecursive), CompileOpts). - -%% Get files which need to be compiled first, i.e. those specified in erl_first_files -%% and parse_transform options. Also produce specific erl_opts for these first -%% files, so that yet to be compiled parse transformations are excluded from it. -erl_first_files(Opts, ErlOpts, Dir, NeededErlFiles) -> - ErlFirstFilesConf = rebar_opts:get(Opts, erl_first_files, []), - valid_erl_first_conf(ErlFirstFilesConf), - NeededSrcDirs = lists:usort(lists:map(fun filename:dirname/1, NeededErlFiles)), - %% NOTE: order of files here is important! - ErlFirstFiles = - [filename:join(Dir, File) || File <- ErlFirstFilesConf, - lists:member(filename:join(Dir, File), NeededErlFiles)], - {ParseTransforms, ParseTransformsErls} = - lists:unzip(lists:flatmap( - fun(PT) -> - PTerls = [filename:join(D, module_to_erl(PT)) || D <- NeededSrcDirs], - [{PT, PTerl} || PTerl <- PTerls, lists:member(PTerl, NeededErlFiles)] - end, proplists:get_all_values(parse_transform, ErlOpts))), - ErlOptsFirst = lists:filter(fun({parse_transform, PT}) -> - not lists:member(PT, ParseTransforms); - (_) -> - true - end, ErlOpts), - {ErlFirstFiles ++ ParseTransformsErls, ErlOptsFirst}. - -split_source_files(SourceFiles, ErlOpts) -> - ParseTransforms = proplists:get_all_values(parse_transform, ErlOpts), - lists:partition(fun(Source) -> - lists:member(filename_to_atom(Source), ParseTransforms) - end, SourceFiles). - -filename_to_atom(F) -> list_to_atom(filename:rootname(filename:basename(F))). - -%% Get subset of SourceFiles which need to be recompiled, respecting -%% dependencies induced by given graph G. -needed_files(G, ErlOpts, RebarOpts, Dir, OutDir, SourceFiles) -> - lists:filter(fun(Source) -> - TargetBase = target_base(OutDir, Source), - Target = TargetBase ++ ".beam", - PrivIncludes = [{i, filename:join(Dir, Src)} - || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])], - AllOpts = [{outdir, filename:dirname(Target)} - ,{i, filename:join(Dir, "include")} - ,{i, Dir}] ++ PrivIncludes ++ ErlOpts, - %% necessary for erlang:function_exported/3 to work as expected - %% called here for clarity as it's required by both opts_changed/2 - %% and erl_compiler_opts_set/0 - _ = code:ensure_loaded(compile), - digraph:vertex(G, Source) > {Source, filelib:last_modified(Target)} - orelse opts_changed(AllOpts, TargetBase) - orelse erl_compiler_opts_set() - end, SourceFiles). - -maybe_rm_beam_and_edge(G, OutDir, Source) -> - %% This is NOT a double check it is the only check that the source file is actually gone - case filelib:is_regular(Source) of - true -> - %% Actually exists, don't delete - false; - false -> - Target = target_base(OutDir, Source) ++ ".beam", - ?DEBUG("Source ~ts is gone, deleting previous beam file if it exists ~ts", [Source, Target]), - file:delete(Target), - digraph:del_vertex(G, Source), - true - end. - -opts_changed(NewOpts, Target) -> - TotalOpts = case erlang:function_exported(compile, env_compiler_options, 0) of - true -> NewOpts ++ compile:env_compiler_options(); - false -> NewOpts - end, - case compile_info(Target) of - {ok, Opts} -> lists:any(fun effects_code_generation/1, lists:usort(TotalOpts) -- lists:usort(Opts)); - _ -> true - end. - -effects_code_generation(Option) -> - case Option of - beam -> false; - report_warnings -> false; - report_errors -> false; - return_errors-> false; - return_warnings-> false; - report -> false; - warnings_as_errors -> false; - binary -> false; - verbose -> false; - {cwd,_} -> false; - {outdir, _} -> false; - _ -> true - end. - -compile_info(Target) -> - case beam_lib:chunks(Target, [compile_info]) of - {ok, {_mod, Chunks}} -> - CompileInfo = proplists:get_value(compile_info, Chunks, []), - {ok, proplists:get_value(options, CompileInfo, [])}; - {error, beam_lib, Reason} -> - ?WARN("Couldn't read debug info from ~p for reason: ~p", [Target, Reason]), - {error, Reason} - end. - -erl_compiler_opts_set() -> - EnvSet = case os:getenv("ERL_COMPILER_OPTIONS") of - false -> false; - _ -> true - end, - %% return false if changed env opts would have been caught in opts_changed/2 - EnvSet andalso not erlang:function_exported(compile, env_compiler_options, 0). - -erlcinfo_file(Dir) -> - filename:join(rebar_dir:local_cache_dir(Dir), ?ERLCINFO_FILE). - -%% Get dependency graph of given Erls files and their dependencies (header files, -%% parse transforms, behaviours etc.) located in their directories or given -%% InclDirs. Note that last modification times stored in vertices already respect -%% dependencies induced by given graph G. -init_erlcinfo(InclDirs, Erls, Dir, OutDir) -> - G = digraph:new([acyclic]), - try restore_erlcinfo(G, InclDirs, Dir) - catch - _:_ -> - ?WARN("Failed to restore ~ts file. Discarding it.~n", [erlcinfo_file(Dir)]), - file:delete(erlcinfo_file(Dir)) - end, - Dirs = source_and_include_dirs(InclDirs, Erls), - %% A source file may have been renamed or deleted. Remove it from the graph - %% and remove any beam file for that source if it exists. - Modified = maybe_rm_beams_and_edges(G, OutDir, Erls), - Modified1 = lists:foldl(update_erlcinfo_fun(G, Dirs), Modified, Erls), - if Modified1 -> store_erlcinfo(G, InclDirs, Dir); not Modified1 -> ok end, - G. - -maybe_rm_beams_and_edges(G, Dir, Files) -> - Vertices = digraph:vertices(G), - case lists:filter(fun(File) -> - case filename:extension(File) =:= ".erl" of - true -> - maybe_rm_beam_and_edge(G, Dir, File); - false -> - false - end - end, lists:sort(Vertices) -- lists:sort(Files)) of - [] -> - false; - _ -> - true - end. - -source_and_include_dirs(InclDirs, Erls) -> - SourceDirs = lists:map(fun filename:dirname/1, Erls), - lists:usort(InclDirs ++ SourceDirs). - -update_erlcinfo(G, Dirs, Source) -> - case digraph:vertex(G, Source) of - {_, LastUpdated} -> - case filelib:last_modified(Source) of - 0 -> - %% The file doesn't exist anymore, - %% erase it from the graph. - %% All the edges will be erased automatically. - digraph:del_vertex(G, Source), - modified; - LastModified when LastUpdated < LastModified -> - modify_erlcinfo(G, Source, LastModified, filename:dirname(Source), Dirs); - _ -> - Modified = lists:foldl( - update_erlcinfo_fun(G, Dirs), - false, digraph:out_neighbours(G, Source)), - MaxModified = update_max_modified_deps(G, Source), - case Modified orelse MaxModified > LastUpdated of - true -> modified; - false -> unmodified - end - end; - false -> - modify_erlcinfo(G, Source, filelib:last_modified(Source), filename:dirname(Source), Dirs) - end. - -update_erlcinfo_fun(G, Dirs) -> - fun(Erl, Modified) -> - case update_erlcinfo(G, Dirs, Erl) of - modified -> true; - unmodified -> Modified - end - end. - -update_max_modified_deps(G, Source) -> - MaxModified = lists:max(lists:map( - fun(File) -> {_, MaxModified} = digraph:vertex(G, File), MaxModified end, - [Source|digraph:out_neighbours(G, Source)])), - digraph:add_vertex(G, Source, MaxModified), - MaxModified. - -modify_erlcinfo(G, Source, LastModified, Dir, Dirs) -> - {ok, Fd} = file:open(Source, [read]), - Incls = parse_attrs(Fd, [], Dir), - AbsIncls = expand_file_names(Incls, Dirs), - ok = file:close(Fd), - digraph:add_vertex(G, Source, LastModified), - digraph:del_edges(G, digraph:out_edges(G, Source)), - lists:foreach( - fun(Incl) -> - update_erlcinfo(G, Dirs, Incl), - digraph:add_edge(G, Source, Incl) - end, AbsIncls), - modified. - -restore_erlcinfo(G, InclDirs, Dir) -> - case file:read_file(erlcinfo_file(Dir)) of - {ok, Data} -> - % Since externally passed InclDirs can influence erlcinfo graph (see - % modify_erlcinfo), we have to check here that they didn't change. - #erlcinfo{vsn=?ERLCINFO_VSN, info={Vs, Es, InclDirs}} = - binary_to_term(Data), - lists:foreach( - fun({V, LastUpdated}) -> - digraph:add_vertex(G, V, LastUpdated) - end, Vs), - lists:foreach( - fun({_, V1, V2, _}) -> - digraph:add_edge(G, V1, V2) - end, Es); - {error, _} -> - ok - end. - -store_erlcinfo(G, InclDirs, Dir) -> - Vs = lists:map(fun(V) -> digraph:vertex(G, V) end, digraph:vertices(G)), - Es = lists:map(fun(E) -> digraph:edge(G, E) end, digraph:edges(G)), - File = erlcinfo_file(Dir), - ok = filelib:ensure_dir(File), - Data = term_to_binary(#erlcinfo{info={Vs, Es, InclDirs}}, [{compressed, 2}]), - file:write_file(File, Data). - -%% NOTE: If, for example, one of the entries in Files, refers to -%% gen_server.erl, that entry will be dropped. It is dropped because -%% such an entry usually refers to the beam file, and we don't pass a -%% list of OTP src dirs for finding gen_server.erl's full path. Also, -%% if gen_server.erl was modified, it's not rebar's task to compile a -%% new version of the beam file. Therefore, it's reasonable to drop -%% such entries. Also see process_attr(behaviour, Form, Includes). --spec expand_file_names([file:filename()], - [file:filename()]) -> [file:filename()]. -expand_file_names(Files, Dirs) -> - %% We check if Files exist by itself or within the directories - %% listed in Dirs. - %% Return the list of files matched. - lists:flatmap( - fun(Incl) -> - case filelib:is_regular(Incl) of - true -> - [Incl]; - false -> - lists:flatmap( - fun(Dir) -> - FullPath = filename:join(Dir, Incl), - case filelib:is_regular(FullPath) of - true -> - [FullPath]; - false -> - [] - end - end, Dirs) - end - end, Files). - --spec internal_erl_compile(rebar_dict(), file:filename(), file:filename(), - file:filename(), list(), rebar_dict()) -> - ok | {ok, any()} | {error, any(), any()}. -internal_erl_compile(Opts, Dir, Module, OutDir, ErlOpts, RebarOpts) -> - Target = target_base(OutDir, Module) ++ ".beam", - ok = filelib:ensure_dir(Target), - PrivIncludes = [{i, filename:join(Dir, Src)} - || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])], - AllOpts = [{outdir, filename:dirname(Target)}] ++ ErlOpts ++ PrivIncludes ++ - [{i, filename:join(Dir, "include")}, {i, Dir}, return], - case compile:file(Module, AllOpts) of - {ok, _Mod} -> - ok; - {ok, _Mod, Ws} -> - FormattedWs = format_error_sources(Ws, Opts), - rebar_base_compiler:ok_tuple(Module, FormattedWs); - {error, Es, Ws} -> - error_tuple(Module, Es, Ws, AllOpts, Opts) - end. - -error_tuple(Module, Es, Ws, AllOpts, Opts) -> - FormattedEs = format_error_sources(Es, Opts), - FormattedWs = format_error_sources(Ws, Opts), - rebar_base_compiler:error_tuple(Module, FormattedEs, FormattedWs, AllOpts). - -format_error_sources(Es, Opts) -> - [{rebar_base_compiler:format_error_source(Src, Opts), Desc} - || {Src, Desc} <- Es]. - -target_base(OutDir, Source) -> - filename:join(OutDir, filename:basename(Source, ".erl")). - --spec compile_mib(rebar_app_info:t()) -> - fun((file:filename(), file:filename(), rebar_dict()) -> 'ok'). -compile_mib(AppInfo) -> - fun(Source, Target, Opts) -> - Dir = filename:dirname(Target), - Mib = filename:rootname(Target), - HrlFilename = Mib ++ ".hrl", - - AppInclude = filename:join([rebar_app_info:dir(AppInfo), "include"]), - - ok = filelib:ensure_dir(Target), - ok = filelib:ensure_dir(filename:join([AppInclude, "dummy.hrl"])), - - AllOpts = [{outdir, Dir} - ,{i, [Dir]}] ++ - rebar_opts:get(Opts, mib_opts, []), - - case snmpc:compile(Source, AllOpts) of - {ok, _} -> - MibToHrlOpts = - case proplists:get_value(verbosity, AllOpts, undefined) of - undefined -> - #options{specific = [], - cwd = rebar_dir:get_cwd()}; - Verbosity -> - #options{specific = [{verbosity, Verbosity}], - cwd = rebar_dir:get_cwd()} - end, - ok = snmpc:mib_to_hrl(Mib, Mib, MibToHrlOpts), - rebar_file_utils:mv(HrlFilename, AppInclude), - ok; - {error, compilation_failed} -> - ?FAIL - end - end. - --spec compile_xrl(file:filename(), file:filename(), - rebar_dict()) -> 'ok'. -compile_xrl(Source, Target, Opts) -> - AllOpts = [{scannerfile, Target} | rebar_opts:get(Opts, xrl_opts, [])], - compile_xrl_yrl(Opts, Source, Target, AllOpts, leex). - --spec compile_yrl(file:filename(), file:filename(), - rebar_dict()) -> 'ok'. -compile_yrl(Source, Target, Opts) -> - AllOpts = [{parserfile, Target} | rebar_opts:get(Opts, yrl_opts, [])], - compile_xrl_yrl(Opts, Source, Target, AllOpts, yecc). - --spec compile_xrl_yrl(rebar_dict(), file:filename(), - file:filename(), list(), module()) -> 'ok'. -compile_xrl_yrl(_Opts, Source, Target, AllOpts, Mod) -> - %% FIX ME: should be the outdir or something - Dir = filename:dirname(filename:dirname(Target)), - AllOpts1 = [{includefile, filename:join(Dir, I)} || {includefile, I} <- AllOpts, - filename:pathtype(I) =:= relative], - case needs_compile(Source, Target) of - true -> - case Mod:file(Source, AllOpts1 ++ [{return, true}]) of - {ok, _} -> - ok; - {ok, _Mod, Ws} -> - rebar_base_compiler:ok_tuple(Source, Ws); - {error, Es, Ws} -> - rebar_base_compiler:error_tuple(Source, - Es, Ws, AllOpts1) - end; - false -> - skipped - end. - -needs_compile(Source, Target) -> - filelib:last_modified(Source) > filelib:last_modified(Target). - - - --spec dirs(file:filename()) -> [file:filename()]. -dirs(Dir) -> - [F || F <- filelib:wildcard(filename:join([Dir, "*"])), filelib:is_dir(F)]. - --spec delete_dir(file:filename(), [string()]) -> 'ok' | {'error', atom()}. -delete_dir(Dir, []) -> - file:del_dir(Dir); -delete_dir(Dir, Subdirs) -> - lists:foreach(fun(D) -> delete_dir(D, dirs(D)) end, Subdirs), - file:del_dir(Dir). - -parse_attrs(Fd, Includes, Dir) -> - case io:parse_erl_form(Fd, "") of - {ok, Form, _Line} -> - case erl_syntax:type(Form) of - attribute -> - NewIncludes = process_attr(Form, Includes, Dir), - parse_attrs(Fd, NewIncludes, Dir); - _ -> - parse_attrs(Fd, Includes, Dir) - end; - {eof, _} -> - Includes; - _Err -> - parse_attrs(Fd, Includes, Dir) - end. - -process_attr(Form, Includes, Dir) -> - AttrName = erl_syntax:atom_value(erl_syntax:attribute_name(Form)), - process_attr(AttrName, Form, Includes, Dir). - -process_attr(import, Form, Includes, _Dir) -> - case erl_syntax_lib:analyze_import_attribute(Form) of - {Mod, _Funs} -> - [module_to_erl(Mod)|Includes]; - Mod -> - [module_to_erl(Mod)|Includes] - end; -process_attr(file, Form, Includes, _Dir) -> - {File, _} = erl_syntax_lib:analyze_file_attribute(Form), - [File|Includes]; -process_attr(include, Form, Includes, _Dir) -> - [FileNode] = erl_syntax:attribute_arguments(Form), - File = erl_syntax:string_value(FileNode), - [File|Includes]; -process_attr(include_lib, Form, Includes, Dir) -> - [FileNode] = erl_syntax:attribute_arguments(Form), - RawFile = erl_syntax:string_value(FileNode), - maybe_expand_include_lib_path(RawFile, Dir) ++ Includes; -process_attr(behavior, Form, Includes, _Dir) -> - process_attr(behaviour, Form, Includes, _Dir); -process_attr(behaviour, Form, Includes, _Dir) -> - [FileNode] = erl_syntax:attribute_arguments(Form), - File = module_to_erl(erl_syntax:atom_value(FileNode)), - [File|Includes]; -process_attr(compile, Form, Includes, _Dir) -> - [Arg] = erl_syntax:attribute_arguments(Form), - case erl_syntax:concrete(Arg) of - {parse_transform, Mod} -> - [module_to_erl(Mod)|Includes]; - {core_transform, Mod} -> - [module_to_erl(Mod)|Includes]; - L when is_list(L) -> - lists:foldl( - fun({parse_transform, Mod}, Acc) -> - [module_to_erl(Mod)|Acc]; - ({core_transform, Mod}, Acc) -> - [module_to_erl(Mod)|Acc]; - (_, Acc) -> - Acc - end, Includes, L); - _ -> - Includes - end; -process_attr(_, _Form, Includes, _Dir) -> - Includes. - -module_to_erl(Mod) -> - atom_to_list(Mod) ++ ".erl". - -%% Given a path like "stdlib/include/erl_compile.hrl", return -%% "OTP_INSTALL_DIR/lib/erlang/lib/stdlib-x.y.z/include/erl_compile.hrl". -%% Usually a simple [Lib, SubDir, File1] = filename:split(File) should -%% work, but to not crash when an unusual include_lib path is used, -%% utilize more elaborate logic. -maybe_expand_include_lib_path(File, Dir) -> - File1 = filename:basename(File), - case filename:split(filename:dirname(File)) of - [_] -> - warn_and_find_path(File, Dir); - [Lib | SubDir] -> - case code:lib_dir(list_to_atom(Lib), list_to_atom(filename:join(SubDir))) of - {error, bad_name} -> - warn_and_find_path(File, Dir); - AppDir -> - [filename:join(AppDir, File1)] - end - end. - -%% The use of -include_lib was probably incorrect by the user but lets try to make it work. -%% We search in the outdir and outdir/../include to see if the header exists. -warn_and_find_path(File, Dir) -> - SrcHeader = filename:join(Dir, File), - case filelib:is_regular(SrcHeader) of - true -> - [SrcHeader]; - false -> - IncludeDir = filename:join(rebar_utils:droplast(filename:split(Dir))++["include"]), - IncludeHeader = filename:join(IncludeDir, File), - case filelib:is_regular(IncludeHeader) of - true -> - [filename:join(IncludeDir, File)]; - false -> - [] - end - end. - -%% -%% Ensure all files in a list are present and abort if one is missing -%% --spec check_files([file:filename()]) -> [file:filename()]. -check_files(FileList) -> - [check_file(F) || F <- FileList]. - -check_file(File) -> - case filelib:is_regular(File) of - false -> ?ABORT("File ~p is missing, aborting\n", [File]); - true -> File - end. - -outdir(RebarOpts) -> - ErlOpts = rebar_opts:erl_opts(RebarOpts), - proplists:get_value(outdir, ErlOpts, ?DEFAULT_OUTDIR). - -include_abs_dirs(ErlOpts, BaseDir) -> - ErlOptIncludes = proplists:get_all_values(i, ErlOpts), - InclDirs = lists:map(fun(Incl) -> filename:absname(Incl) end, ErlOptIncludes), - [filename:join([BaseDir, "include"])|InclDirs]. - -dir_recursive(Opts, Dir, CompileOpts) when is_list(CompileOpts) -> - case proplists:get_value(recursive,CompileOpts) of - undefined -> rebar_dir:recursive(Opts, Dir); - Recursive -> Recursive - end. - -valid_erl_first_conf(FileList) -> - Strs = filter_file_list(FileList), - case rebar_utils:is_list_of_strings(Strs) of - true -> true; - false -> ?ABORT("An invalid file list (~p) was provided as part of your erl_first_files directive", - [FileList]) - end. - -filter_file_list(FileList) -> - Atoms = lists:filter( fun(X) -> is_atom(X) end, FileList), - case Atoms of - [] -> - FileList; - _ -> - atoms_in_erl_first_files_warning(Atoms), - lists:filter( fun(X) -> not(is_atom(X)) end, FileList) - end. - -atoms_in_erl_first_files_warning(Atoms) -> - W = "You have provided atoms as file entries in erl_first_files; " - "erl_first_files only expects lists of filenames as strings. " - "The following modules (~p) may not work as expected and it is advised " - "that you change these entires to string format " - "(e.g., \"src/module.erl\") ", - ?WARN(W, [Atoms]). diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 492d690..a51a557 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -43,7 +43,8 @@ path_from_ancestor/2, canonical_path/1, resolve_link/1, - split_dirname/1]). + split_dirname/1, + ensure_dir/1]). -include("rebar.hrl"). @@ -386,7 +387,7 @@ reset_dir(Path) -> %% delete the directory if it exists _ = ec_file:remove(Path, [recursive]), %% recreate the directory - filelib:ensure_dir(filename:join([Path, "dummy.beam"])). + ensure_dir(Path). %% Linux touch but using erlang functions to work in bot *nix os and @@ -440,6 +441,10 @@ resolve_link(Path) -> split_dirname(Path) -> {filename:dirname(Path), filename:basename(Path)}. +-spec ensure_dir(filelib:dirname_all()) -> ok | {error, file:posix()}. +ensure_dir(Path) -> + filelib:ensure_dir(filename:join(Path, "fake_file")). + %% =================================================================== %% Internal functions %% =================================================================== @@ -505,7 +510,7 @@ cp_r_win32({true, SourceDir}, {false, DestDir}) -> false -> %% Specifying a target directory that doesn't currently exist. %% So let's attempt to create this directory - case filelib:ensure_dir(filename:join(DestDir, "dummy")) of + case ensure_dir(DestDir) of ok -> ok = xcopy_win32(SourceDir, DestDir); {error, Reason} -> diff --git a/src/rebar_otp_app.erl b/src/rebar_otp_app.erl index 1d854da..e14975f 100644 --- a/src/rebar_otp_app.erl +++ b/src/rebar_otp_app.erl @@ -60,7 +60,8 @@ compile(State, App) -> format_error({missing_app_file, Filename}) -> io_lib:format("App file is missing: ~ts", [Filename]); format_error({file_read, AppName, File, Reason}) -> - io_lib:format("Failed to read required ~ts file for processing the application '~ts': ~ts", [File, AppName, file:format_error(Reason)]); + io_lib:format("Failed to read required ~ts file for processing the application '~ts': ~ts", + [File, AppName, file:format_error(Reason)]); format_error({invalid_name, File, AppName}) -> io_lib:format("Invalid ~ts: name of application (~p) must match filename.", [File, AppName]). @@ -125,7 +126,7 @@ preprocess(State, AppInfo, AppSrcFile) -> %% Setup file .app filename and write new contents EbinDir = rebar_app_info:ebin_dir(AppInfo), - filelib:ensure_dir(filename:join(EbinDir, "dummy.beam")), + rebar_file_utils:ensure_dir(EbinDir), AppFile = rebar_app_utils:app_src_to_app(OutDir, AppSrcFile), ok = rebar_file_utils:write_file_if_contents_differ(AppFile, Spec, utf8), @@ -163,32 +164,17 @@ validate_name(AppName, File) -> ebin_modules(AppInfo, Dir) -> Beams = lists:sort(rebar_utils:beams(filename:join(Dir, "ebin"))), - ExtraDirs = extra_dirs(AppInfo), - F = fun(Beam) -> not in_extra_dir(AppInfo, Beam, ExtraDirs) end, - Filtered = lists:filter(F, Beams), + SrcDirs = rebar_dir:src_dirs(rebar_app_info:opts(AppInfo), ["src"]), + FindSourceRules = [{".beam", ".erl", + [{"ebin", SrcDir} || SrcDir <- SrcDirs]}], + Filtered = lists:filter(fun(Beam) -> + rebar_utils:find_source(filename:basename(Beam), + filename:dirname(Beam), + FindSourceRules) + =/= {error, not_found} + end, Beams), [rebar_utils:beam_to_mod(N) || N <- Filtered]. -extra_dirs(State) -> - Extras = rebar_dir:extra_src_dirs(rebar_app_info:opts(State)), - SrcDirs = rebar_dir:src_dirs(rebar_app_info:opts(State), ["src"]), - %% remove any dirs that are defined in `src_dirs` from `extra_src_dirs` - Extras -- SrcDirs. - -in_extra_dir(AppInfo, Beam, Dirs) -> - lists:any(fun(Dir) -> lists:prefix(filename:join([rebar_app_info:out_dir(AppInfo), Dir]), - beam_src(Beam)) end, - Dirs). - -beam_src(Beam) -> - case beam_lib:chunks(Beam, [compile_info]) of - {ok, {_mod, Chunks}} -> - CompileInfo = proplists:get_value(compile_info, Chunks, []), - proplists:get_value(source, CompileInfo, []); - {error, beam_lib, Reason} -> - ?WARN("Couldn't read debug info from ~p for reason: ~p", [Beam, Reason]), - [] - end. - ensure_registered(AppData) -> case lists:keyfind(registered, 1, AppData) of false -> diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 7533a44..7676213 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -60,7 +60,7 @@ get_all_names(State) -> get_package_versions(Dep, Repo, Table, State) -> ?MODULE:verify_table(State), AllowPreRelease = rebar_state:get(State, deps_allow_prerelease, false), - ets:select(Table, [{#package{key={Dep,{'$1', '$2'}, Repo}, + ets:select(Table, [{#package{key={Dep, {'$1', '$2'}, Repo}, _='_'}, [{'==', '$2', {{[],[]}}} || not AllowPreRelease], [{{'$1', '$2'}}]}]). diff --git a/src/rebar_prv_clean.erl b/src/rebar_prv_clean.erl index 4da0a64..3c8a0c3 100644 --- a/src/rebar_prv_clean.erl +++ b/src/rebar_prv_clean.erl @@ -67,11 +67,12 @@ format_error(Reason) -> %% =================================================================== clean_apps(State, Providers, Apps) -> + Compilers = rebar_state:compilers(State), [begin ?INFO("Cleaning out ~ts...", [rebar_app_info:name(AppInfo)]), AppDir = rebar_app_info:dir(AppInfo), AppInfo1 = rebar_hooks:run_all_hooks(AppDir, pre, ?PROVIDER, Providers, AppInfo, State), - rebar_erlc_compiler:clean(AppInfo1), + rebar_compiler:clean(Compilers, AppInfo1), rebar_hooks:run_all_hooks(AppDir, post, ?PROVIDER, Providers, AppInfo1, State) end || AppInfo <- Apps]. diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index edb2d82..54bdee2 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -92,6 +92,12 @@ handle_project_apps(DepsPaths, Providers, State) -> -spec format_error(any()) -> iolist(). format_error({missing_artifact, File}) -> io_lib:format("Missing artifact ~ts", [File]); +format_error({bad_project_builder, Name, Type, Module}) -> + io_lib:format("Error building application ~s:~n Required project builder ~s function " + "~s:build/1 not found", [Name, Type, Module]); +format_error({unknown_project_type, Name, Type}) -> + io_lib:format("Error building application ~s:~n " + "No project builder is configured for type ~s", [Name, Type]); format_error(Reason) -> io_lib:format("~p", [Reason]). @@ -135,10 +141,19 @@ build_extra_dir(State, Dir) -> true -> BaseDir = filename:join([rebar_dir:base_dir(State), "extras"]), OutDir = filename:join([BaseDir, Dir]), - filelib:ensure_dir(filename:join([OutDir, "dummy.beam"])), + rebar_file_utils:ensure_dir(OutDir), copy(rebar_state:dir(State), BaseDir, Dir), - rebar_erlc_compiler:compile_dir(State, BaseDir, OutDir); - false -> ok + + Compilers = rebar_state:compilers(State), + FakeApp = rebar_app_info:new(), + FakeApp1 = rebar_app_info:out_dir(FakeApp, BaseDir), + FakeApp2 = rebar_app_info:ebin_dir(FakeApp1, OutDir), + Opts = rebar_state:opts(State), + FakeApp3 = rebar_app_info:opts(FakeApp2, Opts), + FakeApp4 = rebar_app_info:set(FakeApp3, src_dirs, [OutDir]), + rebar_compiler:compile_all(Compilers, FakeApp4); + false -> + ok end. compile(State, AppInfo) -> @@ -150,7 +165,9 @@ compile(State, Providers, AppInfo) -> AppInfo1 = rebar_hooks:run_all_hooks(AppDir, pre, ?PROVIDER, Providers, AppInfo, State), AppInfo2 = rebar_hooks:run_all_hooks(AppDir, pre, ?ERLC_HOOK, Providers, AppInfo1, State), - rebar_erlc_compiler:compile(AppInfo2), + + build_app(AppInfo2, State), + AppInfo3 = rebar_hooks:run_all_hooks(AppDir, post, ?ERLC_HOOK, Providers, AppInfo2, State), AppInfo4 = rebar_hooks:run_all_hooks(AppDir, pre, ?APP_HOOK, Providers, AppInfo3, State), @@ -179,6 +196,24 @@ compile(State, Providers, AppInfo) -> %% Internal functions %% =================================================================== +build_app(AppInfo, State) -> + case rebar_app_info:project_type(AppInfo) of + Type when Type =:= rebar3 ; Type =:= undefined -> + Compilers = rebar_state:compilers(State), + rebar_compiler:compile_all(Compilers, AppInfo); + Type -> + ProjectBuilders = rebar_state:project_builders(State), + case lists:keyfind(Type, 1, ProjectBuilders) of + {_, Module} -> + %% load plugins since thats where project builders would be + PluginDepsPaths = rebar_state:code_paths(State, all_plugin_deps), + code:add_pathsa(PluginDepsPaths), + Module:build(AppInfo), + rebar_utils:remove_from_code_path(PluginDepsPaths); + _ -> + throw(?PRV_ERROR({unknown_project_type, rebar_app_info:name(AppInfo), Type})) + end + end. update_code_paths(State, ProjectApps, DepsPaths) -> ProjAppsPaths = paths_for_apps(ProjectApps), ExtrasPaths = paths_for_extras(State, ProjectApps), diff --git a/src/rebar_state.erl b/src/rebar_state.erl index ec74eea..c8a9400 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -38,6 +38,11 @@ to_list/1, + compilers/1, compilers/2, + prepend_compilers/2, append_compilers/2, + + project_builders/1, add_project_builder/3, + create_resources/2, resources/1, resources/2, add_resource/2, providers/1, providers/2, add_provider/2, @@ -66,6 +71,8 @@ all_plugin_deps = [] :: [rebar_app_info:t()], all_deps = [] :: [rebar_app_info:t()], + compilers = [] :: [{compiler_type(), extension(), extension(), compile_fun()}], + project_builders = [] :: [{rebar_app_info:project_type(), module()}], resources = [], providers = [], allow_provider_overrides = false :: boolean()}). @@ -74,6 +81,10 @@ -type t() :: #state_t{}. +-type compiler_type() :: atom(). +-type extension() :: string(). +-type compile_fun() :: fun(([file:filename()], rebar_app_info:t(), list()) -> ok). + -spec new() -> t(). new() -> BaseState = base_state(dict:new()), @@ -392,6 +403,32 @@ warn_old_resource(ResourceModule) -> ?WARN("Using custom resource ~s that implements a deprecated api. " "It should be upgraded to rebar_resource_v2.", [ResourceModule]). +compilers(#state_t{compilers=Compilers}) -> + Compilers. + +prepend_compilers(State=#state_t{compilers=Compilers}, NewCompilers) -> + State#state_t{compilers=NewCompilers++Compilers}. + +append_compilers(State=#state_t{compilers=Compilers}, NewCompilers) -> + State#state_t{compilers=Compilers++NewCompilers}. + +compilers(State, Compilers) -> + State#state_t{compilers=Compilers}. + +project_builders(#state_t{project_builders=ProjectBuilders}) -> + ProjectBuilders. + +add_project_builder(State=#state_t{project_builders=ProjectBuilders}, Type, Module) -> + _ = code:ensure_loaded(Module), + case erlang:function_exported(Module, build, 1) of + true -> + State#state_t{project_builders=[{Type, Module} | ProjectBuilders]}; + false -> + ?WARN("Unable to add project builder for type ~s, required function ~s:build/1 not found.", + [Type, Module]), + State + end. + create_resources(Resources, State) -> lists:foldl(fun(R, StateAcc) -> add_resource(StateAcc, R) diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 995d212..419802a 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -37,8 +37,9 @@ escript_foldl/3, find_files/2, find_files/3, + find_files_in_dirs/3, + find_source/3, beam_to_mod/1, - beam_to_mod/2, erl_to_mod/1, beams/1, find_executable/1, @@ -206,6 +207,12 @@ sh(Command0, Options0) -> find_files(Dir, Regex) -> find_files(Dir, Regex, true). +find_files_in_dirs([], _Regex, _Recursive) -> + []; +find_files_in_dirs([Dir | T], Regex, Recursive) -> + find_files(Dir, Regex, Recursive) ++ find_files_in_dirs(T, Regex, Recursive). + + find_files(Dir, Regex, Recursive) -> filelib:fold_files(Dir, Regex, Recursive, fun(F, Acc) -> [F | Acc] end, []). @@ -671,10 +678,6 @@ sh_loop(Port, Fun, Acc) -> end end. -beam_to_mod(Dir, Filename) -> - [Dir | Rest] = filename:split(Filename), - list_to_atom(filename:basename(rebar_string:join(Rest, "."), ".beam")). - beam_to_mod(Filename) -> list_to_atom(filename:basename(Filename, ".beam")). @@ -1074,3 +1077,77 @@ version_pad([Major, Minor, Patch]) -> {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}; version_pad([Major, Minor, Patch | _]) -> {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}. + + +-ifdef(filelib_find_source). +find_source(Filename, Dir, Rules) -> + filelib:find_source(Filename, Dir, Rules). +-else. +%% Looks for a file relative to a given directory + +-type find_file_rule() :: {ObjDirSuffix::string(), SrcDirSuffix::string()}. + +%% Looks for a source file relative to the object file name and directory + +-type find_source_rule() :: {ObjExtension::string(), SrcExtension::string(), + [find_file_rule()]}. + +keep_suffix_search_rules(Rules) -> + [T || {_,_,_}=T <- Rules]. + +-spec find_source(file:filename(), file:filename(), [find_source_rule()]) -> + {ok, file:filename()} | {error, not_found}. +find_source(Filename, Dir, Rules) -> + try_suffix_rules(keep_suffix_search_rules(Rules), Filename, Dir). + +try_suffix_rules(Rules, Filename, Dir) -> + Ext = filename:extension(Filename), + try_suffix_rules(Rules, filename:rootname(Filename, Ext), Dir, Ext). + +try_suffix_rules([{Ext,Src,Rules}|Rest], Root, Dir, Ext) + when is_list(Src), is_list(Rules) -> + case try_dir_rules(add_local_search(Rules), Root ++ Src, Dir) of + {ok, File} -> {ok, File}; + _Other -> + try_suffix_rules(Rest, Root, Dir, Ext) + end; +try_suffix_rules([_|Rest], Root, Dir, Ext) -> + try_suffix_rules(Rest, Root, Dir, Ext); +try_suffix_rules([], _Root, _Dir, _Ext) -> + {error, not_found}. + +%% ensuring we check the directory of the object file before any other directory +add_local_search(Rules) -> + Local = {"",""}, + [Local] ++ lists:filter(fun (X) -> X =/= Local end, Rules). + +try_dir_rules([{From, To}|Rest], Filename, Dir) + when is_list(From), is_list(To) -> + case try_dir_rule(Dir, Filename, From, To) of + {ok, File} -> {ok, File}; + error -> try_dir_rules(Rest, Filename, Dir) + end; +try_dir_rules([], _Filename, _Dir) -> + {error, not_found}. + +try_dir_rule(Dir, Filename, From, To) -> + case lists:suffix(From, Dir) of + true -> + NewDir = lists:sublist(Dir, 1, length(Dir)-length(From))++To, + Src = filename:join(NewDir, Filename), + case filelib:is_regular(Src) of + true -> {ok, Src}; + false -> find_regular_file(filelib:wildcard(Src)) + end; + false -> + error + end. + +find_regular_file([]) -> + error; +find_regular_file([File|Files]) -> + case filelib:is_regular(File) of + true -> {ok, File}; + false -> find_regular_file(Files) + end. +-endif. -- cgit v1.1 From d59b5fdd7b7adc8d814e36f731e98d3cd5eea7b9 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 5 Oct 2018 09:01:57 -0600 Subject: Bump to 3.7.0-rc2 --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 724f921..f17f6a2 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.7.0-rc2"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 21f0a4ebc233120f6634f010d082e71b97a8dd90 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 5 Oct 2018 09:03:29 -0600 Subject: Back to git-based versioning --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index f17f6a2..724f921 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.7.0-rc2"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 4d92f3c040680213127fc3faa9638e1cd34b50fa Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sat, 6 Oct 2018 10:45:06 -0600 Subject: remove unused app_compilers app env setting (#1903) --- src/rebar.app.src | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 724f921..6058efc 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -40,8 +40,8 @@ {pkg, rebar_pkg_resource}, {hg, rebar_hg_resource}]}, - {app_compilers, [{erl, rebar_erlc_compiler}]}, - {compilers, [rebar_compiler_xrl, rebar_compiler_yrl, rebar_compiler_mib, rebar_compiler_erl]}, + {compilers, [rebar_compiler_xrl, rebar_compiler_yrl, + rebar_compiler_mib, rebar_compiler_erl]}, {providers, [rebar_prv_app_discovery, rebar_prv_as, -- cgit v1.1 From 763fd05a0f6be023a4f646c55ddd72ae193e350b Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sat, 6 Oct 2018 18:52:20 -0600 Subject: fix yrl compiler and add test (#1906) --- src/rebar_compiler_yrl.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_compiler_yrl.erl b/src/rebar_compiler_yrl.erl index 765ea36..41d93b1 100644 --- a/src/rebar_compiler_yrl.erl +++ b/src/rebar_compiler_yrl.erl @@ -34,7 +34,7 @@ compile(Source, [{_, OutDir}], _, Opts) -> AllOpts = [{parserfile, Target} | Opts], AllOpts1 = [{includefile, filename:join(OutDir, I)} || {includefile, I} <- AllOpts, filename:pathtype(I) =:= relative], - case yeec:file(Source, AllOpts1 ++ [{return, true}]) of + case yecc:file(Source, AllOpts1 ++ [{return, true}]) of {ok, _} -> ok; {ok, _Mod, Ws} -> -- cgit v1.1 From 2986105bc2a5d4579d3cf174023917f147a6f6da Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sun, 7 Oct 2018 19:35:31 -0400 Subject: Re-add erlc compiler; plugins need it However, we add a once-printed deprecation warning for the new interface. --- src/rebar_erlc_compiler.erl | 841 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 841 insertions(+) create mode 100644 src/rebar_erlc_compiler.erl (limited to 'src') diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl new file mode 100644 index 0000000..e52791c --- /dev/null +++ b/src/rebar_erlc_compiler.erl @@ -0,0 +1,841 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +%% ------------------------------------------------------------------- +%% +%% rebar: Erlang Build Tools +%% +%% Copyright (c) 2009, 2010 Dave Smith (dizzyd@dizzyd.com) +%% +%% Permission is hereby granted, free of charge, to any person obtaining a copy +%% of this software and associated documentation files (the "Software"), to deal +%% in the Software without restriction, including without limitation the rights +%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%% copies of the Software, and to permit persons to whom the Software is +%% furnished to do so, subject to the following conditions: +%% +%% The above copyright notice and this permission notice shall be included in +%% all copies or substantial portions of the Software. +%% +%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%% THE SOFTWARE. +%% ------------------------------------------------------------------- +-module(rebar_erlc_compiler). + +-export([compile/1, compile/2, compile/3, + compile_dir/3, compile_dir/4, + compile_dirs/5, + clean/1]). + +-include("rebar.hrl"). +-include_lib("stdlib/include/erl_compile.hrl"). + +-define(ERLCINFO_VSN, 2). +-define(ERLCINFO_FILE, "erlcinfo"). +-type erlc_info_v() :: {digraph:vertex(), term()} | 'false'. +-type erlc_info_e() :: {digraph:vertex(), digraph:vertex()}. +-type erlc_info() :: {list(erlc_info_v()), list(erlc_info_e()), list(string())}. +-record(erlcinfo, { + vsn = ?ERLCINFO_VSN :: pos_integer(), + info = {[], [], []} :: erlc_info() +}). + +-type compile_opts() :: [compile_opt()]. +-type compile_opt() :: {recursive, boolean()}. + +-define(DEFAULT_OUTDIR, "ebin"). +-define(RE_PREFIX, "^(?!\\._)"). + +%% =================================================================== +%% Public API +%% =================================================================== + +%% Supported configuration variables: +%% +%% * erl_opts - Erlang list of options passed to compile:file/2 +%% It is also possible to specify platform specific +%% options by specifying a pair or a triplet where the +%% first string is a regex that is checked against the +%% string +%% +%% OtpRelease ++ "-" ++ SysArch ++ "-" ++ Words. +%% +%% where +%% +%% OtpRelease = erlang:system_info(otp_release). +%% SysArch = erlang:system_info(system_architecture). +%% Words = integer_to_list(8 * +%% erlang:system_info({wordsize, external})). +%% +%% E.g. to define HAVE_SENDFILE only on systems with +%% sendfile(), to define BACKLOG on Linux/FreeBSD as 128, +%% and to define 'old_inets' for R13 OTP release do: +%% +%% {erl_opts, [{platform_define, +%% "(linux|solaris|freebsd|darwin)", +%% 'HAVE_SENDFILE'}, +%% {platform_define, "(linux|freebsd)", +%% 'BACKLOG', 128}, +%% {platform_define, "R13", +%% 'old_inets'}]}. +%% + +%% @equiv compile(AppInfo, []) +-spec compile(rebar_app_info:t()) -> ok. +compile(AppInfo) when element(1, AppInfo) == app_info_t -> + compile(AppInfo, []). + +%% @doc compile an individual application. +-spec compile(rebar_app_info:t(), compile_opts()) -> ok. +compile(AppInfo, CompileOpts) when element(1, AppInfo) == app_info_t -> + warn_deprecated(), + Dir = rebar_utils:to_list(rebar_app_info:out_dir(AppInfo)), + RebarOpts = rebar_app_info:opts(AppInfo), + + SrcOpts = [check_last_mod, + {recursive, dir_recursive(RebarOpts, "src", CompileOpts)}], + MibsOpts = [check_last_mod, + {recursive, dir_recursive(RebarOpts, "mibs", CompileOpts)}], + + rebar_base_compiler:run(RebarOpts, + check_files([filename:join(Dir, File) + || File <- rebar_opts:get(RebarOpts, xrl_first_files, [])]), + filename:join(Dir, "src"), ".xrl", filename:join(Dir, "src"), ".erl", + fun compile_xrl/3, SrcOpts), + rebar_base_compiler:run(RebarOpts, + check_files([filename:join(Dir, File) + || File <- rebar_opts:get(RebarOpts, yrl_first_files, [])]), + filename:join(Dir, "src"), ".yrl", filename:join(Dir, "src"), ".erl", + fun compile_yrl/3, SrcOpts), + rebar_base_compiler:run(RebarOpts, + check_files([filename:join(Dir, File) + || File <- rebar_opts:get(RebarOpts, mib_first_files, [])]), + filename:join(Dir, "mibs"), ".mib", filename:join([Dir, "priv", "mibs"]), ".bin", + compile_mib(AppInfo), MibsOpts), + + SrcDirs = lists:map(fun(SrcDir) -> filename:join(Dir, SrcDir) end, + rebar_dir:src_dirs(RebarOpts, ["src"])), + OutDir = filename:join(Dir, outdir(RebarOpts)), + compile_dirs(RebarOpts, Dir, SrcDirs, OutDir, CompileOpts), + + ExtraDirs = rebar_dir:extra_src_dirs(RebarOpts), + F = fun(D) -> + case ec_file:is_dir(filename:join([Dir, D])) of + true -> compile_dirs(RebarOpts, Dir, [D], D, CompileOpts); + false -> ok + end + end, + lists:foreach(F, lists:map(fun(SrcDir) -> filename:join(Dir, SrcDir) end, ExtraDirs)). + +%% @hidden +%% these are kept for backwards compatibility but they're bad functions with +%% bad interfaces you probably shouldn't use +%% State/RebarOpts have to have src_dirs set and BaseDir must be the parent +%% directory of those src_dirs + +-spec compile(rebar_dict() | rebar_state:t(), file:name(), file:name()) -> ok. +compile(State, BaseDir, OutDir) when element(1, State) == state_t -> + compile(rebar_state:opts(State), BaseDir, OutDir, [{recursive, false}]); +compile(RebarOpts, BaseDir, OutDir) -> + compile(RebarOpts, BaseDir, OutDir, [{recursive, false}]). + +%% @hidden + +-spec compile(rebar_dict() | rebar_state:t(), file:name(), file:name(), compile_opts()) -> ok. +compile(State, BaseDir, OutDir, CompileOpts) when element(1, State) == state_t -> + compile(rebar_state:opts(State), BaseDir, OutDir, CompileOpts); +compile(RebarOpts, BaseDir, OutDir, CompileOpts) -> + warn_deprecated(), + SrcDirs = lists:map(fun(SrcDir) -> filename:join(BaseDir, SrcDir) end, + rebar_dir:src_dirs(RebarOpts, ["src"])), + compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, CompileOpts), + + ExtraDirs = rebar_dir:extra_src_dirs(RebarOpts), + F = fun(D) -> + case ec_file:is_dir(filename:join([BaseDir, D])) of + true -> compile_dirs(RebarOpts, BaseDir, [D], D, CompileOpts); + false -> ok + end + end, + lists:foreach(F, lists:map(fun(SrcDir) -> filename:join(BaseDir, SrcDir) end, ExtraDirs)). + +%% @equiv compile_dirs(Context, BaseDir, [Dir], Dir, [{recursive, false}]) +-spec compile_dir(rebar_dict() | rebar_state:t(), file:name(), file:name()) -> ok. +compile_dir(State, BaseDir, Dir) when element(1, State) == state_t -> + compile_dir(rebar_state:opts(State), BaseDir, Dir, [{recursive, false}]); +compile_dir(RebarOpts, BaseDir, Dir) -> + compile_dir(RebarOpts, BaseDir, Dir, [{recursive, false}]). + +%% @equiv compile_dirs(Context, BaseDir, [Dir], Dir, Opts) +-spec compile_dir(rebar_dict() | rebar_state:t(), file:name(), file:name(), compile_opts()) -> ok. +compile_dir(State, BaseDir, Dir, Opts) when element(1, State) == state_t -> + compile_dirs(rebar_state:opts(State), BaseDir, [Dir], Dir, Opts); +compile_dir(RebarOpts, BaseDir, Dir, Opts) -> + compile_dirs(RebarOpts, BaseDir, [Dir], Dir, Opts). + +%% @doc compile a list of directories with the given opts. +-spec compile_dirs(rebar_dict() | rebar_state:t(), + file:filename(), + [file:filename()], + file:filename(), + compile_opts()) -> ok. +compile_dirs(State, BaseDir, Dirs, OutDir, CompileOpts) when element(1, State) == state_t -> + compile_dirs(rebar_state:opts(State), BaseDir, Dirs, OutDir, CompileOpts); +compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, CompileOpts) -> + ErlOpts = rebar_opts:erl_opts(RebarOpts), + ?DEBUG("erlopts ~p", [ErlOpts]), + AllErlFiles = gather_src(RebarOpts, BaseDir, SrcDirs, CompileOpts), + ?DEBUG("files to compile ~p", [AllErlFiles]), + + %% Make sure that outdir is on the path + ok = filelib:ensure_dir(filename:join(OutDir, "dummy.beam")), + true = code:add_patha(filename:absname(OutDir)), + + G = init_erlcinfo(include_abs_dirs(ErlOpts, BaseDir), AllErlFiles, BaseDir, OutDir), + + {ParseTransforms, Rest} = split_source_files(AllErlFiles, ErlOpts), + NeededErlFiles = case needed_files(G, ErlOpts, RebarOpts, BaseDir, OutDir, ParseTransforms) of + [] -> needed_files(G, ErlOpts, RebarOpts, BaseDir, OutDir, Rest); + %% at least one parse transform in the opts needs updating, so recompile all + _ -> AllErlFiles + end, + + {ErlFirstFiles, ErlOptsFirst} = erl_first_files(RebarOpts, ErlOpts, BaseDir, NeededErlFiles), + {DepErls, OtherErls} = lists:partition( + fun(Source) -> digraph:in_degree(G, Source) > 0 end, + [File || File <- NeededErlFiles, not lists:member(File, ErlFirstFiles)]), + SubGraph = digraph_utils:subgraph(G, DepErls), + DepErlsOrdered = digraph_utils:topsort(SubGraph), + FirstErls = ErlFirstFiles ++ lists:reverse(DepErlsOrdered), + try + rebar_base_compiler:run( + RebarOpts, FirstErls, OtherErls, + fun(S, C) -> + ErlOpts1 = case lists:member(S, ErlFirstFiles) of + true -> ErlOptsFirst; + false -> ErlOpts + end, + internal_erl_compile(C, BaseDir, S, OutDir, ErlOpts1, RebarOpts) + end) + after + true = digraph:delete(SubGraph), + true = digraph:delete(G) + end, + ok. + +%% @doc remove compiled artifacts from an AppDir. +-spec clean(rebar_app_info:t()) -> 'ok'. +clean(AppInfo) -> + AppDir = rebar_app_info:out_dir(AppInfo), + + MibFiles = rebar_utils:find_files(filename:join([AppDir, "mibs"]), ?RE_PREFIX".*\\.mib\$"), + MIBs = [filename:rootname(filename:basename(MIB)) || MIB <- MibFiles], + rebar_file_utils:delete_each( + [filename:join([AppDir, "include",MIB++".hrl"]) || MIB <- MIBs]), + ok = rebar_file_utils:rm_rf(filename:join([AppDir, "priv/mibs/*.bin"])), + + YrlFiles = rebar_utils:find_files(filename:join([AppDir, "src"]), ?RE_PREFIX".*\\.[x|y]rl\$"), + rebar_file_utils:delete_each( + [rebar_utils:to_list(re:replace(F, "\\.[x|y]rl$", ".erl", [unicode])) + || F <- YrlFiles]), + + BinDirs = ["ebin"|rebar_dir:extra_src_dirs(rebar_app_info:opts(AppInfo))], + ok = clean_dirs(AppDir, BinDirs), + + %% Delete the build graph, if any + rebar_file_utils:rm_rf(erlcinfo_file(AppDir)). + +clean_dirs(_AppDir, []) -> ok; +clean_dirs(AppDir, [Dir|Rest]) -> + ok = rebar_file_utils:rm_rf(filename:join([AppDir, Dir, "*.beam"])), + %% Erlang compilation is recursive, so it's possible that we have a nested + %% directory structure in ebin with .beam files within. As such, we want + %% to scan whatever is left in the app's out_dir directory for sub-dirs which + %% satisfy our criteria. + BeamFiles = rebar_utils:find_files(filename:join([AppDir, Dir]), ?RE_PREFIX".*\\.beam\$"), + rebar_file_utils:delete_each(BeamFiles), + lists:foreach(fun(D) -> delete_dir(D, dirs(D)) end, dirs(filename:join([AppDir, Dir]))), + clean_dirs(AppDir, Rest). + + +%% =================================================================== +%% Internal functions +%% =================================================================== + +gather_src(Opts, BaseDir, Dirs, CompileOpts) -> + gather_src(Opts, filename:split(BaseDir), Dirs, [], CompileOpts). + +gather_src(_Opts, _BaseDirParts, [], Srcs, _CompileOpts) -> Srcs; +gather_src(Opts, BaseDirParts, [Dir|Rest], Srcs, CompileOpts) -> + DirParts = filename:split(Dir), + RelDir = case lists:prefix(BaseDirParts,DirParts) of + true -> + case lists:nthtail(length(BaseDirParts),DirParts) of + [] -> "."; + RestParts -> filename:join(RestParts) + end; + false -> Dir + end, + DirRecursive = dir_recursive(Opts, RelDir, CompileOpts), + gather_src(Opts, BaseDirParts, Rest, Srcs ++ rebar_utils:find_files(Dir, ?RE_PREFIX".*\\.erl\$", DirRecursive), CompileOpts). + +%% Get files which need to be compiled first, i.e. those specified in erl_first_files +%% and parse_transform options. Also produce specific erl_opts for these first +%% files, so that yet to be compiled parse transformations are excluded from it. +erl_first_files(Opts, ErlOpts, Dir, NeededErlFiles) -> + ErlFirstFilesConf = rebar_opts:get(Opts, erl_first_files, []), + valid_erl_first_conf(ErlFirstFilesConf), + NeededSrcDirs = lists:usort(lists:map(fun filename:dirname/1, NeededErlFiles)), + %% NOTE: order of files here is important! + ErlFirstFiles = + [filename:join(Dir, File) || File <- ErlFirstFilesConf, + lists:member(filename:join(Dir, File), NeededErlFiles)], + {ParseTransforms, ParseTransformsErls} = + lists:unzip(lists:flatmap( + fun(PT) -> + PTerls = [filename:join(D, module_to_erl(PT)) || D <- NeededSrcDirs], + [{PT, PTerl} || PTerl <- PTerls, lists:member(PTerl, NeededErlFiles)] + end, proplists:get_all_values(parse_transform, ErlOpts))), + ErlOptsFirst = lists:filter(fun({parse_transform, PT}) -> + not lists:member(PT, ParseTransforms); + (_) -> + true + end, ErlOpts), + {ErlFirstFiles ++ ParseTransformsErls, ErlOptsFirst}. + +split_source_files(SourceFiles, ErlOpts) -> + ParseTransforms = proplists:get_all_values(parse_transform, ErlOpts), + lists:partition(fun(Source) -> + lists:member(filename_to_atom(Source), ParseTransforms) + end, SourceFiles). + +filename_to_atom(F) -> list_to_atom(filename:rootname(filename:basename(F))). + +%% Get subset of SourceFiles which need to be recompiled, respecting +%% dependencies induced by given graph G. +needed_files(G, ErlOpts, RebarOpts, Dir, OutDir, SourceFiles) -> + lists:filter(fun(Source) -> + TargetBase = target_base(OutDir, Source), + Target = TargetBase ++ ".beam", + PrivIncludes = [{i, filename:join(Dir, Src)} + || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])], + AllOpts = [{outdir, filename:dirname(Target)} + ,{i, filename:join(Dir, "include")} + ,{i, Dir}] ++ PrivIncludes ++ ErlOpts, + %% necessary for erlang:function_exported/3 to work as expected + %% called here for clarity as it's required by both opts_changed/2 + %% and erl_compiler_opts_set/0 + _ = code:ensure_loaded(compile), + digraph:vertex(G, Source) > {Source, filelib:last_modified(Target)} + orelse opts_changed(AllOpts, TargetBase) + orelse erl_compiler_opts_set() + end, SourceFiles). + +maybe_rm_beam_and_edge(G, OutDir, Source) -> + %% This is NOT a double check it is the only check that the source file is actually gone + case filelib:is_regular(Source) of + true -> + %% Actually exists, don't delete + false; + false -> + Target = target_base(OutDir, Source) ++ ".beam", + ?DEBUG("Source ~ts is gone, deleting previous beam file if it exists ~ts", [Source, Target]), + file:delete(Target), + digraph:del_vertex(G, Source), + true + end. + +opts_changed(NewOpts, Target) -> + TotalOpts = case erlang:function_exported(compile, env_compiler_options, 0) of + true -> NewOpts ++ compile:env_compiler_options(); + false -> NewOpts + end, + case compile_info(Target) of + {ok, Opts} -> lists:any(fun effects_code_generation/1, lists:usort(TotalOpts) -- lists:usort(Opts)); + _ -> true + end. + +effects_code_generation(Option) -> + case Option of + beam -> false; + report_warnings -> false; + report_errors -> false; + return_errors-> false; + return_warnings-> false; + report -> false; + warnings_as_errors -> false; + binary -> false; + verbose -> false; + {cwd,_} -> false; + {outdir, _} -> false; + _ -> true + end. + +compile_info(Target) -> + case beam_lib:chunks(Target, [compile_info]) of + {ok, {_mod, Chunks}} -> + CompileInfo = proplists:get_value(compile_info, Chunks, []), + {ok, proplists:get_value(options, CompileInfo, [])}; + {error, beam_lib, Reason} -> + ?WARN("Couldn't read debug info from ~p for reason: ~p", [Target, Reason]), + {error, Reason} + end. + +erl_compiler_opts_set() -> + EnvSet = case os:getenv("ERL_COMPILER_OPTIONS") of + false -> false; + _ -> true + end, + %% return false if changed env opts would have been caught in opts_changed/2 + EnvSet andalso not erlang:function_exported(compile, env_compiler_options, 0). + +erlcinfo_file(Dir) -> + filename:join(rebar_dir:local_cache_dir(Dir), ?ERLCINFO_FILE). + +%% Get dependency graph of given Erls files and their dependencies (header files, +%% parse transforms, behaviours etc.) located in their directories or given +%% InclDirs. Note that last modification times stored in vertices already respect +%% dependencies induced by given graph G. +init_erlcinfo(InclDirs, Erls, Dir, OutDir) -> + G = digraph:new([acyclic]), + try restore_erlcinfo(G, InclDirs, Dir) + catch + _:_ -> + ?WARN("Failed to restore ~ts file. Discarding it.~n", [erlcinfo_file(Dir)]), + file:delete(erlcinfo_file(Dir)) + end, + Dirs = source_and_include_dirs(InclDirs, Erls), + %% A source file may have been renamed or deleted. Remove it from the graph + %% and remove any beam file for that source if it exists. + Modified = maybe_rm_beams_and_edges(G, OutDir, Erls), + Modified1 = lists:foldl(update_erlcinfo_fun(G, Dirs), Modified, Erls), + if Modified1 -> store_erlcinfo(G, InclDirs, Dir); not Modified1 -> ok end, + G. + +maybe_rm_beams_and_edges(G, Dir, Files) -> + Vertices = digraph:vertices(G), + case lists:filter(fun(File) -> + case filename:extension(File) =:= ".erl" of + true -> + maybe_rm_beam_and_edge(G, Dir, File); + false -> + false + end + end, lists:sort(Vertices) -- lists:sort(Files)) of + [] -> + false; + _ -> + true + end. + +source_and_include_dirs(InclDirs, Erls) -> + SourceDirs = lists:map(fun filename:dirname/1, Erls), + lists:usort(InclDirs ++ SourceDirs). + +update_erlcinfo(G, Dirs, Source) -> + case digraph:vertex(G, Source) of + {_, LastUpdated} -> + case filelib:last_modified(Source) of + 0 -> + %% The file doesn't exist anymore, + %% erase it from the graph. + %% All the edges will be erased automatically. + digraph:del_vertex(G, Source), + modified; + LastModified when LastUpdated < LastModified -> + modify_erlcinfo(G, Source, LastModified, filename:dirname(Source), Dirs); + _ -> + Modified = lists:foldl( + update_erlcinfo_fun(G, Dirs), + false, digraph:out_neighbours(G, Source)), + MaxModified = update_max_modified_deps(G, Source), + case Modified orelse MaxModified > LastUpdated of + true -> modified; + false -> unmodified + end + end; + false -> + modify_erlcinfo(G, Source, filelib:last_modified(Source), filename:dirname(Source), Dirs) + end. + +update_erlcinfo_fun(G, Dirs) -> + fun(Erl, Modified) -> + case update_erlcinfo(G, Dirs, Erl) of + modified -> true; + unmodified -> Modified + end + end. + +update_max_modified_deps(G, Source) -> + MaxModified = lists:max(lists:map( + fun(File) -> {_, MaxModified} = digraph:vertex(G, File), MaxModified end, + [Source|digraph:out_neighbours(G, Source)])), + digraph:add_vertex(G, Source, MaxModified), + MaxModified. + +modify_erlcinfo(G, Source, LastModified, Dir, Dirs) -> + {ok, Fd} = file:open(Source, [read]), + Incls = parse_attrs(Fd, [], Dir), + AbsIncls = expand_file_names(Incls, Dirs), + ok = file:close(Fd), + digraph:add_vertex(G, Source, LastModified), + digraph:del_edges(G, digraph:out_edges(G, Source)), + lists:foreach( + fun(Incl) -> + update_erlcinfo(G, Dirs, Incl), + digraph:add_edge(G, Source, Incl) + end, AbsIncls), + modified. + +restore_erlcinfo(G, InclDirs, Dir) -> + case file:read_file(erlcinfo_file(Dir)) of + {ok, Data} -> + % Since externally passed InclDirs can influence erlcinfo graph (see + % modify_erlcinfo), we have to check here that they didn't change. + #erlcinfo{vsn=?ERLCINFO_VSN, info={Vs, Es, InclDirs}} = + binary_to_term(Data), + lists:foreach( + fun({V, LastUpdated}) -> + digraph:add_vertex(G, V, LastUpdated) + end, Vs), + lists:foreach( + fun({_, V1, V2, _}) -> + digraph:add_edge(G, V1, V2) + end, Es); + {error, _} -> + ok + end. + +store_erlcinfo(G, InclDirs, Dir) -> + Vs = lists:map(fun(V) -> digraph:vertex(G, V) end, digraph:vertices(G)), + Es = lists:map(fun(E) -> digraph:edge(G, E) end, digraph:edges(G)), + File = erlcinfo_file(Dir), + ok = filelib:ensure_dir(File), + Data = term_to_binary(#erlcinfo{info={Vs, Es, InclDirs}}, [{compressed, 2}]), + file:write_file(File, Data). + +%% NOTE: If, for example, one of the entries in Files, refers to +%% gen_server.erl, that entry will be dropped. It is dropped because +%% such an entry usually refers to the beam file, and we don't pass a +%% list of OTP src dirs for finding gen_server.erl's full path. Also, +%% if gen_server.erl was modified, it's not rebar's task to compile a +%% new version of the beam file. Therefore, it's reasonable to drop +%% such entries. Also see process_attr(behaviour, Form, Includes). +-spec expand_file_names([file:filename()], + [file:filename()]) -> [file:filename()]. +expand_file_names(Files, Dirs) -> + %% We check if Files exist by itself or within the directories + %% listed in Dirs. + %% Return the list of files matched. + lists:flatmap( + fun(Incl) -> + case filelib:is_regular(Incl) of + true -> + [Incl]; + false -> + lists:flatmap( + fun(Dir) -> + FullPath = filename:join(Dir, Incl), + case filelib:is_regular(FullPath) of + true -> + [FullPath]; + false -> + [] + end + end, Dirs) + end + end, Files). + +-spec internal_erl_compile(rebar_dict(), file:filename(), file:filename(), + file:filename(), list(), rebar_dict()) -> + ok | {ok, any()} | {error, any(), any()}. +internal_erl_compile(Opts, Dir, Module, OutDir, ErlOpts, RebarOpts) -> + Target = target_base(OutDir, Module) ++ ".beam", + ok = filelib:ensure_dir(Target), + PrivIncludes = [{i, filename:join(Dir, Src)} + || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])], + AllOpts = [{outdir, filename:dirname(Target)}] ++ ErlOpts ++ PrivIncludes ++ + [{i, filename:join(Dir, "include")}, {i, Dir}, return], + case compile:file(Module, AllOpts) of + {ok, _Mod} -> + ok; + {ok, _Mod, Ws} -> + FormattedWs = format_error_sources(Ws, Opts), + rebar_base_compiler:ok_tuple(Module, FormattedWs); + {error, Es, Ws} -> + error_tuple(Module, Es, Ws, AllOpts, Opts) + end. + +error_tuple(Module, Es, Ws, AllOpts, Opts) -> + FormattedEs = format_error_sources(Es, Opts), + FormattedWs = format_error_sources(Ws, Opts), + rebar_base_compiler:error_tuple(Module, FormattedEs, FormattedWs, AllOpts). + +format_error_sources(Es, Opts) -> + [{rebar_base_compiler:format_error_source(Src, Opts), Desc} + || {Src, Desc} <- Es]. + +target_base(OutDir, Source) -> + filename:join(OutDir, filename:basename(Source, ".erl")). + +-spec compile_mib(rebar_app_info:t()) -> + fun((file:filename(), file:filename(), rebar_dict()) -> 'ok'). +compile_mib(AppInfo) -> + fun(Source, Target, Opts) -> + Dir = filename:dirname(Target), + Mib = filename:rootname(Target), + HrlFilename = Mib ++ ".hrl", + + AppInclude = filename:join([rebar_app_info:dir(AppInfo), "include"]), + + ok = filelib:ensure_dir(Target), + ok = filelib:ensure_dir(filename:join([AppInclude, "dummy.hrl"])), + + AllOpts = [{outdir, Dir} + ,{i, [Dir]}] ++ + rebar_opts:get(Opts, mib_opts, []), + + case snmpc:compile(Source, AllOpts) of + {ok, _} -> + MibToHrlOpts = + case proplists:get_value(verbosity, AllOpts, undefined) of + undefined -> + #options{specific = [], + cwd = rebar_dir:get_cwd()}; + Verbosity -> + #options{specific = [{verbosity, Verbosity}], + cwd = rebar_dir:get_cwd()} + end, + ok = snmpc:mib_to_hrl(Mib, Mib, MibToHrlOpts), + rebar_file_utils:mv(HrlFilename, AppInclude), + ok; + {error, compilation_failed} -> + ?FAIL + end + end. + +-spec compile_xrl(file:filename(), file:filename(), + rebar_dict()) -> 'ok'. +compile_xrl(Source, Target, Opts) -> + AllOpts = [{scannerfile, Target} | rebar_opts:get(Opts, xrl_opts, [])], + compile_xrl_yrl(Opts, Source, Target, AllOpts, leex). + +-spec compile_yrl(file:filename(), file:filename(), + rebar_dict()) -> 'ok'. +compile_yrl(Source, Target, Opts) -> + AllOpts = [{parserfile, Target} | rebar_opts:get(Opts, yrl_opts, [])], + compile_xrl_yrl(Opts, Source, Target, AllOpts, yecc). + +-spec compile_xrl_yrl(rebar_dict(), file:filename(), + file:filename(), list(), module()) -> 'ok'. +compile_xrl_yrl(_Opts, Source, Target, AllOpts, Mod) -> + %% FIX ME: should be the outdir or something + Dir = filename:dirname(filename:dirname(Target)), + AllOpts1 = [{includefile, filename:join(Dir, I)} || {includefile, I} <- AllOpts, + filename:pathtype(I) =:= relative], + case needs_compile(Source, Target) of + true -> + case Mod:file(Source, AllOpts1 ++ [{return, true}]) of + {ok, _} -> + ok; + {ok, _Mod, Ws} -> + rebar_base_compiler:ok_tuple(Source, Ws); + {error, Es, Ws} -> + rebar_base_compiler:error_tuple(Source, + Es, Ws, AllOpts1) + end; + false -> + skipped + end. + +needs_compile(Source, Target) -> + filelib:last_modified(Source) > filelib:last_modified(Target). + + + +-spec dirs(file:filename()) -> [file:filename()]. +dirs(Dir) -> + [F || F <- filelib:wildcard(filename:join([Dir, "*"])), filelib:is_dir(F)]. + +-spec delete_dir(file:filename(), [string()]) -> 'ok' | {'error', atom()}. +delete_dir(Dir, []) -> + file:del_dir(Dir); +delete_dir(Dir, Subdirs) -> + lists:foreach(fun(D) -> delete_dir(D, dirs(D)) end, Subdirs), + file:del_dir(Dir). + +parse_attrs(Fd, Includes, Dir) -> + case io:parse_erl_form(Fd, "") of + {ok, Form, _Line} -> + case erl_syntax:type(Form) of + attribute -> + NewIncludes = process_attr(Form, Includes, Dir), + parse_attrs(Fd, NewIncludes, Dir); + _ -> + parse_attrs(Fd, Includes, Dir) + end; + {eof, _} -> + Includes; + _Err -> + parse_attrs(Fd, Includes, Dir) + end. + +process_attr(Form, Includes, Dir) -> + AttrName = erl_syntax:atom_value(erl_syntax:attribute_name(Form)), + process_attr(AttrName, Form, Includes, Dir). + +process_attr(import, Form, Includes, _Dir) -> + case erl_syntax_lib:analyze_import_attribute(Form) of + {Mod, _Funs} -> + [module_to_erl(Mod)|Includes]; + Mod -> + [module_to_erl(Mod)|Includes] + end; +process_attr(file, Form, Includes, _Dir) -> + {File, _} = erl_syntax_lib:analyze_file_attribute(Form), + [File|Includes]; +process_attr(include, Form, Includes, _Dir) -> + [FileNode] = erl_syntax:attribute_arguments(Form), + File = erl_syntax:string_value(FileNode), + [File|Includes]; +process_attr(include_lib, Form, Includes, Dir) -> + [FileNode] = erl_syntax:attribute_arguments(Form), + RawFile = erl_syntax:string_value(FileNode), + maybe_expand_include_lib_path(RawFile, Dir) ++ Includes; +process_attr(behavior, Form, Includes, _Dir) -> + process_attr(behaviour, Form, Includes, _Dir); +process_attr(behaviour, Form, Includes, _Dir) -> + [FileNode] = erl_syntax:attribute_arguments(Form), + File = module_to_erl(erl_syntax:atom_value(FileNode)), + [File|Includes]; +process_attr(compile, Form, Includes, _Dir) -> + [Arg] = erl_syntax:attribute_arguments(Form), + case erl_syntax:concrete(Arg) of + {parse_transform, Mod} -> + [module_to_erl(Mod)|Includes]; + {core_transform, Mod} -> + [module_to_erl(Mod)|Includes]; + L when is_list(L) -> + lists:foldl( + fun({parse_transform, Mod}, Acc) -> + [module_to_erl(Mod)|Acc]; + ({core_transform, Mod}, Acc) -> + [module_to_erl(Mod)|Acc]; + (_, Acc) -> + Acc + end, Includes, L); + _ -> + Includes + end; +process_attr(_, _Form, Includes, _Dir) -> + Includes. + +module_to_erl(Mod) -> + atom_to_list(Mod) ++ ".erl". + +%% Given a path like "stdlib/include/erl_compile.hrl", return +%% "OTP_INSTALL_DIR/lib/erlang/lib/stdlib-x.y.z/include/erl_compile.hrl". +%% Usually a simple [Lib, SubDir, File1] = filename:split(File) should +%% work, but to not crash when an unusual include_lib path is used, +%% utilize more elaborate logic. +maybe_expand_include_lib_path(File, Dir) -> + File1 = filename:basename(File), + case filename:split(filename:dirname(File)) of + [_] -> + warn_and_find_path(File, Dir); + [Lib | SubDir] -> + case code:lib_dir(list_to_atom(Lib), list_to_atom(filename:join(SubDir))) of + {error, bad_name} -> + warn_and_find_path(File, Dir); + AppDir -> + [filename:join(AppDir, File1)] + end + end. + +%% The use of -include_lib was probably incorrect by the user but lets try to make it work. +%% We search in the outdir and outdir/../include to see if the header exists. +warn_and_find_path(File, Dir) -> + SrcHeader = filename:join(Dir, File), + case filelib:is_regular(SrcHeader) of + true -> + [SrcHeader]; + false -> + IncludeDir = filename:join(rebar_utils:droplast(filename:split(Dir))++["include"]), + IncludeHeader = filename:join(IncludeDir, File), + case filelib:is_regular(IncludeHeader) of + true -> + [filename:join(IncludeDir, File)]; + false -> + [] + end + end. + +%% +%% Ensure all files in a list are present and abort if one is missing +%% +-spec check_files([file:filename()]) -> [file:filename()]. +check_files(FileList) -> + [check_file(F) || F <- FileList]. + +check_file(File) -> + case filelib:is_regular(File) of + false -> ?ABORT("File ~p is missing, aborting\n", [File]); + true -> File + end. + +outdir(RebarOpts) -> + ErlOpts = rebar_opts:erl_opts(RebarOpts), + proplists:get_value(outdir, ErlOpts, ?DEFAULT_OUTDIR). + +include_abs_dirs(ErlOpts, BaseDir) -> + ErlOptIncludes = proplists:get_all_values(i, ErlOpts), + InclDirs = lists:map(fun(Incl) -> filename:absname(Incl) end, ErlOptIncludes), + [filename:join([BaseDir, "include"])|InclDirs]. + +dir_recursive(Opts, Dir, CompileOpts) when is_list(CompileOpts) -> + case proplists:get_value(recursive,CompileOpts) of + undefined -> rebar_dir:recursive(Opts, Dir); + Recursive -> Recursive + end. + +valid_erl_first_conf(FileList) -> + Strs = filter_file_list(FileList), + case rebar_utils:is_list_of_strings(Strs) of + true -> true; + false -> ?ABORT("An invalid file list (~p) was provided as part of your erl_first_files directive", + [FileList]) + end. + +filter_file_list(FileList) -> + Atoms = lists:filter( fun(X) -> is_atom(X) end, FileList), + case Atoms of + [] -> + FileList; + _ -> + atoms_in_erl_first_files_warning(Atoms), + lists:filter( fun(X) -> not(is_atom(X)) end, FileList) + end. + +atoms_in_erl_first_files_warning(Atoms) -> + W = "You have provided atoms as file entries in erl_first_files; " + "erl_first_files only expects lists of filenames as strings. " + "The following modules (~p) may not work as expected and it is advised " + "that you change these entires to string format " + "(e.g., \"src/module.erl\") ", + ?WARN(W, [Atoms]). + +warn_deprecated() -> + case get({deprecate_warn, ?MODULE}) of + undefined -> + ?WARN("Calling deprecated ~p compiler module. This module has been " + "replaced by rebar_compiler and rebar_compiler_erl, but will " + "remain available.", [?MODULE]), + put({deprecate_warn, ?MODULE}, true), + ok; + _ -> + ok + end. -- cgit v1.1 From 1df574a6f55506d606020d64f42c33f8bbf16a7b Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Wed, 10 Oct 2018 11:53:27 -0600 Subject: throw builder error when returned from project build (#1909) --- src/rebar_prv_compile.erl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index 54bdee2..a509704 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -208,8 +208,12 @@ build_app(AppInfo, State) -> %% load plugins since thats where project builders would be PluginDepsPaths = rebar_state:code_paths(State, all_plugin_deps), code:add_pathsa(PluginDepsPaths), - Module:build(AppInfo), - rebar_utils:remove_from_code_path(PluginDepsPaths); + case Module:build(AppInfo) of + ok -> + rebar_utils:remove_from_code_path(PluginDepsPaths); + {error, Reason} -> + throw({error, {Module, Reason}}) + end; _ -> throw(?PRV_ERROR({unknown_project_type, rebar_app_info:name(AppInfo), Type})) end -- cgit v1.1 From 9d788d893620f868b9c0ee00ddec8ae4d5d8fea7 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 5 Oct 2018 07:57:20 -0400 Subject: Abstracted path management Move path management out of rebar_utils manual code path function handling (which we leave there for backwards compat), and centralize them to allow easier coordination of paths between plugins and deps. On top of path handling, do a check of loaded modules to only purge and reload those that actually need it done in order to prevent all kinds of weird interaction and accidental purge kills. It also allows the possible cohabitation of both at once, with a "in case of conflict pick X" as a policy Changing path handling in providers also highlighted a bunch of bugs in some tests and appears to fix some in other providers, specifically around plugins. --- src/rebar_api.erl | 17 ++++ src/rebar_hooks.erl | 8 +- src/rebar_packages.erl | 2 +- src/rebar_paths.erl | 177 ++++++++++++++++++++++++++++++++++++++++++ src/rebar_plugins.erl | 4 +- src/rebar_prv_common_test.erl | 12 ++- src/rebar_prv_compile.erl | 36 ++++----- src/rebar_prv_dialyzer.erl | 5 +- src/rebar_prv_edoc.erl | 4 +- src/rebar_prv_eunit.erl | 8 +- src/rebar_prv_xref.erl | 5 +- 11 files changed, 233 insertions(+), 45 deletions(-) create mode 100644 src/rebar_paths.erl (limited to 'src') diff --git a/src/rebar_api.erl b/src/rebar_api.erl index 4dabe8a..00eb054 100644 --- a/src/rebar_api.erl +++ b/src/rebar_api.erl @@ -9,6 +9,8 @@ expand_env_variable/3, get_arch/0, wordsize/0, + set_paths/2, + unset_paths/2, add_deps_to_path/1, restore_code_path/1, processing_base_dir/1, @@ -67,6 +69,21 @@ get_arch() -> wordsize() -> rebar_utils:wordsize(). +%% @doc Set code paths. Takes arguments of the form +%% `[plugins, deps]' or `[deps, plugins]' and ensures the +%% project's app and dependencies are set in the right order +%% for the next bit of execution +-spec set_paths(rebar_paths:targets(), rebar_state:t()) -> ok. +set_paths(List, State) -> + rebar_paths:set_paths(List, State). + +%% @doc Unsets code paths. Takes arguments of the form +%% `[plugins, deps]' or `[deps, plugins]' and ensures the +%% paths are no longer active. +-spec unset_paths(rebar_paths:targets(), rebar_state:t()) -> ok. +unset_paths(List, State) -> + rebar_paths:unset_paths(List, State). + %% @doc Add deps to the code path -spec add_deps_to_path(rebar_state:t()) -> ok. add_deps_to_path(State) -> diff --git a/src/rebar_hooks.erl b/src/rebar_hooks.erl index ec6fe31..f2ef8a3 100644 --- a/src/rebar_hooks.erl +++ b/src/rebar_hooks.erl @@ -42,8 +42,9 @@ run_provider_hooks_(Dir, Type, Command, Providers, TypeHooks, State) -> [] -> State; HookProviders -> - PluginDepsPaths = lists:usort(rebar_state:code_paths(State, all_plugin_deps)), - code:add_pathsa(PluginDepsPaths), + %PluginDepsPaths = lists:usort(rebar_state:code_paths(State, all_plugin_deps)), + %code:add_pathsa(PluginDepsPaths), + rebar_paths:set_paths([plugins, deps], State), Providers1 = rebar_state:providers(State), State1 = rebar_state:providers(rebar_state:dir(State, Dir), Providers++Providers1), case rebar_core:do(HookProviders, State1) of @@ -51,7 +52,8 @@ run_provider_hooks_(Dir, Type, Command, Providers, TypeHooks, State) -> ?DEBUG(format_error({bad_provider, Type, Command, ProviderName}), []), throw(?PRV_ERROR({bad_provider, Type, Command, ProviderName})); {ok, State2} -> - rebar_utils:remove_from_code_path(PluginDepsPaths), + %rebar_utils:remove_from_code_path(PluginDepsPaths), + rebar_paths:set_paths([deps, plugins], State2), State2 end end. diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 7676213..096948c 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -82,7 +82,7 @@ get_package(Dep, Vsn, Hash, Repos, Table, State) -> not_found end. -new_package_table() -> +new_package_table() -> ?PACKAGE_TABLE = ets:new(?PACKAGE_TABLE, [named_table, public, ordered_set, {keypos, 2}]), ets:insert(?PACKAGE_TABLE, {?PACKAGE_INDEX_VERSION, package_index_version}). diff --git a/src/rebar_paths.erl b/src/rebar_paths.erl new file mode 100644 index 0000000..bb43897 --- /dev/null +++ b/src/rebar_paths.erl @@ -0,0 +1,177 @@ +-module(rebar_paths). +-include("rebar.hrl"). + +-type target() :: deps | plugins. +-type targets() :: [target(), ...]. +-export_type([target/0, targets/0]). +-export([set_paths/2, unset_paths/2]). + +-spec set_paths(targets(), rebar_state:t()) -> ok. +set_paths(UserTargets, State) -> + Targets = normalize_targets(UserTargets), + GroupPaths = path_groups(Targets, State), + Paths = lists:append([P || {_, P} <- GroupPaths]), + [code:del_path(P) || P <- Paths], + code:add_pathsa(lists:reverse(Paths)), + % set path breaks with escripts + %true = code:set_path(lists:append([P || {_, P} <- GroupPaths])), + AppGroups = app_groups(Targets, State), + purge_and_load(AppGroups, code:all_loaded(), sets:new()), + ok. + +-spec unset_paths(targets(), rebar_state:t()) -> ok. +unset_paths(UserTargets, State) -> + Targets = normalize_targets(UserTargets), + GroupPaths = path_groups(Targets, State), + Paths = lists:append([P || {_, P} <- GroupPaths]), + [code:del_path(P) || P <- Paths], + purge(Paths, code:all_loaded()), + ok. + + +%% The paths are to be set in the reverse order; i.e. the default +%% path is always last when possible (minimize cases where a build +%% tool version clashes with an app's), and put the highest priorities +%% first. +-spec normalize_targets(targets()) -> targets(). +normalize_targets(List) -> + %% Plan for the eventuality of getting values piped in + %% from future versions of rebar3, possibly from plugins and so on, + %% which means we'd risk failing kind of violently. We only support + %% deps and plugins + TmpList = lists:foldl( + fun(deps, [deps | _] = Acc) -> Acc; + (plugins, [plugins | _] = Acc) -> Acc; + (deps, Acc) -> [deps | Acc -- [deps]]; + (plugins, Acc) -> [plugins | Acc -- [plugins]]; + (_, Acc) -> Acc + end, + [], + List + ), + lists:reverse(TmpList). + +purge_and_load([], _, _) -> + ok; +purge_and_load([{_Group, Apps}|Rest], ModPaths, Seen) -> + %% We have: a list of all applications in the current priority group, + %% a list of all loaded modules with their active path, and a list of + %% seen applications. + %% + %% We do the following: + %% 1. identify the apps that have not been solved yet + %% 2. find the paths for all apps in the current group + %% 3. unload and reload apps that may have changed paths in order + %% to get updated module lists and specs + %% (we ignore started apps and apps that have not run for this) + %% 4. create a list of modules to check from that app list + %% 5. check the modules to match their currently loaded paths with + %% the path set from the apps in the current group; modules + %% that differ must be purged; others can stay + + %% 1) + AppNames = [AppName || App <- Apps, + AppName <- [rebar_app_info:name(App)], + not sets:is_element(AppName, Seen)], + GoodApps = [App || AppName <- AppNames, + App <- Apps, + rebar_app_info:name(App) =:= AppName], + %% 2) + %% TODO: add extra dirs (and test), and possibly the stdlib + GoodAppPaths = [rebar_app_info:ebin_dir(App) || App <- GoodApps], + %% ++ [code:lib_dir()], + %% 3) + [begin + AtomApp = binary_to_atom(AppName, utf8), + %% blind load/unload won't interrupt an already-running app, + %% preventing odd errors, maybe! + case application:unload(AtomApp) of + ok -> application:load(AtomApp); + _ -> ok + end + end || AppName <- AppNames, + %% Shouldn't unload ourselves; rebar runs without ever + %% being started and unloading breaks logging! + AppName =/= <<"rebar">>], + + %% 4) + CandidateMods = lists:append( + %% Start by asking the currently loaded app (if loaded) + %% since it would be the primary source of conflicting modules + [case application:get_key(AppName, modules) of + {ok, Mods} -> + Mods; + undefined -> + %% if not found, parse the app file on disk, in case + %% the app's modules are used without it being loaded + case rebar_app_info:app_details(App) of + [] -> []; + Details -> proplists:get_value(modules, Details, []) + end + end || App <- GoodApps, + AppName <- [binary_to_atom(rebar_app_info:name(App), utf8)]] + ), + + %% 5) + Mods = misloaded_modules(CandidateMods, GoodAppPaths, ModPaths), + [purge_mod(Mod) || Mod <- Mods], + purge_and_load(Rest, ModPaths, + sets:union(Seen, sets:from_list(AppNames))). + + +purge(Paths, ModPaths) -> + lists:map(fun purge_mod/1, lists:usort( + [Mod || {Mod, Path} <- ModPaths, + is_list(Path), % not 'preloaded' or mocked + any_prefix(Path, Paths)] + )). + +misloaded_modules(Mods, GoodAppPaths, ModPaths) -> + %% Identify paths that are invalid; i.e. app paths that cover an + %% app in the desired group, but are not in the desired group. + lists:usort( + [purge_mod(Mod) + || Mod <- Mods, + {_, Path} <- [lists:keyfind(Mod, 1, ModPaths)], + is_list(Path), % not 'preloaded' or mocked + not any_prefix(Path, GoodAppPaths)] + ). + +any_prefix(Path, Paths) -> + lists:any(fun(P) -> lists:prefix(P, Path) end, Paths). + +%% assume paths currently set are good; only unload a module so next call +%% uses the correctly set paths +purge_mod(Mod) -> + case erlang:check_process_code(self(), Mod) of + false -> + code:purge(Mod), + code:delete(Mod); + _ -> + %% cannot purge safely without killing ourselves + code:soft_purge(Mod) andalso + code:delete(Mod) + end. + +path_groups(Targets, State) -> + [{Target, get_paths(Target, State)} || Target <- Targets]. + +app_groups(Targets, State) -> + [{Target, get_apps(Target, State)} || Target <- Targets]. + +get_paths(deps, State) -> + rebar_state:code_paths(State, all_deps); +get_paths(plugins, State) -> + rebar_state:code_paths(State, all_plugin_deps). + +get_apps(deps, State) -> + %% The code paths for deps also include the top level apps + %% and the extras, which we don't have here; we have to + %% add the apps by hand + rebar_state:all_deps(State) ++ + case rebar_state:project_apps(State) of + undefined -> []; + List -> List + end; +get_apps(plugins, State) -> + rebar_state:all_plugin_deps(State). diff --git a/src/rebar_plugins.erl b/src/rebar_plugins.erl index d58675d..a9c7e00 100644 --- a/src/rebar_plugins.erl +++ b/src/rebar_plugins.erl @@ -122,12 +122,10 @@ handle_plugin(Profile, Plugin, State, Upgrade) -> %% Add newly built deps and plugin to code path State3 = rebar_state:update_all_plugin_deps(State2, Apps), NewCodePaths = [rebar_app_info:ebin_dir(A) || A <- ToBuild], - AllPluginEbins = filelib:wildcard(filename:join([rebar_dir:plugins_dir(State), "*", "ebin"])), - CodePaths = PreBuiltPaths++(AllPluginEbins--ToBuild), - code:add_pathsa(NewCodePaths++CodePaths), %% Store plugin code paths so we can remove them when compiling project apps State4 = rebar_state:update_code_paths(State3, all_plugin_deps, PreBuiltPaths++NewCodePaths), + rebar_paths:set_paths([plugins, deps], State4), {plugin_providers(Plugin), State4} catch diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index 9e71ee7..3d3bd8a 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -58,7 +58,7 @@ do(State) -> do(State, Tests) -> ?INFO("Running Common Test suites...", []), - rebar_utils:update_code(rebar_state:code_paths(State, all_deps), [soft_purge]), + rebar_paths:set_paths([deps, plugins], State), %% Run ct provider prehooks Providers = rebar_state:providers(State), @@ -73,14 +73,14 @@ do(State, Tests) -> ok -> %% Run ct provider post hooks for all project apps and top level project hooks rebar_hooks:run_project_and_app_hooks(Cwd, post, ?PROVIDER, Providers, State), - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), + rebar_paths:set_paths([plugins, deps], State), {ok, State}; Error -> - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), + rebar_paths:set_paths([plugins, deps], State), Error end; Error -> - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), + rebar_paths:set_paths([plugins, deps], State), Error end. @@ -250,11 +250,9 @@ select_tests(State, ProjectApps, CmdOpts, CfgOpts) -> end, SysConfigs), %% NB: load the applications (from user directories too) to support OTP < 17 %% to our best ability. - OldPath = code:get_path(), - code:add_pathsa(rebar_state:code_paths(State, all_deps)), + rebar_paths:set_paths([deps, plugins], State), [application:load(Application) || Config <- Configs, {Application, _} <- Config], rebar_utils:reread_config(Configs), - code:set_path(OldPath), Opts = merge_opts(CmdOpts,CfgOpts), discover_tests(State, ProjectApps, Opts). diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index a509704..405f73a 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -37,10 +37,7 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> IsDepsOnly = is_deps_only(State), - DepsPaths = rebar_state:code_paths(State, all_deps), - PluginDepsPaths = rebar_state:code_paths(State, all_plugin_deps), - rebar_utils:remove_from_code_path(PluginDepsPaths), - code:add_pathsa(DepsPaths), + rebar_paths:set_paths([deps, plugins], State), Providers = rebar_state:providers(State), Deps = rebar_state:deps_to_build(State), @@ -50,11 +47,10 @@ do(State) -> true -> State; false -> - handle_project_apps(DepsPaths, Providers, State) + handle_project_apps(Providers, State) end, - rebar_utils:cleanup_code_path(rebar_state:code_paths(State1, default) - ++ rebar_state:code_paths(State, all_plugin_deps)), + rebar_paths:set_paths([plugins, deps], State1), {ok, State1}. @@ -62,7 +58,7 @@ is_deps_only(State) -> {Args, _} = rebar_state:command_parsed_args(State), proplists:get_value(deps_only, Args, false). -handle_project_apps(DepsPaths, Providers, State) -> +handle_project_apps(Providers, State) -> Cwd = rebar_state:dir(State), ProjectApps = rebar_state:project_apps(State), {ok, ProjectApps1} = rebar_digraph:compile_order(ProjectApps), @@ -76,7 +72,7 @@ handle_project_apps(DepsPaths, Providers, State) -> %% projects with structures like /apps/foo,/apps/bar,/test build_extra_dirs(State, ProjectApps2), - State3 = update_code_paths(State2, ProjectApps2, DepsPaths), + State3 = update_code_paths(State2, ProjectApps2), rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State2), case rebar_state:has_all_artifacts(State3) of @@ -176,11 +172,10 @@ compile(State, Providers, AppInfo) -> %% The rebar_otp_app compilation step is safe regarding the %% overall path management, so we can just load all plugins back %% in memory. - PluginDepsPaths = rebar_state:code_paths(State, all_plugin_deps), - code:add_pathsa(PluginDepsPaths), + rebar_paths:set_paths([plugins, deps], State), AppFileCompileResult = rebar_otp_app:compile(State, AppInfo4), %% Clean up after ourselves, leave things as they were. - rebar_utils:remove_from_code_path(PluginDepsPaths), + rebar_paths:set_paths([deps, plugins], State), case AppFileCompileResult of {ok, AppInfo5} -> @@ -206,21 +201,22 @@ build_app(AppInfo, State) -> case lists:keyfind(Type, 1, ProjectBuilders) of {_, Module} -> %% load plugins since thats where project builders would be - PluginDepsPaths = rebar_state:code_paths(State, all_plugin_deps), - code:add_pathsa(PluginDepsPaths), - case Module:build(AppInfo) of - ok -> - rebar_utils:remove_from_code_path(PluginDepsPaths); - {error, Reason} -> - throw({error, {Module, Reason}}) + rebar_paths:set_paths([plugins, deps], State), + Res = Module:build(AppInfo), + rebar_paths:set_paths([deps, plugins], State), + case Res of + ok -> ok; + {error, Reason} -> throw({error, {Module, Reason}}) end; _ -> throw(?PRV_ERROR({unknown_project_type, rebar_app_info:name(AppInfo), Type})) end end. -update_code_paths(State, ProjectApps, DepsPaths) -> + +update_code_paths(State, ProjectApps) -> ProjAppsPaths = paths_for_apps(ProjectApps), ExtrasPaths = paths_for_extras(State, ProjectApps), + DepsPaths = rebar_state:code_paths(State, all_deps), rebar_state:code_paths(State, all_deps, DepsPaths ++ ProjAppsPaths ++ ExtrasPaths). paths_for_apps(Apps) -> paths_for_apps(Apps, []). diff --git a/src/rebar_prv_dialyzer.erl b/src/rebar_prv_dialyzer.erl index 99a7698..585051c 100644 --- a/src/rebar_prv_dialyzer.erl +++ b/src/rebar_prv_dialyzer.erl @@ -85,7 +85,8 @@ short_desc() -> do(State) -> maybe_fix_env(), ?INFO("Dialyzer starting, this may take a while...", []), - code:add_pathsa(rebar_state:code_paths(State, all_deps)), + rebar_paths:unset_paths([plugins], State), % no plugins in analysis + rebar_paths:set_paths([deps], State), Plt = get_plt(State), try @@ -104,7 +105,7 @@ do(State) -> throw:{output_file_error, _, _} = Error -> ?PRV_ERROR(Error) after - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)) + rebar_paths:set_paths([plugins,deps], State) end. %% This is used to workaround dialyzer quirk discussed here diff --git a/src/rebar_prv_edoc.erl b/src/rebar_prv_edoc.erl index 9517335..c78296a 100644 --- a/src/rebar_prv_edoc.erl +++ b/src/rebar_prv_edoc.erl @@ -32,7 +32,7 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()} | {error, {module(), any()}}. do(State) -> - code:add_pathsa(rebar_state:code_paths(State, all_deps)), + rebar_paths:set_paths([deps, plugins], State), ProjectApps = rebar_state:project_apps(State), Providers = rebar_state:providers(State), EdocOpts = rebar_state:get(State, edoc_opts, []), @@ -64,7 +64,7 @@ do(State) -> {app_failed, AppName} end, rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State), - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), + rebar_paths:set_paths([plugins, deps], State), case Res of {app_failed, App} -> ?PRV_ERROR({app_failed, App}); diff --git a/src/rebar_prv_eunit.erl b/src/rebar_prv_eunit.erl index 4b71416..f120926 100644 --- a/src/rebar_prv_eunit.erl +++ b/src/rebar_prv_eunit.erl @@ -54,7 +54,7 @@ do(State, Tests) -> ?INFO("Performing EUnit tests...", []), setup_name(State), - rebar_utils:update_code(rebar_state:code_paths(State, all_deps), [soft_purge]), + rebar_paths:set_paths([deps, plugins], State), %% Run eunit provider prehooks Providers = rebar_state:providers(State), @@ -67,14 +67,14 @@ do(State, Tests) -> {ok, State1} -> %% Run eunit provider posthooks rebar_hooks:run_project_and_app_hooks(Cwd, post, ?PROVIDER, Providers, State1), - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), + rebar_paths:set_paths([plugins, deps], State), {ok, State1}; Error -> - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), + rebar_paths:set_paths([plugins, deps], State), Error end; Error -> - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), + rebar_paths:set_paths([plugins, deps], State), Error end. diff --git a/src/rebar_prv_xref.erl b/src/rebar_prv_xref.erl index 2405ebb..a6b1f73 100644 --- a/src/rebar_prv_xref.erl +++ b/src/rebar_prv_xref.erl @@ -36,8 +36,7 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - OldPath = code:get_path(), - code:add_pathsa(rebar_state:code_paths(State, all_deps)), + rebar_paths:set_paths([deps, plugins], State), XrefChecks = prepare(State), XrefIgnores = rebar_state:get(State, xref_ignores, []), %% Run xref checks @@ -48,7 +47,7 @@ do(State) -> QueryChecks = rebar_state:get(State, xref_queries, []), QueryResults = lists:foldl(fun check_query/2, [], QueryChecks), stopped = xref:stop(xref), - rebar_utils:cleanup_code_path(OldPath), + rebar_paths:set_paths([plugins, deps], State), case XrefResults =:= [] andalso QueryResults =:= [] of true -> {ok, State}; -- cgit v1.1 From 311ee6b1371c3eea3611dc5d7945b1b5667c75bd Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sat, 6 Oct 2018 11:38:33 -0400 Subject: Fix a bug in compiler path handling Also handle some formatting --- src/rebar_compiler.erl | 7 +++---- src/rebar_paths.erl | 13 ++++++------- src/rebar_prv_compile.erl | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-) (limited to 'src') diff --git a/src/rebar_compiler.erl b/src/rebar_compiler.erl index c2c514a..6e94cb2 100644 --- a/src/rebar_compiler.erl +++ b/src/rebar_compiler.erl @@ -34,11 +34,10 @@ -define(RE_PREFIX, "^(?!\\._)"). compile_all(Compilers, AppInfo) -> - OutDir = rebar_utils:to_list(rebar_app_info:out_dir(AppInfo)), - + EbinDir = rebar_utils:to_list(rebar_app_info:ebin_dir(AppInfo)), %% Make sure that outdir is on the path - ok = rebar_file_utils:ensure_dir(OutDir), - true = code:add_patha(filename:absname(OutDir)), + ok = rebar_file_utils:ensure_dir(EbinDir), + true = code:add_patha(filename:absname(EbinDir)), %% necessary for erlang:function_exported/3 to work as expected %% called here for clarity as it's required by both opts_changed/2 diff --git a/src/rebar_paths.erl b/src/rebar_paths.erl index bb43897..71f0016 100644 --- a/src/rebar_paths.erl +++ b/src/rebar_paths.erl @@ -130,11 +130,10 @@ misloaded_modules(Mods, GoodAppPaths, ModPaths) -> %% Identify paths that are invalid; i.e. app paths that cover an %% app in the desired group, but are not in the desired group. lists:usort( - [purge_mod(Mod) - || Mod <- Mods, - {_, Path} <- [lists:keyfind(Mod, 1, ModPaths)], - is_list(Path), % not 'preloaded' or mocked - not any_prefix(Path, GoodAppPaths)] + [Mod || Mod <- Mods, + {_, Path} <- [lists:keyfind(Mod, 1, ModPaths)], + is_list(Path), % not 'preloaded' or mocked + not any_prefix(Path, GoodAppPaths)] ). any_prefix(Path, Paths) -> @@ -168,10 +167,10 @@ get_apps(deps, State) -> %% The code paths for deps also include the top level apps %% and the extras, which we don't have here; we have to %% add the apps by hand - rebar_state:all_deps(State) ++ case rebar_state:project_apps(State) of undefined -> []; List -> List - end; + end ++ + rebar_state:all_deps(State); get_apps(plugins, State) -> rebar_state:all_plugin_deps(State). diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index 405f73a..4b36636 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -115,7 +115,7 @@ copy_and_build_project_apps(State, Providers, Apps) -> rebar_app_info:dir(AppInfo), rebar_app_info:out_dir(AppInfo)) || AppInfo <- Apps], - code:add_pathsa([rebar_app_info:out_dir(AppInfo) || AppInfo <- Apps]), + code:add_pathsa([rebar_app_info:ebin_dir(AppInfo) || AppInfo <- Apps]), [compile(State, Providers, AppInfo) || AppInfo <- Apps]. -- cgit v1.1 From af5cecd8eec9692f43d04ad53c8f28734012b873 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 8 Oct 2018 12:41:30 -0400 Subject: Clean path code, add tests, add clash detection Some finishing touch to that code --- src/rebar_paths.erl | 47 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/rebar_paths.erl b/src/rebar_paths.erl index 71f0016..23fc755 100644 --- a/src/rebar_paths.erl +++ b/src/rebar_paths.erl @@ -5,6 +5,11 @@ -type targets() :: [target(), ...]. -export_type([target/0, targets/0]). -export([set_paths/2, unset_paths/2]). +-export([clashing_apps/2]). + +-ifdef(TEST). +-export([misloaded_modules/3]). +-endif. -spec set_paths(targets(), rebar_state:t()) -> ok. set_paths(UserTargets, State) -> @@ -13,8 +18,8 @@ set_paths(UserTargets, State) -> Paths = lists:append([P || {_, P} <- GroupPaths]), [code:del_path(P) || P <- Paths], code:add_pathsa(lists:reverse(Paths)), - % set path breaks with escripts - %true = code:set_path(lists:append([P || {_, P} <- GroupPaths])), + % set path breaks with escripts; we gotta do it by hand + % true = code:set_path(lists:append([P || {_, P} <- GroupPaths])), AppGroups = app_groups(Targets, State), purge_and_load(AppGroups, code:all_loaded(), sets:new()), ok. @@ -28,6 +33,16 @@ unset_paths(UserTargets, State) -> purge(Paths, code:all_loaded()), ok. +clashing_apps(Targets, State) -> + AppGroups = app_groups(Targets, State), + AppNames = [{G, sets:from_list( + [rebar_app_info:name(App) || App <- Apps] + )} || {G, Apps} <- AppGroups], + clashing_app_names(sets:new(), AppNames, []). + +%%%%%%%%%%%%%%% +%%% PRIVATE %%% +%%%%%%%%%%%%%%% %% The paths are to be set in the reverse order; i.e. the default %% path is always last when possible (minimize cases where a build @@ -77,9 +92,9 @@ purge_and_load([{_Group, Apps}|Rest], ModPaths, Seen) -> App <- Apps, rebar_app_info:name(App) =:= AppName], %% 2) - %% TODO: add extra dirs (and test), and possibly the stdlib + %% (no need for extra_src_dirs since those get put into ebin; + %% also no need for OTP libs; we want to allow overtaking them) GoodAppPaths = [rebar_app_info:ebin_dir(App) || App <- GoodApps], - %% ++ [code:lib_dir()], %% 3) [begin AtomApp = binary_to_atom(AppName, utf8), @@ -118,12 +133,12 @@ purge_and_load([{_Group, Apps}|Rest], ModPaths, Seen) -> purge_and_load(Rest, ModPaths, sets:union(Seen, sets:from_list(AppNames))). - purge(Paths, ModPaths) -> + SortedPaths = lists:sort(Paths), lists:map(fun purge_mod/1, lists:usort( [Mod || {Mod, Path} <- ModPaths, is_list(Path), % not 'preloaded' or mocked - any_prefix(Path, Paths)] + any_prefix(Path, SortedPaths)] )). misloaded_modules(Mods, GoodAppPaths, ModPaths) -> @@ -152,6 +167,26 @@ purge_mod(Mod) -> code:delete(Mod) end. + +%% This is a tricky O(n²) check since we want to +%% know whether an app clashes with any of the top priority groups. +%% +%% For example, let's say we have `[deps, plugins]', then we want +%% to find the plugins that clash with deps: +%% +%% `[{deps, [ClashingPlugins]}, {plugins, []}]' +%% +%% In case we'd ever have alternative or additional types, we can +%% find all clashes from other 'groups'. +clashing_app_names(_, [], Acc) -> + lists:reverse(Acc); +clashing_app_names(PrevNames, [{G,AppNames} | Rest], Acc) -> + CurrentNames = sets:subtract(AppNames, PrevNames), + NextNames = sets:subtract(sets:union([A || {_, A} <- Rest]), PrevNames), + Clashes = sets:intersection(CurrentNames, NextNames), + NewAcc = [{G, sets:to_list(Clashes)} | Acc], + clashing_app_names(sets:union(PrevNames, CurrentNames), Rest, NewAcc). + path_groups(Targets, State) -> [{Target, get_paths(Target, State)} || Target <- Targets]. -- cgit v1.1 From dada4e36e6d9a5c4b41bbe1f68389520e7c59ace Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 11 Oct 2018 08:38:37 -0400 Subject: Optimize path handling - Only set paths that need to be put as a priority - Clean up paths before leaving API mode The first point accounted for some performance cost, but the latter one explains the 40% overhead in test runs: since rebar3 calls rebar3 a lot with a bunch of fake apps, and that the new mechanism for path handling by default does not _remove_ paths, it just _orders_ them, we would end up in a situation where as the tests ran, more and more fake paths would get added to the VM. By the time the run was over, all path handling would take longer since more paths needed filtering every time. By resetting paths at the end of an API run, we prevent a given 'project' from polluting another one's runtime and performance once the API successfully returns. --- src/rebar3.erl | 15 +++++++++++- src/rebar_hooks.erl | 7 ++---- src/rebar_paths.erl | 59 +++++++++++++++++++++++------------------------ src/rebar_plugins.erl | 2 +- src/rebar_prv_compile.erl | 12 +++++----- src/rebar_prv_xref.erl | 3 +-- 6 files changed, 53 insertions(+), 45 deletions(-) (limited to 'src') diff --git a/src/rebar3.erl b/src/rebar3.erl index c931a85..059d530 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -175,7 +175,20 @@ run_aux(State, RawArgs) -> State10 = rebar_state:code_paths(State9, default, code:get_path()), - rebar_core:init_command(rebar_state:command_args(State10, Args), Task). + case rebar_core:init_command(rebar_state:command_args(State10, Args), Task) of + {ok, State11} -> + case rebar_state:get(State11, caller, command_line) of + api -> + rebar_paths:unset_paths([deps, plugins], State11), + {ok, State11}; + _ -> + {ok, State11} + end; + Other -> + Other + end. + + %% @doc set up base configuration having to do with verbosity, where %% to find config files, and so on, and return an internal rebar3 state term. diff --git a/src/rebar_hooks.erl b/src/rebar_hooks.erl index f2ef8a3..358458e 100644 --- a/src/rebar_hooks.erl +++ b/src/rebar_hooks.erl @@ -42,9 +42,7 @@ run_provider_hooks_(Dir, Type, Command, Providers, TypeHooks, State) -> [] -> State; HookProviders -> - %PluginDepsPaths = lists:usort(rebar_state:code_paths(State, all_plugin_deps)), - %code:add_pathsa(PluginDepsPaths), - rebar_paths:set_paths([plugins, deps], State), + rebar_paths:set_paths([plugins], State), Providers1 = rebar_state:providers(State), State1 = rebar_state:providers(rebar_state:dir(State, Dir), Providers++Providers1), case rebar_core:do(HookProviders, State1) of @@ -52,8 +50,7 @@ run_provider_hooks_(Dir, Type, Command, Providers, TypeHooks, State) -> ?DEBUG(format_error({bad_provider, Type, Command, ProviderName}), []), throw(?PRV_ERROR({bad_provider, Type, Command, ProviderName})); {ok, State2} -> - %rebar_utils:remove_from_code_path(PluginDepsPaths), - rebar_paths:set_paths([deps, plugins], State2), + rebar_paths:set_paths([deps], State2), State2 end end. diff --git a/src/rebar_paths.erl b/src/rebar_paths.erl index 23fc755..900443d 100644 --- a/src/rebar_paths.erl +++ b/src/rebar_paths.erl @@ -1,3 +1,5 @@ +%% BEFORE THIS FIX: rebar3 ct 266.78s user 144.06s system 144% cpu 4:33.70 total +%% CURRENT TIME: rebar3 ct 419.30s user 301.00s system 152% cpu 7:51.98 total -module(rebar_paths). -include("rebar.hrl"). @@ -8,20 +10,17 @@ -export([clashing_apps/2]). -ifdef(TEST). --export([misloaded_modules/3]). +-export([misloaded_modules/2]). -endif. -spec set_paths(targets(), rebar_state:t()) -> ok. set_paths(UserTargets, State) -> Targets = normalize_targets(UserTargets), GroupPaths = path_groups(Targets, State), - Paths = lists:append([P || {_, P} <- GroupPaths]), - [code:del_path(P) || P <- Paths], - code:add_pathsa(lists:reverse(Paths)), - % set path breaks with escripts; we gotta do it by hand - % true = code:set_path(lists:append([P || {_, P} <- GroupPaths])), + Paths = lists:append(lists:reverse([P || {_, P} <- GroupPaths])), + code:add_pathsa(Paths), AppGroups = app_groups(Targets, State), - purge_and_load(AppGroups, code:all_loaded(), sets:new()), + purge_and_load(AppGroups, sets:new()), ok. -spec unset_paths(targets(), rebar_state:t()) -> ok. @@ -33,6 +32,7 @@ unset_paths(UserTargets, State) -> purge(Paths, code:all_loaded()), ok. +-spec clashing_apps(targets(), rebar_state:t()) -> [{target(), [binary()]}]. clashing_apps(Targets, State) -> AppGroups = app_groups(Targets, State), AppNames = [{G, sets:from_list( @@ -66,9 +66,9 @@ normalize_targets(List) -> ), lists:reverse(TmpList). -purge_and_load([], _, _) -> +purge_and_load([], _) -> ok; -purge_and_load([{_Group, Apps}|Rest], ModPaths, Seen) -> +purge_and_load([{_Group, Apps}|Rest], Seen) -> %% We have: a list of all applications in the current priority group, %% a list of all loaded modules with their active path, and a list of %% seen applications. @@ -79,7 +79,12 @@ purge_and_load([{_Group, Apps}|Rest], ModPaths, Seen) -> %% 3. unload and reload apps that may have changed paths in order %% to get updated module lists and specs %% (we ignore started apps and apps that have not run for this) - %% 4. create a list of modules to check from that app list + %% This part turns out to be the bottleneck of this module, so + %% to speed it up, using clash detection proves useful: + %% only reload apps that clashed since others are unlikely to + %% conflict in significant ways + %% 4. create a list of modules to check from that app list—only loaded + %% modules make sense to check. %% 5. check the modules to match their currently loaded paths with %% the path set from the apps in the current group; modules %% that differ must be purged; others can stay @@ -126,27 +131,29 @@ purge_and_load([{_Group, Apps}|Rest], ModPaths, Seen) -> end || App <- GoodApps, AppName <- [binary_to_atom(rebar_app_info:name(App), utf8)]] ), + ModPaths = [{Mod,Path} || Mod <- CandidateMods, + erlang:function_exported(Mod, module_info, 0), + {file, Path} <- [code:is_loaded(Mod)]], %% 5) - Mods = misloaded_modules(CandidateMods, GoodAppPaths, ModPaths), + Mods = misloaded_modules(GoodAppPaths, ModPaths), [purge_mod(Mod) || Mod <- Mods], - purge_and_load(Rest, ModPaths, - sets:union(Seen, sets:from_list(AppNames))). + + purge_and_load(Rest, sets:union(Seen, sets:from_list(AppNames))). purge(Paths, ModPaths) -> SortedPaths = lists:sort(Paths), - lists:map(fun purge_mod/1, lists:usort( - [Mod || {Mod, Path} <- ModPaths, - is_list(Path), % not 'preloaded' or mocked - any_prefix(Path, SortedPaths)] - )). + lists:map(fun purge_mod/1, + [Mod || {Mod, Path} <- ModPaths, + is_list(Path), % not 'preloaded' or mocked + any_prefix(Path, SortedPaths)] + ). -misloaded_modules(Mods, GoodAppPaths, ModPaths) -> +misloaded_modules(GoodAppPaths, ModPaths) -> %% Identify paths that are invalid; i.e. app paths that cover an %% app in the desired group, but are not in the desired group. lists:usort( - [Mod || Mod <- Mods, - {_, Path} <- [lists:keyfind(Mod, 1, ModPaths)], + [Mod || {Mod, Path} <- ModPaths, is_list(Path), % not 'preloaded' or mocked not any_prefix(Path, GoodAppPaths)] ). @@ -157,15 +164,7 @@ any_prefix(Path, Paths) -> %% assume paths currently set are good; only unload a module so next call %% uses the correctly set paths purge_mod(Mod) -> - case erlang:check_process_code(self(), Mod) of - false -> - code:purge(Mod), - code:delete(Mod); - _ -> - %% cannot purge safely without killing ourselves - code:soft_purge(Mod) andalso - code:delete(Mod) - end. + code:soft_purge(Mod) andalso code:delete(Mod). %% This is a tricky O(n²) check since we want to diff --git a/src/rebar_plugins.erl b/src/rebar_plugins.erl index a9c7e00..2a78c6e 100644 --- a/src/rebar_plugins.erl +++ b/src/rebar_plugins.erl @@ -125,7 +125,7 @@ handle_plugin(Profile, Plugin, State, Upgrade) -> %% Store plugin code paths so we can remove them when compiling project apps State4 = rebar_state:update_code_paths(State3, all_plugin_deps, PreBuiltPaths++NewCodePaths), - rebar_paths:set_paths([plugins, deps], State4), + rebar_paths:set_paths([plugins], State4), {plugin_providers(Plugin), State4} catch diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index 4b36636..ee96d9f 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -37,7 +37,7 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> IsDepsOnly = is_deps_only(State), - rebar_paths:set_paths([deps, plugins], State), + rebar_paths:set_paths([deps], State), Providers = rebar_state:providers(State), Deps = rebar_state:deps_to_build(State), @@ -50,7 +50,7 @@ do(State) -> handle_project_apps(Providers, State) end, - rebar_paths:set_paths([plugins, deps], State1), + rebar_paths:set_paths([plugins], State1), {ok, State1}. @@ -172,10 +172,10 @@ compile(State, Providers, AppInfo) -> %% The rebar_otp_app compilation step is safe regarding the %% overall path management, so we can just load all plugins back %% in memory. - rebar_paths:set_paths([plugins, deps], State), + rebar_paths:set_paths([plugins], State), AppFileCompileResult = rebar_otp_app:compile(State, AppInfo4), - %% Clean up after ourselves, leave things as they were. - rebar_paths:set_paths([deps, plugins], State), + %% Clean up after ourselves, leave things as they were with deps first + rebar_paths:set_paths([deps], State), case AppFileCompileResult of {ok, AppInfo5} -> @@ -203,7 +203,7 @@ build_app(AppInfo, State) -> %% load plugins since thats where project builders would be rebar_paths:set_paths([plugins, deps], State), Res = Module:build(AppInfo), - rebar_paths:set_paths([deps, plugins], State), + rebar_paths:set_paths([deps], State), case Res of ok -> ok; {error, Reason} -> throw({error, {Module, Reason}}) diff --git a/src/rebar_prv_xref.erl b/src/rebar_prv_xref.erl index a6b1f73..12063d5 100644 --- a/src/rebar_prv_xref.erl +++ b/src/rebar_prv_xref.erl @@ -36,7 +36,7 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - rebar_paths:set_paths([deps, plugins], State), + rebar_paths:set_paths([deps], State), XrefChecks = prepare(State), XrefIgnores = rebar_state:get(State, xref_ignores, []), %% Run xref checks @@ -47,7 +47,6 @@ do(State) -> QueryChecks = rebar_state:get(State, xref_queries, []), QueryResults = lists:foldl(fun check_query/2, [], QueryChecks), stopped = xref:stop(xref), - rebar_paths:set_paths([plugins, deps], State), case XrefResults =:= [] andalso QueryResults =:= [] of true -> {ok, State}; -- cgit v1.1 From a80d1988963698717548ba269d7e5f915fca4e27 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sun, 14 Oct 2018 09:37:50 -0600 Subject: fix finding transitive deps with prerelease versions (#1914) --- src/rebar_packages.erl | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 7676213..6faa5e1 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -55,11 +55,13 @@ get_all_names(State) -> _='_'}, [], ['$1']}])). --spec get_package_versions(unicode:unicode_binary(), unicode:unicode_binary(), +-spec get_package_versions(unicode:unicode_binary(), ec_semver:semver(), + unicode:unicode_binary(), ets:tid(), rebar_state:t()) -> [vsn()]. -get_package_versions(Dep, Repo, Table, State) -> +get_package_versions(Dep, {_, AlphaInfo}, Repo, Table, State) -> ?MODULE:verify_table(State), - AllowPreRelease = rebar_state:get(State, deps_allow_prerelease, false), + AllowPreRelease = rebar_state:get(State, deps_allow_prerelease, false) + orelse AlphaInfo =/= {[],[]}, ets:select(Table, [{#package{key={Dep, {'$1', '$2'}, Repo}, _='_'}, [{'==', '$2', {{[],[]}}} || not AllowPreRelease], [{{'$1', '$2'}}]}]). @@ -180,7 +182,7 @@ find_highest_matching(Dep, Constraint, Repo, Table, State) -> end. find_highest_matching_(Dep, Constraint, #{name := Repo}, Table, State) -> - try get_package_versions(Dep, Repo, Table, State) of + try get_package_versions(Dep, Constraint, Repo, Table, State) of [Vsn] -> handle_single_vsn(Vsn, Constraint); Vsns -> @@ -292,7 +294,7 @@ resolve_version(Dep, DepVsn, Hash, HexRegistry, State) when is_binary(Hash) -> end; resolve_version(Dep, undefined, Hash, HexRegistry, State) -> Fun = fun(Repo) -> - case highest_matching(Dep, "0", Repo, HexRegistry, State) of + case highest_matching(Dep, {0,{[],[]}}, Repo, HexRegistry, State) of none -> not_found; {ok, Vsn} -> @@ -360,9 +362,9 @@ resolve_version_(Dep, DepVsn, Repo, HexRegistry, State) -> end. rm_ws(<<" ", R/binary>>) -> - rm_ws(R); + ec_semver:parse(rm_ws(R)); rm_ws(R) -> - R. + ec_semver:parse(R). valid_vsn(Vsn) -> %% Regepx from https://github.com/sindresorhus/semver-regex/blob/master/index.js @@ -375,7 +377,7 @@ highest_matching(Dep, Vsn, Repo, HexRegistry, State) -> find_highest_matching_(Dep, Vsn, #{name => Repo}, HexRegistry, State). cmp(Dep, Vsn, Repo, HexRegistry, State, CmpFun) -> - case get_package_versions(Dep, Repo, HexRegistry, State) of + case get_package_versions(Dep, Vsn, Repo, HexRegistry, State) of [] -> none; Vsns -> @@ -398,7 +400,7 @@ cmp_(BestMatch, MinVsn, [Vsn | R], CmpFun) -> %% We need to treat this differently since we want a version that is LOWER but %% the higest possible one. cmpl(Dep, Vsn, Repo, HexRegistry, State, CmpFun) -> - case get_package_versions(Dep, Repo, HexRegistry, State) of + case get_package_versions(Dep, Vsn, Repo, HexRegistry, State) of [] -> none; Vsns -> -- cgit v1.1 From fb6de6e0e5dc0da1c4e64c166bcb69327420cb60 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sun, 14 Oct 2018 12:43:28 -0400 Subject: Drop outdated comments --- src/rebar_paths.erl | 2 -- 1 file changed, 2 deletions(-) (limited to 'src') diff --git a/src/rebar_paths.erl b/src/rebar_paths.erl index 900443d..82c0218 100644 --- a/src/rebar_paths.erl +++ b/src/rebar_paths.erl @@ -1,5 +1,3 @@ -%% BEFORE THIS FIX: rebar3 ct 266.78s user 144.06s system 144% cpu 4:33.70 total -%% CURRENT TIME: rebar3 ct 419.30s user 301.00s system 152% cpu 7:51.98 total -module(rebar_paths). -include("rebar.hrl"). -- cgit v1.1 From 86519cf743204eab1d922ca8133fbf00c66f9ee8 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sun, 14 Oct 2018 10:50:16 -0600 Subject: fix resolving versions from vcs (#1915) app_info was turning any vsn into a binary which the vcs_vsn function interprets as being an actual version and not something like <<"git">>. original_vsn shouldn't be converted as it is then not longer "original". --- src/rebar_app_info.erl | 10 +++++----- src/rebar_pkg_resource.erl | 2 +- src/rebar_state.erl | 6 +++++- src/rebar_utils.erl | 11 ++++++++++- 4 files changed, 21 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index b1c6243..9dfe278 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -124,14 +124,14 @@ new(AppName) -> {ok, t()}. new(AppName, Vsn) -> {ok, #app_info_t{name=rebar_utils:to_binary(AppName), - original_vsn=rebar_utils:to_binary(Vsn)}}. + original_vsn=Vsn}}. %% @doc build a complete version of the app info with all fields set. -spec new(atom() | binary() | string(), binary() | string(), file:name()) -> {ok, t()}. new(AppName, Vsn, Dir) -> {ok, #app_info_t{name=rebar_utils:to_binary(AppName), - original_vsn=rebar_utils:to_binary(Vsn), + original_vsn=Vsn, dir=rebar_utils:to_list(Dir), out_dir=rebar_utils:to_list(Dir), ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin")}}. @@ -141,7 +141,7 @@ new(AppName, Vsn, Dir) -> {ok, t()}. new(AppName, Vsn, Dir, Deps) -> {ok, #app_info_t{name=rebar_utils:to_binary(AppName), - original_vsn=rebar_utils:to_binary(Vsn), + original_vsn=Vsn, dir=rebar_utils:to_list(Dir), out_dir=rebar_utils:to_list(Dir), ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin"), @@ -153,7 +153,7 @@ new(AppName, Vsn, Dir, Deps) -> new(Parent, AppName, Vsn, Dir, Deps) -> {ok, #app_info_t{name=rebar_utils:to_binary(AppName), parent=Parent, - original_vsn=rebar_utils:to_binary(Vsn), + original_vsn=Vsn, dir=rebar_utils:to_list(Dir), out_dir=rebar_utils:to_list(Dir), ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin"), @@ -390,7 +390,7 @@ original_vsn(#app_info_t{original_vsn=Vsn}) -> %% asking for a semver) -spec original_vsn(t(), binary() | string()) -> t(). original_vsn(AppInfo=#app_info_t{}, Vsn) -> - AppInfo#app_info_t{original_vsn=rebar_utils:to_binary(Vsn)}. + AppInfo#app_info_t{original_vsn=Vsn}. %% @doc returns the list of applications the app depends on. -spec applications(t()) -> list(). diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 97eabb6..823b7fc 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -60,7 +60,7 @@ lock(AppInfo, _) -> Res :: boolean(). needs_update(AppInfo, _) -> {pkg, _Name, Vsn, _Hash, _} = rebar_app_info:source(AppInfo), - case rebar_app_info:original_vsn(AppInfo) =:= rebar_utils:to_binary(Vsn) of + case rebar_utils:to_binary(rebar_app_info:original_vsn(AppInfo)) =:= rebar_utils:to_binary(Vsn) of true -> false; false -> diff --git a/src/rebar_state.erl b/src/rebar_state.erl index c8a9400..31d3a08 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -43,7 +43,7 @@ project_builders/1, add_project_builder/3, - create_resources/2, + create_resources/2, set_resources/2, resources/1, resources/2, add_resource/2, providers/1, providers/2, add_provider/2, allow_provider_overrides/1, allow_provider_overrides/2 @@ -368,6 +368,10 @@ namespace(State=#state_t{}, Namespace) -> resources(#state_t{resources=Resources}) -> Resources. +-spec set_resources(t(), [{rebar_resource_v2:type(), module()}]) -> t(). +set_resources(State, Resources) -> + State#state_t{resources=Resources}. + -spec resources(t(), [{rebar_resource_v2:type(), module()}]) -> t(). resources(State, NewResources) -> lists:foldl(fun(Resource, StateAcc) -> diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 419802a..1769b79 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -715,6 +715,15 @@ escript_foldl(Fun, Acc, File) -> Error end. +%% TODO: this is just for rebar3_hex and maybe other plugins +%% but eventually it should be dropped +vcs_vsn(OriginalVsn, Dir, Resources) when is_list(Dir) , + is_list(Resources) -> + ?WARN("Using deprecated rebar_utils:vcs_vsn/3. Please upgrade your plugins.", []), + FakeState = rebar_state:new(), + {ok, AppInfo} = rebar_app_info:new(fake, OriginalVsn, Dir), + vcs_vsn(AppInfo, OriginalVsn, + rebar_state:set_resources(FakeState, Resources)); vcs_vsn(AppInfo, Vcs, State) -> case vcs_vsn_cmd(AppInfo, Vcs, State) of {plain, VsnString} -> @@ -728,7 +737,7 @@ vcs_vsn(AppInfo, Vcs, State) -> end. %% Temp work around for repos like relx that use "semver" -vcs_vsn_cmd(_AppInfo, Vsn, _) when is_binary(Vsn) -> +vcs_vsn_cmd(_, Vsn, _) when is_binary(Vsn) -> {plain, Vsn}; vcs_vsn_cmd(AppInfo, VCS, State) when VCS =:= semver ; VCS =:= "semver" -> vcs_vsn_cmd(AppInfo, git, State); -- cgit v1.1