diff options
Diffstat (limited to 'src')
32 files changed, 2242 insertions, 798 deletions
diff --git a/src/rebar.app.src b/src/rebar.app.src index 5ab3ddd..58fee02 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -25,8 +25,11 @@ bbmustache, ssl_verify_hostname, certifi, + cth_readable, relx, - inets]}, + cf, + inets, + eunit_formatters]}, {env, [ %% Default log level {log_level, warn}, diff --git a/src/rebar.hrl b/src/rebar.hrl index 8ad0faa..f4e7f5e 100644 --- a/src/rebar.hrl +++ b/src/rebar.hrl @@ -22,8 +22,9 @@ -define(DEFAULT_TEST_DEPS_DIR, "test/lib"). -define(DEFAULT_RELEASE_DIR, "rel"). -define(DEFAULT_CONFIG_FILE, "rebar.config"). --define(DEFAULT_CDN, "https://s3.amazonaws.com/s3.hex.pm/tarballs"). --define(DEFAULT_HEX_REGISTRY, "https://s3.amazonaws.com/s3.hex.pm/registry.ets.gz"). +-define(DEFAULT_CDN, "https://s3.amazonaws.com/s3.hex.pm/"). +-define(REMOTE_PACKAGE_DIR, "tarballs"). +-define(REMOTE_REGISTRY_FILE, "registry.ets.gz"). -define(LOCK_FILE, "rebar.lock"). -define(PACKAGE_INDEX_VERSION, 3). diff --git a/src/rebar3.erl b/src/rebar3.erl index 2b73844..879378e 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -105,25 +105,35 @@ run_aux(State, RawArgs) -> rebar_state:apply_profiles(State, [list_to_atom(Profile)]) end, + rebar_utils:check_min_otp_version(rebar_state:get(State1, minimum_otp_vsn, undefined)), + rebar_utils:check_blacklisted_otp_versions(rebar_state:get(State1, blacklisted_otp_vsns, undefined)), + + State2 = case os:getenv("HEX_CDN") of + false -> + State1; + CDN -> + rebar_state:set(State1, rebar_packages_cdn, CDN) + end, + %% bootstrap test profile - State2 = rebar_state:add_to_profile(State1, test, test_state(State1)), + State3 = rebar_state:add_to_profile(State2, test, test_state(State1)), %% Process each command, resetting any state between each one BaseDir = rebar_state:get(State, base_dir, ?DEFAULT_BASE_DIR), - State3 = rebar_state:set(State2, base_dir, - filename:join(filename:absname(rebar_state:dir(State2)), BaseDir)), + State4 = rebar_state:set(State3, base_dir, + filename:join(filename:absname(rebar_state:dir(State3)), BaseDir)), {ok, Providers} = application:get_env(rebar, providers), %% Providers can modify profiles stored in opts, so set default after initializing providers - State4 = rebar_state:create_logic_providers(Providers, State3), - State5 = rebar_plugins:project_apps_install(State4), - State6 = rebar_state:default(State5, rebar_state:opts(State5)), + State5 = rebar_state:create_logic_providers(Providers, State4), + State6 = rebar_plugins:top_level_install(State5), + State7 = rebar_state:default(State6, rebar_state:opts(State6)), {Task, Args} = parse_args(RawArgs), - State7 = rebar_state:code_paths(State6, default, code:get_path()), + State8 = rebar_state:code_paths(State7, default, code:get_path()), - rebar_core:init_command(rebar_state:command_args(State7, Args), Task). + rebar_core:init_command(rebar_state:command_args(State8, Args), Task). init_config() -> %% Initialize logging system @@ -339,4 +349,4 @@ safe_define_test_macro(Opts) -> test_defined([{d, 'TEST'}|_]) -> true; test_defined([{d, 'TEST', true}|_]) -> true; test_defined([_|Rest]) -> test_defined(Rest); -test_defined([]) -> false.
\ No newline at end of file +test_defined([]) -> false. diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index 95b4624..9fee4e0 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -23,6 +23,7 @@ original_vsn/1, original_vsn/2, ebin_dir/1, + priv_dir/1, applications/1, applications/2, profiles/1, @@ -361,6 +362,10 @@ out_dir(AppInfo=#app_info_t{}, OutDir) -> ebin_dir(#app_info_t{out_dir=OutDir}) -> ec_cnv:to_list(filename:join(OutDir, "ebin")). +-spec priv_dir(t()) -> file:name(). +priv_dir(#app_info_t{out_dir=OutDir}) -> + ec_cnv:to_list(filename:join(OutDir, "priv")). + -spec resource_type(t(), pkg | src) -> t(). resource_type(AppInfo=#app_info_t{}, Type) -> AppInfo#app_info_t{resource_type=Type}. diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index 602fd42..d3ef841 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -118,14 +118,14 @@ parse_dep(Dep, Parent, DepsDir, State, Locks, Level) -> end. parse_dep(Parent, {Name, Vsn, {pkg, PkgName}}, DepsDir, IsLock, State) -> - {PkgName1, PkgVsn} = parse_goal(ec_cnv:to_binary(PkgName), ec_cnv:to_binary(Vsn)), + {PkgName1, PkgVsn} = {ec_cnv:to_binary(PkgName), ec_cnv:to_binary(Vsn)}, dep_to_app(Parent, DepsDir, Name, PkgVsn, {pkg, PkgName1, PkgVsn}, IsLock, State); parse_dep(Parent, {Name, {pkg, PkgName}}, DepsDir, IsLock, State) -> %% Package dependency with different package name from app name dep_to_app(Parent, DepsDir, Name, undefined, {pkg, ec_cnv:to_binary(PkgName), undefined}, IsLock, State); parse_dep(Parent, {Name, Vsn}, DepsDir, IsLock, State) when is_list(Vsn); is_binary(Vsn) -> %% Versioned Package dependency - {PkgName, PkgVsn} = parse_goal(ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)), + {PkgName, PkgVsn} = {ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}, dep_to_app(Parent, DepsDir, PkgName, PkgVsn, {pkg, PkgName, PkgVsn}, IsLock, State); parse_dep(Parent, Name, DepsDir, IsLock, State) when is_atom(Name); is_binary(Name) -> %% Unversioned package dependency @@ -166,23 +166,26 @@ dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> Overrides = rebar_state:get(State, overrides, []), AppInfo2 = rebar_app_info:set(AppInfo1, overrides, rebar_app_info:get(AppInfo, overrides, [])++Overrides), AppInfo3 = rebar_app_info:apply_overrides(rebar_app_info:get(AppInfo2, overrides, []), AppInfo2), - rebar_app_info:is_lock(AppInfo3, IsLock). + AppInfo4 = rebar_app_info:apply_profiles(AppInfo3, [default, prod]), + AppInfo5 = rebar_app_info:profiles(AppInfo4, [default]), + rebar_app_info:is_lock(AppInfo5, IsLock). -update_source(AppInfo, {pkg, PkgName, undefined}, State) -> - {PkgName1, PkgVsn1} = get_package(PkgName, State), +update_source(AppInfo, {pkg, PkgName, PkgVsn}, State) -> + {PkgName1, PkgVsn1} = case PkgVsn of + undefined -> + get_package(PkgName, "0", State); + <<"~>", Vsn/binary>> -> + [Vsn1] = binary:split(Vsn, [<<" ">>], [trim_all, global]), + get_package(PkgName, Vsn1, State); + _ -> + {PkgName, PkgVsn} + end, AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName1, PkgVsn1}), Deps = rebar_packages:deps(PkgName1 ,PkgVsn1 ,State), AppInfo2 = rebar_app_info:resource_type(rebar_app_info:deps(AppInfo1, Deps), pkg), rebar_app_info:original_vsn(AppInfo2, PkgVsn1); -update_source(AppInfo, {pkg, PkgName, PkgVsn}, State) -> - AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName, PkgVsn}), - Deps = rebar_packages:deps(PkgName - ,PkgVsn - ,State), - AppInfo2 = rebar_app_info:resource_type(rebar_app_info:deps(AppInfo1, Deps), pkg), - rebar_app_info:original_vsn(AppInfo2, PkgVsn); update_source(AppInfo, Source, _State) -> rebar_app_info:source(AppInfo, Source). @@ -198,19 +201,8 @@ format_error(Error) -> %% Internal functions %% =================================================================== --spec parse_goal(binary(), binary()) -> {binary(), binary()} | {binary(), binary(), binary()}. -parse_goal(Name, Constraint) -> - case re:run(Constraint, "([^\\d]*)(\\d.*)", [{capture, [1,2], binary}]) of - {match, [<<>>, Vsn]} -> - {Name, Vsn}; - {match, [Op, Vsn]} -> - {Name, Vsn, binary_to_atom(Op, utf8)}; - nomatch -> - throw(?PRV_ERROR({bad_constraint, Name, Constraint})) - end. - -get_package(Dep, State) -> - case rebar_packages:find_highest_matching(Dep, "0", ?PACKAGE_TABLE, State) of +get_package(Dep, Vsn, State) -> + case rebar_packages:find_highest_matching(Dep, Vsn, ?PACKAGE_TABLE, State) of {ok, HighestDepVsn} -> {Dep, HighestDepVsn}; none -> diff --git a/src/rebar_dir.erl b/src/rebar_dir.erl index 09e3114..3729704 100644 --- a/src/rebar_dir.erl +++ b/src/rebar_dir.erl @@ -121,8 +121,37 @@ processing_base_dir(State, Dir) -> AbsDir = filename:absname(Dir), AbsDir =:= rebar_state:get(State, base_dir). +make_absolute_path(Path) -> + case filename:pathtype(Path) of + absolute -> + Path; + relative -> + {ok, Dir} = file:get_cwd(), + filename:join([Dir, Path]); + volumerelative -> + Volume = hd(filename:split(Path)), + {ok, Dir} = file:get_cwd(Volume), + filename:join([Dir, Path]) + end. + +make_normalized_path(Path) -> + AbsPath = make_absolute_path(Path), + Components = filename:split(AbsPath), + make_normalized_path(Components, []). + +make_normalized_path([], NormalizedPath) -> + filename:join(lists:reverse(NormalizedPath)); +make_normalized_path([H|T], NormalizedPath) -> + case H of + "." -> make_normalized_path(T, NormalizedPath); + ".." -> make_normalized_path(T, tl(NormalizedPath)); + _ -> make_normalized_path(T, [H|NormalizedPath]) + end. + make_relative_path(Source, Target) -> - do_make_relative_path(filename:split(Source), filename:split(Target)). + AbsSource = make_normalized_path(Source), + AbsTarget = make_normalized_path(Target), + do_make_relative_path(filename:split(AbsSource), filename:split(AbsTarget)). do_make_relative_path([H|T1], [H|T2]) -> do_make_relative_path(T1, T2); diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index 57b7387..3480cf6 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -116,7 +116,7 @@ compile(AppInfo, CompileOpts) when element(1, AppInfo) == app_info_t -> check_files([filename:join(Dir, File) || File <- rebar_opts:get(RebarOpts, mib_first_files, [])]), filename:join(Dir, "mibs"), ".mib", filename:join([Dir, "priv", "mibs"]), ".bin", - fun compile_mib/3), + compile_mib(AppInfo)), SrcDirs = lists:map(fun(SrcDir) -> filename:join(Dir, SrcDir) end, rebar_dir:src_dirs(RebarOpts, ["src"])), @@ -304,7 +304,8 @@ needed_files(G, ErlOpts, Dir, OutDir, SourceFiles) -> TargetBase = target_base(OutDir, Source), Target = TargetBase ++ ".beam", AllOpts = [{outdir, filename:dirname(Target)} - ,{i, filename:join(Dir, "include")}] ++ ErlOpts, + ,{i, filename:join(Dir, "include")} + ,{i, Dir}] ++ ErlOpts, digraph:vertex(G, Source) > {Source, filelib:last_modified(Target)} orelse opts_changed(AllOpts, TargetBase) end, SourceFiles). @@ -503,7 +504,7 @@ internal_erl_compile(_Opts, Dir, Module, OutDir, ErlOpts) -> Target = target_base(OutDir, Module) ++ ".beam", ok = filelib:ensure_dir(Target), AllOpts = [{outdir, filename:dirname(Target)}] ++ ErlOpts ++ - [{i, filename:join(Dir, "include")}, return], + [{i, filename:join(Dir, "include")}, {i, Dir}, return], case compile:file(Module, AllOpts) of {ok, _Mod} -> ok; @@ -516,32 +517,38 @@ internal_erl_compile(_Opts, Dir, Module, OutDir, ErlOpts) -> target_base(OutDir, Source) -> filename:join(OutDir, filename:basename(Source, ".erl")). --spec compile_mib(file:filename(), file:filename(), - rebar_dict()) -> 'ok'. -compile_mib(Source, Target, Opts) -> - Dir = filename:dirname(Target), - ok = filelib:ensure_dir(Target), - ok = filelib:ensure_dir(filename:join([Dir, "include", "dummy.hrl"])), - AllOpts = [{outdir, Dir} - ,{i, [Dir]}] ++ - rebar_opts:get(Opts, mib_opts, []), - - case snmpc:compile(Source, AllOpts) of - {ok, _} -> - Mib = filename:rootname(Target), - MibToHrlOpts = - case proplists:get_value(verbosity, AllOpts, undefined) of - undefined -> - #options{specific = []}; - Verbosity -> - #options{specific = [{verbosity, Verbosity}]} - end, - ok = snmpc:mib_to_hrl(Mib, Mib, MibToHrlOpts), - Hrl_filename = Mib ++ ".hrl", - rebar_file_utils:mv(Hrl_filename, "include"), - ok; - {error, compilation_failed} -> - ?FAIL +-spec compile_mib(rebar_app_info:t()) -> + fun((file:filename(), file:filename(), rebar_dict()) -> 'ok'). +compile_mib(AppInfo) -> + fun(Source, Target, Opts) -> + Dir = filename:dirname(Target), + Mib = filename:rootname(Target), + HrlFilename = Mib ++ ".hrl", + + AppInclude = filename:join([rebar_app_info:dir(AppInfo), "include"]), + + ok = filelib:ensure_dir(Target), + ok = filelib:ensure_dir(filename:join([AppInclude, "dummy.hrl"])), + + AllOpts = [{outdir, Dir} + ,{i, [Dir]}] ++ + rebar_opts:get(Opts, mib_opts, []), + + case snmpc:compile(Source, AllOpts) of + {ok, _} -> + MibToHrlOpts = + case proplists:get_value(verbosity, AllOpts, undefined) of + undefined -> + #options{specific = []}; + Verbosity -> + #options{specific = [{verbosity, Verbosity}]} + end, + ok = snmpc:mib_to_hrl(Mib, Mib, MibToHrlOpts), + rebar_file_utils:mv(HrlFilename, AppInclude), + ok; + {error, compilation_failed} -> + ?FAIL + end end. -spec compile_xrl(file:filename(), file:filename(), @@ -688,7 +695,7 @@ warn_and_find_path(File, Dir) -> true -> [SrcHeader]; false -> - IncludeDir = filename:join(filename:join(rebar_utils:droplast(filename:split(Dir))), "include"), + IncludeDir = filename:join(rebar_utils:droplast(filename:split(Dir))++["include"]), IncludeHeader = filename:join(IncludeDir, File), case filelib:is_regular(IncludeHeader) of true -> diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index ea1a6a2..0f84520 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -139,7 +139,7 @@ cp_r(Sources, Dest) -> {unix, _} -> EscSources = [rebar_utils:escape_chars(Src) || Src <- Sources], SourceStr = string:join(EscSources, " "), - {ok, []} = rebar_utils:sh(?FMT("cp -R ~s \"~s\"", + {ok, []} = rebar_utils:sh(?FMT("cp -Rp ~s \"~s\"", [SourceStr, rebar_utils:escape_double_quotes(Dest)]), [{use_stdout, false}, abort_on_error]), ok; @@ -262,9 +262,11 @@ path_from_ancestor_(_, _) -> {error, badparent}. %% reduce a filepath by removing all incidences of `.' and `..' -spec canonical_path(string()) -> string(). -canonical_path(Dir) -> canonical_path([], filename:split(filename:absname(Dir))). +canonical_path(Dir) -> + Canon = canonical_path([], filename:split(filename:absname(Dir))), + filename:nativename(Canon). -canonical_path([], []) -> filename:nativename("/"); +canonical_path([], []) -> filename:absname("/"); canonical_path(Acc, []) -> filename:join(lists:reverse(Acc)); canonical_path(Acc, ["."|Rest]) -> canonical_path(Acc, Rest); canonical_path([_|Acc], [".."|Rest]) -> canonical_path(Acc, Rest); @@ -283,13 +285,19 @@ delete_each_dir_win32([Dir | Rest]) -> delete_each_dir_win32(Rest). xcopy_win32(Source,Dest)-> - %% "xcopy \"~s\" \"~s\" /q /y /e 2> nul", Chanegd to robocopy to + %% "xcopy \"~s\" \"~s\" /q /y /e 2> nul", Changed to robocopy to %% handle long names. May have issues with older windows. Cmd = case filelib:is_dir(Source) of true -> + %% For robocopy, copying /a/b/c/ to /d/e/f/ recursively does not + %% create /d/e/f/c/*, but rather copies all files to /d/e/f/*. + %% The usage we make here expects the former, not the later, so we + %% must manually add the last fragment of a directory to the `Dest` + %% in order to properly replicate POSIX platforms + NewDest = filename:join([Dest, filename:basename(Source)]), ?FMT("robocopy \"~s\" \"~s\" /e /is 1> nul", [rebar_utils:escape_double_quotes(filename:nativename(Source)), - rebar_utils:escape_double_quotes(filename:nativename(Dest))]); + rebar_utils:escape_double_quotes(filename:nativename(NewDest))]); false -> ?FMT("robocopy \"~s\" \"~s\" \"~s\" /e /is 1> nul", [rebar_utils:escape_double_quotes(filename:nativename(filename:dirname(Source))), diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl index bea74a2..876d047 100644 --- a/src/rebar_git_resource.erl +++ b/src/rebar_git_resource.erl @@ -45,7 +45,7 @@ needs_update(Dir, {git, Url, {branch, Branch}}) -> not ((Current =:= []) andalso compare_url(Dir, Url)); needs_update(Dir, {git, Url, "master"}) -> needs_update(Dir, {git, Url, {branch, "master"}}); -needs_update(Dir, {git, Url, Ref}) -> +needs_update(Dir, {git, _, Ref}) -> {ok, Current} = rebar_utils:sh(?FMT("git rev-parse -q HEAD", []), [{cd, Dir}]), Current1 = string:strip(string:strip(Current, both, $\n), both, $\r), @@ -64,7 +64,7 @@ needs_update(Dir, {git, Url, Ref}) -> end, ?DEBUG("Comparing git ref ~s with ~s", [Ref1, Current1]), - not ((Current1 =:= Ref2) andalso compare_url(Dir, Url)). + (Current1 =/= Ref2). compare_url(Dir, Url) -> {ok, CurrentUrl} = rebar_utils:sh(?FMT("git config --get remote.origin.url", []), diff --git a/src/rebar_opts.erl b/src/rebar_opts.erl index 47451c5..b02a504 100644 --- a/src/rebar_opts.erl +++ b/src/rebar_opts.erl @@ -111,6 +111,12 @@ merge_opts(NewOpts, OldOpts) -> NewValue; (profiles, NewValue, OldValue) -> dict:to_list(merge_opts(dict:from_list(NewValue), dict:from_list(OldValue))); + (mib_first_files, Value, Value) -> + Value; + (mib_first_files, NewValue, OldValue) -> + OldValue ++ NewValue; + (relx, NewValue, OldValue) -> + rebar_utils:tup_umerge(OldValue, NewValue); (_Key, NewValue, OldValue) when is_list(NewValue) -> case io_lib:printable_list(NewValue) of true when NewValue =:= [] -> diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 7be3372..c56009e 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -28,7 +28,17 @@ packages(State) -> ok; false -> ?DEBUG("Error loading package index.", []), - ?ERROR("Bad packages index, try to fix with `rebar3 update`", []), + handle_bad_index(State) + end. + +handle_bad_index(State) -> + ?ERROR("Bad packages index. Trying to fix by updating the registry.", []), + {ok, State1} = rebar_prv_update:do(State), + case load_and_verify_version(State1) of + true -> + ok; + false -> + %% Still unable to load after an update, create an empty registry ets:new(?PACKAGE_TABLE, [named_table, public]) end. @@ -36,7 +46,7 @@ close_packages() -> catch ets:delete(?PACKAGE_TABLE). load_and_verify_version(State) -> - RegistryDir = registry_dir(State), + {ok, RegistryDir} = registry_dir(State), case ets:file2tab(filename:join(RegistryDir, ?INDEX_FILE)) of {ok, _} -> case ets:lookup_element(?PACKAGE_TABLE, package_index_version, 2) of @@ -52,10 +62,24 @@ load_and_verify_version(State) -> deps(Name, Vsn, State) -> try - ?MODULE:verify_table(State), - ets:lookup_element(?PACKAGE_TABLE, {ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}, 2) + deps_(Name, Vsn, State) + catch + _:_ -> + handle_missing_package(Name, Vsn, State) + end. + +deps_(Name, Vsn, State) -> + ?MODULE:verify_table(State), + ets:lookup_element(?PACKAGE_TABLE, {ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}, 2). + +handle_missing_package(Name, Vsn, State) -> + ?INFO("Package ~s-~s not found. Fetching registry updates and trying again...", [Name, Vsn]), + {ok, State1} = rebar_prv_update:do(State), + try + deps_(Name, Vsn, State1) catch _:_ -> + %% Even after an update the package is still missing, time to error out throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)})) end. @@ -65,21 +89,30 @@ registry_dir(State) -> ?DEFAULT_CDN -> RegistryDir = filename:join([CacheDir, "hex", "default"]), ok = filelib:ensure_dir(filename:join(RegistryDir, "placeholder")), - RegistryDir; + {ok, RegistryDir}; CDN -> - {ok, {_, _, Host, _, Path, _}} = http_uri:parse(CDN), - CDNHostPath = lists:reverse(string:tokens(Host, ".")), - CDNPath = tl(filename:split(Path)), - RegistryDir = filename:join([CacheDir, "hex"] ++ CDNHostPath ++ CDNPath), - ok = filelib:ensure_dir(filename:join(RegistryDir, "placeholder")), - RegistryDir + case rebar_utils:url_append_path(CDN, ?REMOTE_PACKAGE_DIR) of + {ok, Parsed} -> + {ok, {_, _, Host, _, Path, _}} = http_uri:parse(Parsed), + CDNHostPath = lists:reverse(string:tokens(Host, ".")), + CDNPath = tl(filename:split(Path)), + RegistryDir = filename:join([CacheDir, "hex"] ++ CDNHostPath ++ CDNPath), + ok = filelib:ensure_dir(filename:join(RegistryDir, "placeholder")), + {ok, RegistryDir}; + _ -> + {uri_parse_error, CDN} + end end. package_dir(State) -> - RegistryDir = registry_dir(State), - PackageDir = filename:join([RegistryDir, "packages"]), - ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")), - PackageDir. + case registry_dir(State) of + {ok, RegistryDir} -> + PackageDir = filename:join([RegistryDir, "packages"]), + ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")), + {ok, PackageDir}; + Error -> + Error + end. registry_checksum({pkg, Name, Vsn}, State) -> try @@ -138,12 +171,12 @@ handle_single_vsn(Dep, Vsn, Constraint) -> {ok, Vsn}; false -> ?WARN("Only existing version of ~s is ~s which does not match constraint ~~> ~s. " - "Using anyway, but it is not guarenteed to work.", [Dep, Vsn, Constraint]), + "Using anyway, but it is not guaranteed to work.", [Dep, Vsn, Constraint]), {ok, Vsn} end. format_error({missing_package, Package, Version}) -> - io_lib:format("Package not found in registry: ~s-~s. Try to fix with `rebar3 update`", [Package, Version]). + io_lib:format("Package not found in registry: ~s-~s.", [Package, Version]). verify_table(State) -> ets:info(?PACKAGE_TABLE, named_table) =:= true orelse load_and_verify_version(State). diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 4f55ad1..ec7e09d 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -30,11 +30,15 @@ needs_update(Dir, {pkg, _Name, Vsn}) -> download(TmpDir, Pkg={pkg, Name, Vsn}, State) -> CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), - PackageDir = rebar_packages:package_dir(State), + {ok, PackageDir} = rebar_packages:package_dir(State), Package = binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>), CachePath = filename:join(PackageDir, Package), - Url = string:join([CDN, Package], "/"), - cached_download(TmpDir, CachePath, Pkg, Url, etag(CachePath), State). + case rebar_utils:url_append_path(CDN, filename:join(?REMOTE_PACKAGE_DIR, Package)) of + {ok, Url} -> + cached_download(TmpDir, CachePath, Pkg, Url, etag(CachePath), State); + _ -> + {fetch_fail, Name, Vsn} + end. cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn}, Url, ETag, State) -> case request(Url, ETag) of @@ -100,7 +104,7 @@ make_vsn(_) -> {error, "Replacing version of type pkg not supported."}. request(Url, ETag) -> - case httpc:request(get, {Url, [{"if-none-match", ETag} || ETag =/= false]}, + case httpc:request(get, {Url, [{"if-none-match", ETag} || ETag =/= false]++[{"User-Agent", rebar_utils:user_agent()}]}, [{ssl, ssl_opts(Url)}, {relaxed, true}], [{body_format, binary}], rebar) of diff --git a/src/rebar_plugins.erl b/src/rebar_plugins.erl index f2d3977..3c33498 100644 --- a/src/rebar_plugins.erl +++ b/src/rebar_plugins.erl @@ -3,7 +3,8 @@ -module(rebar_plugins). --export([project_apps_install/1 +-export([top_level_install/1 + ,project_apps_install/1 ,install/2 ,handle_plugins/3 ,handle_plugins/4]). @@ -14,11 +15,18 @@ %% Public API %% =================================================================== +-spec top_level_install(rebar_state:t()) -> rebar_state:t(). +top_level_install(State) -> + Profiles = rebar_state:current_profiles(State), + lists:foldl(fun(Profile, StateAcc) -> + Plugins = rebar_state:get(State, {plugins, Profile}, []), + handle_plugins(Profile, Plugins, StateAcc) + end, State, Profiles). + -spec project_apps_install(rebar_state:t()) -> rebar_state:t(). project_apps_install(State) -> Profiles = rebar_state:current_profiles(State), ProjectApps = rebar_state:project_apps(State), - lists:foldl(fun(Profile, StateAcc) -> Plugins = rebar_state:get(State, {plugins, Profile}, []), StateAcc1 = handle_plugins(Profile, Plugins, StateAcc), @@ -34,10 +42,20 @@ project_apps_install(State) -> -spec install(rebar_state:t(), rebar_app_info:t()) -> rebar_state:t(). install(State, AppInfo) -> Profiles = rebar_state:current_profiles(State), - lists:foldl(fun(Profile, StateAcc) -> - Plugins = rebar_app_info:get(AppInfo, {plugins, Profile}, []), - handle_plugins(Profile, Plugins, StateAcc) - end, State, Profiles). + + %% don't lose the overrides of the dep we are processing plugins for + Overrides = rebar_app_info:get(AppInfo, overrides, []), + StateOverrides = rebar_state:get(State, overrides, []), + AllOverrides = Overrides ++ StateOverrides, + State1 = rebar_state:set(State, overrides, AllOverrides), + + State2 = lists:foldl(fun(Profile, StateAcc) -> + Plugins = rebar_app_info:get(AppInfo, {plugins, Profile}, []), + handle_plugins(Profile, Plugins, StateAcc) + end, State1, Profiles), + + %% Reset the overrides after processing the dep + rebar_state:set(State2, overrides, StateOverrides). handle_plugins(Profile, Plugins, State) -> handle_plugins(Profile, Plugins, State, false). diff --git a/src/rebar_prv_app_discovery.erl b/src/rebar_prv_app_discovery.erl index 5449f82..1954214 100644 --- a/src/rebar_prv_app_discovery.erl +++ b/src/rebar_prv_app_discovery.erl @@ -36,7 +36,8 @@ do(State) -> LibDirs = rebar_dir:lib_dirs(State), try State1 = rebar_app_discover:do(State, LibDirs), - {ok, State1} + State2 = rebar_plugins:project_apps_install(State1), + {ok, State2} catch throw:{error, {rebar_packages, Error}} -> {error, {rebar_packages, Error}}; diff --git a/src/rebar_prv_clean.erl b/src/rebar_prv_clean.erl index 7f952e3..8f31fdd 100644 --- a/src/rebar_prv_clean.erl +++ b/src/rebar_prv_clean.erl @@ -27,32 +27,35 @@ init(State) -> {example, "rebar3 clean"}, {short_desc, "Remove compiled beam files from apps."}, {desc, "Remove compiled beam files from apps."}, - {opts, [{all, $a, "all", undefined, "Clean all apps include deps"}]}])), + {opts, [{all, $a, "all", undefined, "Clean all apps include deps"}, + {profile, $p, "profile", string, "Clean under profile. Equivalent to `rebar3 as <profile> clean`"}]}])), {ok, State1}. -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> Providers = rebar_state:providers(State), - {all, All} = handle_args(State), + {All, Profiles} = handle_args(State), + + State1 = rebar_state:apply_profiles(State, [list_to_atom(X) || X <- Profiles]), Cwd = rebar_dir:get_cwd(), - rebar_hooks:run_all_hooks(Cwd, pre, ?PROVIDER, Providers, State), + rebar_hooks:run_all_hooks(Cwd, pre, ?PROVIDER, Providers, State1), case All of true -> - DepsDir = rebar_dir:deps_dir(State), + DepsDir = rebar_dir:deps_dir(State1), AllApps = rebar_app_discover:find_apps([filename:join(DepsDir, "*")], all), - clean_apps(State, Providers, AllApps); + clean_apps(State1, Providers, AllApps); false -> - ProjectApps = rebar_state:project_apps(State), - clean_apps(State, Providers, ProjectApps) + ProjectApps = rebar_state:project_apps(State1), + clean_apps(State1, Providers, ProjectApps) end, - clean_extras(State), + clean_extras(State1), - rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State), + rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State1), - {ok, State}. + {ok, State1}. -spec format_error(any()) -> iolist(). format_error(Reason) -> @@ -78,4 +81,5 @@ clean_extras(State) -> handle_args(State) -> {Args, _} = rebar_state:command_parsed_args(State), All = proplists:get_value(all, Args, false), - {all, All}. + Profiles = proplists:get_all_values(profile, Args), + {All, Profiles}. diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index 1f4c02d..4be50d8 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -2,19 +2,21 @@ %% ex: ts=4 sw=4 et -module(rebar_prv_common_test). + -behaviour(provider). -export([init/1, do/1, format_error/1]). %% exported for test purposes, consider private --export([setup_ct/1]). +-export([compile/2, prepare_tests/1, translate_paths/2]). -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). -define(PROVIDER, ct). --define(DEPS, [compile]). +%% we need to modify app_info state before compile +-define(DEPS, [lock]). %% =================================================================== %% Public API @@ -31,77 +33,461 @@ init(State) -> {desc, "Run Common Tests."}, {opts, ct_opts(State)}, {profiles, [test]}]), - State1 = rebar_state:add_provider(State, Provider), - State2 = rebar_state:add_to_profile(State1, test, test_state(State1)), - {ok, State2}. + {ok, rebar_state:add_provider(State, Provider)}. -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> + Tests = prepare_tests(State), + case compile(State, Tests) of + %% successfully compiled apps + {ok, S} -> do(S, Tests); + %% this should look like a compiler error, not a ct error + Error -> Error + end. + +do(State, Tests) -> ?INFO("Running Common Test suites...", []), - rebar_utils:update_code(rebar_state:code_paths(State, all_deps)), + rebar_utils:update_code(rebar_state:code_paths(State, all_deps), [soft_purge]), %% Run ct provider prehooks Providers = rebar_state:providers(State), Cwd = rebar_dir:get_cwd(), rebar_hooks:run_all_hooks(Cwd, pre, ?PROVIDER, Providers, State), - try run_test(State) of - {ok, State1} = Result -> - %% Run ct provider posthooks - rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State1), - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), - Result; - ?PRV_ERROR(_) = Error -> + case Tests of + {ok, T} -> + case run_tests(State, T) of + ok -> + %% Run ct provider posthooks + rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State), + rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), + {ok, State}; + Error -> + rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), + Error + end; + Error -> rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), Error - catch - throw:{error, Reason} -> - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), - ?PRV_ERROR(Reason) end. +run_tests(State, Opts) -> + T = translate_paths(State, Opts), + Opts1 = setup_logdir(State, T), + Opts2 = turn_off_auto_compile(Opts1), + ?DEBUG("ct_opts ~p", [Opts2]), + {RawOpts, _} = rebar_state:command_parsed_args(State), + Result = case proplists:get_value(verbose, RawOpts, false) of + true -> run_test_verbose(Opts2); + false -> run_test_quiet(Opts2) + end, + ok = maybe_write_coverdata(State), + Result. + -spec format_error(any()) -> iolist(). -format_error({multiple_dirs_and_suites, Opts}) -> - io_lib:format("Multiple dirs declared alongside suite in opts: ~p", [Opts]); -format_error({bad_dir_or_suite, Opts}) -> - io_lib:format("Bad value for dir or suite in opts: ~p", [Opts]); +format_error({error, Reason}) -> + io_lib:format("Error running tests:~n ~p", [Reason]); +format_error({error_running_tests, Reason}) -> + format_error({error, Reason}); format_error({failures_running_tests, {Failed, AutoSkipped}}) -> io_lib:format("Failures occured running tests: ~b", [Failed+AutoSkipped]); -format_error({error_running_tests, Reason}) -> - io_lib:format("Error running tests: ~p", [Reason]); -format_error(suite_at_project_root) -> - io_lib:format("Test suites can't be located in project root", []); -format_error({error, Reason}) -> - io_lib:format("Unknown error: ~p", [Reason]). +format_error({badconfig, {Msg, {Value, Key}}}) -> + io_lib:format(Msg, [Value, Key]); +format_error({badconfig, Msg}) -> + io_lib:format(Msg, []); +format_error({multiple_errors, Errors}) -> + io_lib:format(lists:concat(["Error running tests:"] ++ + lists:map(fun(Error) -> "~n " ++ Error end, Errors)), []). %% =================================================================== %% Internal functions %% =================================================================== -run_test(State) -> - case setup_ct(State) of - {error, {no_tests_specified, Opts}} -> - ?WARN("No tests specified in opts: ~p", [Opts]), - {ok, State}; - Opts -> - Opts1 = setup_logdir(State, Opts), - ?DEBUG("common test opts: ~p", [Opts1]), - run_test(State, Opts1) - end. +prepare_tests(State) -> + %% command line test options + CmdOpts = cmdopts(State), + %% rebar.config test options + CfgOpts = cfgopts(State), + ProjectApps = rebar_state:project_apps(State), + %% prioritize tests to run first trying any command line specified + %% tests falling back to tests specified in the config file finally + %% running a default set if no other tests are present + select_tests(State, ProjectApps, CmdOpts, CfgOpts). -run_test(State, Opts) -> +cmdopts(State) -> {RawOpts, _} = rebar_state:command_parsed_args(State), - ok = rebar_prv_cover:maybe_cover_compile(State, apps), - Result = case proplists:get_value(verbose, RawOpts, false) of - true -> run_test_verbose(Opts); - false -> run_test_quiet(Opts) + %% filter out opts common_test doesn't know about and convert + %% to ct acceptable forms + transform_opts(RawOpts, []). + +transform_opts([], Acc) -> lists:reverse(Acc); +transform_opts([{dir, Dirs}|Rest], Acc) -> + transform_opts(Rest, [{dir, split_string(Dirs)}|Acc]); +transform_opts([{suite, Suites}|Rest], Acc) -> + transform_opts(Rest, [{suite, split_string(Suites)}|Acc]); +transform_opts([{group, Groups}|Rest], Acc) -> + transform_opts(Rest, [{group, split_string(Groups)}|Acc]); +transform_opts([{testcase, Cases}|Rest], Acc) -> + transform_opts(Rest, [{testcase, split_string(Cases)}|Acc]); +transform_opts([{config, Configs}|Rest], Acc) -> + transform_opts(Rest, [{config, split_string(Configs)}|Acc]); +transform_opts([{logopts, LogOpts}|Rest], Acc) -> + transform_opts(Rest, [{logopts, lists:map(fun(P) -> list_to_atom(P) end, split_string(LogOpts))}|Acc]); +transform_opts([{force_stop, "true"}|Rest], Acc) -> + transform_opts(Rest, [{force_stop, true}|Acc]); +transform_opts([{force_stop, "false"}|Rest], Acc) -> + transform_opts(Rest, [{force_stop, false}|Acc]); +transform_opts([{force_stop, "skip_rest"}|Rest], Acc) -> + transform_opts(Rest, [{force_stop, skip_rest}|Acc]); +transform_opts([{create_priv_dir, CreatePrivDir}|Rest], Acc) -> + transform_opts(Rest, [{create_priv_dir, list_to_atom(CreatePrivDir)}|Acc]); +%% drop cover from opts, ct doesn't care about it +transform_opts([{cover, _}|Rest], Acc) -> + transform_opts(Rest, Acc); +%% drop verbose from opts, ct doesn't care about it +transform_opts([{verbose, _}|Rest], Acc) -> + transform_opts(Rest, Acc); +%% getopt should handle anything else +transform_opts([Opt|Rest], Acc) -> + transform_opts(Rest, [Opt|Acc]). + +split_string(String) -> + string:tokens(String, [$,]). + +cfgopts(State) -> + case rebar_state:get(State, ct_opts, []) of + Opts when is_list(Opts) -> + ensure_opts(add_hooks(Opts, State), []); + Wrong -> + %% probably a single non list term + ?PRV_ERROR({badconfig, {"Value `~p' of option `~p' must be a list", {Wrong, ct_opts}}}) + end. + +ensure_opts([], Acc) -> lists:reverse(Acc); +ensure_opts([{test_spec, _}|_Rest], _Acc) -> + ?PRV_ERROR({badconfig, "Test specs not supported"}); +ensure_opts([{auto_compile, _}|_Rest], _Acc) -> + ?PRV_ERROR({badconfig, "Auto compile not supported"}); +ensure_opts([{suite, Suite}|Rest], Acc) when is_integer(hd(Suite)) -> + ensure_opts(Rest, [{suite, Suite}|Acc]); +ensure_opts([{suite, Suite}|Rest], Acc) when is_atom(Suite) -> + ensure_opts(Rest, [{suite, atom_to_list(Suite)}|Acc]); +ensure_opts([{suite, Suites}|Rest], Acc) -> + NewSuites = {suite, lists:map(fun(S) when is_atom(S) -> atom_to_list(S); + (S) when is_list(S) -> S + end, + Suites)}, + ensure_opts(Rest, [NewSuites|Acc]); +ensure_opts([{K, V}|Rest], Acc) -> + ensure_opts(Rest, [{K, V}|Acc]); +ensure_opts([V|_Rest], _Acc) -> + ?PRV_ERROR({badconfig, {"Member `~p' of option `~p' must be a 2-tuple", {V, ct_opts}}}). + +add_hooks(Opts, State) -> + case {readable(State), lists:keyfind(ct_hooks, 1, Opts)} of + {false, _} -> + Opts; + {true, false} -> + [{ct_hooks, [cth_readable_failonly, cth_readable_shell]} | Opts]; + {true, {ct_hooks, Hooks}} -> + %% Make sure hooks are there once only. + ReadableHooks = [cth_readable_failonly, cth_readable_shell], + NewHooks = (Hooks -- ReadableHooks) ++ ReadableHooks, + lists:keyreplace(ct_hooks, 1, Opts, {ct_hooks, NewHooks}) + end. + +select_tests(_, _, {error, _} = Error, _) -> Error; +select_tests(_, _, _, {error, _} = Error) -> Error; +select_tests(State, ProjectApps, CmdOpts, CfgOpts) -> + Merged = lists:ukeymerge(1, + lists:ukeysort(1, CmdOpts), + lists:ukeysort(1, CfgOpts)), + %% make sure `dir` and/or `suite` from command line go in as + %% a pair overriding both `dir` and `suite` from config if + %% they exist + Opts = case {proplists:get_value(suite, CmdOpts), proplists:get_value(dir, CmdOpts)} of + {undefined, undefined} -> Merged; + {_Suite, undefined} -> lists:keydelete(dir, 1, Merged); + {undefined, _Dir} -> lists:keydelete(suite, 1, Merged); + {_Suite, _Dir} -> Merged end, - ok = rebar_prv_cover:maybe_write_coverdata(State, ?PROVIDER), - case Result of - ok -> {ok, State}; - Error -> Error + discover_tests(State, ProjectApps, Opts). + +discover_tests(State, ProjectApps, Opts) -> + case {proplists:get_value(suite, Opts), proplists:get_value(dir, Opts)} of + %% no dirs or suites defined, try using `$APP/test` and `$ROOT/test` + %% as suites + {undefined, undefined} -> {ok, [default_tests(State, ProjectApps)|Opts]}; + {_, _} -> {ok, Opts} + end. + +default_tests(State, ProjectApps) -> + BareTest = filename:join([rebar_state:dir(State), "test"]), + F = fun(App) -> rebar_app_info:dir(App) == rebar_state:dir(State) end, + AppTests = application_dirs(ProjectApps, []), + case filelib:is_dir(BareTest) andalso not lists:any(F, ProjectApps) of + %% `test` dir at root of project is already scheduled to be + %% included or `test` does not exist + false -> {dir, AppTests}; + %% need to add `test` dir at root to dirs to be included + true -> {dir, AppTests ++ [BareTest]} + end. + +application_dirs([], []) -> []; +application_dirs([], Acc) -> lists:reverse(Acc); +application_dirs([App|Rest], Acc) -> + TestDir = filename:join([rebar_app_info:dir(App), "test"]), + case filelib:is_dir(TestDir) of + true -> application_dirs(Rest, [TestDir|Acc]); + false -> application_dirs(Rest, Acc) + end. + +compile(State, {ok, _} = Tests) -> + %% inject `ct_first_files` and `ct_compile_opts` into the applications + %% to be compiled + case inject_ct_state(State, Tests) of + {ok, NewState} -> do_compile(NewState); + Error -> Error + end; +%% maybe compile even in the face of errors? +compile(_State, Error) -> Error. + +do_compile(State) -> + case rebar_prv_compile:do(State) of + %% successfully compiled apps + {ok, S} -> + ok = maybe_cover_compile(S), + {ok, S}; + %% this should look like a compiler error, not an eunit error + Error -> Error + end. + +inject_ct_state(State, {ok, Tests}) -> + Apps = rebar_state:project_apps(State), + case inject_ct_state(State, Apps, []) of + {ok, {NewState, ModdedApps}} -> + test_dirs(NewState, ModdedApps, Tests); + {error, _} = Error -> Error + end; +inject_ct_state(_State, Error) -> Error. + +inject_ct_state(State, [App|Rest], Acc) -> + case inject(rebar_app_info:opts(App), State) of + {error, _} = Error -> Error; + NewOpts -> + NewApp = rebar_app_info:opts(App, NewOpts), + inject_ct_state(State, Rest, [NewApp|Acc]) + end; +inject_ct_state(State, [], Acc) -> + case inject(rebar_state:opts(State), State) of + {error, _} = Error -> Error; + NewOpts -> + {ok, {rebar_state:opts(State, NewOpts), lists:reverse(Acc)}} + end. + +opts(Opts, Key, Default) -> + case rebar_opts:get(Opts, Key, Default) of + Vs when is_list(Vs) -> Vs; + Wrong -> + ?PRV_ERROR({badconfig, {"Value `~p' of option `~p' must be a list", {Wrong, Key}}}) + end. + +inject(Opts, State) -> erl_opts(Opts, State). + +erl_opts(Opts, State) -> + %% append `ct_compile_opts` to app defined `erl_opts` + ErlOpts = opts(Opts, erl_opts, []), + CTOpts = opts(Opts, ct_compile_opts, []), + case add_transforms(append(CTOpts, ErlOpts), State) of + {error, Error} -> {error, Error}; + NewErlOpts -> first_files(rebar_opts:set(Opts, erl_opts, NewErlOpts)) + end. + +first_files(Opts) -> + %% append `ct_first_files` to app defined `erl_first_files` + FirstFiles = opts(Opts, erl_first_files, []), + CTFirstFiles = opts(Opts, ct_first_files, []), + case append(CTFirstFiles, FirstFiles) of + {error, _} = Error -> Error; + NewFirstFiles -> rebar_opts:set(Opts, erl_first_files, NewFirstFiles) + end. + +append({error, _} = Error, _) -> Error; +append(_, {error, _} = Error) -> Error; +append(A, B) -> A ++ B. + +add_transforms(CTOpts, State) when is_list(CTOpts) -> + case readable(State) of + true -> + ReadableTransform = [{parse_transform, cth_readable_transform}], + (CTOpts -- ReadableTransform) ++ ReadableTransform; + false -> + CTOpts + end; +add_transforms({error, _} = Error, _State) -> Error. + +readable(State) -> + {RawOpts, _} = rebar_state:command_parsed_args(State), + case proplists:get_value(readable, RawOpts) of + true -> true; + false -> false; + undefined -> rebar_state:get(State, ct_readable, true) + end. + +test_dirs(State, Apps, Opts) -> + case {proplists:get_value(suite, Opts), proplists:get_value(dir, Opts)} of + {Suites, undefined} -> set_compile_dirs(State, Apps, {suite, Suites}); + {undefined, Dirs} -> set_compile_dirs(State, Apps, {dir, Dirs}); + {Suites, Dir} when is_integer(hd(Dir)) -> + set_compile_dirs(State, Apps, join(Suites, Dir)); + {Suites, [Dir]} when is_integer(hd(Dir)) -> + set_compile_dirs(State, Apps, join(Suites, Dir)); + {_Suites, _Dirs} -> {error, "Only a single directory may be specified when specifying suites"} + end. + +join(Suite, Dir) when is_integer(hd(Suite)) -> + {suite, [filename:join([Dir, Suite])]}; +join(Suites, Dir) -> + {suite, lists:map(fun(S) -> filename:join([Dir, S]) end, Suites)}. + +set_compile_dirs(State, Apps, {dir, Dir}) when is_integer(hd(Dir)) -> + %% single directory + %% insert `Dir` into an app if relative, or the base state if not + %% app relative but relative to the root or not at all if outside + %% project scope + {NewState, NewApps} = maybe_inject_test_dir(State, [], Apps, Dir), + {ok, rebar_state:project_apps(NewState, NewApps)}; +set_compile_dirs(State, Apps, {dir, Dirs}) -> + %% multiple directories + F = fun(Dir, {S, A}) -> maybe_inject_test_dir(S, [], A, Dir) end, + {NewState, NewApps} = lists:foldl(F, {State, Apps}, Dirs), + {ok, rebar_state:project_apps(NewState, NewApps)}; +set_compile_dirs(State, Apps, {suite, Suites}) -> + %% suites with dir component + Dirs = find_suite_dirs(Suites), + F = fun(Dir, {S, A}) -> maybe_inject_test_dir(S, [], A, Dir) end, + {NewState, NewApps} = lists:foldl(F, {State, Apps}, Dirs), + {ok, rebar_state:project_apps(NewState, NewApps)}. + +find_suite_dirs(Suites) -> + AllDirs = lists:map(fun(S) -> filename:dirname(filename:absname(S)) end, Suites), + %% eliminate duplicates + lists:usort(AllDirs). + +maybe_inject_test_dir(State, AppAcc, [App|Rest], Dir) -> + case rebar_file_utils:path_from_ancestor(Dir, rebar_app_info:dir(App)) of + {ok, []} -> + %% normal operation involves copying the entire directory a + %% suite exists in but if the suite is in the app root directory + %% the current compiler tries to compile all subdirs including priv + %% instead copy only files ending in `.erl' and directories + %% ending in `_SUITE_data' into the `_build/PROFILE/lib/APP' dir + ok = copy_bare_suites(Dir, rebar_app_info:out_dir(App)), + Opts = inject_test_dir(rebar_state:opts(State), rebar_app_info:out_dir(App)), + {rebar_state:opts(State, Opts), AppAcc ++ [App]}; + {ok, Path} -> + Opts = inject_test_dir(rebar_app_info:opts(App), Path), + {State, AppAcc ++ [rebar_app_info:opts(App, Opts)] ++ Rest}; + {error, badparent} -> + maybe_inject_test_dir(State, AppAcc ++ [App], Rest, Dir) + end; +maybe_inject_test_dir(State, AppAcc, [], Dir) -> + case rebar_file_utils:path_from_ancestor(Dir, rebar_state:dir(State)) of + {ok, []} -> + %% normal operation involves copying the entire directory a + %% suite exists in but if the suite is in the root directory + %% that results in a loop as we copy `_build' into itself + %% instead copy only files ending in `.erl' and directories + %% ending in `_SUITE_data' in the `_build/PROFILE/extras' dir + ExtrasDir = filename:join([rebar_dir:base_dir(State), "extras"]), + ok = copy_bare_suites(Dir, ExtrasDir), + Opts = inject_test_dir(rebar_state:opts(State), ExtrasDir), + {rebar_state:opts(State, Opts), AppAcc}; + {ok, Path} -> + Opts = inject_test_dir(rebar_state:opts(State), Path), + {rebar_state:opts(State, Opts), AppAcc}; + {error, badparent} -> + {State, AppAcc} + end. + +copy_bare_suites(From, To) -> + filelib:ensure_dir(filename:join([To, "dummy.txt"])), + SrcFiles = rebar_utils:find_files(From, ".*\\.[e|h]rl\$", false), + DataDirs = lists:filter(fun filelib:is_dir/1, + filelib:wildcard(filename:join([From, "*_SUITE_data"]))), + ok = rebar_file_utils:cp_r(SrcFiles, To), + rebar_file_utils:cp_r(DataDirs, To). + +inject_test_dir(Opts, Dir) -> + %% append specified test targets to app defined `extra_src_dirs` + ExtraSrcDirs = rebar_opts:get(Opts, extra_src_dirs, []), + rebar_opts:set(Opts, extra_src_dirs, ExtraSrcDirs ++ [Dir]). + +translate_paths(State, Opts) -> + case {proplists:get_value(suite, Opts), proplists:get_value(dir, Opts)} of + {_Suites, undefined} -> translate_suites(State, Opts, []); + {undefined, _Dirs} -> translate_dirs(State, Opts, []); + %% both dirs and suites are defined, only translate dir paths + _ -> translate_dirs(State, Opts, []) + end. + +translate_dirs(_State, [], Acc) -> lists:reverse(Acc); +translate_dirs(State, [{dir, Dir}|Rest], Acc) when is_integer(hd(Dir)) -> + %% single dir + Apps = rebar_state:project_apps(State), + translate_dirs(State, Rest, [{dir, translate(State, Apps, Dir)}|Acc]); +translate_dirs(State, [{dir, Dirs}|Rest], Acc) -> + %% multiple dirs + Apps = rebar_state:project_apps(State), + NewDirs = {dir, lists:map(fun(Dir) -> translate(State, Apps, Dir) end, Dirs)}, + translate_dirs(State, Rest, [NewDirs|Acc]); +translate_dirs(State, [Test|Rest], Acc) -> + translate_dirs(State, Rest, [Test|Acc]). + +translate_suites(_State, [], Acc) -> lists:reverse(Acc); +translate_suites(State, [{suite, Suite}|Rest], Acc) when is_integer(hd(Suite)) -> + %% single suite + Apps = rebar_state:project_apps(State), + translate_suites(State, Rest, [{suite, translate_suite(State, Apps, Suite)}|Acc]); +translate_suites(State, [{suite, Suites}|Rest], Acc) -> + %% multiple suites + Apps = rebar_state:project_apps(State), + NewSuites = {suite, lists:map(fun(Suite) -> translate_suite(State, Apps, Suite) end, Suites)}, + translate_suites(State, Rest, [NewSuites|Acc]); +translate_suites(State, [Test|Rest], Acc) -> + translate_suites(State, Rest, [Test|Acc]). + +translate_suite(State, Apps, Suite) -> + Dirname = filename:dirname(Suite), + Basename = filename:basename(Suite), + case Dirname of + "." -> Suite; + _ -> filename:join([translate(State, Apps, Dirname), Basename]) end. +translate(State, [App|Rest], Path) -> + case rebar_file_utils:path_from_ancestor(Path, rebar_app_info:dir(App)) of + {ok, P} -> filename:join([rebar_app_info:out_dir(App), P]); + {error, badparent} -> translate(State, Rest, Path) + end; +translate(State, [], Path) -> + case rebar_file_utils:path_from_ancestor(Path, rebar_state:dir(State)) of + {ok, P} -> filename:join([rebar_dir:base_dir(State), "extras", P]); + %% not relative, leave as is + {error, badparent} -> Path + end. + +setup_logdir(State, Opts) -> + Logdir = case proplists:get_value(logdir, Opts) of + undefined -> filename:join([rebar_dir:base_dir(State), "logs"]); + Dir -> Dir + end, + filelib:ensure_dir(filename:join([Logdir, "dummy.beam"])), + [{logdir, Logdir}|lists:keydelete(logdir, 1, Opts)]. + +turn_off_auto_compile(Opts) -> + [{auto_compile, false}|lists:keydelete(auto_compile, 1, Opts)]. + run_test_verbose(Opts) -> handle_results(ct:run_test(Opts)). run_test_quiet(Opts) -> @@ -171,272 +557,47 @@ format_skipped({0, 0}) -> format_skipped({User, Auto}) -> io_lib:format("Skipped ~p (~p, ~p) tests. ", [User+Auto, User, Auto]). -test_state(State) -> - TestOpts = case rebar_state:get(State, ct_compile_opts, []) of - [] -> []; - Opts -> [{erl_opts, Opts}] - end, - [first_files(State)|TestOpts]. - -first_files(State) -> - CTFirst = rebar_state:get(State, ct_first_files, []), - {erl_first_files, CTFirst}. - -setup_ct(State) -> - Opts = resolve_ct_opts(State), - Opts1 = discover_tests(State, Opts), - copy_and_compile_tests(State, Opts1). - -resolve_ct_opts(State) -> - {RawOpts, _} = rebar_state:command_parsed_args(State), - CmdOpts = transform_opts(RawOpts), - CfgOpts = rebar_state:get(State, ct_opts, []), - Merged = lists:ukeymerge(1, - lists:ukeysort(1, CmdOpts), - lists:ukeysort(1, CfgOpts)), - %% make sure `dir` and/or `suite` from command line go in as - %% a pair overriding both `dir` and `suite` from config if - %% they exist - case {proplists:get_value(suite, CmdOpts), proplists:get_value(dir, CmdOpts)} of - {undefined, undefined} -> Merged; - {_Suite, undefined} -> lists:keydelete(dir, 1, Merged); - {undefined, _Dir} -> lists:keydelete(suite, 1, Merged); - {_Suite, _Dir} -> Merged - end. - -discover_tests(State, Opts) -> - case proplists:get_value(spec, Opts) of - undefined -> discover_dirs_and_suites(State, Opts); - TestSpec -> discover_testspec(TestSpec, Opts) - end. - -discover_dirs_and_suites(State, Opts) -> - case {proplists:get_value(dir, Opts), proplists:get_value(suite, Opts)} of - %% no dirs or suites defined, try using `$APP/test` and `$ROOT/test` - %% as suites - {undefined, undefined} -> test_dirs(State, Opts); - %% no dirs defined - {undefined, _} -> Opts; - %% no suites defined - {_, undefined} -> Opts; - %% a single dir defined, this is ok - {Dirs, Suites} when is_integer(hd(Dirs)), is_list(Suites) -> Opts; - %% still a single dir defined, adjust to make acceptable to ct - {[Dir], Suites} when is_integer(hd(Dir)), is_list(Suites) -> - [{dir, Dir}|lists:keydelete(dir, 1, Opts)]; - %% multiple dirs and suites, error now to simplify later steps - {_, _} -> throw({error, {multiple_dirs_and_suites, Opts}}) - end. - -discover_testspec(_TestSpec, Opts) -> - lists:keydelete(auto_compile, 1, Opts). - -copy_and_compile_tests(State, Opts) -> - %% possibly enable cover +maybe_cover_compile(State) -> {RawOpts, _} = rebar_state:command_parsed_args(State), State1 = case proplists:get_value(cover, RawOpts, false) of true -> rebar_state:set(State, cover_enabled, true); false -> State end, - copy_and_compile_test_suites(State1, Opts). - -copy_and_compile_test_suites(State, Opts) -> - case proplists:get_value(suite, Opts) of - %% no suites, try dirs - undefined -> copy_and_compile_test_dirs(State, Opts); - Suites -> - Dir = proplists:get_value(dir, Opts, undefined), - AllSuites = join(Dir, Suites), - Dirs = find_suite_dirs(AllSuites), - lists:foreach(fun(S) -> - NewPath = copy(State, S), - compile_dir(State, NewPath) - end, Dirs), - NewSuites = lists:map(fun(S) -> retarget_path(State, S) end, AllSuites), - [{suite, NewSuites}|lists:keydelete(suite, 1, Opts)] - end. - -copy_and_compile_test_dirs(State, Opts) -> - case proplists:get_value(dir, Opts) of - undefined -> {error, {no_tests_specified, Opts}}; - %% dir is a single directory - Dir when is_list(Dir), is_integer(hd(Dir)) -> - NewPath = copy(State, Dir), - [{dir, compile_dir(State, NewPath)}|lists:keydelete(dir, 1, Opts)]; - %% dir is a list of directories - Dirs when is_list(Dirs) -> - NewDirs = lists:map(fun(Dir) -> - NewPath = copy(State, Dir), - compile_dir(State, NewPath) - end, Dirs), - [{dir, NewDirs}|lists:keydelete(dir, 1, Opts)] - end. - -join(undefined, Suites) -> Suites; -join(Dir, Suites) when is_list(Dir), is_integer(hd(Dir)) -> - lists:map(fun(S) -> filename:join([Dir, S]) end, Suites); -%% multiple dirs or a bad dir argument, try to continue anyways -join(_, Suites) -> Suites. - -find_suite_dirs(Suites) -> - AllDirs = lists:map(fun(S) -> filename:dirname(filename:absname(S)) end, Suites), - %% eliminate duplicates - lists:usort(AllDirs). - -copy(State, Dir) -> - From = reduce_path(Dir), - retarget_path(State, From). - -compile_dir(State, Dir) -> - NewState = replace_src_dirs(State, [filename:absname(Dir)]), - ok = rebar_erlc_compiler:compile(rebar_state:opts(NewState), rebar_dir:base_dir(State), Dir), - ok = maybe_cover_compile(State, Dir), - Dir. - -retarget_path(State, Path) -> - ProjectApps = rebar_state:project_apps(State), - retarget_path(State, Path, ProjectApps). - -%% not relative to any apps in project, check to see it's relative to -%% project root -retarget_path(State, Path, []) -> - case relative_path(reduce_path(Path), rebar_state:dir(State)) of - {ok, NewPath} -> filename:join([rebar_dir:base_dir(State), NewPath]); - %% not relative to project root, don't modify - {error, not_relative} -> Path - end; -%% relative to current app, retarget to the same dir relative to -%% the app's out_dir -retarget_path(State, Path, [App|Rest]) -> - case relative_path(reduce_path(Path), rebar_app_info:dir(App)) of - {ok, NewPath} -> filename:join([rebar_app_info:out_dir(App), NewPath]); - {error, not_relative} -> retarget_path(State, Path, Rest) - end. - -relative_path(Target, To) -> - relative_path1(filename:split(filename:absname(Target)), - filename:split(filename:absname(To))). - -relative_path1([Part|Target], [Part|To]) -> relative_path1(Target, To); -relative_path1([], []) -> {ok, ""}; -relative_path1(Target, []) -> {ok, filename:join(Target)}; -relative_path1(_, _) -> {error, not_relative}. - -reduce_path(Dir) -> reduce_path([], filename:split(filename:absname(Dir))). - -reduce_path([], []) -> filename:nativename("/"); -reduce_path(Acc, []) -> filename:join(lists:reverse(Acc)); -reduce_path(Acc, ["."|Rest]) -> reduce_path(Acc, Rest); -reduce_path([_|Acc], [".."|Rest]) -> reduce_path(Acc, Rest); -reduce_path([], [".."|Rest]) -> reduce_path([], Rest); -reduce_path(Acc, [Component|Rest]) -> reduce_path([Component|Acc], Rest). - -replace_src_dirs(State, Dirs) -> - %% replace any `src_dirs` with the test dirs - ErlOpts = rebar_state:get(State, erl_opts, []), - StrippedErlOpts = filter_src_dirs(ErlOpts), - State1 = rebar_state:set(State, erl_opts, StrippedErlOpts), - State2 = rebar_state:set(State1, src_dirs, []), - rebar_state:set(State2, extra_src_dirs, Dirs). - -filter_src_dirs(ErlOpts) -> - lists:filter(fun({src_dirs, _}) -> false; ({extra_src_dirs, _}) -> false; (_) -> true end, ErlOpts). + rebar_prv_cover:maybe_cover_compile(State1). -test_dirs(State, Opts) -> - BareTest = filename:join([rebar_state:dir(State), "test"]), - F = fun(App) -> rebar_app_info:dir(App) == rebar_state:dir(State) end, - TestApps = project_apps(State), - case filelib:is_dir(BareTest) andalso not lists:any(F, TestApps) of - %% `test` dir at root of project is already scheduled to be - %% included or `test` does not exist - false -> application_dirs(TestApps, Opts, []); - %% need to add `test` dir at root to dirs to be included - true -> application_dirs(TestApps, Opts, [BareTest]) - end. - -project_apps(State) -> - filter_checkouts(rebar_state:project_apps(State)). - -filter_checkouts(Apps) -> filter_checkouts(Apps, []). - -filter_checkouts([], Acc) -> lists:reverse(Acc); -filter_checkouts([App|Rest], Acc) -> - case rebar_app_info:is_checkout(App) of - true -> filter_checkouts(Rest, Acc); - false -> filter_checkouts(Rest, [App|Acc]) - end. - -application_dirs([], Opts, []) -> Opts; -application_dirs([], Opts, [Acc]) -> [{dir, Acc}|Opts]; -application_dirs([], Opts, Acc) -> [{dir, lists:reverse(Acc)}|Opts]; -application_dirs([App|Rest], Opts, Acc) -> - TestDir = filename:join([rebar_app_info:dir(App), "test"]), - case filelib:is_dir(TestDir) of - true -> application_dirs(Rest, Opts, [TestDir|Acc]); - false -> application_dirs(Rest, Opts, Acc) - end. - -setup_logdir(State, Opts) -> - Logdir = case proplists:get_value(logdir, Opts) of - undefined -> filename:join([rebar_dir:base_dir(State), "logs"]); - Dir -> Dir - end, - ensure_dir([Logdir]), - [{logdir, Logdir}|lists:keydelete(logdir, 1, Opts)]. - -ensure_dir([]) -> ok; -ensure_dir([Dir|Rest]) -> - case ec_file:is_dir(Dir) of - true -> - ok; - false -> - ec_file:mkdir_path(Dir) - end, - ensure_dir(Rest). - -maybe_cover_compile(State, Dir) -> - {Opts, _} = rebar_state:command_parsed_args(State), - State1 = case proplists:get_value(cover, Opts, false) of +maybe_write_coverdata(State) -> + {RawOpts, _} = rebar_state:command_parsed_args(State), + State1 = case proplists:get_value(cover, RawOpts, false) of true -> rebar_state:set(State, cover_enabled, true); false -> State end, - rebar_prv_cover:maybe_cover_compile(State1, [Dir]). + rebar_prv_cover:maybe_write_coverdata(State1, ?PROVIDER). ct_opts(_State) -> [{dir, undefined, "dir", string, help(dir)}, %% comma-seperated list {suite, undefined, "suite", string, help(suite)}, %% comma-seperated list {group, undefined, "group", string, help(group)}, %% comma-seperated list {testcase, undefined, "case", string, help(testcase)}, %% comma-seperated list - {spec, undefined, "spec", string, help(spec)}, %% comma-seperated list - {join_specs, undefined, "join_specs", boolean, help(join_specs)}, %% Boolean {label, undefined, "label", string, help(label)}, %% String {config, undefined, "config", string, help(config)}, %% comma-seperated list - {userconfig, undefined, "userconfig", string, help(userconfig)}, %% [{CallbackMod, CfgStrings}] | {CallbackMod, CfgStrings} {allow_user_terms, undefined, "allow_user_terms", boolean, help(allow_user_terms)}, %% Bool {logdir, undefined, "logdir", string, help(logdir)}, %% dir - {logopts, undefined, "logopts", string, help(logopts)}, %% enum, no_nl | no_src - {verbosity, undefined, "verbosity", string, help(verbosity)}, %% Integer OR [{Category, VLevel}] - {silent_connections, undefined, "silent_connections", string, - help(silent_connections)}, % all OR %% comma-seperated list - {stylesheet, undefined, "stylesheet", string, help(stylesheet)}, %% file + {logopts, undefined, "logopts", string, help(logopts)}, %% comma seperated list + {verbosity, undefined, "verbosity", integer, help(verbosity)}, %% Integer {cover, $c, "cover", {boolean, false}, help(cover)}, - {cover_spec, undefined, "cover_spec", string, help(cover_spec)}, %% file - {cover_stop, undefined, "cover_stop", boolean, help(cover_stop)}, %% Boolean - {event_handler, undefined, "event_handler", string, help(event_handler)}, %% EH | [EH] WHERE EH atom() | {atom(), InitArgs} | {[atom()], InitArgs} - {include, undefined, "include", string, help(include)}, % comma-seperated list - {abort_if_missing_suites, undefined, "abort_if_missing_suites", {boolean, true}, - help(abort_if_missing_suites)}, %% boolean - {multiply_timetraps, undefined, "multiply_timetraps", integer, - help(multiply_timetraps)}, %% integer - {scale_timetraps, undefined, "scale_timetraps", boolean, help(scale_timetraps)}, %% Boolean - {create_priv_dir, undefined, "create_priv_dir", string, help(create_priv_dir)}, %% enum: auto_per_run | auto_per_tc | manual_per_tc {repeat, undefined, "repeat", integer, help(repeat)}, %% integer {duration, undefined, "duration", string, help(duration)}, % format: HHMMSS {until, undefined, "until", string, help(until)}, %% format: YYMoMoDD[HHMMSS] - {force_stop, undefined, "force_stop", string, help(force_stop)}, % enum: skip_rest, bool - {basic_html, undefined, "basic_html", boolean, help(basic_html)}, %% Booloean - {ct_hooks, undefined, "ct_hooks", string, help(ct_hooks)}, %% List: [CTHModule | {CTHModule, CTHInitArgs}] where CTHModule is atom CthInitArgs is term - {auto_compile, undefined, "auto_compile", {boolean, false}, help(auto_compile)}, + {force_stop, undefined, "force_stop", string, help(force_stop)}, %% String + {basic_html, undefined, "basic_html", boolean, help(basic_html)}, %% Boolean + {stylesheet, undefined, "stylesheet", string, help(stylesheet)}, %% String + {decrypt_key, undefined, "decrypt_key", string, help(decrypt_key)}, %% String + {decrypt_file, undefined, "decrypt_file", string, help(decrypt_file)}, %% String + {abort_if_missing_suites, undefined, "abort_if_missing_suites", {boolean, true}, help(abort_if_missing_suites)}, %% Boolean + {multiply_timetraps, undefined, "multiply_timetraps", integer, help(multiple_timetraps)}, %% Integer + {scale_timetraps, undefined, "scale_timetraps", boolean, help(scale_timetraps)}, + {create_priv_dir, undefined, "create_priv_dir", string, help(create_priv_dir)}, + {readable, undefined, "readable", boolean, help(readable)}, {verbose, $v, "verbose", boolean, help(verbose)} ]. @@ -448,28 +609,20 @@ help(group) -> "List of test groups to run"; help(testcase) -> "List of test cases to run"; -help(spec) -> - "List of test specs to run"; help(label) -> "Test label"; help(config) -> "List of config files"; +help(allow_user_terms) -> + "Allow user defined config values in config files"; help(logdir) -> "Log folder"; +help(logopts) -> + "Options for common test logging"; help(verbosity) -> "Verbosity"; -help(stylesheet) -> - "Stylesheet to use for test results"; help(cover) -> "Generate cover data"; -help(cover_spec) -> - "Cover file to use"; -help(event_handler) -> - "Event handlers to attach to the runner"; -help(include) -> - "Include folder"; -help(abort_if_missing_suites) -> - "Abort if suites are missing"; help(repeat) -> "How often to repeat tests"; help(duration) -> @@ -477,85 +630,27 @@ help(duration) -> help(until) -> "Run until (format: HHMMSS)"; help(force_stop) -> - "Force stop after time"; + "Force stop on test timeout (true | false | skip_rest)"; help(basic_html) -> "Show basic HTML"; +help(stylesheet) -> + "CSS stylesheet to apply to html output"; +help(decrypt_key) -> + "Path to key for decrypting config"; +help(decrypt_file) -> + "Path to file containing key for decrypting config"; +help(abort_if_missing_suites) -> + "Abort if suites are missing"; +help(multiply_timetraps) -> + "Multiply timetraps"; +help(scale_timetraps) -> + "Scale timetraps"; +help(create_priv_dir) -> + "Create priv dir (auto_per_run | auto_per_tc | manual_per_tc)"; +help(readable) -> + "Shows test case names and only displays logs to shell on failures"; help(verbose) -> "Verbose output"; help(_) -> "". -transform_opts(Opts) -> - transform_opts(Opts, []). - -transform_opts([], Acc) -> Acc; -%% drop `cover` and `verbose` so they're not passed as an option to common_test -transform_opts([{cover, _}|Rest], Acc) -> - transform_opts(Rest, Acc); -transform_opts([{cover_spec, CoverSpec}|Rest], Acc) -> - transform_opts(Rest, [{cover, CoverSpec}|Acc]); -transform_opts([{verbose, _}|Rest], Acc) -> - transform_opts(Rest, Acc); -transform_opts([{ct_hooks, CtHooks}|Rest], Acc) -> - transform_opts(Rest, [{ct_hooks, parse_term(CtHooks)}|Acc]); -transform_opts([{force_stop, "skip_rest"}|Rest], Acc) -> - transform_opts(Rest, [{force_stop, skip_rest}|Acc]); -transform_opts([{force_stop, _}|Rest], Acc) -> - transform_opts(Rest, [{force_stop, true}|Acc]); -transform_opts([{repeat, Repeat}|Rest], Acc) -> - transform_opts(Rest, [{repeat, - ec_cnv:to_integer(Repeat)}|Acc]); -transform_opts([{create_priv_dir, CreatePrivDir}|Rest], Acc) -> - transform_opts(Rest, [{create_priv_dir, - to_atoms(CreatePrivDir)}|Acc]); -transform_opts([{multiply_timetraps, MultiplyTimetraps}|Rest], Acc) -> - transform_opts(Rest, [{multiply_timetraps, - ec_cnv:to_integer(MultiplyTimetraps)}|Acc]); -transform_opts([{event_handler, EventHandler}|Rest], Acc) -> - transform_opts(Rest, [{event_handler, parse_term(EventHandler)}|Acc]); -transform_opts([{silent_connections, "all"}|Rest], Acc) -> - transform_opts(Rest, [{silent_connections, all}|Acc]); -transform_opts([{silent_connections, SilentConnections}|Rest], Acc) -> - transform_opts(Rest, [{silent_connections, - to_atoms(split_string(SilentConnections))}|Acc]); -transform_opts([{verbosity, Verbosity}|Rest], Acc) -> - transform_opts(Rest, [{verbosity, parse_term(Verbosity)}|Acc]); -transform_opts([{logopts, LogOpts}|Rest], Acc) -> - transform_opts(Rest, [{logopts, to_atoms(split_string(LogOpts))}|Acc]); -transform_opts([{userconfig, UserConfig}|Rest], Acc) -> - transform_opts(Rest, [{userconfig, parse_term(UserConfig)}|Acc]); -transform_opts([{testcase, Testcase}|Rest], Acc) -> - transform_opts(Rest, [{testcase, to_atoms(split_string(Testcase))}|Acc]); -transform_opts([{group, Group}|Rest], Acc) -> % @TODO handle "" - % Input is a list or an atom. It can also be a nested list. - transform_opts(Rest, [{group, parse_term(Group)}|Acc]); -transform_opts([{suite, Suite}|Rest], Acc) -> - transform_opts(Rest, [{suite, split_string(Suite)}|Acc]); -transform_opts([{Key, Val}|Rest], Acc) when is_list(Val) -> - % Default to splitting a string on comma, that works fine for both flat - % lists of which there are many and single-items. - Val1 = case split_string(Val) of - [Val2] -> - Val2; - Val2 -> - Val2 - end, - transform_opts(Rest, [{Key, Val1}|Acc]); -transform_opts([{Key, Val}|Rest], Acc) -> - transform_opts(Rest, [{Key, Val}|Acc]). - -to_atoms(List) -> - lists:map(fun(X) -> list_to_atom(X) end, List). - -split_string(String) -> - string:tokens(String, ","). - -parse_term(String) -> - String1 = "[" ++ String ++ "].", - {ok, Tokens, _} = erl_scan:string(String1), - case erl_parse:parse_term(Tokens) of - {ok, [Terms]} -> - Terms; - Term -> - Term - end. diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index 2996aee..30af90b 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -217,6 +217,10 @@ copy(OldAppDir, AppDir, Dir) -> %% TODO: use ec_file:copy/2 to do this, it preserves timestamps and %% may prevent recompilation of files in extra dirs +copy(Source, Source) -> + %% allow users to specify a directory in _build as a directory + %% containing additional source/tests + ok; copy(Source, Target) -> %% important to do this so no files are copied onto themselves %% which truncates them to zero length on some platforms @@ -243,6 +247,21 @@ resolve_src_dirs(Opts) -> %% in src_dirs also exist in extra_src_dirs normalize_src_dirs(SrcDirs, ExtraDirs) -> S = lists:usort(SrcDirs), - E = lists:usort(ExtraDirs), - {S, lists:subtract(E, S)}. + E = lists:subtract(lists:usort(ExtraDirs), S), + ok = warn_on_problematic_directories(S ++ E), + {S, E}. + +%% warn when directories called `eunit' and `ct' are added to compile dirs +warn_on_problematic_directories(AllDirs) -> + F = fun(Dir) -> + case is_a_problem(Dir) of + true -> ?WARN("Possible name clash with directory ~p.", [Dir]); + false -> ok + end + end, + lists:foreach(F, AllDirs). + +is_a_problem("eunit") -> true; +is_a_problem("common_test") -> true; +is_a_problem(_) -> false. diff --git a/src/rebar_prv_cover.erl b/src/rebar_prv_cover.erl index 0b9b9bb..c915141 100644 --- a/src/rebar_prv_cover.erl +++ b/src/rebar_prv_cover.erl @@ -207,6 +207,8 @@ format_table(Stats, CoverFiles) -> MaxLength = max(lists:foldl(fun max_length/2, 0, Stats), 20), Header = header(MaxLength), Seperator = seperator(MaxLength), + TotalLabel = format("total", MaxLength), + TotalCov = format(calculate_total(Stats), 8), [io_lib:format("~ts~n~ts~n~ts~n", [Seperator, Header, Seperator]), lists:map(fun({Mod, Coverage}) -> Name = format(Mod, MaxLength), @@ -214,6 +216,8 @@ format_table(Stats, CoverFiles) -> io_lib:format(" | ~ts | ~ts |~n", [Name, Cov]) end, Stats), io_lib:format("~ts~n", [Seperator]), + io_lib:format(" | ~ts | ~ts |~n", [TotalLabel, TotalCov]), + io_lib:format("~ts~n", [Seperator]), io_lib:format(" coverage calculated from:~n", []), lists:map(fun(File) -> io_lib:format(" ~ts~n", [File]) @@ -234,6 +238,18 @@ seperator(Width) -> format(String, Width) -> io_lib:format("~*.ts", [Width, String]). +calculate_total(Stats) when length(Stats) =:= 0 -> + "0%"; +calculate_total(Stats) -> + TotalStats = length(Stats), + TotalCovInt = round(lists:foldl( + fun({_Mod, Coverage, _File}, Acc) -> + Acc + (list_to_integer(string:strip(Coverage, right, $%)) / TotalStats); + ({_Mod, Coverage}, Acc) -> + Acc + (list_to_integer(string:strip(Coverage, right, $%)) / TotalStats) + end, 0, Stats)), + integer_to_list(TotalCovInt) ++ "%". + write_index(State, Coverage) -> CoverDir = cover_dir(State), FileName = filename:join([CoverDir, "index.html"]), @@ -265,6 +281,8 @@ write_index_section(F, [{Section, DataFile, Mods}|Rest]) -> [strip_coverdir(Report), Mod, Cov]) end, lists:foreach(fun(M) -> ok = file:write(F, FmtLink(M)) end, Mods), + ok = file:write(F, ?FMT("<tr><td><strong>Total</strong></td><td>~ts</td>\n", + [calculate_total(Mods)])), ok = file:write(F, "</table>\n"), write_index_section(F, Rest). @@ -279,21 +297,26 @@ cover_compile(State, apps) -> Apps = filter_checkouts(rebar_state:project_apps(State)), AppDirs = app_dirs(Apps), ExtraDirs = extra_src_dirs(State, Apps), - cover_compile(State, AppDirs ++ ExtraDirs); + cover_compile(State, lists:filter(fun(D) -> ec_file:is_dir(D) end, AppDirs ++ ExtraDirs)); cover_compile(State, Dirs) -> %% start the cover server if necessary {ok, CoverPid} = start_cover(), %% redirect cover output true = redirect_cover_output(State, CoverPid), - CompileResult = compile(Dirs, []), - %% print any warnings about modules that failed to cover compile - lists:foreach(fun print_cover_warnings/1, lists:flatten(CompileResult)). - -compile([], Acc) -> lists:reverse(Acc); -compile([Dir|Rest], Acc) -> - ?INFO("covering ~p", [Dir]), - Result = cover:compile_beam_directory(Dir), - compile(Rest, [Result|Acc]). + lists:foreach(fun(Dir) -> + ?DEBUG("cover compiling ~p", [Dir]), + case catch(cover:compile_beam_directory(Dir)) of + {error, eacces} -> + ?WARN("Directory ~p not readable, modules will not be included in coverage", [Dir]); + {error, enoent} -> + ?WARN("Directory ~p not found", [Dir]); + {'EXIT', {Reason, _}} -> + ?WARN("Cover compilation for directory ~p failed: ~p", [Dir, Reason]); + Results -> + %% print any warnings about modules that failed to cover compile + lists:foreach(fun print_cover_warnings/1, lists:flatten(Results)) + end + end, Dirs). app_dirs(Apps) -> lists:foldl(fun app_ebin_dirs/2, [], Apps). @@ -302,7 +325,7 @@ app_ebin_dirs(App, Acc) -> AppDir = rebar_app_info:ebin_dir(App), ExtraDirs = rebar_dir:extra_src_dirs(rebar_app_info:opts(App), []), OutDir = rebar_app_info:out_dir(App), - [filename:join([OutDir, D]) || D <- [AppDir|ExtraDirs]] ++ Acc. + [AppDir] ++ [filename:join([OutDir, D]) || D <- ExtraDirs] ++ Acc. extra_src_dirs(State, Apps) -> BaseDir = rebar_state:dir(State), @@ -339,9 +362,8 @@ redirect_cover_output(State, CoverPid) -> group_leader(F, CoverPid). print_cover_warnings({ok, _}) -> ok; -print_cover_warnings({error, File}) -> - ?WARN("Cover compilation of ~p failed, module is not included in cover data.", - [File]). +print_cover_warnings({error, Error}) -> + ?WARN("Cover compilation failed: ~p", [Error]). write_coverdata(State, Task) -> DataDir = cover_dir(State), diff --git a/src/rebar_prv_dialyzer.erl b/src/rebar_prv_dialyzer.erl index 487e9d1..834eb98 100644 --- a/src/rebar_prv_dialyzer.erl +++ b/src/rebar_prv_dialyzer.erl @@ -173,7 +173,7 @@ do_update_proj_plt(State, Plt, Output) -> case read_plt(State, Plt) of {ok, OldFiles} -> check_plt(State, Plt, Output, OldFiles, Files); - {error, no_such_file} -> + error -> build_proj_plt(State, Plt, Output, Files) end. @@ -252,14 +252,25 @@ read_plt(_State, Plt) -> case dialyzer:plt_info(Plt) of {ok, Info} -> Files = proplists:get_value(files, Info, []), - {ok, Files}; - {error, no_such_file} = Error -> - Error; + read_plt_files(Plt, Files); + {error, no_such_file} -> + error; {error, read_error} -> Error = io_lib:format("Could not read the PLT file ~p", [Plt]), throw({dialyzer_error, Error}) end. +%% If any file no longer exists dialyzer will fail when updating the PLT. +read_plt_files(Plt, Files) -> + case [File || File <- Files, not filelib:is_file(File)] of + [] -> + {ok, Files}; + Missing -> + ?INFO("Could not find ~p files in ~p...", [length(Missing), Plt]), + ?DEBUG("Could not find files: ~p", [Missing]), + error + end. + check_plt(State, Plt, Output, OldList, FilesList) -> Old = sets:from_list(OldList), Files = sets:from_list(FilesList), @@ -337,7 +348,7 @@ update_base_plt(State, BasePlt, Output, BaseFiles) -> case read_plt(State, BasePlt) of {ok, OldBaseFiles} -> check_plt(State, BasePlt, Output, OldBaseFiles, BaseFiles); - {error, no_such_file} -> + error -> _ = filelib:ensure_dir(BasePlt), build_plt(State, BasePlt, Output, BaseFiles) end. @@ -386,7 +397,7 @@ run_dialyzer(State, Opts, Output) -> case proplists:get_bool(get_warnings, Opts) of true -> WarningsList = get_config(State, warnings, []), - Opts2 = [{warnings, WarningsList}, + Opts2 = [{warnings, legacy_warnings(WarningsList)}, {check_plt, false} | Opts], ?DEBUG("Running dialyzer with options: ~p~n", [Opts2]), @@ -401,6 +412,14 @@ run_dialyzer(State, Opts, Output) -> {0, State} end. +legacy_warnings(Warnings) -> + case dialyzer_version() of + TupleVsn when TupleVsn < {2, 8, 0} -> + [Warning || Warning <- Warnings, Warning =/= unknown]; + _ -> + Warnings + end. + format_warnings(Output, Warnings) -> Warnings1 = rebar_dialyzer_format:format_warnings(Warnings), console_warnings(Warnings1), @@ -464,3 +483,16 @@ collect_nested_dependent_apps(App, Seen) -> end end end. + +dialyzer_version() -> + _ = application:load(dialyzer), + {ok, Vsn} = application:get_key(dialyzer, vsn), + case string:tokens(Vsn, ".") of + [Major, Minor] -> + version_tuple(Major, Minor, "0"); + [Major, Minor, Patch | _] -> + version_tuple(Major, Minor, Patch) + end. + +version_tuple(Major, Minor, Patch) -> + {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}. diff --git a/src/rebar_prv_eunit.erl b/src/rebar_prv_eunit.erl index 2c687ac..a1a4408 100644 --- a/src/rebar_prv_eunit.erl +++ b/src/rebar_prv_eunit.erl @@ -9,7 +9,7 @@ do/1, format_error/1]). %% exported solely for tests --export([compile/2, prepare_tests/1, eunit_opts/1]). +-export([prepare_tests/1, eunit_opts/1, validate_tests/2]). -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). @@ -39,24 +39,26 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> Tests = prepare_tests(State), - case compile(State, Tests) of + %% inject `eunit_first_files`, `eunit_compile_opts` and any + %% directories required by tests into the applications + NewState = inject_eunit_state(State, Tests), + case compile(NewState) of %% successfully compiled apps {ok, S} -> do(S, Tests); - %% this should look like a compiler error, not an eunit error Error -> Error end. do(State, Tests) -> ?INFO("Performing EUnit tests...", []), - rebar_utils:update_code(rebar_state:code_paths(State, all_deps)), + rebar_utils:update_code(rebar_state:code_paths(State, all_deps), [soft_purge]), %% Run eunit provider prehooks Providers = rebar_state:providers(State), Cwd = rebar_dir:get_cwd(), rebar_hooks:run_all_hooks(Cwd, pre, ?PROVIDER, Providers, State), - case Tests of + case validate_tests(State, Tests) of {ok, T} -> case run_tests(State, T) of {ok, State1} -> @@ -95,6 +97,8 @@ format_error({error_running_tests, Reason}) -> format_error({eunit_test_errors, Errors}) -> io_lib:format(lists:concat(["Error Running EUnit Tests:"] ++ lists:map(fun(Error) -> "~n " ++ Error end, Errors)), []); +format_error({badconfig, {Msg, {Value, Key}}}) -> + io_lib:format(Msg, [Value, Key]); format_error({error, Error}) -> format_error({error_running_tests, Error}). @@ -102,45 +106,150 @@ format_error({error, Error}) -> %% Internal functions %% =================================================================== -compile(State, {ok, Tests}) -> - %% inject `eunit_first_files`, `eunit_compile_opts` and any - %% directories required by tests into the applications - NewState = inject_eunit_state(State, Tests), +prepare_tests(State) -> + %% parse and translate command line tests + CmdTests = resolve_tests(State), + CfgTests = cfg_tests(State), + ProjectApps = rebar_state:project_apps(State), + %% prioritize tests to run first trying any command line specified + %% tests falling back to tests specified in the config file finally + %% running a default set if no other tests are present + select_tests(State, ProjectApps, CmdTests, CfgTests). - case rebar_prv_compile:do(NewState) of - %% successfully compiled apps - {ok, S} -> - ok = maybe_cover_compile(S), - {ok, S}; - %% this should look like a compiler error, not an eunit error - Error -> Error - end; -%% maybe compile even in the face of errors? -compile(_State, Error) -> Error. +resolve_tests(State) -> + {RawOpts, _} = rebar_state:command_parsed_args(State), + Apps = resolve(app, application, RawOpts), + Applications = resolve(application, RawOpts), + Dirs = resolve(dir, RawOpts), + Files = resolve(file, RawOpts), + Modules = resolve(module, RawOpts), + Suites = resolve(suite, module, RawOpts), + Apps ++ Applications ++ Dirs ++ Files ++ Modules ++ Suites. -inject_eunit_state(State, Tests) -> +resolve(Flag, RawOpts) -> resolve(Flag, Flag, RawOpts). + +resolve(Flag, EUnitKey, RawOpts) -> + case proplists:get_value(Flag, RawOpts) of + undefined -> []; + Args -> lists:map(fun(Arg) -> normalize(EUnitKey, Arg) end, string:tokens(Args, [$,])) + end. + +normalize(Key, Value) when Key == dir; Key == file -> {Key, Value}; +normalize(Key, Value) -> {Key, list_to_atom(Value)}. + +cfg_tests(State) -> + case rebar_state:get(State, eunit_tests, []) of + Tests when is_list(Tests) -> + lists:map(fun({app, App}) -> {application, App}; (T) -> T end, Tests); + Wrong -> + %% probably a single non list term + ?PRV_ERROR({badconfig, {"Value `~p' of option `~p' must be a list", {Wrong, eunit_tests}}}) + end. + +select_tests(_State, _ProjectApps, {error, _} = Error, _) -> Error; +select_tests(_State, _ProjectApps, _, {error, _} = Error) -> Error; +select_tests(State, ProjectApps, [], []) -> {ok, default_tests(State, ProjectApps)}; +select_tests(_State, _ProjectApps, [], Tests) -> {ok, Tests}; +select_tests(_State, _ProjectApps, Tests, _) -> {ok, Tests}. + +default_tests(State, Apps) -> + %% use `{application, App}` for each app in project + AppTests = set_apps(Apps), + %% additional test modules in `test` dir of each app + ModTests = set_modules(Apps, State), + AppTests ++ ModTests. + +set_apps(Apps) -> set_apps(Apps, []). + +set_apps([], Acc) -> Acc; +set_apps([App|Rest], Acc) -> + AppName = list_to_atom(binary_to_list(rebar_app_info:name(App))), + set_apps(Rest, [{application, AppName}|Acc]). + +set_modules(Apps, State) -> set_modules(Apps, State, {[], []}). + +set_modules([], State, {AppAcc, TestAcc}) -> + TestSrc = gather_src([filename:join([rebar_state:dir(State), "test"])]), + dedupe_tests({AppAcc, TestAcc ++ TestSrc}); +set_modules([App|Rest], State, {AppAcc, TestAcc}) -> + F = fun(Dir) -> filename:join([rebar_app_info:dir(App), Dir]) end, + AppDirs = lists:map(F, rebar_dir:src_dirs(rebar_app_info:opts(App), ["src"])), + AppSrc = gather_src(AppDirs), + TestDirs = [filename:join([rebar_app_info:dir(App), "test"])], + TestSrc = gather_src(TestDirs), + set_modules(Rest, State, {AppSrc ++ AppAcc, TestSrc ++ TestAcc}). + +gather_src(Dirs) -> gather_src(Dirs, []). + +gather_src([], Srcs) -> Srcs; +gather_src([Dir|Rest], Srcs) -> + gather_src(Rest, Srcs ++ rebar_utils:find_files(Dir, "^[^._].*\\.erl\$", true)). + +dedupe_tests({AppMods, TestMods}) -> + %% for each modules in TestMods create a test if there is not a module + %% in AppMods that will trigger it + F = fun(Mod) -> + M = filename:basename(Mod, ".erl"), + MatchesTest = fun(Dir) -> filename:basename(Dir, ".erl") ++ "_tests" == M end, + case lists:any(MatchesTest, AppMods) of + false -> {true, {module, list_to_atom(M)}}; + true -> false + end + end, + lists:usort(rebar_utils:filtermap(F, TestMods)). + +inject_eunit_state(State, {ok, Tests}) -> Apps = rebar_state:project_apps(State), - ModdedApps = lists:map(fun(App) -> - NewOpts = inject(rebar_app_info:opts(App), State), - rebar_app_info:opts(App, NewOpts) - end, Apps), - NewOpts = inject(rebar_state:opts(State), State), - NewState = rebar_state:opts(State, NewOpts), - test_dirs(NewState, ModdedApps, Tests). - -inject(Opts, State) -> + case inject_eunit_state(State, Apps, []) of + {ok, {NewState, ModdedApps}} -> + test_dirs(NewState, ModdedApps, Tests); + {error, _} = Error -> Error + end; +inject_eunit_state(_State, Error) -> Error. + +inject_eunit_state(State, [App|Rest], Acc) -> + case inject(rebar_app_info:opts(App)) of + {error, _} = Error -> Error; + NewOpts -> + NewApp = rebar_app_info:opts(App, NewOpts), + inject_eunit_state(State, Rest, [NewApp|Acc]) + end; +inject_eunit_state(State, [], Acc) -> + case inject(rebar_state:opts(State)) of + {error, _} = Error -> Error; + NewOpts -> {ok, {rebar_state:opts(State, NewOpts), lists:reverse(Acc)}} + end. + +opts(Opts, Key, Default) -> + case rebar_opts:get(Opts, Key, Default) of + Vs when is_list(Vs) -> Vs; + Wrong -> + ?PRV_ERROR({badconfig, {"Value `~p' of option `~p' must be a list", {Wrong, Key}}}) + end. + +inject(Opts) -> erl_opts(Opts). + +erl_opts(Opts) -> %% append `eunit_compile_opts` to app defined `erl_opts` - ErlOpts = rebar_opts:get(Opts, erl_opts, []), - EUnitOpts = rebar_state:get(State, eunit_compile_opts, []), - NewErlOpts = EUnitOpts ++ ErlOpts, + ErlOpts = opts(Opts, erl_opts, []), + EUnitOpts = opts(Opts, eunit_compile_opts, []), + case append(EUnitOpts, ErlOpts) of + {error, _} = Error -> Error; + NewErlOpts -> first_files(rebar_opts:set(Opts, erl_opts, NewErlOpts)) + end. + +first_files(Opts) -> %% append `eunit_first_files` to app defined `erl_first_files` - FirstFiles = rebar_opts:get(Opts, erl_first_files, []), - EUnitFirstFiles = rebar_state:get(State, eunit_first_files, []), - NewFirstFiles = EUnitFirstFiles ++ FirstFiles, - %% insert the new keys into the opts - lists:foldl(fun({K, V}, NewOpts) -> rebar_opts:set(NewOpts, K, V) end, - Opts, - [{erl_opts, NewErlOpts}, {erl_first_files, NewFirstFiles}]). + FirstFiles = opts(Opts, erl_first_files, []), + EUnitFirstFiles = opts(Opts, eunit_first_files, []), + case append(EUnitFirstFiles, FirstFiles) of + {error, _} = Error -> Error; + NewFirstFiles -> rebar_opts:set(Opts, erl_first_files, NewFirstFiles) + end. + +append({error, _} = Error, _) -> Error; +append(_, {error, _} = Error) -> Error; +append(A, B) -> A ++ B. test_dirs(State, Apps, []) -> rebar_state:project_apps(State, Apps); test_dirs(State, Apps, [{dir, Dir}|Rest]) -> @@ -174,67 +283,23 @@ maybe_inject_test_dir(State, AppAcc, [], Dir) -> inject_test_dir(Opts, Dir) -> %% append specified test targets to app defined `extra_src_dirs` - ExtraSrcDirs = rebar_opts:get(Opts, extra_src_dirs, []), + ExtraSrcDirs = rebar_dir:extra_src_dirs(Opts), rebar_opts:set(Opts, extra_src_dirs, ExtraSrcDirs ++ [Dir]). -prepare_tests(State) -> - %% parse and translate command line tests - CmdTests = resolve_tests(State), - CfgTests = rebar_state:get(State, eunit_tests, []), - ProjectApps = rebar_state:project_apps(State), - %% prioritize tests to run first trying any command line specified - %% tests falling back to tests specified in the config file finally - %% running a default set if no other tests are present - Tests = select_tests(State, ProjectApps, CmdTests, CfgTests), - %% check applications for existence in project, modules for existence - %% in applications, files and dirs for existence on disk and allow - %% any unrecognized tests through for eunit to handle - validate_tests(State, ProjectApps, Tests). - -resolve_tests(State) -> - {RawOpts, _} = rebar_state:command_parsed_args(State), - Apps = resolve(app, application, RawOpts), - Applications = resolve(application, RawOpts), - Dirs = resolve(dir, RawOpts), - Files = resolve(file, RawOpts), - Modules = resolve(module, RawOpts), - Suites = resolve(suite, module, RawOpts), - Apps ++ Applications ++ Dirs ++ Files ++ Modules ++ Suites. - -resolve(Flag, RawOpts) -> resolve(Flag, Flag, RawOpts). - -resolve(Flag, EUnitKey, RawOpts) -> - case proplists:get_value(Flag, RawOpts) of - undefined -> []; - Args -> lists:map(fun(Arg) -> normalize(EUnitKey, Arg) end, string:tokens(Args, [$,])) - end. - -normalize(Key, Value) when Key == dir; Key == file -> {Key, Value}; -normalize(Key, Value) -> {Key, list_to_atom(Value)}. - -select_tests(State, ProjectApps, [], []) -> default_tests(State, ProjectApps); -select_tests(_State, _ProjectApps, [], Tests) -> Tests; -select_tests(_State, _ProjectApps, Tests, _) -> Tests. - -default_tests(State, Apps) -> - Tests = set_apps(Apps, []), - BareTest = filename:join([rebar_state:dir(State), "test"]), - F = fun(App) -> rebar_app_info:dir(App) == rebar_state:dir(State) end, - case filelib:is_dir(BareTest) andalso not lists:any(F, Apps) of - %% `test` dir at root of project is already scheduled to be - %% included or `test` does not exist - false -> lists:reverse(Tests); - %% need to add `test` dir at root to dirs to be included - true -> lists:reverse([{dir, BareTest}|Tests]) +compile({error, _} = Error) -> Error; +compile(State) -> + case rebar_prv_compile:do(State) of + %% successfully compiled apps + {ok, S} -> + ok = maybe_cover_compile(S), + {ok, S}; + %% this should look like a compiler error, not an eunit error + Error -> Error end. -set_apps([], Acc) -> Acc; -set_apps([App|Rest], Acc) -> - AppName = list_to_atom(binary_to_list(rebar_app_info:name(App))), - set_apps(Rest, [{application, AppName}|Acc]). - -validate_tests(State, ProjectApps, Tests) -> - gather_tests(fun(Elem) -> validate(State, ProjectApps, Elem) end, Tests, []). +validate_tests(State, {ok, Tests}) -> + gather_tests(fun(Elem) -> validate(State, Elem) end, Tests, []); +validate_tests(_State, Error) -> Error. gather_tests(_F, [], Acc) -> {ok, lists:reverse(Acc)}; gather_tests(F, [Test|Rest], Acc) -> @@ -251,27 +316,31 @@ gather_errors(F, [Test|Rest], Acc) -> {error, Error} -> gather_errors(F, Rest, [Error|Acc]) end. -validate(State, ProjectApps, {application, App}) -> - validate_app(State, ProjectApps, App); -validate(State, _ProjectApps, {dir, Dir}) -> +validate(State, {application, App}) -> + validate_app(State, App); +validate(State, {dir, Dir}) -> validate_dir(State, Dir); -validate(State, _ProjectApps, {file, File}) -> +validate(State, {file, File}) -> validate_file(State, File); -validate(State, _ProjectApps, {module, Module}) -> +validate(State, {module, Module}) -> validate_module(State, Module); -validate(State, _ProjectApps, {suite, Module}) -> +validate(State, {suite, Module}) -> validate_module(State, Module); -validate(State, _ProjectApps, Module) when is_atom(Module) -> +validate(State, Module) when is_atom(Module) -> validate_module(State, Module); -validate(State, ProjectApps, Path) when is_list(Path) -> +validate(State, Path) when is_list(Path) -> case ec_file:is_dir(Path) of - true -> validate(State, ProjectApps, {dir, Path}); - false -> validate(State, ProjectApps, {file, Path}) + true -> validate(State, {dir, Path}); + false -> validate(State, {file, Path}) end; %% unrecognized tests should be included. if they're invalid eunit will error %% and rebar.config may contain arbitrarily complex tests that are effectively %% unvalidatable -validate(_State, _ProjectApps, _Test) -> ok. +validate(_State, _Test) -> ok. + +validate_app(State, AppName) -> + ProjectApps = rebar_state:project_apps(State), + validate_app(State, ProjectApps, AppName). validate_app(_State, [], AppName) -> {error, lists:concat(["Application `", AppName, "' not found in project."])}; @@ -294,18 +363,29 @@ validate_file(State, File) -> end. validate_module(_State, Module) -> - Path = code:which(Module), - case beam_lib:chunks(Path, [exports]) of - {ok, _} -> ok; - {error, beam_lib, _} -> {error, lists:concat(["Module `", Module, "' not found in project."])} + case code:which(Module) of + non_existing -> {error, lists:concat(["Module `", Module, "' not found in project."])}; + _ -> ok end. resolve_eunit_opts(State) -> {Opts, _} = rebar_state:command_parsed_args(State), EUnitOpts = rebar_state:get(State, eunit_opts, []), - case proplists:get_value(verbose, Opts, false) of - true -> set_verbose(EUnitOpts); - false -> EUnitOpts + EUnitOpts1 = case proplists:get_value(verbose, Opts, false) of + true -> set_verbose(EUnitOpts); + false -> EUnitOpts + end, + IsVerbose = lists:member(verbose, EUnitOpts1), + case proplists:get_value(eunit_formatters, EUnitOpts1, not IsVerbose) of + true -> custom_eunit_formatters(EUnitOpts1); + false -> EUnitOpts1 + end. + +custom_eunit_formatters(Opts) -> + %% If `report` is already set then treat that like `eunit_formatters` is false + case lists:keymember(report, 1, Opts) of + true -> Opts; + false -> [no_tty, {report, {eunit_progress, [colored, profile]}} | Opts] end. set_verbose(Opts) -> @@ -318,26 +398,37 @@ set_verbose(Opts) -> translate_paths(State, Tests) -> translate_paths(State, Tests, []). translate_paths(_State, [], Acc) -> lists:reverse(Acc); -translate_paths(State, [{dir, Dir}|Rest], Acc) -> - Apps = rebar_state:project_apps(State), - translate_paths(State, Rest, [translate(State, Apps, Dir)|Acc]); -translate_paths(State, [{file, File}|Rest], Acc) -> - Dir = filename:dirname(File), +translate_paths(State, [{K, _} = Path|Rest], Acc) when K == file; K == dir -> Apps = rebar_state:project_apps(State), - translate_paths(State, Rest, [translate(State, Apps, Dir)|Acc]); + translate_paths(State, Rest, [translate(State, Apps, Path)|Acc]); translate_paths(State, [Test|Rest], Acc) -> translate_paths(State, Rest, [Test|Acc]). -translate(State, [App|Rest], Dir) -> +translate(State, [App|Rest], {dir, Dir}) -> case rebar_file_utils:path_from_ancestor(Dir, rebar_app_info:dir(App)) of {ok, Path} -> {dir, filename:join([rebar_app_info:out_dir(App), Path])}; - {error, badparent} -> translate(State, Rest, Dir) + {error, badparent} -> translate(State, Rest, {dir, Dir}) end; -translate(State, [], Dir) -> +translate(State, [App|Rest], {file, FilePath}) -> + Dir = filename:dirname(FilePath), + File = filename:basename(FilePath), + case rebar_file_utils:path_from_ancestor(Dir, rebar_app_info:dir(App)) of + {ok, Path} -> {file, filename:join([rebar_app_info:out_dir(App), Path, File])}; + {error, badparent} -> translate(State, Rest, {file, FilePath}) + end; +translate(State, [], {dir, Dir}) -> case rebar_file_utils:path_from_ancestor(Dir, rebar_state:dir(State)) of {ok, Path} -> {dir, filename:join([rebar_dir:base_dir(State), "extras", Path])}; %% not relative, leave as is {error, badparent} -> {dir, Dir} + end; +translate(State, [], {file, FilePath}) -> + Dir = filename:dirname(FilePath), + File = filename:basename(FilePath), + case rebar_file_utils:path_from_ancestor(Dir, rebar_app_info:dir(State)) of + {ok, Path} -> {file, filename:join([rebar_dir:base_dir(State), "extras", Path, File])}; + %% not relative, leave as is + {error, badparent} -> {file, FilePath} end. maybe_cover_compile(State) -> diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index 118d799..a484c5f 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -251,13 +251,12 @@ update_unseen_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Loc -spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer()) -> {rebar_app_info:t(), [rebar_app_info:t()], rebar_state:t()}. handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) -> - Profiles = rebar_state:current_profiles(State), Name = rebar_app_info:name(AppInfo), C = rebar_config:consult(rebar_app_info:dir(AppInfo)), AppInfo0 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), C), AppInfo1 = rebar_app_info:apply_overrides(rebar_app_info:get(AppInfo, overrides, []), AppInfo0), - AppInfo2 = rebar_app_info:apply_profiles(AppInfo1, Profiles), + AppInfo2 = rebar_app_info:apply_profiles(AppInfo1, [default, prod]), Plugins = rebar_app_info:get(AppInfo2, plugins, []), AppInfo3 = rebar_app_info:set(AppInfo2, {plugins, Profile}, Plugins), @@ -269,7 +268,7 @@ handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) -> State1 = rebar_plugins:install(State, AppInfo3), %% Upgrade lock level to be the level the dep will have in this dep tree - Deps = rebar_app_info:get(AppInfo3, {deps, default}, []), + Deps = rebar_app_info:get(AppInfo3, {deps, default}, []) ++ rebar_app_info:get(AppInfo3, {deps, prod}, []), AppInfo4 = rebar_app_info:deps(AppInfo3, rebar_state:deps_names(Deps)), %% Keep all overrides from the global config and this dep when parsing its deps diff --git a/src/rebar_prv_local_install.erl b/src/rebar_prv_local_install.erl index 65468a3..1b58859 100644 --- a/src/rebar_prv_local_install.erl +++ b/src/rebar_prv_local_install.erl @@ -15,7 +15,7 @@ -include_lib("kernel/include/file.hrl"). -define(PROVIDER, install). --define(NAMESPACE, unstable). +-define(NAMESPACE, local). -define(DEPS, []). %% =================================================================== @@ -60,7 +60,7 @@ format_error(Reason) -> bin_contents(OutputDir) -> <<"#!/usr/bin/env sh -erl -pz ", (ec_cnv:to_binary(OutputDir))/binary,"/*/ebin +sbtu +A0 -noshell -boot start_clean -s rebar3 main -extra \"$@\" +erl -pz ", (ec_cnv:to_binary(OutputDir))/binary,"/*/ebin +sbtu +A0 -noshell -boot start_clean -s rebar3 main $REBAR3_ERL_ARGS -extra \"$@\" ">>. extract_escript(State, ScriptPath) -> diff --git a/src/rebar_prv_local_upgrade.erl b/src/rebar_prv_local_upgrade.erl index bdfc232..aa9ee44 100644 --- a/src/rebar_prv_local_upgrade.erl +++ b/src/rebar_prv_local_upgrade.erl @@ -14,7 +14,7 @@ -include_lib("kernel/include/file.hrl"). -define(PROVIDER, upgrade). --define(NAMESPACE, unstable). +-define(NAMESPACE, local). -define(DEPS, []). %% =================================================================== diff --git a/src/rebar_prv_new.erl b/src/rebar_prv_new.erl index 6574aca..064315e 100644 --- a/src/rebar_prv_new.erl +++ b/src/rebar_prv_new.erl @@ -7,6 +7,7 @@ format_error/1]). -include("rebar.hrl"). +-include_lib("providers/include/providers.hrl"). -define(PROVIDER, new). -define(DEPS, []). @@ -32,19 +33,19 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - case rebar_state:command_args(State) of + case strip_flags(rebar_state:command_args(State)) of ["help"] -> ?CONSOLE("Call `rebar3 new help <template>` for a detailed description~n", []), - show_short_templates(rebar_templater:list_templates(State)), + show_short_templates(list_templates(State)), {ok, State}; ["help", TemplateName] -> - case lists:keyfind(TemplateName, 1, rebar_templater:list_templates(State)) of + case lists:keyfind(TemplateName, 1, list_templates(State)) of false -> ?CONSOLE("template not found.", []); Term -> show_template(Term) end, {ok, State}; [TemplateName | Opts] -> - case lists:keyfind(TemplateName, 1, rebar_templater:list_templates(State)) of + case lists:keyfind(TemplateName, 1, list_templates(State)) of false -> ?CONSOLE("template not found.", []); _ -> @@ -53,11 +54,13 @@ do(State) -> end, {ok, State}; [] -> - show_short_templates(rebar_templater:list_templates(State)), + show_short_templates(list_templates(State)), {ok, State} end. -spec format_error(any()) -> iolist(). +format_error({consult, File, Reason}) -> + io_lib:format("Error consulting file at ~s for reason ~p", [File, Reason]); format_error(Reason) -> io_lib:format("~p", [Reason]). @@ -65,6 +68,15 @@ format_error(Reason) -> %% Internal functions %% =================================================================== +list_templates(State) -> + lists:foldl(fun({error, {consult, File, Reason}}, Acc) -> + ?WARN("Error consulting template file ~s for reason ~p", + [File, Reason]), + Acc + ; (Tpl, Acc) -> + [Tpl|Acc] + end, [], lists:reverse(rebar_templater:list_templates(State))). + info() -> io_lib:format( "Create rebar3 project based on template and vars.~n" @@ -72,6 +84,10 @@ info() -> "Valid command line options:~n" " <template> [var=foo,...]~n", []). +strip_flags([]) -> []; +strip_flags(["-"++_|Opts]) -> strip_flags(Opts); +strip_flags([Opt | Opts]) -> [Opt | strip_flags(Opts)]. + is_forced(State) -> {Args, _} = rebar_state:command_parsed_args(State), case proplists:get_value(force, Args) of @@ -116,10 +132,13 @@ show_template({Name, Type, Location, Description, Vars}) -> format_vars(Vars)]). format_type(escript) -> "built-in"; +format_type(plugin) -> "plugin"; format_type(file) -> "custom". format_type(escript, _) -> "built-in template"; +format_type(plugin, Loc) -> + io_lib:format("plugin template (~s)", [Loc]); format_type(file, Loc) -> io_lib:format("custom template (~s)", [Loc]). diff --git a/src/rebar_prv_plugins.erl b/src/rebar_prv_plugins.erl index 20bc1ea..7e6b88e 100644 --- a/src/rebar_prv_plugins.erl +++ b/src/rebar_prv_plugins.erl @@ -34,34 +34,37 @@ do(State) -> GlobalConfigFile = rebar_dir:global_config(), GlobalConfig = rebar_state:new(rebar_config:consult_file(GlobalConfigFile)), GlobalPlugins = rebar_state:get(GlobalConfig, plugins, []), - GlobalPluginsDir = filename:join(rebar_dir:global_cache_dir(rebar_state:opts(State)), "plugins"), - display_plugins("Global plugins", GlobalPluginsDir, GlobalPlugins), + GlobalPluginsDir = filename:join([rebar_dir:global_cache_dir(rebar_state:opts(State)), "plugins", "*"]), + GlobalApps = rebar_app_discover:find_apps([GlobalPluginsDir], all), + display_plugins("Global plugins", GlobalApps, GlobalPlugins), Plugins = rebar_state:get(State, plugins, []), - PluginsDir =rebar_dir:plugins_dir(State), - display_plugins("Local plugins", PluginsDir, Plugins), + PluginsDir = filename:join(rebar_dir:plugins_dir(State), "*"), + CheckoutsDir = filename:join(rebar_dir:checkouts_dir(State), "*"), + Apps = rebar_app_discover:find_apps([CheckoutsDir, PluginsDir], all), + display_plugins("Local plugins", Apps, Plugins), {ok, State}. -spec format_error(any()) -> iolist(). format_error(Reason) -> io_lib:format("~p", [Reason]). -display_plugins(_Header, _Dir, []) -> +display_plugins(_Header, _Apps, []) -> ok; -display_plugins(Header, Dir, Plugins) -> +display_plugins(Header, Apps, Plugins) -> ?CONSOLE("--- ~s ---", [Header]), - display_plugins(Dir, Plugins), + display_plugins(Apps, Plugins), ?CONSOLE("", []). -display_plugins(Dir, Plugins) -> +display_plugins(Apps, Plugins) -> lists:foreach(fun(Plugin) -> - Name = if is_atom(Plugin) -> Plugin; - is_tuple(Plugin) -> element(1, Plugin) + Name = if is_atom(Plugin) -> ec_cnv:to_binary(Plugin); + is_tuple(Plugin) -> ec_cnv:to_binary(element(1, Plugin)) end, - case rebar_app_info:discover(filename:join(Dir, Name)) of + case rebar_app_utils:find(Name, Apps) of {ok, _App} -> ?CONSOLE("~s", [Name]); - not_found -> + error -> ?DEBUG("Unable to find plugin ~s", [Name]) end end, Plugins). diff --git a/src/rebar_prv_shell.erl b/src/rebar_prv_shell.erl index 4cf1e04..ea759fc 100644 --- a/src/rebar_prv_shell.erl +++ b/src/rebar_prv_shell.erl @@ -27,6 +27,7 @@ -module(rebar_prv_shell). -author("Kresten Krab Thorup <krab@trifork.com>"). +-author("Fred Hebert <mononcqc@ferd.ca>"). -behaviour(provider). @@ -56,12 +57,23 @@ init(State) -> {short_desc, "Run shell with project apps and deps in path."}, {desc, info()}, {opts, [{config, undefined, "config", string, - "Path to the config file to use. Defaults to the " - "sys_config defined for relx, if present."}, + "Path to the config file to use. Defaults to " + "{shell, [{config, File}]} and then the relx " + "sys.config file if not specified."}, {name, undefined, "name", atom, "Gives a long name to the node."}, {sname, undefined, "sname", atom, - "Gives a short name to the node."}]} + "Gives a short name to the node."}, + {script_file, undefined, "script", string, + "Path to an escript file to run before " + "starting the project apps. Defaults to " + "rebar.config {shell, [{script_file, File}]} " + "if not specified."}, + {apps, undefined, "apps", string, + "A list of apps to boot before starting the " + "shell. (E.g. --apps app1,app2,app3) Defaults " + "to rebar.config {shell, [{apps, Apps}]} or " + "relx apps if not specified."}]} ]) ), {ok, State1}. @@ -86,6 +98,7 @@ shell(State) -> setup_name(State), setup_paths(State), setup_shell(), + maybe_run_script(State), %% apps must be started after the change in shell because otherwise %% their application masters never gets the new group leader (held in %% their internal state) @@ -99,22 +112,64 @@ info() -> "Start a shell with project and deps preloaded similar to~n'erl -pa ebin -pa deps/*/ebin'.~n". setup_shell() -> - %% scan all processes for any with references to the old user and save them to - %% update later - NeedsUpdate = [Pid || Pid <- erlang:processes(), - proplists:get_value(group_leader, erlang:process_info(Pid)) == whereis(user) - ], - %% terminate the current user + OldUser = kill_old_user(), + %% Test for support here + NewUser = try erlang:open_port({spawn,'tty_sl -c -e'}, []) of + Port when is_port(Port) -> + true = port_close(Port), + setup_new_shell() + catch + error:_ -> + setup_old_shell() + end, + rewrite_leaders(OldUser, NewUser). + +kill_old_user() -> + OldUser = whereis(user), + %% terminate the current user's port, in a way that makes it shut down, + %% but without taking down the supervision tree so that the escript doesn't + %% fully die + [P] = [P || P <- element(2,process_info(whereis(user), links)), is_port(P)], + user ! {'EXIT', P, normal}, % pretend the port died, then the port can die! + OldUser. + +setup_new_shell() -> + %% terminate the current user supervision structure ok = supervisor:terminate_child(kernel_sup, user), %% start a new shell (this also starts a new user under the correct group) _ = user_drv:start(), %% wait until user_drv and user have been registered (max 3 seconds) ok = wait_until_user_started(3000), + whereis(user). + +setup_old_shell() -> + %% scan all processes for any with references to the old user and save them to + %% update later + NewUser = rebar_user:start(), % hikack IO stuff with fake user + NewUser = whereis(user), + NewUser. + +rewrite_leaders(OldUser, NewUser) -> %% set any process that had a reference to the old user's group leader to the %% new user process. Catch the race condition when the Pid exited after the %% liveness check. - _ = [catch erlang:group_leader(whereis(user), Pid) || Pid <- NeedsUpdate, - is_process_alive(Pid)], + _ = [catch erlang:group_leader(NewUser, Pid) + || Pid <- erlang:processes(), + proplists:get_value(group_leader, erlang:process_info(Pid)) == OldUser, + is_process_alive(Pid)], + %% Application masters have the same problem, but they hold the old group + %% leader in their state and hold on to it. Re-point the processes whose + %% leaders are application masters. This can mess up a few things around + %% shutdown time, but is nicer than the current lock-up. + OldMasters = [Pid + || Pid <- erlang:processes(), + Pid < NewUser, % only change old masters + {_,Dict} <- [erlang:process_info(Pid, dictionary)], + {application_master,init,4} == proplists:get_value('$initial_call', Dict)], + _ = [catch erlang:group_leader(NewUser, Pid) + || Pid <- erlang:processes(), + lists:member(proplists:get_value(group_leader, erlang:process_info(Pid)), + OldMasters)], try %% enable error_logger's tty output error_logger:swap_handler(tty), @@ -128,12 +183,58 @@ setup_shell() -> hope_for_best end. + setup_paths(State) -> %% Add deps to path code:add_pathsa(rebar_state:code_paths(State, all_deps)), %% add project app test paths ok = add_test_paths(State). +maybe_run_script(State) -> + case first_value([fun find_script_option/1, + fun find_script_rebar/1], State) of + no_value -> + ?DEBUG("No script_file specified.", []), + ok; + "none" -> + ?DEBUG("Shell script execution skipped (--script none).", []), + ok; + RelFile -> + File = filename:absname(RelFile), + try run_script_file(File) + catch + C:E -> + ?ABORT("Couldn't run shell escript ~p - ~p:~p~nStack: ~p", + [File, C, E, erlang:get_stacktrace()]) + end + end. + +-spec find_script_option(rebar_state:t()) -> no_value | list(). +find_script_option(State) -> + {Opts, _} = rebar_state:command_parsed_args(State), + debug_get_value(script_file, Opts, no_value, + "Found script file from command line option."). + +-spec find_script_rebar(rebar_state:t()) -> no_value | list(). +find_script_rebar(State) -> + Config = rebar_state:get(State, shell, []), + %% Either a string, or undefined + debug_get_value(script_file, Config, no_value, + "Found script file from rebar config file."). + +run_script_file(File) -> + ?DEBUG("Extracting escript from ~p", [File]), + {ok, Script} = escript:extract(File, [compile_source]), + Beam = proplists:get_value(source, Script), + Mod = proplists:get_value(module, beam_lib:info(Beam)), + ?DEBUG("Compiled escript as ~p", [Mod]), + FakeFile = "/fake_path/" ++ atom_to_list(Mod), + {module, Mod} = code:load_binary(Mod, FakeFile, Beam), + ?DEBUG("Evaling ~p:main([]).", [Mod]), + Result = Mod:main([]), + ?DEBUG("Result: ~p", [Result]), + Result. + maybe_boot_apps(State) -> case find_apps_to_boot(State) of undefined -> @@ -172,17 +273,42 @@ check_epmd(_) -> find_apps_to_boot(State) -> %% Try the shell_apps option - case rebar_state:get(State, shell_apps, undefined) of - undefined -> - %% Get to the relx tuple instead - case lists:keyfind(release, 1, rebar_state:get(State, relx, [])) of - {_, _, Apps} -> Apps; - false -> undefined - end; + case first_value([fun find_apps_option/1, + fun find_apps_rebar/1, + fun find_apps_relx/1], State) of + no_value -> + undefined; Apps -> Apps end. +-spec find_apps_option(rebar_state:t()) -> no_value | [atom()]. +find_apps_option(State) -> + {Opts, _} = rebar_state:command_parsed_args(State), + case debug_get_value(apps, Opts, no_value, + "Found shell apps from command line option.") of + no_value -> no_value; + AppsStr -> + [ list_to_atom(AppStr) + || AppStr <- string:tokens(AppsStr, " ,:") ] + end. + +-spec find_apps_rebar(rebar_state:t()) -> no_value | list(). +find_apps_rebar(State) -> + ShellOpts = rebar_state:get(State, shell, []), + debug_get_value(apps, ShellOpts, no_value, + "Found shell opts from command line option."). + +-spec find_apps_relx(rebar_state:t()) -> no_value | list(). +find_apps_relx(State) -> + case lists:keyfind(release, 1, rebar_state:get(State, relx, [])) of + {_, _, Apps} -> + ?DEBUG("Found shell apps from relx.", []), + Apps; + false -> + no_value + end. + load_apps(Apps) -> [case application:load(App) of ok -> @@ -199,9 +325,14 @@ reread_config(State) -> no_config -> ok; ConfigList -> - _ = [application:set_env(Application, Key, Val) + try + [application:set_env(Application, Key, Val) || {Application, Items} <- ConfigList, - {Key, Val} <- Items], + {Key, Val} <- Items] + catch _:_ -> + ?ERROR("The configuration file submitted could not be read " + "and will be ignored.", []) + end, ok end. @@ -261,31 +392,51 @@ add_test_paths(State) -> % First try the --config flag, then try the relx sys_config -spec find_config(rebar_state:t()) -> [tuple()] | no_config. find_config(State) -> - case find_config_option(State) of - no_config -> - find_config_relx(State); - Result -> - Result + case first_value([fun find_config_option/1, + fun find_config_rebar/1, + fun find_config_relx/1], State) of + no_value -> + no_config; + Filename when is_list(Filename) -> + consult_config(State, Filename) + end. + +-spec first_value([Fun], State) -> no_value | Value when + Value :: any(), + State :: rebar_state:t(), + Fun :: fun ((State) -> no_value | Value). +first_value([], _) -> no_value; +first_value([Fun | Rest], State) -> + case Fun(State) of + no_value -> + first_value(Rest, State); + Value -> + Value + end. + +debug_get_value(Key, List, Default, Description) -> + case proplists:get_value(Key, List, Default) of + Default -> Default; + Value -> + ?DEBUG(Description, []), + Value end. --spec find_config_option(rebar_state:t()) -> [tuple()] | no_config. +-spec find_config_option(rebar_state:t()) -> Filename::list() | no_value. find_config_option(State) -> {Opts, _} = rebar_state:command_parsed_args(State), - case proplists:get_value(config, Opts) of - undefined -> - no_config; - Filename -> - consult_config(State, Filename) - end. + debug_get_value(config, Opts, no_value, + "Found config from command line option."). + +-spec find_config_rebar(rebar_state:t()) -> [tuple()] | no_value. +find_config_rebar(State) -> + debug_get_value(config, rebar_state:get(State, shell, []), no_value, + "Found config from rebar config file."). --spec find_config_relx(rebar_state:t()) -> [tuple()] | no_config. +-spec find_config_relx(rebar_state:t()) -> [tuple()] | no_value. find_config_relx(State) -> - case proplists:get_value(sys_config, rebar_state:get(State, relx, [])) of - undefined -> - no_config; - Filename -> - consult_config(State, Filename) - end. + debug_get_value(sys_config, rebar_state:get(State, relx, []), no_value, + "Found config from relx."). -spec consult_config(rebar_state:t(), string()) -> [tuple()]. consult_config(State, Filename) -> diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index 6637ebe..0e3b9a0 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -36,26 +36,36 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> try - RegistryDir = rebar_packages:registry_dir(State), - filelib:ensure_dir(filename:join(RegistryDir, "dummy")), - HexFile = filename:join(RegistryDir, "registry"), - ?INFO("Updating package registry...", []), - TmpDir = ec_file:insecure_mkdtemp(), - TmpFile = filename:join(TmpDir, "packages.gz"), + case rebar_packages:registry_dir(State) of + {ok, RegistryDir} -> + filelib:ensure_dir(filename:join(RegistryDir, "dummy")), + HexFile = filename:join(RegistryDir, "registry"), + ?INFO("Updating package registry...", []), + TmpDir = ec_file:insecure_mkdtemp(), + TmpFile = filename:join(TmpDir, "packages.gz"), - Url = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_HEX_REGISTRY), - case httpc:request(get, {Url, []}, - [], [{stream, TmpFile}, {sync, true}], - rebar) of - {ok, saved_to_file} -> - {ok, Data} = file:read_file(TmpFile), - Unzipped = zlib:gunzip(Data), - ok = file:write_file(HexFile, Unzipped), - ?INFO("Writing registry to ~s", [HexFile]), - hex_to_index(State), - {ok, State}; - _ -> - ?PRV_ERROR(package_index_download) + CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), + case rebar_utils:url_append_path(CDN, ?REMOTE_REGISTRY_FILE) of + {ok, Url} -> + ?DEBUG("Fetching registry from ~p", [Url]), + case httpc:request(get, {Url, [{"User-Agent", rebar_utils:user_agent()}]}, + [], [{stream, TmpFile}, {sync, true}], + rebar) of + {ok, saved_to_file} -> + {ok, Data} = file:read_file(TmpFile), + Unzipped = zlib:gunzip(Data), + ok = file:write_file(HexFile, Unzipped), + ?INFO("Writing registry to ~s", [HexFile]), + hex_to_index(State), + {ok, State}; + _ -> + ?PRV_ERROR(package_index_download) + end; + _ -> + ?PRV_ERROR({package_parse_cdn, CDN}) + end; + {uri_parse_error, CDN} -> + ?PRV_ERROR({package_parse_cdn, CDN}) end catch _E:C -> @@ -64,6 +74,8 @@ do(State) -> end. -spec format_error(any()) -> iolist(). +format_error({package_parse_cdn, Uri}) -> + io_lib:format("Failed to parse CDN url: ~p", [Uri]); format_error(package_index_download) -> "Failed to download package index."; format_error(package_index_write) -> @@ -75,7 +87,7 @@ is_supported(<<"rebar3">>) -> true; is_supported(_) -> false. hex_to_index(State) -> - RegistryDir = rebar_packages:registry_dir(State), + {ok, RegistryDir} = rebar_packages:registry_dir(State), HexFile = filename:join(RegistryDir, "registry"), try ets:file2tab(HexFile) of {ok, Registry} -> @@ -92,12 +104,24 @@ hex_to_index(State) -> false -> true end; - ({Pkg, [Vsns]}, _) when is_binary(Pkg) -> - ets:insert(?PACKAGE_TABLE, {Pkg, Vsns}); (_, _) -> true end, true, Registry), + ets:foldl(fun({Pkg, [[]]}, _) when is_binary(Pkg) -> + true; + ({Pkg, [Vsns=[Vsn | _Rest]]}, _) when is_binary(Pkg) -> + %% Verify the package is of the right build tool by checking if the first + %% version exists in the table from the foldl above + case ets:member(?PACKAGE_TABLE, {Pkg, Vsn}) of + true -> + ets:insert(?PACKAGE_TABLE, {Pkg, Vsns}); + false -> + true + end; + (_, _) -> + true + end, true, Registry), ets:insert(?PACKAGE_TABLE, {package_index_version, ?PACKAGE_INDEX_VERSION}), ?INFO("Writing index to ~s", [PackageIndex]), ets:tab2file(?PACKAGE_TABLE, PackageIndex), diff --git a/src/rebar_relx.erl b/src/rebar_relx.erl index 36a24f0..5d29258 100644 --- a/src/rebar_relx.erl +++ b/src/rebar_relx.erl @@ -30,7 +30,7 @@ do(Module, Command, Provider, State) -> relx:main([{lib_dirs, LibDirs} ,{caller, api} | output_dir(OutputDir, Options)], AllOptions); Config -> - Config1 = update_config(Config), + Config1 = merge_overlays(Config), relx:main([{lib_dirs, LibDirs} ,{config, Config1} ,{caller, api} | output_dir(OutputDir, Options)], AllOptions) @@ -46,27 +46,16 @@ do(Module, Command, Provider, State) -> format_error(Reason) -> io_lib:format("~p", [Reason]). -%% To handle profiles rebar3 expects the provider to use the first entry -%% in a configuration key-value list as the value of a key if dups exist. -%% This does not work with relx. Some config options must not lose their -%% order (release which has an extends option is one). So here we pull out -%% options that are special so we can reverse the rest so what we expect -%% from a rebar3 profile is what we get on the relx side. --define(SPECIAL_KEYS, [release, vm_args, sys_config, overlay_vars, lib_dirs]). - -update_config(Config) -> - {Special, Other} = - lists:foldl(fun(Tuple, {SpecialAcc, OtherAcc}) when is_tuple(Tuple) -> - case lists:member(element(1, Tuple), ?SPECIAL_KEYS) of - true -> - {[Tuple | SpecialAcc], OtherAcc}; - false -> - {SpecialAcc, [Tuple | OtherAcc]} - end - end, {[], []}, Config), - lists:reverse(Special) ++ Other. - %% Don't override output_dir if the user passed one on the command line output_dir(OutputDir, Options) -> [{output_dir, OutputDir} || not(lists:member("-o", Options)) andalso not(lists:member("--output-dir", Options))]. + +merge_overlays(Config) -> + {Overlays, Others} = + lists:partition(fun(C) when element(1, C) =:= overlay -> true; + (_) -> false + end, Config), + %% Have profile overlay entries come before others to match how profiles work elsewhere + NewOverlay = lists:reverse(lists:flatmap(fun({overlay, Overlay}) -> Overlay end, Overlays)), + [{overlay, NewOverlay} | Others]. diff --git a/src/rebar_state.erl b/src/rebar_state.erl index 176a80b..0c07b2a 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -370,7 +370,24 @@ providers(State, NewProviders) -> -spec add_provider(t(), providers:t()) -> t(). add_provider(State=#state_t{providers=Providers}, Provider) -> - State#state_t{providers=[Provider | Providers]}. + Name = providers:impl(Provider), + Namespace = providers:namespace(Provider), + Module = providers:module(Provider), + case lists:any(fun(P) -> + case {providers:impl(P), providers:namespace(P)} of + {Name, Namespace} -> + ?DEBUG("Not adding provider ~p ~p from module ~p because it already exists from module ~p", + [Namespace, Name, providers:module(P), Module]), + true; + _ -> + false + end + end, Providers) of + true -> + State; + false -> + State#state_t{providers=[Provider | Providers]} + end. create_logic_providers(ProviderModules, State0) -> try diff --git a/src/rebar_templater.erl b/src/rebar_templater.erl index c7fb2af..2f33bfc 100644 --- a/src/rebar_templater.erl +++ b/src/rebar_templater.erl @@ -59,10 +59,14 @@ list_templates(State) -> %% Expand a single template's value list_template(Files, {Name, Type, File}, State) -> - TemplateTerms = consult(load_file(Files, Type, File)), - {Name, Type, File, - get_template_description(TemplateTerms), - get_template_vars(TemplateTerms, State)}. + case consult(load_file(Files, Type, File)) of + {error, Reason} -> + {error, {consult, File, Reason}}; + TemplateTerms -> + {Name, Type, File, + get_template_description(TemplateTerms), + get_template_vars(TemplateTerms, State)} + end. %% Load up the template description out from a list of attributes read in %% a .template file. @@ -155,9 +159,34 @@ drop_var_docs([{K,V}|Rest]) -> [{K,V} | drop_var_docs(Rest)]. create({Template, Type, File}, Files, UserVars, Force, State) -> TemplateTerms = consult(load_file(Files, Type, File)), Vars = drop_var_docs(override_vars(UserVars, get_template_vars(TemplateTerms, State))), + maybe_warn_about_name(Vars), TemplateCwd = filename:dirname(File), execute_template(TemplateTerms, Files, {Template, Type, TemplateCwd}, Vars, Force). +maybe_warn_about_name(Vars) -> + Name = proplists:get_value(name, Vars, "valid"), + case validate_atom(Name) of + invalid -> + ?WARN("The 'name' variable is often associated with Erlang " + "module names and/or file names. The value submitted " + "(~s) isn't an unquoted Erlang atom. Templates " + "generated may contain errors.", + [Name]); + valid -> + ok + end. + +validate_atom(Str) -> + case io_lib:fread("~a", unicode:characters_to_list(Str)) of + {ok, [Atom], ""} -> + case io_lib:write_atom(Atom) of + "'" ++ _ -> invalid; % quoted + _ -> valid % unquoted + end; + _ -> + invalid + end. + %% Run template instructions one at a time. execute_template([], _, {Template,_,_}, _, _) -> ?DEBUG("Template ~s applied", [Template]), @@ -235,6 +264,7 @@ replace_var([H|T], Acc, Vars) -> %% Load a list of all the files in the escript and on disk find_templates(State) -> DiskTemplates = find_disk_templates(State), + PluginTemplates = find_plugin_templates(State), {MainTemplates, Files} = case rebar_state:escript_path(State) of undefined -> @@ -245,19 +275,23 @@ find_templates(State) -> F = cache_escript_files(State), {find_escript_templates(F), F} end, - AvailTemplates = find_available_templates(DiskTemplates, - MainTemplates), + AvailTemplates = find_available_templates([MainTemplates, + PluginTemplates, + DiskTemplates]), ?DEBUG("Available templates: ~p\n", [AvailTemplates]), {AvailTemplates, Files}. -find_available_templates(TemplateList1, TemplateList2) -> - AvailTemplates = prioritize_templates( - tag_names(TemplateList1), - tag_names(TemplateList2)), - +find_available_templates(TemplateListList) -> + AvailTemplates = prioritize_templates(TemplateListList), ?DEBUG("Available templates: ~p\n", [AvailTemplates]), AvailTemplates. +prioritize_templates([TemplateList]) -> + tag_names(TemplateList); +prioritize_templates([TemplateList | TemplateListList]) -> + prioritize_templates(tag_names(TemplateList), + prioritize_templates(TemplateListList)). + %% Scan the current escript for available files cache_escript_files(State) -> {ok, Files} = rebar_utils:escript_foldl( @@ -295,6 +329,14 @@ find_other_templates(State) -> rebar_utils:find_files(TemplateDir, ?TEMPLATE_RE) end. +%% Fetch template indexes that sit on disk in plugins +find_plugin_templates(State) -> + [{plugin, File} + || App <- rebar_state:all_plugin_deps(State), + Priv <- [rebar_app_info:priv_dir(App)], + Priv =/= undefined, + File <- rebar_utils:find_files(Priv, ?TEMPLATE_RE)]. + %% Take an existing list of templates and tag them by name the way %% the user would enter it from the CLI tag_names(List) -> @@ -312,6 +354,10 @@ prioritize_templates([{Name, Type, File} | Rest], Valid) -> ?DEBUG("Skipping template ~p, due to presence of a built-in " "template with the same name", [Name]), prioritize_templates(Rest, Valid); + {_, plugin, _} -> + ?DEBUG("Skipping template ~p, due to presence of a plugin " + "template with the same name", [Name]), + prioritize_templates(Rest, Valid); {_, file, _} -> ?DEBUG("Skipping template ~p, due to presence of a custom " "template at ~s", [Name, File]), @@ -323,6 +369,9 @@ prioritize_templates([{Name, Type, File} | Rest], Valid) -> load_file(Files, escript, Name) -> {Name, Bin} = lists:keyfind(Name, 1, Files), Bin; +load_file(_Files, plugin, Name) -> + {ok, Bin} = file:read_file(Name), + Bin; load_file(_Files, file, Name) -> {ok, Bin} = file:read_file(Name), Bin. @@ -338,8 +387,10 @@ consult(Cont, Str, Acc) -> {done, Result, Remaining} -> case Result of {ok, Tokens, _} -> - {ok, Term} = erl_parse:parse_term(Tokens), - consult([], Remaining, [Term | Acc]); + case erl_parse:parse_term(Tokens) of + {ok, Term} -> consult([], Remaining, [Term | Acc]); + {error, Reason} -> {error, Reason} + end; {eof, _Other} -> lists:reverse(Acc); {error, Info, _} -> diff --git a/src/rebar_user.erl b/src/rebar_user.erl new file mode 100644 index 0000000..f20142d --- /dev/null +++ b/src/rebar_user.erl @@ -0,0 +1,757 @@ +%%% This file is a literal copy of Erlang/OTP's user.erl module, renamed +%%% to rebar_user.erl and modified in a few place to force a shell to always +%%% boot or to remove dead comments. +%%% +%%% Its usage is required because unlike the standard (new) shell, it is +%%% not possible to get rid of the old one without killing the rebar3 escript +%%% at the same time. As such, this module is being used to duplicate +%%% the old shell while stealing the usage of the IO driver {fd,0,1} +%%% (stdio) and then booting our own shell with paths and stuff in it. + +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(rebar_user). +-compile(inline). + +%% Basic standard i/o server for user interface port. + +-export([start/0, start/1, start_out/0]). +-export([interfaces/1]). + +-define(NAME, user). + +%% Defines for control ops +-define(CTRL_OP_GET_WINSIZE,100). + +%% +%% The basic server and start-up. +%% + +start() -> + start_port([eof,binary]). + +start([Mod,Fun|Args]) -> + %% Mod,Fun,Args should return a pid. That process is supposed to act + %% as the io port. + Pid = apply(Mod, Fun, Args), % This better work! + Id = spawn(fun() -> server(Pid) end), + register(?NAME, Id), + Id. + +start_out() -> + %% Output-only version of start/0 + start_port([out,binary]). + +start_port(PortSettings) -> + Id = spawn(fun() -> server({fd,0,1}, PortSettings) end), + register(?NAME, Id), + Id. + +%% Return the pid of the shell process. +%% Note: We can't ask the user process for this info since it +%% may be busy waiting for data from the port. +interfaces(User) -> + case process_info(User, dictionary) of + {dictionary,Dict} -> + case lists:keysearch(shell, 1, Dict) of + {value,Sh={shell,Shell}} when is_pid(Shell) -> + [Sh]; + _ -> + [] + end; + _ -> + [] + end. + +server(Pid) when is_pid(Pid) -> + process_flag(trap_exit, true), + link(Pid), + run(Pid). + +server(PortName,PortSettings) -> + process_flag(trap_exit, true), + Port = open_port(PortName,PortSettings), + run(Port). + +run(P) -> + put(read_mode,list), + put(encoding,latin1), + group_leader(self(), self()), + catch_loop(P, start_init_shell()). + +catch_loop(Port, Shell) -> + catch_loop(Port, Shell, queue:new()). + +catch_loop(Port, Shell, Q) -> + case catch server_loop(Port, Q) of + new_shell -> + exit(Shell, kill), + catch_loop(Port, start_new_shell()); + {unknown_exit,{Shell,Reason},_} -> % shell has exited + case Reason of + normal -> + put_port(<<"*** ">>, Port); + _ -> + put_port(<<"*** ERROR: ">>, Port) + end, + put_port(<<"Shell process terminated! ***\n">>, Port), + catch_loop(Port, start_new_shell()); + {unknown_exit,_,Q1} -> + catch_loop(Port, Shell, Q1); + {'EXIT',R} -> + exit(R) + end. + +link_and_save_shell(Shell) -> + link(Shell), + put(shell, Shell), + Shell. + +start_init_shell() -> + link_and_save_shell(shell:start(init)). + +start_new_shell() -> + link_and_save_shell(shell:start()). + +server_loop(Port, Q) -> + receive + {io_request,From,ReplyAs,Request} when is_pid(From) -> + server_loop(Port, do_io_request(Request, From, ReplyAs, Port, Q)); + {Port,{data,Bytes}} -> + case get(shell) of + noshell -> + server_loop(Port, queue:snoc(Q, Bytes)); + _ -> + case contains_ctrl_g_or_ctrl_c(Bytes) of + false -> + server_loop(Port, queue:snoc(Q, Bytes)); + _ -> + throw(new_shell) + end + end; + {Port, eof} -> + put(eof, true), + server_loop(Port, Q); + + %% Ignore messages from port here. + {'EXIT',Port,badsig} -> % Ignore badsig errors + server_loop(Port, Q); + {'EXIT',Port,What} -> % Port has exited + exit(What); + + %% Check if shell has exited + {'EXIT',SomePid,What} -> + case get(shell) of + noshell -> + server_loop(Port, Q); % Ignore + _ -> + throw({unknown_exit,{SomePid,What},Q}) + end; + + _Other -> % Ignore other messages + server_loop(Port, Q) + end. + + +get_fd_geometry(Port) -> + case (catch port_control(Port,?CTRL_OP_GET_WINSIZE,[])) of + List when length(List) =:= 8 -> + <<W:32/native,H:32/native>> = list_to_binary(List), + {W,H}; + _ -> + error + end. + + +%% NewSaveBuffer = io_request(Request, FromPid, ReplyAs, Port, SaveBuffer) + +do_io_request(Req, From, ReplyAs, Port, Q0) -> + case io_request(Req, Port, Q0) of + {_Status,Reply,Q1} -> + _ = io_reply(From, ReplyAs, Reply), + Q1; + {exit,What} -> + ok = send_port(Port, close), + exit(What) + end. + +%% New in R13B +%% Encoding option (unicode/latin1) +io_request({put_chars,unicode,Chars}, Port, Q) -> % Binary new in R9C + case wrap_characters_to_binary(Chars, unicode, get(encoding)) of + error -> + {error,{error,put_chars},Q}; + Bin -> + put_chars(Bin, Port, Q) + end; +io_request({put_chars,unicode,Mod,Func,Args}, Port, Q) -> + case catch apply(Mod,Func,Args) of + Data when is_list(Data); is_binary(Data) -> + case wrap_characters_to_binary(Data, unicode, get(encoding)) of + Bin when is_binary(Bin) -> + put_chars(Bin, Port, Q); + error -> + {error,{error,put_chars},Q} + end; + Undef -> + put_chars(Undef, Port, Q) + end; +io_request({put_chars,latin1,Chars}, Port, Q) -> % Binary new in R9C + case catch unicode:characters_to_binary(Chars, latin1, get(encoding)) of + Data when is_binary(Data) -> + put_chars(Data, Port, Q); + _ -> + {error,{error,put_chars},Q} + end; +io_request({put_chars,latin1,Mod,Func,Args}, Port, Q) -> + case catch apply(Mod,Func,Args) of + Data when is_list(Data); is_binary(Data) -> + case + catch unicode:characters_to_binary(Data,latin1,get(encoding)) + of + Bin when is_binary(Bin) -> + put_chars(Bin, Port, Q); + _ -> + {error,{error,put_chars},Q} + end; + Undef -> + put_chars(Undef, Port, Q) + end; +io_request({get_chars,Enc,Prompt,N}, Port, Q) -> % New in R9C + get_chars(Prompt, io_lib, collect_chars, N, Port, Q, Enc); +io_request({get_line,Enc,Prompt}, Port, Q) -> + case get(read_mode) of + binary -> + get_line_bin(Prompt,Port,Q,Enc); + _ -> + get_chars(Prompt, io_lib, collect_line, [], Port, Q, Enc) + end; +io_request({get_until,Enc,Prompt,M,F,As}, Port, Q) -> + get_chars(Prompt, io_lib, get_until, {M,F,As}, Port, Q, Enc); +%% End New in R13B +io_request(getopts, Port, Q) -> + getopts(Port, Q); +io_request({setopts,Opts}, Port, Q) when is_list(Opts) -> + setopts(Opts, Port, Q); +io_request({requests,Reqs}, Port, Q) -> + io_requests(Reqs, {ok,ok,Q}, Port); + +%% New in R12 +io_request({get_geometry,columns},Port,Q) -> + case get_fd_geometry(Port) of + {W,_H} -> + {ok,W,Q}; + _ -> + {error,{error,enotsup},Q} + end; +io_request({get_geometry,rows},Port,Q) -> + case get_fd_geometry(Port) of + {_W,H} -> + {ok,H,Q}; + _ -> + {error,{error,enotsup},Q} + end; +%% BC with pre-R13 nodes +io_request({put_chars,Chars}, Port, Q) -> + io_request({put_chars,latin1,Chars}, Port, Q); +io_request({put_chars,Mod,Func,Args}, Port, Q) -> + io_request({put_chars,latin1,Mod,Func,Args}, Port, Q); +io_request({get_chars,Prompt,N}, Port, Q) -> + io_request({get_chars,latin1,Prompt,N}, Port, Q); +io_request({get_line,Prompt}, Port, Q) -> + io_request({get_line,latin1,Prompt}, Port, Q); +io_request({get_until,Prompt,M,F,As}, Port, Q) -> + io_request({get_until,latin1,Prompt,M,F,As}, Port, Q); + +io_request(R, _Port, Q) -> %Unknown request + {error,{error,{request,R}},Q}. %Ignore but give error (?) + +%% Status = io_requests(RequestList, PrevStat, Port) +%% Process a list of output requests as long as the previous status is 'ok'. + +io_requests([R|Rs], {ok,_Res,Q}, Port) -> + io_requests(Rs, io_request(R, Port, Q), Port); +io_requests([_|_], Error, _) -> + Error; +io_requests([], Stat, _) -> + Stat. + +%% put_port(DeepList, Port) +%% Take a deep list of characters, flatten and output them to the +%% port. + +put_port(List, Port) -> + send_port(Port, {command, List}). + +%% send_port(Port, Command) + +send_port(Port, Command) -> + Port ! {self(),Command}, + ok. + +%% io_reply(From, ReplyAs, Reply) +%% The function for sending i/o command acknowledgement. +%% The ACK contains the return value. + +io_reply(From, ReplyAs, Reply) -> + From ! {io_reply,ReplyAs,Reply}. + +%% put_chars +put_chars(Chars, Port, Q) when is_binary(Chars) -> + ok = put_port(Chars, Port), + {ok,ok,Q}; +put_chars(Chars, Port, Q) -> + case catch list_to_binary(Chars) of + Binary when is_binary(Binary) -> + put_chars(Binary, Port, Q); + _ -> + {error,{error,put_chars},Q} + end. + +expand_encoding([]) -> + []; +expand_encoding([latin1 | T]) -> + [{encoding,latin1} | expand_encoding(T)]; +expand_encoding([unicode | T]) -> + [{encoding,unicode} | expand_encoding(T)]; +expand_encoding([H|T]) -> + [H|expand_encoding(T)]. + +%% setopts +setopts(Opts0,Port,Q) -> + Opts = proplists:unfold( + proplists:substitute_negations( + [{list,binary}], + expand_encoding(Opts0))), + case check_valid_opts(Opts) of + true -> + do_setopts(Opts,Port,Q); + false -> + {error,{error,enotsup},Q} + end. +check_valid_opts([]) -> + true; +check_valid_opts([{binary,_}|T]) -> + check_valid_opts(T); +check_valid_opts([{encoding,Valid}|T]) when Valid =:= latin1; Valid =:= utf8; Valid =:= unicode -> + check_valid_opts(T); +check_valid_opts(_) -> + false. + +do_setopts(Opts, _Port, Q) -> + case proplists:get_value(encoding,Opts) of + Valid when Valid =:= unicode; Valid =:= utf8 -> + put(encoding,unicode); + latin1 -> + put(encoding,latin1); + undefined -> + ok + end, + case proplists:get_value(binary, Opts) of + true -> + put(read_mode,binary), + {ok,ok,Q}; + false -> + put(read_mode,list), + {ok,ok,Q}; + _ -> + {ok,ok,Q} + end. + +getopts(_Port,Q) -> + Bin = {binary, get(read_mode) =:= binary}, + Uni = {encoding, get(encoding)}, + {ok,[Bin,Uni],Q}. + +get_line_bin(Prompt,Port,Q, Enc) -> + case prompt(Port, Prompt) of + error -> + {error,{error,get_line},Q}; + ok -> + case {get(eof),queue:is_empty(Q)} of + {true,true} -> + {ok,eof,Q}; + _ -> + get_line(Prompt,Port, Q, [], Enc) + end + end. + +get_line(Prompt, Port, Q, Acc, Enc) -> + case queue:is_empty(Q) of + true -> + receive + {Port,{data,Bytes}} -> + get_line_bytes(Prompt, Port, Q, Acc, Bytes, Enc); + {Port, eof} -> + put(eof, true), + {ok, eof, []}; + {io_request,From,ReplyAs,{get_geometry,_}=Req} when is_pid(From) -> + do_io_request(Req, From, ReplyAs, Port, + queue:new()), + %% No prompt. + get_line(Prompt, Port, Q, Acc, Enc); + {io_request,From,ReplyAs,Request} when is_pid(From) -> + do_io_request(Request, From, ReplyAs, Port, queue:new()), + case prompt(Port, Prompt) of + error -> + {error,{error,get_line},Q}; + ok -> + get_line(Prompt, Port, Q, Acc, Enc) + end; + {'EXIT',From,What} when node(From) =:= node() -> + {exit,What} + end; + false -> + get_line_doit(Prompt, Port, Q, Acc, Enc) + end. + +get_line_bytes(Prompt, Port, Q, Acc, Bytes, Enc) -> + case get(shell) of + noshell -> + get_line_doit(Prompt, Port, queue:snoc(Q, Bytes),Acc,Enc); + _ -> + case contains_ctrl_g_or_ctrl_c(Bytes) of + false -> + get_line_doit(Prompt, Port, queue:snoc(Q, Bytes), Acc, Enc); + _ -> + throw(new_shell) + end + end. +is_cr_at(Pos,Bin) -> + case Bin of + <<_:Pos/binary,$\r,_/binary>> -> + true; + _ -> + false + end. +srch(<<>>,_,_) -> + nomatch; +srch(<<X:8,_/binary>>,X,N) -> + {match,[{N,1}]}; +srch(<<_:8,T/binary>>,X,N) -> + srch(T,X,N+1). + +get_line_doit(Prompt, Port, Q, Accu, Enc) -> + case queue:is_empty(Q) of + true -> + case get(eof) of + true -> + case Accu of + [] -> + {ok,eof,Q}; + _ -> + {ok,binrev(Accu,[]),Q} + end; + _ -> + get_line(Prompt, Port, Q, Accu, Enc) + end; + false -> + Bin = queue:head(Q), + case srch(Bin,$\n,0) of + nomatch -> + X = byte_size(Bin)-1, + case is_cr_at(X,Bin) of + true -> + <<D:X/binary,_/binary>> = Bin, + get_line_doit(Prompt, Port, queue:tail(Q), + [<<$\r>>,D|Accu], Enc); + false -> + get_line_doit(Prompt, Port, queue:tail(Q), + [Bin|Accu], Enc) + end; + {match,[{Pos,1}]} -> + %% We are done + PosPlus = Pos + 1, + case Accu of + [] -> + {Head,Tail} = + case is_cr_at(Pos - 1,Bin) of + false -> + <<H:PosPlus/binary, + T/binary>> = Bin, + {H,T}; + true -> + PosMinus = Pos - 1, + <<H:PosMinus/binary, + _,_,T/binary>> = Bin, + {binrev([],[H,$\n]),T} + end, + case Tail of + <<>> -> + {ok, cast(Head,Enc), queue:tail(Q)}; + _ -> + {ok, cast(Head,Enc), + queue:cons(Tail, queue:tail(Q))} + end; + [<<$\r>>|Stack1] when Pos =:= 0 -> + <<_:PosPlus/binary,Tail/binary>> = Bin, + case Tail of + <<>> -> + {ok, cast(binrev(Stack1, [$\n]),Enc), + queue:tail(Q)}; + _ -> + {ok, cast(binrev(Stack1, [$\n]),Enc), + queue:cons(Tail, queue:tail(Q))} + end; + _ -> + {Head,Tail} = + case is_cr_at(Pos - 1,Bin) of + false -> + <<H:PosPlus/binary, + T/binary>> = Bin, + {H,T}; + true -> + PosMinus = Pos - 1, + <<H:PosMinus/binary, + _,_,T/binary>> = Bin, + {[H,$\n],T} + end, + case Tail of + <<>> -> + {ok, cast(binrev(Accu,[Head]),Enc), + queue:tail(Q)}; + _ -> + {ok, cast(binrev(Accu,[Head]),Enc), + queue:cons(Tail, queue:tail(Q))} + end + end + end + end. + +binrev(L, T) -> + list_to_binary(lists:reverse(L, T)). + +%% Entry function. +get_chars(Prompt, M, F, Xa, Port, Q, Enc) -> + case prompt(Port, Prompt) of + error -> + {error,{error,get_chars},Q}; + ok -> + case {get(eof),queue:is_empty(Q)} of + {true,true} -> + {ok,eof,Q}; + _ -> + get_chars(Prompt, M, F, Xa, Port, Q, start, Enc) + end + end. + +%% First loop. Wait for port data. Respond to output requests. +get_chars(Prompt, M, F, Xa, Port, Q, State, Enc) -> + case queue:is_empty(Q) of + true -> + receive + {Port,{data,Bytes}} -> + get_chars_bytes(State, M, F, Xa, Port, Q, Bytes, Enc); + {Port, eof} -> + put(eof, true), + {ok, eof, []}; + {io_request,From,ReplyAs,{get_geometry,_}=Req} when is_pid(From) -> + do_io_request(Req, From, ReplyAs, Port, + queue:new()), %Keep Q over this call + %% No prompt. + get_chars(Prompt, M, F, Xa, Port, Q, State, Enc); + {io_request,From,ReplyAs,Request} when is_pid(From) -> + get_chars_req(Prompt, M, F, Xa, Port, Q, State, + Request, From, ReplyAs, Enc); + {'EXIT',From,What} when node(From) =:= node() -> + {exit,What} + end; + false -> + get_chars_apply(State, M, F, Xa, Port, Q, Enc) + end. + +get_chars_req(Prompt, M, F, XtraArg, Port, Q, State, + Req, From, ReplyAs, Enc) -> + do_io_request(Req, From, ReplyAs, Port, queue:new()), %Keep Q over this call + case prompt(Port, Prompt) of + error -> + {error,{error,get_chars},Q}; + ok -> + get_chars(Prompt, M, F, XtraArg, Port, Q, State, Enc) + end. + +%% Second loop. Pass data to client as long as it wants more. +%% A ^G in data interrupts loop if 'noshell' is not undefined. +get_chars_bytes(State, M, F, Xa, Port, Q, Bytes, Enc) -> + case get(shell) of + noshell -> + get_chars_apply(State, M, F, Xa, Port, queue:snoc(Q, Bytes),Enc); + _ -> + case contains_ctrl_g_or_ctrl_c(Bytes) of + false -> + get_chars_apply(State, M, F, Xa, Port, + queue:snoc(Q, Bytes),Enc); + _ -> + throw(new_shell) + end + end. + +get_chars_apply(State0, M, F, Xa, Port, Q, Enc) -> + case catch M:F(State0, cast(queue:head(Q),Enc), Enc, Xa) of + {stop,Result,<<>>} -> + {ok,Result,queue:tail(Q)}; + {stop,Result,[]} -> + {ok,Result,queue:tail(Q)}; + {stop,Result,eof} -> + {ok,Result,queue:tail(Q)}; + {stop,Result,Buf} -> + {ok,Result,queue:cons(Buf, queue:tail(Q))}; + {'EXIT',_Why} -> + {error,{error,err_func(M, F, Xa)},queue:new()}; + State1 -> + get_chars_more(State1, M, F, Xa, Port, queue:tail(Q), Enc) + end. + +get_chars_more(State, M, F, Xa, Port, Q, Enc) -> + case queue:is_empty(Q) of + true -> + case get(eof) of + undefined -> + receive + {Port,{data,Bytes}} -> + get_chars_bytes(State, M, F, Xa, Port, Q, Bytes, Enc); + {Port,eof} -> + put(eof, true), + get_chars_apply(State, M, F, Xa, Port, + queue:snoc(Q, eof), Enc); + {'EXIT',From,What} when node(From) =:= node() -> + {exit,What} + end; + _ -> + get_chars_apply(State, M, F, Xa, Port, queue:snoc(Q, eof), Enc) + end; + false -> + get_chars_apply(State, M, F, Xa, Port, Q, Enc) + end. + +%% common case, reduces execution time by 20% +prompt(_Port, '') -> ok; +prompt(Port, Prompt) -> + Encoding = get(encoding), + PromptString = io_lib:format_prompt(Prompt, Encoding), + case wrap_characters_to_binary(PromptString, unicode, Encoding) of + Bin when is_binary(Bin) -> + put_port(Bin, Port); + error -> + error + end. + +%% Convert error code to make it look as before +err_func(io_lib, get_until, {_,F,_}) -> + F; +err_func(_, F, _) -> + F. + +%% using regexp reduces execution time by >50% compared to old code +%% running two regexps in sequence is much faster than \\x03|\\x07 +contains_ctrl_g_or_ctrl_c(BinOrList)-> + case {re:run(BinOrList, <<3>>),re:run(BinOrList, <<7>>)} of + {nomatch, nomatch} -> false; + _ -> true + end. + +%% Convert a buffer between list and binary +cast(Data, _Encoding) when is_atom(Data) -> + Data; +cast(Data, Encoding) -> + IoEncoding = get(encoding), + cast(Data, get(read_mode), IoEncoding, Encoding). + +cast(B, binary, latin1, latin1) when is_binary(B) -> + B; +cast(L, binary, latin1, latin1) -> + case catch erlang:iolist_to_binary(L) of + Bin when is_binary(Bin) -> Bin; + _ -> exit({no_translation, latin1, latin1}) + end; +cast(Data, binary, unicode, latin1) when is_binary(Data); is_list(Data) -> + case catch unicode:characters_to_binary(Data, unicode, latin1) of + Bin when is_binary(Bin) -> Bin; + _ -> exit({no_translation, unicode, latin1}) + end; +cast(Data, binary, latin1, unicode) when is_binary(Data); is_list(Data) -> + case catch unicode:characters_to_binary(Data, latin1, unicode) of + Bin when is_binary(Bin) -> Bin; + _ -> exit({no_translation, latin1, unicode}) + end; +cast(B, binary, unicode, unicode) when is_binary(B) -> + B; +cast(L, binary, unicode, unicode) -> + case catch unicode:characters_to_binary(L, unicode) of + Bin when is_binary(Bin) -> Bin; + _ -> exit({no_translation, unicode, unicode}) + end; +cast(B, list, latin1, latin1) when is_binary(B) -> + binary_to_list(B); +cast(L, list, latin1, latin1) -> + case catch erlang:iolist_to_binary(L) of + Bin when is_binary(Bin) -> binary_to_list(Bin); + _ -> exit({no_translation, latin1, latin1}) + end; +cast(Data, list, unicode, latin1) when is_binary(Data); is_list(Data) -> + case catch unicode:characters_to_list(Data, unicode) of + Chars when is_list(Chars) -> + [ case X of + High when High > 255 -> + exit({no_translation, unicode, latin1}); + Low -> + Low + end || X <- Chars ]; + _ -> + exit({no_translation, unicode, latin1}) + end; +cast(Data, list, latin1, unicode) when is_binary(Data); is_list(Data) -> + case catch unicode:characters_to_list(Data, latin1) of + Chars when is_list(Chars) -> Chars; + _ -> exit({no_translation, latin1, unicode}) + end; +cast(Data, list, unicode, unicode) when is_binary(Data); is_list(Data) -> + case catch unicode:characters_to_list(Data, unicode) of + Chars when is_list(Chars) -> Chars; + _ -> exit({no_translation, unicode, unicode}) + end. + +wrap_characters_to_binary(Chars, unicode, latin1) -> + case catch unicode:characters_to_binary(Chars, unicode, latin1) of + Bin when is_binary(Bin) -> + Bin; + _ -> + case catch unicode:characters_to_list(Chars, unicode) of + L when is_list(L) -> + list_to_binary( + [ case X of + High when High > 255 -> + ["\\x{",erlang:integer_to_list(X, 16),$}]; + Low -> + Low + end || X <- L ]); + _ -> + error + end + end; +wrap_characters_to_binary(Bin, From, From) when is_binary(Bin) -> + Bin; +wrap_characters_to_binary(Chars, From, To) -> + case catch unicode:characters_to_binary(Chars, From, To) of + Bin when is_binary(Bin) -> + Bin; + _ -> + error + end. + diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index d00a46f..56a3940 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -47,6 +47,7 @@ deprecated/4, indent/1, update_code/1, + update_code/2, remove_from_code_path/1, cleanup_code_path/1, args_to_tasks/1, @@ -60,13 +61,15 @@ tup_find/2, line_count/1, set_httpc_options/0, + url_append_path/2, escape_chars/1, escape_double_quotes/1, escape_double_quotes_weak/1, check_min_otp_version/1, check_blacklisted_otp_versions/1, info_useless/2, - list_dir/1]). + list_dir/1, + user_agent/0]). %% for internal use only -export([otp_release/0]). @@ -284,7 +287,18 @@ tup_umerge(NewList, OldList) -> tup_umerge_([], Olds) -> Olds; tup_umerge_([New|News], Olds) -> - lists:reverse(umerge(News, Olds, [], New)). + tup_umerge_dedup_(umerge(new, News, Olds, [], New), []). + +%% removes 100% identical duplicate elements so that +%% `[a,{a,b},a,{a,c},a]' returns `[a,{a,b},{a,c}]'. +%% Operates on a reverted list that gets reversed as part of this pass +tup_umerge_dedup_([], Acc) -> + Acc; +tup_umerge_dedup_([H|T], Acc) -> + case lists:member(H,T) of + true -> tup_umerge_dedup_(T, Acc); + false -> tup_umerge_dedup_(T, [H|Acc]) + end. tup_find(_Elem, []) -> false; @@ -300,35 +314,58 @@ tup_find(Elem, [Elem1 | Elems]) when is_tuple(Elem1) -> tup_find(Elem, [_Elem | Elems]) -> tup_find(Elem, Elems). -%% This is equivalent to umerge2_2 in the stdlib, except we use the expanded -%% value/key only to compare -umerge(News, [Old|Olds], Merged, Cmp) when element(1, Cmp) == element(1, Old); - element(1, Cmp) == Old; - Cmp == element(1, Old); - Cmp =< Old -> - umerge(News, Olds, [Cmp | Merged], Cmp, Old); -umerge(News, [Old|Olds], Merged, Cmp) -> - umerge(News, Olds, [Old | Merged], Cmp); -umerge(News, [], Merged, Cmp) -> - lists:reverse(News, [Cmp | Merged]). - -%% Similar to stdlib's umerge2_1 in the stdlib, except that when the expanded -%% value/keys compare equal, we check if the element is a full dupe to clear it -%% (like the stdlib function does) or otherwise keep the duplicate around in -%% an order that prioritizes 'New' elements. -umerge([New|News], Olds, Merged, CmpMerged, Cmp) when CmpMerged == Cmp -> - umerge(News, Olds, Merged, New); -umerge([New|News], Olds, Merged, _CmpMerged, Cmp) when element(1,New) == element(1, Cmp); - element(1,New) == Cmp; - New == element(1, Cmp); - New =< Cmp -> - umerge(News, Olds, [New | Merged], New, Cmp); -umerge([New|News], Olds, Merged, _CmpMerged, Cmp) -> % > - umerge(News, Olds, [Cmp | Merged], New); -umerge([], Olds, Merged, CmpMerged, Cmp) when CmpMerged == Cmp -> - lists:reverse(Olds, Merged); -umerge([], Olds, Merged, _CmpMerged, Cmp) -> - lists:reverse(Olds, [Cmp | Merged]). +-spec umerge(new|old, News, Olds, Acc, Current) -> Merged when + News :: [term()], + Olds :: [term()], + Acc :: [term()], + Current :: term(), + Merged :: [term()]. +umerge(_, [], [], Acc, Current) -> + [Current | Acc]; +umerge(new, News, [], Acc, Current) -> + %% only news left + lists:reverse(News, [Current|Acc]); +umerge(old, [], Olds, Acc, Current) -> + %% only olds left + lists:reverse(Olds, [Current|Acc]); +umerge(new, News, [Old|Olds], Acc, Current) -> + {Dir, Merged, NewCurrent} = compare({new, Current}, {old, Old}), + umerge(Dir, News, Olds, [Merged|Acc], NewCurrent); +umerge(old, [New|News], Olds, Acc, Current) -> + {Dir, Merged, NewCurrent} = compare({new, New}, {old, Current}), + umerge(Dir, News, Olds, [Merged|Acc], NewCurrent). + +-spec compare({Priority, term()}, {Secondary, term()}) -> + {NextPriority, Merged, Larger} when + Priority :: new | old, + Secondary :: new | old, + NextPriority :: new | old, + Merged :: term(), + Larger :: term(). +compare({Priority, A}, {Secondary, B}) when is_tuple(A), is_tuple(B) -> + KA = element(1,A), + KB = element(1,B), + if KA == KB -> {Secondary, A, B}; + KA < KB -> {Secondary, A, B}; + KA > KB -> {Priority, B, A} + end; +compare({Priority, A}, {Secondary, B}) when not is_tuple(A), not is_tuple(B) -> + if A == B -> {Secondary, A, B}; + A < B -> {Secondary, A, B}; + A > B -> {Priority, B, A} + end; +compare({Priority, A}, {Secondary, B}) when is_tuple(A), not is_tuple(B) -> + KA = element(1,A), + if KA == B -> {Secondary, A, B}; + KA < B -> {Secondary, A, B}; + KA > B -> {Priority, B, A} + end; +compare({Priority, A}, {Secondary, B}) when not is_tuple(A), is_tuple(B) -> + KB = element(1,B), + if A == KB -> {Secondary, A, B}; + A < KB -> {Secondary, A, B}; + A > KB -> {Priority, B, A} + end. %% Implements wc -l functionality used to determine patchcount from git output line_count(PatchLines) -> @@ -371,6 +408,10 @@ abort_if_blacklisted(BlacklistedRegex, OtpRelease) -> [OtpRelease, BlacklistedRegex]) end. +user_agent() -> + {ok, Vsn} = application:get_key(rebar, vsn), + ?FMT("Rebar/~s (OTP/~s)", [Vsn, otp_release()]). + %% ==================================================================== %% Internal functions %% ==================================================================== @@ -644,7 +685,9 @@ indent(Amount) when erlang:is_integer(Amount) -> %% Replace code paths with new paths for existing apps and %% purge code of the old modules from those apps. -update_code(Paths) -> +update_code(Paths) -> update_code(Paths, []). + +update_code(Paths, Opts) -> lists:foreach(fun(Path) -> Name = filename:basename(Path, "/ebin"), App = list_to_atom(Name), @@ -654,19 +697,18 @@ update_code(Paths) -> code:add_patha(Path), ok; {ok, Modules} -> - %% stick rebar ebin dir before purging to prevent - %% inadvertent termination - RebarBin = code:lib_dir(rebar, ebin), - ok = code:stick_dir(RebarBin), %% replace_path causes problems when running %% tests in projects like erlware_commons that rebar3 %% also includes %code:replace_path(App, Path), code:del_path(App), code:add_patha(Path), - [begin code:purge(M), code:delete(M) end || M <- Modules], - %% unstick rebar dir - ok = code:unstick_dir(RebarBin) + case lists:member(soft_purge, Opts) of + true -> + [begin code:soft_purge(M), code:delete(M) end || M <- Modules]; + false -> + [begin code:purge(M), code:delete(M) end || M <- Modules] + end end end, Paths). @@ -762,6 +804,15 @@ set_httpc_options(Scheme, Proxy) -> {ok, {_, _, Host, Port, _, _}} = http_uri:parse(Proxy), httpc:set_options([{Scheme, {{Host, Port}, []}}], rebar). +url_append_path(Url, ExtraPath) -> + case http_uri:parse(Url) of + {ok, {Scheme, UserInfo, Host, Port, Path, Query}} -> + {ok, lists:append([atom_to_list(Scheme), "://", UserInfo, Host, ":", integer_to_list(Port), + filename:join(Path, ExtraPath), "?", Query])}; + _ -> + error + end. + %% escape\ as\ a\ shell\? escape_chars(Str) when is_atom(Str) -> escape_chars(atom_to_list(Str)); @@ -782,8 +833,11 @@ info_useless(Old, New) -> not lists:member(Name, New)], ok. --ifdef(no_list_dir_all). -list_dir(Dir) -> file:list_dir(Dir). --else. -list_dir(Dir) -> file:list_dir_all(Dir). --endif. +list_dir(Dir) -> + %% `list_dir_all` returns raw files which are unsupported + %% prior to R16 so just fall back to `list_dir` if running + %% on an earlier vm + case erlang:function_exported(file, list_dir_all, 1) of + true -> file:list_dir_all(Dir); + false -> file:list_dir(Dir) + end. |