%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- %% ex: ts=4 sw=4 et -module(rebar_prv_common_test). -behaviour(provider). -export([init/1, do/1, format_error/1]). %% exported for test purposes, consider private -export([setup_ct/1]). -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). -define(PROVIDER, ct). -define(DEPS, [compile]). %% =================================================================== %% Public API %% =================================================================== -spec init(rebar_state:t()) -> {ok, rebar_state:t()}. init(State) -> Provider = providers:create([{name, ?PROVIDER}, {module, ?MODULE}, {deps, ?DEPS}, {bare, true}, {example, "rebar3 ct"}, {short_desc, "Run Common Tests."}, {desc, "Run Common Tests."}, {opts, ct_opts(State)}, {profiles, [test]}]), State1 = rebar_state:add_provider(State, Provider), State2 = rebar_state:add_to_profile(State1, test, test_state(State1)), {ok, State2}. -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> ?INFO("Running Common Test suites...", []), rebar_utils:update_code(rebar_state:code_paths(State, all_deps), [soft_purge]), %% Run ct provider prehooks Providers = rebar_state:providers(State), Cwd = rebar_dir:get_cwd(), rebar_hooks:run_all_hooks(Cwd, pre, ?PROVIDER, Providers, State), try run_test(State) of {ok, State1} = Result -> %% Run ct provider posthooks rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State1), rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), Result; ?PRV_ERROR(_) = Error -> rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), Error catch throw:{error, Reason} -> rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), ?PRV_ERROR(Reason) end. -spec format_error(any()) -> iolist(). format_error({multiple_dirs_and_suites, Opts}) -> io_lib:format("Multiple dirs declared alongside suite in opts: ~p", [Opts]); format_error({bad_dir_or_suite, Opts}) -> io_lib:format("Bad value for dir or suite in opts: ~p", [Opts]); format_error({failures_running_tests, {Failed, AutoSkipped}}) -> io_lib:format("Failures occured running tests: ~b", [Failed+AutoSkipped]); format_error({error_running_tests, Reason}) -> io_lib:format("Error running tests: ~p", [Reason]); format_error(suite_at_project_root) -> io_lib:format("Test suites can't be located in project root", []); format_error({error, Reason}) -> io_lib:format("Unknown error: ~p", [Reason]). %% =================================================================== %% Internal functions %% =================================================================== run_test(State) -> case setup_ct(State) of {error, {no_tests_specified, Opts}} -> ?WARN("No tests specified in opts: ~p", [Opts]), {ok, State}; Opts -> Opts1 = setup_logdir(State, Opts), ?DEBUG("common test opts: ~p", [Opts1]), run_test(State, Opts1) end. run_test(State, Opts) -> {RawOpts, _} = rebar_state:command_parsed_args(State), ok = rebar_prv_cover:maybe_cover_compile(State, apps), Result = case proplists:get_value(verbose, RawOpts, false) of true -> run_test_verbose(Opts); false -> run_test_quiet(Opts) end, ok = rebar_prv_cover:maybe_write_coverdata(State, ?PROVIDER), case Result of ok -> {ok, State}; Error -> Error end. run_test_verbose(Opts) -> handle_results(ct:run_test(Opts)). run_test_quiet(Opts) -> Pid = self(), Ref = erlang:make_ref(), LogDir = proplists:get_value(logdir, Opts), {_, Monitor} = erlang:spawn_monitor(fun() -> {ok, F} = file:open(filename:join([LogDir, "ct.latest.log"]), [write]), true = group_leader(F, self()), Pid ! {Ref, ct:run_test(Opts)} end), receive {Ref, Result} -> handle_quiet_results(Opts, Result); {'DOWN', Monitor, _, _, Reason} -> handle_results(?PRV_ERROR(Reason)) end. handle_results(Results) when is_list(Results) -> Result = lists:foldl(fun sum_results/2, {0, 0, {0,0}}, Results), handle_results(Result); handle_results({_, Failed, {_, AutoSkipped}}) when Failed > 0 orelse AutoSkipped > 0 -> ?PRV_ERROR({failures_running_tests, {Failed, AutoSkipped}}); handle_results({error, Reason}) -> ?PRV_ERROR({error_running_tests, Reason}); handle_results(_) -> ok. sum_results({Passed, Failed, {UserSkipped, AutoSkipped}}, {Passed2, Failed2, {UserSkipped2, AutoSkipped2}}) -> {Passed+Passed2, Failed+Failed2, {UserSkipped+UserSkipped2, AutoSkipped+AutoSkipped2}}. handle_quiet_results(_, {error, _} = Result) -> handle_results(Result); handle_quiet_results(CTOpts, Results) when is_list(Results) -> _ = [format_result(Result) || Result <- Results], case handle_results(Results) of ?PRV_ERROR({failures_running_tests, _}) = Error -> LogDir = proplists:get_value(logdir, CTOpts), Index = filename:join([LogDir, "index.html"]), ?CONSOLE("Results written to ~p.", [Index]), Error; Other -> Other end; handle_quiet_results(CTOpts, Result) -> handle_quiet_results(CTOpts, [Result]). format_result({Passed, 0, {0, 0}}) -> ?CONSOLE("All ~p tests passed.", [Passed]); format_result({Passed, Failed, Skipped}) -> Format = [format_failed(Failed), format_skipped(Skipped), format_passed(Passed)], ?CONSOLE("~s", [Format]). format_failed(0) -> []; format_failed(Failed) -> io_lib:format("Failed ~p tests. ", [Failed]). format_passed(Passed) -> io_lib:format("Passed ~p tests. ", [Passed]). format_skipped({0, 0}) -> []; format_skipped({User, Auto}) -> io_lib:format("Skipped ~p (~p, ~p) tests. ", [User+Auto, User, Auto]). test_state(State) -> TestOpts = case rebar_state:get(State, ct_compile_opts, []) of [] -> []; Opts -> [{erl_opts, Opts}] end, [first_files(State)|TestOpts]. first_files(State) -> CTFirst = rebar_state:get(State, ct_first_files, []), {erl_first_files, CTFirst}. setup_ct(State) -> Opts = resolve_ct_opts(State), Opts1 = discover_tests(State, Opts), copy_and_compile_tests(State, Opts1). resolve_ct_opts(State) -> {RawOpts, _} = rebar_state:command_parsed_args(State), CmdOpts = transform_opts(RawOpts), CfgOpts = rebar_state:get(State, ct_opts, []), Merged = lists:ukeymerge(1, lists:ukeysort(1, CmdOpts), lists:ukeysort(1, CfgOpts)), %% make sure `dir` and/or `suite` from command line go in as %% a pair overriding both `dir` and `suite` from config if %% they exist case {proplists:get_value(suite, CmdOpts), proplists:get_value(dir, CmdOpts)} of {undefined, undefined} -> Merged; {_Suite, undefined} -> lists:keydelete(dir, 1, Merged); {undefined, _Dir} -> lists:keydelete(suite, 1, Merged); {_Suite, _Dir} -> Merged end. discover_tests(State, Opts) -> case proplists:get_value(spec, Opts) of undefined -> discover_dirs_and_suites(State, Opts); TestSpec -> discover_testspec(TestSpec, Opts) end. discover_dirs_and_suites(State, Opts) -> case {proplists:get_value(dir, Opts), proplists:get_value(suite, Opts)} of %% no dirs or suites defined, try using `$APP/test` and `$ROOT/test` %% as suites {undefined, undefined} -> test_dirs(State, Opts); %% no dirs defined {undefined, _} -> Opts; %% no suites defined {_, undefined} -> Opts; %% a single dir defined, this is ok {Dirs, Suites} when is_integer(hd(Dirs)), is_list(Suites) -> Opts; %% still a single dir defined, adjust to make acceptable to ct {[Dir], Suites} when is_integer(hd(Dir)), is_list(Suites) -> [{dir, Dir}|lists:keydelete(dir, 1, Opts)]; %% multiple dirs and suites, error now to simplify later steps {_, _} -> throw({error, {multiple_dirs_and_suites, Opts}}) end. discover_testspec(_TestSpec, Opts) -> lists:keydelete(auto_compile, 1, Opts). copy_and_compile_tests(State, Opts) -> %% possibly enable cover {RawOpts, _} = rebar_state:command_parsed_args(State), State1 = case proplists:get_value(cover, RawOpts, false) of true -> rebar_state:set(State, cover_enabled, true); false -> State end, copy_and_compile_test_suites(State1, Opts). copy_and_compile_test_suites(State, Opts) -> case proplists:get_value(suite, Opts) of %% no suites, try dirs undefined -> copy_and_compile_test_dirs(State, Opts); Suites -> Dir = proplists:get_value(dir, Opts, undefined), AllSuites = join(Dir, Suites), Dirs = find_suite_dirs(AllSuites), lists:foreach(fun(S) -> NewPath = copy(State, S), compile_dir(State, NewPath) end, Dirs), NewSuites = lists:map(fun(S) -> retarget_path(State, S) end, AllSuites), [{suite, NewSuites}|lists:keydelete(suite, 1, Opts)] end. copy_and_compile_test_dirs(State, Opts) -> case proplists:get_value(dir, Opts) of undefined -> {error, {no_tests_specified, Opts}}; %% dir is a single directory Dir when is_list(Dir), is_integer(hd(Dir)) -> NewPath = copy(State, Dir), [{dir, compile_dir(State, NewPath)}|lists:keydelete(dir, 1, Opts)]; %% dir is a list of directories Dirs when is_list(Dirs) -> NewDirs = lists:map(fun(Dir) -> NewPath = copy(State, Dir), compile_dir(State, NewPath) end, Dirs), [{dir, NewDirs}|lists:keydelete(dir, 1, Opts)] end. join(undefined, Suites) -> Suites; join(Dir, Suites) when is_list(Dir), is_integer(hd(Dir)) -> lists:map(fun(S) -> filename:join([Dir, S]) end, Suites); %% multiple dirs or a bad dir argument, try to continue anyways join(_, Suites) -> Suites. find_suite_dirs(Suites) -> AllDirs = lists:map(fun(S) -> filename:dirname(filename:absname(S)) end, Suites), %% eliminate duplicates lists:usort(AllDirs). copy(State, Dir) -> From = reduce_path(Dir), retarget_path(State, From). compile_dir(State, Dir) -> NewState = replace_src_dirs(State, [filename:absname(Dir)]), ok = rebar_erlc_compiler:compile(rebar_state:opts(NewState), rebar_dir:base_dir(State), Dir), ok = maybe_cover_compile(State, Dir), Dir. retarget_path(State, Path) -> ProjectApps = rebar_state:project_apps(State), retarget_path(State, Path, ProjectApps). %% not relative to any apps in project, check to see it's relative to %% project root retarget_path(State, Path, []) -> case relative_path(reduce_path(Path), rebar_state:dir(State)) of {ok, NewPath} -> filename:join([rebar_dir:base_dir(State), NewPath]); %% not relative to project root, don't modify {error, not_relative} -> Path end; %% relative to current app, retarget to the same dir relative to %% the app's out_dir retarget_path(State, Path, [App|Rest]) -> case relative_path(reduce_path(Path), rebar_app_info:dir(App)) of {ok, NewPath} -> filename:join([rebar_app_info:out_dir(App), NewPath]); {error, not_relative} -> retarget_path(State, Path, Rest) end. relative_path(Target, To) -> relative_path1(filename:split(filename:absname(Target)), filename:split(filename:absname(To))). relative_path1([Part|Target], [Part|To]) -> relative_path1(Target, To); relative_path1([], []) -> {ok, ""}; relative_path1(Target, []) -> {ok, filename:join(Target)}; relative_path1(_, _) -> {error, not_relative}. reduce_path(Dir) -> reduce_path([], filename:split(filename:absname(Dir))). reduce_path([], []) -> filename:nativename("/"); reduce_path(Acc, []) -> filename:join(lists:reverse(Acc)); reduce_path(Acc, ["."|Rest]) -> reduce_path(Acc, Rest); reduce_path([_|Acc], [".."|Rest]) -> reduce_path(Acc, Rest); reduce_path([], [".."|Rest]) -> reduce_path([], Rest); reduce_path(Acc, [Component|Rest]) -> reduce_path([Component|Acc], Rest). replace_src_dirs(State, Dirs) -> %% replace any `src_dirs` with the test dirs ErlOpts = rebar_state:get(State, erl_opts, []), StrippedErlOpts = filter_src_dirs(ErlOpts), State1 = rebar_state:set(State, erl_opts, StrippedErlOpts), State2 = rebar_state:set(State1, src_dirs, []), rebar_state:set(State2, extra_src_dirs, Dirs). filter_src_dirs(ErlOpts) -> lists:filter(fun({src_dirs, _}) -> false; ({extra_src_dirs, _}) -> false; (_) -> true end, ErlOpts). test_dirs(State, Opts) -> BareTest = filename:join([rebar_state:dir(State), "test"]), F = fun(App) -> rebar_app_info:dir(App) == rebar_state:dir(State) end, TestApps = project_apps(State), case filelib:is_dir(BareTest) andalso not lists:any(F, TestApps) of %% `test` dir at root of project is already scheduled to be %% included or `test` does not exist false -> application_dirs(TestApps, Opts, []); %% need to add `test` dir at root to dirs to be included true -> application_dirs(TestApps, Opts, [BareTest]) end. project_apps(State) -> filter_checkouts(rebar_state:project_apps(State)). filter_checkouts(Apps) -> filter_checkouts(Apps, []). filter_checkouts([], Acc) -> lists:reverse(Acc); filter_checkouts([App|Rest], Acc) -> case rebar_app_info:is_checkout(App) of true -> filter_checkouts(Rest, Acc); false -> filter_checkouts(Rest, [App|Acc]) end. application_dirs([], Opts, []) -> Opts; application_dirs([], Opts, [Acc]) -> [{dir, Acc}|Opts]; application_dirs([], Opts, Acc) -> [{dir, lists:reverse(Acc)}|Opts]; application_dirs([App|Rest], Opts, Acc) -> TestDir = filename:join([rebar_app_info:dir(App), "test"]), case filelib:is_dir(TestDir) of true -> application_dirs(Rest, Opts, [TestDir|Acc]); false -> application_dirs(Rest, Opts, Acc) end. setup_logdir(State, Opts) -> Logdir = case proplists:get_value(logdir, Opts) of undefined -> filename:join([rebar_dir:base_dir(State), "logs"]); Dir -> Dir end, ensure_dir([Logdir]), [{logdir, Logdir}|lists:keydelete(logdir, 1, Opts)]. ensure_dir([]) -> ok; ensure_dir([Dir|Rest]) -> case ec_file:is_dir(Dir) of true -> ok; false -> ec_file:mkdir_path(Dir) end, ensure_dir(Rest). maybe_cover_compile(State, Dir) -> {Opts, _} = rebar_state:command_parsed_args(State), State1 = case proplists:get_value(cover, Opts, false) of true -> rebar_state:set(State, cover_enabled, true); false -> State end, rebar_prv_cover:maybe_cover_compile(State1, [Dir]). ct_opts(_State) -> [{dir, undefined, "dir", string, help(dir)}, %% comma-seperated list {suite, undefined, "suite", string, help(suite)}, %% comma-seperated list {group, undefined, "group", string, help(group)}, %% comma-seperated list {testcase, undefined, "case", string, help(testcase)}, %% comma-seperated list {spec, undefined, "spec", string, help(spec)}, %% comma-seperated list {join_specs, undefined, "join_specs", boolean, help(join_specs)}, %% Boolean {label, undefined, "label", string, help(label)}, %% String {config, undefined, "config", string, help(config)}, %% comma-seperated list {userconfig, undefined, "userconfig", string, help(userconfig)}, %% [{CallbackMod, CfgStrings}] | {CallbackMod, CfgStrings} {allow_user_terms, undefined, "allow_user_terms", boolean, help(allow_user_terms)}, %% Bool {logdir, undefined, "logdir", string, help(logdir)}, %% dir {logopts, undefined, "logopts", string, help(logopts)}, %% enum, no_nl | no_src {verbosity, undefined, "verbosity", string, help(verbosity)}, %% Integer OR [{Category, VLevel}] {silent_connections, undefined, "silent_connections", string, help(silent_connections)}, % all OR %% comma-seperated list {stylesheet, undefined, "stylesheet", string, help(stylesheet)}, %% file {cover, $c, "cover", {boolean, false}, help(cover)}, {cover_spec, undefined, "cover_spec", string, help(cover_spec)}, %% file {cover_stop, undefined, "cover_stop", boolean, help(cover_stop)}, %% Boolean {event_handler, undefined, "event_handler", string, help(event_handler)}, %% EH | [EH] WHERE EH atom() | {atom(), InitArgs} | {[atom()], InitArgs} {include, undefined, "include", string, help(include)}, % comma-seperated list {abort_if_missing_suites, undefined, "abort_if_missing_suites", {boolean, true}, help(abort_if_missing_suites)}, %% boolean {multiply_timetraps, undefined, "multiply_timetraps", integer, help(multiply_timetraps)}, %% integer {scale_timetraps, undefined, "scale_timetraps", boolean, help(scale_timetraps)}, %% Boolean {create_priv_dir, undefined, "create_priv_dir", string, help(create_priv_dir)}, %% enum: auto_per_run | auto_per_tc | manual_per_tc {repeat, undefined, "repeat", integer, help(repeat)}, %% integer {duration, undefined, "duration", string, help(duration)}, % format: HHMMSS {until, undefined, "until", string, help(until)}, %% format: YYMoMoDD[HHMMSS] {force_stop, undefined, "force_stop", string, help(force_stop)}, % enum: skip_rest, bool {basic_html, undefined, "basic_html", boolean, help(basic_html)}, %% Booloean {ct_hooks, undefined, "ct_hooks", string, help(ct_hooks)}, %% List: [CTHModule | {CTHModule, CTHInitArgs}] where CTHModule is atom CthInitArgs is term {auto_compile, undefined, "auto_compile", {boolean, false}, help(auto_compile)}, {verbose, $v, "verbose", boolean, help(verbose)} ]. help(dir) -> "List of additional directories containing test suites"; help(suite) -> "List of test suites to run"; help(group) -> "List of test groups to run"; help(testcase) -> "List of test cases to run"; help(spec) -> "List of test specs to run"; help(label) -> "Test label"; help(config) -> "List of config files"; help(logdir) -> "Log folder"; help(verbosity) -> "Verbosity"; help(stylesheet) -> "Stylesheet to use for test results"; help(cover) -> "Generate cover data"; help(cover_spec) -> "Cover file to use"; help(event_handler) -> "Event handlers to attach to the runner"; help(include) -> "Include folder"; help(abort_if_missing_suites) -> "Abort if suites are missing"; help(repeat) -> "How often to repeat tests"; help(duration) -> "Max runtime (format: HHMMSS)"; help(until) -> "Run until (format: HHMMSS)"; help(force_stop) -> "Force stop after time"; help(basic_html) -> "Show basic HTML"; help(verbose) -> "Verbose output"; help(_) -> "". transform_opts(Opts) -> transform_opts(Opts, []). transform_opts([], Acc) -> Acc; %% drop `cover` and `verbose` so they're not passed as an option to common_test transform_opts([{cover, _}|Rest], Acc) -> transform_opts(Rest, Acc); transform_opts([{cover_spec, CoverSpec}|Rest], Acc) -> transform_opts(Rest, [{cover, CoverSpec}|Acc]); transform_opts([{verbose, _}|Rest], Acc) -> transform_opts(Rest, Acc); transform_opts([{ct_hooks, CtHooks}|Rest], Acc) -> transform_opts(Rest, [{ct_hooks, parse_term(CtHooks)}|Acc]); transform_opts([{force_stop, "skip_rest"}|Rest], Acc) -> transform_opts(Rest, [{force_stop, skip_rest}|Acc]); transform_opts([{force_stop, _}|Rest], Acc) -> transform_opts(Rest, [{force_stop, true}|Acc]); transform_opts([{repeat, Repeat}|Rest], Acc) -> transform_opts(Rest, [{repeat, ec_cnv:to_integer(Repeat)}|Acc]); transform_opts([{create_priv_dir, CreatePrivDir}|Rest], Acc) -> transform_opts(Rest, [{create_priv_dir, to_atoms(CreatePrivDir)}|Acc]); transform_opts([{multiply_timetraps, MultiplyTimetraps}|Rest], Acc) -> transform_opts(Rest, [{multiply_timetraps, ec_cnv:to_integer(MultiplyTimetraps)}|Acc]); transform_opts([{event_handler, EventHandler}|Rest], Acc) -> transform_opts(Rest, [{event_handler, parse_term(EventHandler)}|Acc]); transform_opts([{silent_connections, "all"}|Rest], Acc) -> transform_opts(Rest, [{silent_connections, all}|Acc]); transform_opts([{silent_connections, SilentConnections}|Rest], Acc) -> transform_opts(Rest, [{silent_connections, to_atoms(split_string(SilentConnections))}|Acc]); transform_opts([{verbosity, Verbosity}|Rest], Acc) -> transform_opts(Rest, [{verbosity, parse_term(Verbosity)}|Acc]); transform_opts([{logopts, LogOpts}|Rest], Acc) -> transform_opts(Rest, [{logopts, to_atoms(split_string(LogOpts))}|Acc]); transform_opts([{userconfig, UserConfig}|Rest], Acc) -> transform_opts(Rest, [{userconfig, parse_term(UserConfig)}|Acc]); transform_opts([{testcase, Testcase}|Rest], Acc) -> transform_opts(Rest, [{testcase, to_atoms(split_string(Testcase))}|Acc]); transform_opts([{group, Group}|Rest], Acc) -> % @TODO handle "" % Input is a list or an atom. It can also be a nested list. transform_opts(Rest, [{group, parse_term(Group)}|Acc]); transform_opts([{suite, Suite}|Rest], Acc) -> transform_opts(Rest, [{suite, split_string(Suite)}|Acc]); transform_opts([{Key, Val}|Rest], Acc) when is_list(Val) -> % Default to splitting a string on comma, that works fine for both flat % lists of which there are many and single-items. Val1 = case split_string(Val) of [Val2] -> Val2; Val2 -> Val2 end, transform_opts(Rest, [{Key, Val1}|Acc]); transform_opts([{Key, Val}|Rest], Acc) -> transform_opts(Rest, [{Key, Val}|Acc]). to_atoms(List) -> lists:map(fun(X) -> list_to_atom(X) end, List). split_string(String) -> string:tokens(String, ","). parse_term(String) -> String1 = "[" ++ String ++ "].", {ok, Tokens, _} = erl_scan:string(String1), case erl_parse:parse_term(Tokens) of {ok, [Terms]} -> Terms; Term -> Term end.