diff options
| -rw-r--r-- | src/cth_fail_fast.erl | 118 | ||||
| -rw-r--r-- | src/rebar_prv_common_test.erl | 26 |
2 files changed, 140 insertions, 4 deletions
diff --git a/src/cth_fail_fast.erl b/src/cth_fail_fast.erl new file mode 100644 index 0000000..13b3557 --- /dev/null +++ b/src/cth_fail_fast.erl @@ -0,0 +1,118 @@ +-module(cth_fail_fast). + +%% Callbacks +-export([id/1]). +-export([init/2]). + +-export([pre_init_per_suite/3]). +-export([post_init_per_suite/4]). +-export([pre_end_per_suite/3]). +-export([post_end_per_suite/4]). + +-export([pre_init_per_group/3]). +-export([post_init_per_group/4]). +-export([pre_end_per_group/3]). +-export([post_end_per_group/4]). + +-export([pre_init_per_testcase/3]). +-export([post_end_per_testcase/4]). + +-export([on_tc_fail/3]). +-export([on_tc_skip/3, on_tc_skip/4]). + +-export([terminate/1]). + +%% We work by setting an 'abort' variable on each test case that fails +%% and then triggering the failure before starting the next test. This +%% ensures that all other hooks have run for the same event, and +%% simplifies error reporting. +-record(state, {abort=false}). + +%% @doc Return a unique id for this CTH. +id(_Opts) -> + {?MODULE, make_ref()}. + +%% @doc Always called before any other callback function. Use this to initiate +%% any common state. +init(_Id, _Opts) -> + {ok, #state{}}. + +%% @doc Called before init_per_suite is called. +pre_init_per_suite(_Suite,_Config,#state{abort=true}) -> + abort(); +pre_init_per_suite(_Suite,Config,State) -> + {Config, State}. + +%% @doc Called after init_per_suite. +post_init_per_suite(_Suite,_Config,Return,State) -> + {Return, State}. + +%% @doc Called before end_per_suite. +pre_end_per_suite(_Suite,_Config,#state{abort=true}) -> + abort(); +pre_end_per_suite(_Suite,Config,State) -> + {Config, State}. + +%% @doc Called after end_per_suite. +post_end_per_suite(_Suite,_Config,Return,State) -> + {Return, State}. + +%% @doc Called before each init_per_group. +pre_init_per_group(_Group,_Config,#state{abort=true}) -> + abort(); +pre_init_per_group(_Group,Config,State) -> + {Config, State}. + +%% @doc Called after each init_per_group. +post_init_per_group(_Group,_Config,Return, State) -> + {Return, State}. + +%% @doc Called after each end_per_group. +pre_end_per_group(_Group,_Config,#state{abort=true}) -> + abort(); +pre_end_per_group(_Group,Config,State) -> + {Config, State}. + +%% @doc Called after each end_per_group. +post_end_per_group(_Group,_Config,Return, State) -> + {Return, State}. + +%% @doc Called before each test case. +pre_init_per_testcase(_TC,_Config,#state{abort=true}) -> + abort(); +pre_init_per_testcase(_TC,Config,State) -> + {Config, State}. + +%% @doc Called after each test case. +post_end_per_testcase(_TC,_Config,ok,State) -> + {ok, State}; +post_end_per_testcase(_TC,_Config,Error,State) -> + {Error, State#state{abort=true}}. + +%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group, +%% post_end_per_group and post_end_per_testcase if the suite, group or test case failed. +on_tc_fail(_TC, _Reason, State) -> + State. + +%% @doc Called when a test case is skipped by either user action +%% or due to an init function failing. (>= 19.3) +on_tc_skip(_Suite, _TC, {tc_auto_skip, _}, State) -> + State#state{abort=true}; +on_tc_skip(_Suite, _TC, _Reason, State) -> + State. + +%% @doc Called when a test case is skipped by either user action +%% or due to an init function failing. (Pre-19.3) +on_tc_skip(_TC, {tc_auto_skip, _}, State) -> + State#state{abort=true}; +on_tc_skip(_TC, _Reason, State) -> + State. + +%% @doc Called when the scope of the CTH is done +terminate(#state{}) -> + ok. + +%%% Helpers +abort() -> + io:format(user, "Detected test failure. Aborting~n", []), + halt(1). diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index 3d3bd8a..c31c060 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -171,6 +171,9 @@ transform_opts([{cover, _}|Rest], Acc) -> %% drop verbose from opts, ct doesn't care about it transform_opts([{verbose, _}|Rest], Acc) -> transform_opts(Rest, Acc); +%% drop fail_fast from opts, ct doesn't care about it +transform_opts([{fail_fast, _}|Rest], Acc) -> + transform_opts(Rest, Acc); %% getopt should handle anything else transform_opts([Opt|Rest], Acc) -> transform_opts(Rest, [Opt|Acc]). @@ -224,15 +227,21 @@ ensure_opts([V|Rest], Acc) -> ensure_opts(Rest, [V|Acc]). add_hooks(Opts, State) -> + FailFast = case fails_fast(State) of + true -> [cth_fail_fast]; + false -> [] + end, case {readable(State), lists:keyfind(ct_hooks, 1, Opts)} of {false, _} -> Opts; {Other, false} -> - [{ct_hooks, [cth_readable_failonly, readable_shell_type(Other), cth_retry]} | Opts]; + [{ct_hooks, [cth_readable_failonly, readable_shell_type(Other), + cth_retry] ++ FailFast} | Opts]; {Other, {ct_hooks, Hooks}} -> %% Make sure hooks are there once only. - ReadableHooks = [cth_readable_failonly, readable_shell_type(Other), cth_retry], - AllReadableHooks = [cth_readable_failonly, cth_retry, + ReadableHooks = [cth_readable_failonly, readable_shell_type(Other), + cth_retry] ++ FailFast, + AllReadableHooks = [cth_readable_failonly, cth_retry, cth_fail_fast, cth_readable_shell, cth_readable_compact_shell], NewHooks = (Hooks -- AllReadableHooks) ++ ReadableHooks, lists:keyreplace(ct_hooks, 1, Opts, {ct_hooks, NewHooks}) @@ -445,6 +454,10 @@ readable(State) -> undefined -> rebar_state:get(State, ct_readable, compact) end. +fails_fast(State) -> + {RawOpts, _} = rebar_state:command_parsed_args(State), + proplists:get_value(fail_fast, RawOpts) == true. + test_dirs(State, Apps, Opts) -> case proplists:get_value(spec, Opts) of undefined -> @@ -773,7 +786,8 @@ ct_opts(_State) -> {setcookie, undefined, "setcookie", atom, help(setcookie)}, {sys_config, undefined, "sys_config", string, help(sys_config)}, %% comma-separated list {compile_only, undefined, "compile_only", boolean, help(compile_only)}, - {retry, undefined, "retry", boolean, help(retry)} + {retry, undefined, "retry", boolean, help(retry)}, + {fail_fast, undefined, "fail_fast", {boolean, false}, help(fail_fast)} ]. help(compile_only) -> @@ -846,5 +860,9 @@ help(setcookie) -> "Sets the cookie if the node is distributed"; help(retry) -> "Experimental feature. If any specification for previously failing test is found, runs them."; +help(fail_fast) -> + "Experimental feature. If any test fails, the run is aborted. Since common test does not " + "support this natively, we abort the rebar3 run on a failure. This May break CT's disk logging and " + "other rebar3 features."; help(_) -> "". |