summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorFred Hebert <mononcqc@ferd.ca>2018-12-18 11:27:17 -0500
committerGitHub <noreply@github.com>2018-12-18 11:27:17 -0500
commit968458b43592ee112ea85addc674af3beb476da6 (patch)
tree50686e7fb1647980a82a71a2d067bb1ddb4e967d
parenta9a7a05813e8b30e74cde173a80dd647a81fb8a3 (diff)
parent76d8803a5dd0940054b5a22b16cbe43b6c677c9e (diff)
Merge pull request #1979 from ferd/ct-failfast
Add a --fail_fast switch to common test
-rw-r--r--src/cth_fail_fast.erl118
-rw-r--r--src/rebar_prv_common_test.erl26
2 files changed, 140 insertions, 4 deletions
diff --git a/src/cth_fail_fast.erl b/src/cth_fail_fast.erl
new file mode 100644
index 0000000..13b3557
--- /dev/null
+++ b/src/cth_fail_fast.erl
@@ -0,0 +1,118 @@
+-module(cth_fail_fast).
+
+%% Callbacks
+-export([id/1]).
+-export([init/2]).
+
+-export([pre_init_per_suite/3]).
+-export([post_init_per_suite/4]).
+-export([pre_end_per_suite/3]).
+-export([post_end_per_suite/4]).
+
+-export([pre_init_per_group/3]).
+-export([post_init_per_group/4]).
+-export([pre_end_per_group/3]).
+-export([post_end_per_group/4]).
+
+-export([pre_init_per_testcase/3]).
+-export([post_end_per_testcase/4]).
+
+-export([on_tc_fail/3]).
+-export([on_tc_skip/3, on_tc_skip/4]).
+
+-export([terminate/1]).
+
+%% We work by setting an 'abort' variable on each test case that fails
+%% and then triggering the failure before starting the next test. This
+%% ensures that all other hooks have run for the same event, and
+%% simplifies error reporting.
+-record(state, {abort=false}).
+
+%% @doc Return a unique id for this CTH.
+id(_Opts) ->
+ {?MODULE, make_ref()}.
+
+%% @doc Always called before any other callback function. Use this to initiate
+%% any common state.
+init(_Id, _Opts) ->
+ {ok, #state{}}.
+
+%% @doc Called before init_per_suite is called.
+pre_init_per_suite(_Suite,_Config,#state{abort=true}) ->
+ abort();
+pre_init_per_suite(_Suite,Config,State) ->
+ {Config, State}.
+
+%% @doc Called after init_per_suite.
+post_init_per_suite(_Suite,_Config,Return,State) ->
+ {Return, State}.
+
+%% @doc Called before end_per_suite.
+pre_end_per_suite(_Suite,_Config,#state{abort=true}) ->
+ abort();
+pre_end_per_suite(_Suite,Config,State) ->
+ {Config, State}.
+
+%% @doc Called after end_per_suite.
+post_end_per_suite(_Suite,_Config,Return,State) ->
+ {Return, State}.
+
+%% @doc Called before each init_per_group.
+pre_init_per_group(_Group,_Config,#state{abort=true}) ->
+ abort();
+pre_init_per_group(_Group,Config,State) ->
+ {Config, State}.
+
+%% @doc Called after each init_per_group.
+post_init_per_group(_Group,_Config,Return, State) ->
+ {Return, State}.
+
+%% @doc Called after each end_per_group.
+pre_end_per_group(_Group,_Config,#state{abort=true}) ->
+ abort();
+pre_end_per_group(_Group,Config,State) ->
+ {Config, State}.
+
+%% @doc Called after each end_per_group.
+post_end_per_group(_Group,_Config,Return, State) ->
+ {Return, State}.
+
+%% @doc Called before each test case.
+pre_init_per_testcase(_TC,_Config,#state{abort=true}) ->
+ abort();
+pre_init_per_testcase(_TC,Config,State) ->
+ {Config, State}.
+
+%% @doc Called after each test case.
+post_end_per_testcase(_TC,_Config,ok,State) ->
+ {ok, State};
+post_end_per_testcase(_TC,_Config,Error,State) ->
+ {Error, State#state{abort=true}}.
+
+%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group,
+%% post_end_per_group and post_end_per_testcase if the suite, group or test case failed.
+on_tc_fail(_TC, _Reason, State) ->
+ State.
+
+%% @doc Called when a test case is skipped by either user action
+%% or due to an init function failing. (>= 19.3)
+on_tc_skip(_Suite, _TC, {tc_auto_skip, _}, State) ->
+ State#state{abort=true};
+on_tc_skip(_Suite, _TC, _Reason, State) ->
+ State.
+
+%% @doc Called when a test case is skipped by either user action
+%% or due to an init function failing. (Pre-19.3)
+on_tc_skip(_TC, {tc_auto_skip, _}, State) ->
+ State#state{abort=true};
+on_tc_skip(_TC, _Reason, State) ->
+ State.
+
+%% @doc Called when the scope of the CTH is done
+terminate(#state{}) ->
+ ok.
+
+%%% Helpers
+abort() ->
+ io:format(user, "Detected test failure. Aborting~n", []),
+ halt(1).
diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl
index 3d3bd8a..c31c060 100644
--- a/src/rebar_prv_common_test.erl
+++ b/src/rebar_prv_common_test.erl
@@ -171,6 +171,9 @@ transform_opts([{cover, _}|Rest], Acc) ->
%% drop verbose from opts, ct doesn't care about it
transform_opts([{verbose, _}|Rest], Acc) ->
transform_opts(Rest, Acc);
+%% drop fail_fast from opts, ct doesn't care about it
+transform_opts([{fail_fast, _}|Rest], Acc) ->
+ transform_opts(Rest, Acc);
%% getopt should handle anything else
transform_opts([Opt|Rest], Acc) ->
transform_opts(Rest, [Opt|Acc]).
@@ -224,15 +227,21 @@ ensure_opts([V|Rest], Acc) ->
ensure_opts(Rest, [V|Acc]).
add_hooks(Opts, State) ->
+ FailFast = case fails_fast(State) of
+ true -> [cth_fail_fast];
+ false -> []
+ end,
case {readable(State), lists:keyfind(ct_hooks, 1, Opts)} of
{false, _} ->
Opts;
{Other, false} ->
- [{ct_hooks, [cth_readable_failonly, readable_shell_type(Other), cth_retry]} | Opts];
+ [{ct_hooks, [cth_readable_failonly, readable_shell_type(Other),
+ cth_retry] ++ FailFast} | Opts];
{Other, {ct_hooks, Hooks}} ->
%% Make sure hooks are there once only.
- ReadableHooks = [cth_readable_failonly, readable_shell_type(Other), cth_retry],
- AllReadableHooks = [cth_readable_failonly, cth_retry,
+ ReadableHooks = [cth_readable_failonly, readable_shell_type(Other),
+ cth_retry] ++ FailFast,
+ AllReadableHooks = [cth_readable_failonly, cth_retry, cth_fail_fast,
cth_readable_shell, cth_readable_compact_shell],
NewHooks = (Hooks -- AllReadableHooks) ++ ReadableHooks,
lists:keyreplace(ct_hooks, 1, Opts, {ct_hooks, NewHooks})
@@ -445,6 +454,10 @@ readable(State) ->
undefined -> rebar_state:get(State, ct_readable, compact)
end.
+fails_fast(State) ->
+ {RawOpts, _} = rebar_state:command_parsed_args(State),
+ proplists:get_value(fail_fast, RawOpts) == true.
+
test_dirs(State, Apps, Opts) ->
case proplists:get_value(spec, Opts) of
undefined ->
@@ -773,7 +786,8 @@ ct_opts(_State) ->
{setcookie, undefined, "setcookie", atom, help(setcookie)},
{sys_config, undefined, "sys_config", string, help(sys_config)}, %% comma-separated list
{compile_only, undefined, "compile_only", boolean, help(compile_only)},
- {retry, undefined, "retry", boolean, help(retry)}
+ {retry, undefined, "retry", boolean, help(retry)},
+ {fail_fast, undefined, "fail_fast", {boolean, false}, help(fail_fast)}
].
help(compile_only) ->
@@ -846,5 +860,9 @@ help(setcookie) ->
"Sets the cookie if the node is distributed";
help(retry) ->
"Experimental feature. If any specification for previously failing test is found, runs them.";
+help(fail_fast) ->
+ "Experimental feature. If any test fails, the run is aborted. Since common test does not "
+ "support this natively, we abort the rebar3 run on a failure. This May break CT's disk logging and "
+ "other rebar3 features.";
help(_) ->
"".