diff options
| -rw-r--r-- | priv/shell-completion/bash/rebar3 | 23 | ||||
| -rw-r--r-- | priv/shell-completion/fish/rebar3.fish | 22 | ||||
| -rw-r--r-- | priv/shell-completion/zsh/_rebar3 | 24 | ||||
| -rw-r--r-- | src/rebar_prv_common_test.erl | 734 | ||||
| -rw-r--r-- | test/rebar_ct_SUITE.erl | 698 |
5 files changed, 933 insertions, 568 deletions
diff --git a/priv/shell-completion/bash/rebar3 b/priv/shell-completion/bash/rebar3 index 87ee9eb..771297c 100644 --- a/priv/shell-completion/bash/rebar3 +++ b/priv/shell-completion/bash/rebar3 @@ -54,11 +54,7 @@ _rebar3() --suite \ --group \ --case \ - --spec \ - --join_specs \ - --label \ --config \ - --userconfig \ --allow_user_terms \ --logdir \ --logopts \ @@ -66,21 +62,20 @@ _rebar3() --silent_connections \ --stylesheet \ --cover \ - --cover_spec \ - --cover_stop \ - --event_handler \ - --include \ - --abort_if_missing_suites \ - --multiply_timetraps \ - --scale_timetraps \ - --create_priv_dir \ --repeat \ --duration \ --until \ --force_stop \ --basic_html \ - --ct_hooks \ - --verbose" + --stylesheet \ + --decrypt_key \ + --decrypt_file \ + --abort_if_missing_suites \ + --multiply_timetraps \ + --scale_timetraps \ + --create_priv_dir \ + --verbose" \ + --auto_compile elif [[ ${prev} == deps ]] ; then : elif [[ ${prev} == dialyzer ]] ; then diff --git a/priv/shell-completion/fish/rebar3.fish b/priv/shell-completion/fish/rebar3.fish index 0d5d302..7b63e20 100644 --- a/priv/shell-completion/fish/rebar3.fish +++ b/priv/shell-completion/fish/rebar3.fish @@ -90,10 +90,28 @@ complete -f -c 'rebar3' -n '__fish_rebar3_needs_command' -a ct -d "Run Common Te complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l dir -d "Compile and run all test suites in the specified directories." complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l suites -d "Compile and run all test suites specified. Must be specified by full path, either absolute or relative to the current directory." complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l group -d "Test groups to run." +complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l label -d "Test label." complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l config -d "Config files to use when running tests." +complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l allow_user_terms -d "Allow user defined terms in config files." complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l logdir -d "The directory in which test logs will be written. Default: _build/test/logs" -complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -s v -l verbose -d "Enable verbose output. Default: false" -complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -s c -l cover -d "Generate cover data" +complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l logopts -d "Options for common test logging." +complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l verbosity -d "Verbosity." +complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -s c -l cover -d "Generate cover data." +complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l include -d "Include folders." +complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l repeat -d "How often to repeat tests." +complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l duration -d "Max runtime (format: HHMMSS)." +complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l until -d "Run until (format: HHMMSS)." +complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l force_stop -d "Force stop on test timeout." +complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l basic_html -d "Show basic HTML." +complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l stylesheet -d "CSS stylesheet to apply to html output." +complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l decrypt_key -d "Path to key for decrypting config." +complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l decrypt_file -d "Path to file containing key for decrypting config." +complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l abort_if_missing_suites -d "Abort if suites are missing." +complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l multiply_timetraps -d "Multiply timetraps." +complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l scale_timetraps -d "Scale timetraps." +complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l create_priv_dir -d "Create priv dir (auto_per_run | auto_per_tc | manual_per_tc)." +complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -s v -l verbose -d "Enable verbose output. Default: false." +complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l auto_compile -d "Let common test compile test suites instead of rebar3." complete -f -c 'rebar3' -n '__fish_rebar3_needs_command' -a deps -d "List dependencies" diff --git a/priv/shell-completion/zsh/_rebar3 b/priv/shell-completion/zsh/_rebar3 index d4e1c35..f0fb351 100644 --- a/priv/shell-completion/zsh/_rebar3 +++ b/priv/shell-completion/zsh/_rebar3 @@ -43,33 +43,27 @@ _rebar3 () { '(--suite)--suite[List of test suites to run]:suites' \ '(--group)--group[List of test groups to run]:groups' \ '(--case)--case[List of test cases to run]:cases' \ - '(--spec)--spec[List of test specs to run]:specs' \ - '(--join_specs)--join_specs' \ '(--label)--label[Test label]:label' \ '(--config)--config[List of config files]:config files:_files' \ - '(--userconfig)--userconfig' \ '(--allow_user_terms)--allow_user_terms' \ '(--logdir)--logdir[Log folder]:log folder:_files -/' \ '(--logopts)--logopts' \ '(--verbosity)--verbosity[Verbosity]:verbosity' \ - '(--silent_connections)--silent_connections' \ - '(--stylesheet)--stylesheet[Stylesheet to use for test results]:stylesheet:_files' \ '(-c --cover)'{-c,--cover}'[Generate cover data]' \ - '(--cover_spec)--cover_spec[Cover file to use]:cover file:_files' \ - '(--cover_stop)--cover_stop' \ - '(--event_handler)--event_handler[Event handlers to attach to the runner]:event handlers' \ - '(--include)--include[Include folder]:include directory:_files -/' \ - '(--abort_if_missing_suites)--abort_if_missing_suites[Abort if suites are missing]:abort missing suites:(true false)' \ - '(--multiply_timetraps)--multiply_timetraps' \ - '(--scale_timetraps)--scale_timetraps' \ - '(--create_priv_dir)--create_priv_dir' \ '(--repeat)--repeat[How often to repeat tests]:repeat test count' \ '(--duration)--duration[Max runtime (format: HHMMSS)]:max run time' \ '(--until)--until[Run until (format: HHMMSS)]:run until time' \ - '(--force_stop)--force_stop[Force stop after time]' \ + '(--force_stop)--force_stop[Force stop on test timeout]:skip_rest' \ '(--basic_html)--basic_html[Show basic HTML]' \ - '(--ct_hooks)--ct_hooks:ct hooks' \ + '(--stylesheet)--stylesheet[Stylesheet to use for test results]:stylesheet:_files' \ + '(--decrypt_key)--decrypt_key[Path to key for decrypting config]:decrypt key:_files' \ + '(--decrypt_file)--decrypt_file[Path to file containing key for decrypting config]:decrypt file:_files' \ + '(--abort_if_missing_suites)--abort_if_missing_suites[Abort if suites are missing]:abort missing suites:(true false)' \ + '(--multiply_timetraps)--multiply_timetraps' \ + '(--scale_timetraps)--scale_timetraps' \ + '(--create_priv_dir)--create_priv_dir' \ '(-v --verbose)'{-v,--verbose}'[Print coverage analysis]' \ + '(--auto_compile)--auto_compile' \ && ret=0 ;; (deps) diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index 32d4433..7a088e5 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -2,19 +2,21 @@ %% ex: ts=4 sw=4 et -module(rebar_prv_common_test). + -behaviour(provider). -export([init/1, do/1, format_error/1]). %% exported for test purposes, consider private --export([setup_ct/1]). +-export([compile/2, prepare_tests/1, translate_paths/2]). -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). -define(PROVIDER, ct). --define(DEPS, [compile]). +%% we need to modify app_info state before compile +-define(DEPS, [lock]). %% =================================================================== %% Public API @@ -31,12 +33,19 @@ init(State) -> {desc, "Run Common Tests."}, {opts, ct_opts(State)}, {profiles, [test]}]), - State1 = rebar_state:add_provider(State, Provider), - State2 = rebar_state:add_to_profile(State1, test, test_state(State1)), - {ok, State2}. + {ok, rebar_state:add_provider(State, Provider)}. -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> + Tests = prepare_tests(State), + case compile(State, Tests) of + %% successfully compiled apps + {ok, S} -> do(S, Tests); + %% this should look like a compiler error, not a ct error + Error -> Error + end. + +do(State, Tests) -> ?INFO("Running Common Test suites...", []), rebar_utils:update_code(rebar_state:code_paths(State, all_deps), [soft_purge]), @@ -45,63 +54,332 @@ do(State) -> Cwd = rebar_dir:get_cwd(), rebar_hooks:run_all_hooks(Cwd, pre, ?PROVIDER, Providers, State), - try run_test(State) of - {ok, State1} = Result -> - %% Run ct provider posthooks - rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State1), - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), - Result; - ?PRV_ERROR(_) = Error -> + case Tests of + {ok, T} -> + case run_tests(State, T) of + ok -> + %% Run ct provider posthooks + rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State), + rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), + {ok, State}; + Error -> + rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), + Error + end; + Error -> rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), Error - catch - throw:{error, Reason} -> - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), - ?PRV_ERROR(Reason) + end. + +run_tests(State, Opts) -> + T = translate_paths(State, Opts), + Opts1 = setup_logdir(State, T), + Opts2 = turn_off_auto_compile(Opts1), + ?DEBUG("ct_opts ~p", [Opts2]), + {RawOpts, _} = rebar_state:command_parsed_args(State), + ok = maybe_write_coverdata(State), + case proplists:get_value(verbose, RawOpts, false) of + true -> run_test_verbose(Opts2); + false -> run_test_quiet(Opts2) end. -spec format_error(any()) -> iolist(). -format_error({multiple_dirs_and_suites, Opts}) -> - io_lib:format("Multiple dirs declared alongside suite in opts: ~p", [Opts]); -format_error({bad_dir_or_suite, Opts}) -> - io_lib:format("Bad value for dir or suite in opts: ~p", [Opts]); +format_error({error, Reason}) -> + io_lib:format("Error running tests:~n ~p", [Reason]); +format_error({error_running_tests, Reason}) -> + format_error({error, Reason}); format_error({failures_running_tests, {Failed, AutoSkipped}}) -> io_lib:format("Failures occured running tests: ~b", [Failed+AutoSkipped]); -format_error({error_running_tests, Reason}) -> - io_lib:format("Error running tests: ~p", [Reason]); -format_error(suite_at_project_root) -> - io_lib:format("Test suites can't be located in project root", []); -format_error({error, Reason}) -> - io_lib:format("Unknown error: ~p", [Reason]). +format_error({multiple_errors, Errors}) -> + io_lib:format(lists:concat(["Error running tests:"] ++ + lists:map(fun(Error) -> "~n " ++ Error end, Errors)), []). %% =================================================================== %% Internal functions %% =================================================================== -run_test(State) -> - case setup_ct(State) of - {error, {no_tests_specified, Opts}} -> - ?WARN("No tests specified in opts: ~p", [Opts]), - {ok, State}; - Opts -> - Opts1 = setup_logdir(State, Opts), - ?DEBUG("common test opts: ~p", [Opts1]), - run_test(State, Opts1) - end. +prepare_tests(State) -> + %% command line test options + CmdOpts = cmdopts(State), + %% rebar.config test options + CfgOpts = cfgopts(State), + ProjectApps = rebar_state:project_apps(State), + %% prioritize tests to run first trying any command line specified + %% tests falling back to tests specified in the config file finally + %% running a default set if no other tests are present + select_tests(State, ProjectApps, CmdOpts, CfgOpts). -run_test(State, Opts) -> +cmdopts(State) -> {RawOpts, _} = rebar_state:command_parsed_args(State), - ok = rebar_prv_cover:maybe_cover_compile(State, apps), - Result = case proplists:get_value(verbose, RawOpts, false) of - true -> run_test_verbose(Opts); - false -> run_test_quiet(Opts) + %% filter out opts common_test doesn't know about and convert + %% to ct acceptable forms + transform_opts(RawOpts, []). + +transform_opts([], Acc) -> lists:reverse(Acc); +transform_opts([{dir, Dirs}|Rest], Acc) -> + transform_opts(Rest, [{dir, split_string(Dirs)}|Acc]); +transform_opts([{suite, Suites}|Rest], Acc) -> + transform_opts(Rest, [{suite, split_string(Suites)}|Acc]); +transform_opts([{group, Groups}|Rest], Acc) -> + transform_opts(Rest, [{group, split_string(Groups)}|Acc]); +transform_opts([{testcase, Cases}|Rest], Acc) -> + transform_opts(Rest, [{testcase, split_string(Cases)}|Acc]); +transform_opts([{config, Configs}|Rest], Acc) -> + transform_opts(Rest, [{config, split_string(Configs)}|Acc]); +transform_opts([{logopts, LogOpts}|Rest], Acc) -> + transform_opts(Rest, [{logopts, lists:map(fun(P) -> list_to_atom(P) end, split_string(LogOpts))}|Acc]); +transform_opts([{force_stop, "true"}|Rest], Acc) -> + transform_opts(Rest, [{force_stop, true}|Acc]); +transform_opts([{force_stop, "false"}|Rest], Acc) -> + transform_opts(Rest, [{force_stop, false}|Acc]); +transform_opts([{force_stop, "skip_rest"}|Rest], Acc) -> + transform_opts(Rest, [{force_stop, skip_rest}|Acc]); +transform_opts([{create_priv_dir, CreatePrivDir}|Rest], Acc) -> + transform_opts(Rest, [{create_priv_dir, list_to_atom(CreatePrivDir)}|Acc]); +%% drop cover from opts, ct doesn't care about it +transform_opts([{cover, _}|Rest], Acc) -> + transform_opts(Rest, Acc); +%% drop verbose from opts, ct doesn't care about it +transform_opts([{verbose, _}|Rest], Acc) -> + transform_opts(Rest, Acc); +%% getopt should handle anything else +transform_opts([Opt|Rest], Acc) -> + transform_opts(Rest, [Opt|Acc]). + +split_string(String) -> + string:tokens(String, [$,]). + +cfgopts(State) -> + Opts = rebar_state:get(State, ct_opts, []), + rebar_utils:filtermap(fun filter_opts/1, Opts). + +filter_opts({test_spec, _}) -> + ?WARN("Test specs not supported", []), + false; +filter_opts({auto_compile, _}) -> + ?WARN("Auto compile not supported", []), + false; +filter_opts({suite, Suite}) when is_integer(hd(Suite)) -> true; +filter_opts({suite, Suite}) when is_atom(Suite) -> + {true, {suite, atom_to_list(Suite)}}; +filter_opts({suite, Suites}) -> + {true, {suite, lists:map(fun(S) when is_atom(S) -> atom_to_list(S); + (S) when is_list(S) -> S + end, + Suites)}}; +filter_opts(_) -> true. + +select_tests(State, ProjectApps, CmdOpts, CfgOpts) -> + FixedOpts = lists:filter(fun({_, _}) -> true; (V) -> ?WARN("`~p` is not a valid option for `ct_opts`", [V]) end, CfgOpts), + Merged = lists:ukeymerge(1, + lists:ukeysort(1, CmdOpts), + lists:ukeysort(1, FixedOpts)), + %% make sure `dir` and/or `suite` from command line go in as + %% a pair overriding both `dir` and `suite` from config if + %% they exist + Opts = case {proplists:get_value(suite, CmdOpts), proplists:get_value(dir, CmdOpts)} of + {undefined, undefined} -> Merged; + {_Suite, undefined} -> lists:keydelete(dir, 1, Merged); + {undefined, _Dir} -> lists:keydelete(suite, 1, Merged); + {_Suite, _Dir} -> Merged end, - ok = rebar_prv_cover:maybe_write_coverdata(State, ?PROVIDER), - case Result of - ok -> {ok, State}; - Error -> Error + discover_tests(State, ProjectApps, Opts). + +discover_tests(State, ProjectApps, Opts) -> + case {proplists:get_value(suite, Opts), proplists:get_value(dir, Opts)} of + %% no dirs or suites defined, try using `$APP/test` and `$ROOT/test` + %% as suites + {undefined, undefined} -> {ok, [default_tests(State, ProjectApps)|Opts]}; + {_, _} -> {ok, Opts} + end. + +default_tests(State, ProjectApps) -> + BareTest = filename:join([rebar_state:dir(State), "test"]), + F = fun(App) -> rebar_app_info:dir(App) == rebar_state:dir(State) end, + AppTests = application_dirs(ProjectApps, []), + case filelib:is_dir(BareTest) andalso not lists:any(F, ProjectApps) of + %% `test` dir at root of project is already scheduled to be + %% included or `test` does not exist + false -> {dir, AppTests}; + %% need to add `test` dir at root to dirs to be included + true -> {dir, AppTests ++ [BareTest]} + end. + +application_dirs([], []) -> []; +application_dirs([], Acc) -> lists:reverse(Acc); +application_dirs([App|Rest], Acc) -> + TestDir = filename:join([rebar_app_info:dir(App), "test"]), + case filelib:is_dir(TestDir) of + true -> application_dirs(Rest, [TestDir|Acc]); + false -> application_dirs(Rest, Acc) + end. + +compile(State, {ok, Tests}) -> + %% inject `ct_first_files` and `ct_compile_opts` into the applications + %% to be compiled + case inject_ct_state(State, Tests) of + {ok, NewState} -> do_compile(NewState); + Error -> Error + end; +%% maybe compile even in the face of errors? +compile(_State, Error) -> Error. + +do_compile(State) -> + case rebar_prv_compile:do(State) of + %% successfully compiled apps + {ok, S} -> + ok = maybe_cover_compile(S), + {ok, S}; + %% this should look like a compiler error, not an eunit error + Error -> Error + end. + +inject_ct_state(State, Tests) -> + Apps = rebar_state:project_apps(State), + ModdedApps = lists:map(fun(App) -> + NewOpts = inject(rebar_app_info:opts(App), State), + rebar_app_info:opts(App, NewOpts) + end, Apps), + NewOpts = inject(rebar_state:opts(State), State), + NewState = rebar_state:opts(State, NewOpts), + test_dirs(NewState, ModdedApps, Tests). + +inject(Opts, State) -> + %% append `ct_compile_opts` to app defined `erl_opts` + ErlOpts = rebar_opts:get(Opts, erl_opts, []), + CTOpts = rebar_state:get(State, ct_compile_opts, []), + NewErlOpts = CTOpts ++ ErlOpts, + %% append `ct_first_files` to app defined `erl_first_files` + FirstFiles = rebar_opts:get(Opts, erl_first_files, []), + CTFirstFiles = rebar_state:get(State, ct_first_files, []), + NewFirstFiles = CTFirstFiles ++ FirstFiles, + %% insert the new keys into the opts + lists:foldl(fun({K, V}, NewOpts) -> rebar_opts:set(NewOpts, K, V) end, + Opts, + [{erl_opts, NewErlOpts}, {erl_first_files, NewFirstFiles}]). + +test_dirs(State, Apps, Opts) -> + case {proplists:get_value(suite, Opts), proplists:get_value(dir, Opts)} of + {Suites, undefined} -> set_compile_dirs(State, Apps, {suite, Suites}); + {undefined, Dirs} -> set_compile_dirs(State, Apps, {dir, Dirs}); + {Suites, Dir} when is_integer(hd(Dir)) -> + set_compile_dirs(State, Apps, join(Suites, Dir)); + {Suites, [Dir]} when is_integer(hd(Dir)) -> + set_compile_dirs(State, Apps, join(Suites, Dir)); + {_Suites, _Dirs} -> {error, "Only a single directory may be specified when specifying suites"} + end. + +join(Suite, Dir) when is_integer(hd(Suite)) -> + {suite, [filename:join([Dir, Suite])]}; +join(Suites, Dir) -> + {suite, lists:map(fun(S) -> filename:join([Dir, S]) end, Suites)}. + +set_compile_dirs(State, Apps, {dir, Dir}) when is_integer(hd(Dir)) -> + %% single directory + %% insert `Dir` into an app if relative, or the base state if not + %% app relative but relative to the root or not at all if outside + %% project scope + {NewState, NewApps} = maybe_inject_test_dir(State, [], Apps, Dir), + {ok, rebar_state:project_apps(NewState, NewApps)}; +set_compile_dirs(State, Apps, {dir, Dirs}) -> + %% multiple directories + F = fun(Dir, {S, A}) -> maybe_inject_test_dir(S, [], A, Dir) end, + {NewState, NewApps} = lists:foldl(F, {State, Apps}, Dirs), + {ok, rebar_state:project_apps(NewState, NewApps)}; +set_compile_dirs(State, Apps, {suite, Suites}) -> + %% suites with dir component + Dirs = find_suite_dirs(Suites), + F = fun(Dir, {S, A}) -> maybe_inject_test_dir(S, [], A, Dir) end, + {NewState, NewApps} = lists:foldl(F, {State, Apps}, Dirs), + {ok, rebar_state:project_apps(NewState, NewApps)}. + +find_suite_dirs(Suites) -> + AllDirs = lists:map(fun(S) -> filename:dirname(filename:absname(S)) end, Suites), + %% eliminate duplicates + lists:usort(AllDirs). + +maybe_inject_test_dir(State, AppAcc, [App|Rest], Dir) -> + case rebar_file_utils:path_from_ancestor(Dir, rebar_app_info:dir(App)) of + {ok, Path} -> + Opts = inject_test_dir(rebar_app_info:opts(App), Path), + {State, AppAcc ++ [rebar_app_info:opts(App, Opts)] ++ Rest}; + {error, badparent} -> + maybe_inject_test_dir(State, AppAcc ++ [App], Rest, Dir) + end; +maybe_inject_test_dir(State, AppAcc, [], Dir) -> + case rebar_file_utils:path_from_ancestor(Dir, rebar_state:dir(State)) of + {ok, []} -> + ?WARN("Can't have suites in root of project dir, dropping from tests", []), + {State, AppAcc}; + {ok, Path} -> + Opts = inject_test_dir(rebar_state:opts(State), Path), + {rebar_state:opts(State, Opts), AppAcc}; + {error, badparent} -> + {State, AppAcc} end. +inject_test_dir(Opts, Dir) -> + %% append specified test targets to app defined `extra_src_dirs` + ExtraSrcDirs = rebar_opts:get(Opts, extra_src_dirs, []), + rebar_opts:set(Opts, extra_src_dirs, ExtraSrcDirs ++ [Dir]). + +translate_paths(State, Opts) -> + case {proplists:get_value(suite, Opts), proplists:get_value(dir, Opts)} of + {_Suites, undefined} -> translate_suites(State, Opts, []); + {undefined, _Dirs} -> translate_dirs(State, Opts, []); + %% both dirs and suites are defined, only translate dir paths + _ -> translate_dirs(State, Opts, []) + end. + +translate_dirs(_State, [], Acc) -> lists:reverse(Acc); +translate_dirs(State, [{dir, Dir}|Rest], Acc) when is_integer(hd(Dir)) -> + %% single dir + Apps = rebar_state:project_apps(State), + translate_dirs(State, Rest, [{dir, translate(State, Apps, Dir)}|Acc]); +translate_dirs(State, [{dir, Dirs}|Rest], Acc) -> + %% multiple dirs + Apps = rebar_state:project_apps(State), + NewDirs = {dir, lists:map(fun(Dir) -> translate(State, Apps, Dir) end, Dirs)}, + translate_dirs(State, Rest, [NewDirs|Acc]); +translate_dirs(State, [Test|Rest], Acc) -> + translate_dirs(State, Rest, [Test|Acc]). + +translate_suites(_State, [], Acc) -> lists:reverse(Acc); +translate_suites(State, [{suite, Suite}|Rest], Acc) when is_integer(hd(Suite)) -> + %% single suite + Apps = rebar_state:project_apps(State), + translate_suites(State, Rest, [{suite, translate(State, Apps, Suite)}|Acc]); +translate_suites(State, [{suite, Suites}|Rest], Acc) -> + %% multiple suites + Apps = rebar_state:project_apps(State), + NewSuites = {suite, lists:map(fun(Suite) -> translate(State, Apps, Suite) end, Suites)}, + translate_suites(State, Rest, [NewSuites|Acc]); +translate_suites(State, [Test|Rest], Acc) -> + translate_suites(State, Rest, [Test|Acc]). + +translate(State, [App|Rest], Path) -> + case rebar_file_utils:path_from_ancestor(Path, rebar_app_info:dir(App)) of + {ok, P} -> filename:join([rebar_app_info:out_dir(App), P]); + {error, badparent} -> translate(State, Rest, Path) + end; +translate(State, [], Path) -> + case rebar_file_utils:path_from_ancestor(Path, rebar_state:dir(State)) of + {ok, P} -> filename:join([rebar_dir:base_dir(State), "extras", P]); + %% not relative, leave as is + {error, badparent} -> Path + end. + +setup_logdir(State, Opts) -> + Logdir = case proplists:get_value(logdir, Opts) of + undefined -> filename:join([rebar_dir:base_dir(State), "logs"]); + Dir -> Dir + end, + filelib:ensure_dir(filename:join([Logdir, "dummy.beam"])), + [{logdir, Logdir}|lists:keydelete(logdir, 1, Opts)]. + +turn_off_auto_compile(Opts) -> + [{auto_compile, false}|lists:keydelete(auto_compile, 1, Opts)]. + run_test_verbose(Opts) -> handle_results(ct:run_test(Opts)). run_test_quiet(Opts) -> @@ -171,272 +449,46 @@ format_skipped({0, 0}) -> format_skipped({User, Auto}) -> io_lib:format("Skipped ~p (~p, ~p) tests. ", [User+Auto, User, Auto]). -test_state(State) -> - TestOpts = case rebar_state:get(State, ct_compile_opts, []) of - [] -> []; - Opts -> [{erl_opts, Opts}] - end, - [first_files(State)|TestOpts]. - -first_files(State) -> - CTFirst = rebar_state:get(State, ct_first_files, []), - {erl_first_files, CTFirst}. - -setup_ct(State) -> - Opts = resolve_ct_opts(State), - Opts1 = discover_tests(State, Opts), - copy_and_compile_tests(State, Opts1). - -resolve_ct_opts(State) -> - {RawOpts, _} = rebar_state:command_parsed_args(State), - CmdOpts = transform_opts(RawOpts), - CfgOpts = rebar_state:get(State, ct_opts, []), - Merged = lists:ukeymerge(1, - lists:ukeysort(1, CmdOpts), - lists:ukeysort(1, CfgOpts)), - %% make sure `dir` and/or `suite` from command line go in as - %% a pair overriding both `dir` and `suite` from config if - %% they exist - case {proplists:get_value(suite, CmdOpts), proplists:get_value(dir, CmdOpts)} of - {undefined, undefined} -> Merged; - {_Suite, undefined} -> lists:keydelete(dir, 1, Merged); - {undefined, _Dir} -> lists:keydelete(suite, 1, Merged); - {_Suite, _Dir} -> Merged - end. - -discover_tests(State, Opts) -> - case proplists:get_value(spec, Opts) of - undefined -> discover_dirs_and_suites(State, Opts); - TestSpec -> discover_testspec(TestSpec, Opts) - end. - -discover_dirs_and_suites(State, Opts) -> - case {proplists:get_value(dir, Opts), proplists:get_value(suite, Opts)} of - %% no dirs or suites defined, try using `$APP/test` and `$ROOT/test` - %% as suites - {undefined, undefined} -> test_dirs(State, Opts); - %% no dirs defined - {undefined, _} -> Opts; - %% no suites defined - {_, undefined} -> Opts; - %% a single dir defined, this is ok - {Dirs, Suites} when is_integer(hd(Dirs)), is_list(Suites) -> Opts; - %% still a single dir defined, adjust to make acceptable to ct - {[Dir], Suites} when is_integer(hd(Dir)), is_list(Suites) -> - [{dir, Dir}|lists:keydelete(dir, 1, Opts)]; - %% multiple dirs and suites, error now to simplify later steps - {_, _} -> throw({error, {multiple_dirs_and_suites, Opts}}) - end. - -discover_testspec(_TestSpec, Opts) -> - lists:keydelete(auto_compile, 1, Opts). - -copy_and_compile_tests(State, Opts) -> - %% possibly enable cover +maybe_cover_compile(State) -> {RawOpts, _} = rebar_state:command_parsed_args(State), State1 = case proplists:get_value(cover, RawOpts, false) of true -> rebar_state:set(State, cover_enabled, true); false -> State end, - copy_and_compile_test_suites(State1, Opts). - -copy_and_compile_test_suites(State, Opts) -> - case proplists:get_value(suite, Opts) of - %% no suites, try dirs - undefined -> copy_and_compile_test_dirs(State, Opts); - Suites -> - Dir = proplists:get_value(dir, Opts, undefined), - AllSuites = join(Dir, Suites), - Dirs = find_suite_dirs(AllSuites), - lists:foreach(fun(S) -> - NewPath = copy(State, S), - compile_dir(State, NewPath) - end, Dirs), - NewSuites = lists:map(fun(S) -> retarget_path(State, S) end, AllSuites), - [{suite, NewSuites}|lists:keydelete(suite, 1, Opts)] - end. - -copy_and_compile_test_dirs(State, Opts) -> - case proplists:get_value(dir, Opts) of - undefined -> {error, {no_tests_specified, Opts}}; - %% dir is a single directory - Dir when is_list(Dir), is_integer(hd(Dir)) -> - NewPath = copy(State, Dir), - [{dir, compile_dir(State, NewPath)}|lists:keydelete(dir, 1, Opts)]; - %% dir is a list of directories - Dirs when is_list(Dirs) -> - NewDirs = lists:map(fun(Dir) -> - NewPath = copy(State, Dir), - compile_dir(State, NewPath) - end, Dirs), - [{dir, NewDirs}|lists:keydelete(dir, 1, Opts)] - end. - -join(undefined, Suites) -> Suites; -join(Dir, Suites) when is_list(Dir), is_integer(hd(Dir)) -> - lists:map(fun(S) -> filename:join([Dir, S]) end, Suites); -%% multiple dirs or a bad dir argument, try to continue anyways -join(_, Suites) -> Suites. - -find_suite_dirs(Suites) -> - AllDirs = lists:map(fun(S) -> filename:dirname(filename:absname(S)) end, Suites), - %% eliminate duplicates - lists:usort(AllDirs). - -copy(State, Dir) -> - From = reduce_path(Dir), - retarget_path(State, From). - -compile_dir(State, Dir) -> - NewState = replace_src_dirs(State, [filename:absname(Dir)]), - ok = rebar_erlc_compiler:compile(rebar_state:opts(NewState), rebar_dir:base_dir(State), Dir), - ok = maybe_cover_compile(State, Dir), - Dir. - -retarget_path(State, Path) -> - ProjectApps = rebar_state:project_apps(State), - retarget_path(State, Path, ProjectApps). - -%% not relative to any apps in project, check to see it's relative to -%% project root -retarget_path(State, Path, []) -> - case relative_path(reduce_path(Path), rebar_state:dir(State)) of - {ok, NewPath} -> filename:join([rebar_dir:base_dir(State), NewPath]); - %% not relative to project root, don't modify - {error, not_relative} -> Path - end; -%% relative to current app, retarget to the same dir relative to -%% the app's out_dir -retarget_path(State, Path, [App|Rest]) -> - case relative_path(reduce_path(Path), rebar_app_info:dir(App)) of - {ok, NewPath} -> filename:join([rebar_app_info:out_dir(App), NewPath]); - {error, not_relative} -> retarget_path(State, Path, Rest) - end. - -relative_path(Target, To) -> - relative_path1(filename:split(filename:absname(Target)), - filename:split(filename:absname(To))). - -relative_path1([Part|Target], [Part|To]) -> relative_path1(Target, To); -relative_path1([], []) -> {ok, ""}; -relative_path1(Target, []) -> {ok, filename:join(Target)}; -relative_path1(_, _) -> {error, not_relative}. - -reduce_path(Dir) -> reduce_path([], filename:split(filename:absname(Dir))). - -reduce_path([], []) -> filename:nativename("/"); -reduce_path(Acc, []) -> filename:join(lists:reverse(Acc)); -reduce_path(Acc, ["."|Rest]) -> reduce_path(Acc, Rest); -reduce_path([_|Acc], [".."|Rest]) -> reduce_path(Acc, Rest); -reduce_path([], [".."|Rest]) -> reduce_path([], Rest); -reduce_path(Acc, [Component|Rest]) -> reduce_path([Component|Acc], Rest). - -replace_src_dirs(State, Dirs) -> - %% replace any `src_dirs` with the test dirs - ErlOpts = rebar_state:get(State, erl_opts, []), - StrippedErlOpts = filter_src_dirs(ErlOpts), - State1 = rebar_state:set(State, erl_opts, StrippedErlOpts), - State2 = rebar_state:set(State1, src_dirs, []), - rebar_state:set(State2, extra_src_dirs, Dirs). - -filter_src_dirs(ErlOpts) -> - lists:filter(fun({src_dirs, _}) -> false; ({extra_src_dirs, _}) -> false; (_) -> true end, ErlOpts). - -test_dirs(State, Opts) -> - BareTest = filename:join([rebar_state:dir(State), "test"]), - F = fun(App) -> rebar_app_info:dir(App) == rebar_state:dir(State) end, - TestApps = project_apps(State), - case filelib:is_dir(BareTest) andalso not lists:any(F, TestApps) of - %% `test` dir at root of project is already scheduled to be - %% included or `test` does not exist - false -> application_dirs(TestApps, Opts, []); - %% need to add `test` dir at root to dirs to be included - true -> application_dirs(TestApps, Opts, [BareTest]) - end. - -project_apps(State) -> - filter_checkouts(rebar_state:project_apps(State)). - -filter_checkouts(Apps) -> filter_checkouts(Apps, []). - -filter_checkouts([], Acc) -> lists:reverse(Acc); -filter_checkouts([App|Rest], Acc) -> - case rebar_app_info:is_checkout(App) of - true -> filter_checkouts(Rest, Acc); - false -> filter_checkouts(Rest, [App|Acc]) - end. - -application_dirs([], Opts, []) -> Opts; -application_dirs([], Opts, [Acc]) -> [{dir, Acc}|Opts]; -application_dirs([], Opts, Acc) -> [{dir, lists:reverse(Acc)}|Opts]; -application_dirs([App|Rest], Opts, Acc) -> - TestDir = filename:join([rebar_app_info:dir(App), "test"]), - case filelib:is_dir(TestDir) of - true -> application_dirs(Rest, Opts, [TestDir|Acc]); - false -> application_dirs(Rest, Opts, Acc) - end. - -setup_logdir(State, Opts) -> - Logdir = case proplists:get_value(logdir, Opts) of - undefined -> filename:join([rebar_dir:base_dir(State), "logs"]); - Dir -> Dir - end, - ensure_dir([Logdir]), - [{logdir, Logdir}|lists:keydelete(logdir, 1, Opts)]. + rebar_prv_cover:maybe_cover_compile(State1). -ensure_dir([]) -> ok; -ensure_dir([Dir|Rest]) -> - case ec_file:is_dir(Dir) of - true -> - ok; - false -> - ec_file:mkdir_path(Dir) - end, - ensure_dir(Rest). - -maybe_cover_compile(State, Dir) -> - {Opts, _} = rebar_state:command_parsed_args(State), - State1 = case proplists:get_value(cover, Opts, false) of +maybe_write_coverdata(State) -> + {RawOpts, _} = rebar_state:command_parsed_args(State), + State1 = case proplists:get_value(cover, RawOpts, false) of true -> rebar_state:set(State, cover_enabled, true); false -> State end, - rebar_prv_cover:maybe_cover_compile(State1, [Dir]). + rebar_prv_cover:maybe_write_coverdata(State1, ?PROVIDER). ct_opts(_State) -> [{dir, undefined, "dir", string, help(dir)}, %% comma-seperated list {suite, undefined, "suite", string, help(suite)}, %% comma-seperated list {group, undefined, "group", string, help(group)}, %% comma-seperated list {testcase, undefined, "case", string, help(testcase)}, %% comma-seperated list - {spec, undefined, "spec", string, help(spec)}, %% comma-seperated list - {join_specs, undefined, "join_specs", boolean, help(join_specs)}, %% Boolean {label, undefined, "label", string, help(label)}, %% String {config, undefined, "config", string, help(config)}, %% comma-seperated list - {userconfig, undefined, "userconfig", string, help(userconfig)}, %% [{CallbackMod, CfgStrings}] | {CallbackMod, CfgStrings} {allow_user_terms, undefined, "allow_user_terms", boolean, help(allow_user_terms)}, %% Bool {logdir, undefined, "logdir", string, help(logdir)}, %% dir - {logopts, undefined, "logopts", string, help(logopts)}, %% enum, no_nl | no_src - {verbosity, undefined, "verbosity", string, help(verbosity)}, %% Integer OR [{Category, VLevel}] - {silent_connections, undefined, "silent_connections", string, - help(silent_connections)}, % all OR %% comma-seperated list - {stylesheet, undefined, "stylesheet", string, help(stylesheet)}, %% file + {logopts, undefined, "logopts", string, help(logopts)}, %% comma seperated list + {verbosity, undefined, "verbosity", integer, help(verbosity)}, %% Integer {cover, $c, "cover", {boolean, false}, help(cover)}, - {cover_spec, undefined, "cover_spec", string, help(cover_spec)}, %% file - {cover_stop, undefined, "cover_stop", boolean, help(cover_stop)}, %% Boolean - {event_handler, undefined, "event_handler", string, help(event_handler)}, %% EH | [EH] WHERE EH atom() | {atom(), InitArgs} | {[atom()], InitArgs} - {include, undefined, "include", string, help(include)}, % comma-seperated list - {abort_if_missing_suites, undefined, "abort_if_missing_suites", {boolean, true}, - help(abort_if_missing_suites)}, %% boolean - {multiply_timetraps, undefined, "multiply_timetraps", integer, - help(multiply_timetraps)}, %% integer - {scale_timetraps, undefined, "scale_timetraps", boolean, help(scale_timetraps)}, %% Boolean - {create_priv_dir, undefined, "create_priv_dir", string, help(create_priv_dir)}, %% enum: auto_per_run | auto_per_tc | manual_per_tc {repeat, undefined, "repeat", integer, help(repeat)}, %% integer {duration, undefined, "duration", string, help(duration)}, % format: HHMMSS {until, undefined, "until", string, help(until)}, %% format: YYMoMoDD[HHMMSS] - {force_stop, undefined, "force_stop", string, help(force_stop)}, % enum: skip_rest, bool - {basic_html, undefined, "basic_html", boolean, help(basic_html)}, %% Booloean - {ct_hooks, undefined, "ct_hooks", string, help(ct_hooks)}, %% List: [CTHModule | {CTHModule, CTHInitArgs}] where CTHModule is atom CthInitArgs is term - {auto_compile, undefined, "auto_compile", {boolean, false}, help(auto_compile)}, + {force_stop, undefined, "force_stop", string, help(force_stop)}, %% String + {basic_html, undefined, "basic_html", boolean, help(basic_html)}, %% Boolean + {stylesheet, undefined, "stylesheet", string, help(stylesheet)}, %% String + {decrypt_key, undefined, "decrypt_key", string, help(decrypt_key)}, %% String + {decrypt_file, undefined, "decrypt_file", string, help(decrypt_file)}, %% String + {abort_if_missing_suites, undefined, "abort_if_missing_suites", {boolean, true}, help(abort_if_missing_suites)}, %% Boolean + {multiply_timetraps, undefined, "multiply_timetraps", integer, help(multiple_timetraps)}, %% Integer + {scale_timetraps, undefined, "scale_timetraps", boolean, help(scale_timetraps)}, + {create_priv_dir, undefined, "create_priv_dir", string, help(create_priv_dir)}, {verbose, $v, "verbose", boolean, help(verbose)} ]. @@ -448,28 +500,20 @@ help(group) -> "List of test groups to run"; help(testcase) -> "List of test cases to run"; -help(spec) -> - "List of test specs to run"; help(label) -> "Test label"; help(config) -> "List of config files"; +help(allow_user_terms) -> + "Allow user defined config values in config files"; help(logdir) -> "Log folder"; +help(logopts) -> + "Options for common test logging"; help(verbosity) -> "Verbosity"; -help(stylesheet) -> - "Stylesheet to use for test results"; help(cover) -> "Generate cover data"; -help(cover_spec) -> - "Cover file to use"; -help(event_handler) -> - "Event handlers to attach to the runner"; -help(include) -> - "Include folder"; -help(abort_if_missing_suites) -> - "Abort if suites are missing"; help(repeat) -> "How often to repeat tests"; help(duration) -> @@ -477,85 +521,25 @@ help(duration) -> help(until) -> "Run until (format: HHMMSS)"; help(force_stop) -> - "Force stop after time"; + "Force stop on test timeout (true | false | skip_rest)"; help(basic_html) -> "Show basic HTML"; +help(stylesheet) -> + "CSS stylesheet to apply to html output"; +help(decrypt_key) -> + "Path to key for decrypting config"; +help(decrypt_file) -> + "Path to file containing key for decrypting config"; +help(abort_if_missing_suites) -> + "Abort if suites are missing"; +help(multiply_timetraps) -> + "Multiply timetraps"; +help(scale_timetraps) -> + "Scale timetraps"; +help(create_priv_dir) -> + "Create priv dir (auto_per_run | auto_per_tc | manual_per_tc)"; help(verbose) -> "Verbose output"; help(_) -> "". -transform_opts(Opts) -> - transform_opts(Opts, []). - -transform_opts([], Acc) -> Acc; -%% drop `cover` and `verbose` so they're not passed as an option to common_test -transform_opts([{cover, _}|Rest], Acc) -> - transform_opts(Rest, Acc); -transform_opts([{cover_spec, CoverSpec}|Rest], Acc) -> - transform_opts(Rest, [{cover, CoverSpec}|Acc]); -transform_opts([{verbose, _}|Rest], Acc) -> - transform_opts(Rest, Acc); -transform_opts([{ct_hooks, CtHooks}|Rest], Acc) -> - transform_opts(Rest, [{ct_hooks, parse_term(CtHooks)}|Acc]); -transform_opts([{force_stop, "skip_rest"}|Rest], Acc) -> - transform_opts(Rest, [{force_stop, skip_rest}|Acc]); -transform_opts([{force_stop, _}|Rest], Acc) -> - transform_opts(Rest, [{force_stop, true}|Acc]); -transform_opts([{repeat, Repeat}|Rest], Acc) -> - transform_opts(Rest, [{repeat, - ec_cnv:to_integer(Repeat)}|Acc]); -transform_opts([{create_priv_dir, CreatePrivDir}|Rest], Acc) -> - transform_opts(Rest, [{create_priv_dir, - to_atoms(CreatePrivDir)}|Acc]); -transform_opts([{multiply_timetraps, MultiplyTimetraps}|Rest], Acc) -> - transform_opts(Rest, [{multiply_timetraps, - ec_cnv:to_integer(MultiplyTimetraps)}|Acc]); -transform_opts([{event_handler, EventHandler}|Rest], Acc) -> - transform_opts(Rest, [{event_handler, parse_term(EventHandler)}|Acc]); -transform_opts([{silent_connections, "all"}|Rest], Acc) -> - transform_opts(Rest, [{silent_connections, all}|Acc]); -transform_opts([{silent_connections, SilentConnections}|Rest], Acc) -> - transform_opts(Rest, [{silent_connections, - to_atoms(split_string(SilentConnections))}|Acc]); -transform_opts([{verbosity, Verbosity}|Rest], Acc) -> - transform_opts(Rest, [{verbosity, parse_term(Verbosity)}|Acc]); -transform_opts([{logopts, LogOpts}|Rest], Acc) -> - transform_opts(Rest, [{logopts, to_atoms(split_string(LogOpts))}|Acc]); -transform_opts([{userconfig, UserConfig}|Rest], Acc) -> - transform_opts(Rest, [{userconfig, parse_term(UserConfig)}|Acc]); -transform_opts([{testcase, Testcase}|Rest], Acc) -> - transform_opts(Rest, [{testcase, to_atoms(split_string(Testcase))}|Acc]); -transform_opts([{group, Group}|Rest], Acc) -> % @TODO handle "" - % Input is a list or an atom. It can also be a nested list. - transform_opts(Rest, [{group, parse_term(Group)}|Acc]); -transform_opts([{suite, Suite}|Rest], Acc) -> - transform_opts(Rest, [{suite, split_string(Suite)}|Acc]); -transform_opts([{Key, Val}|Rest], Acc) when is_list(Val) -> - % Default to splitting a string on comma, that works fine for both flat - % lists of which there are many and single-items. - Val1 = case split_string(Val) of - [Val2] -> - Val2; - Val2 -> - Val2 - end, - transform_opts(Rest, [{Key, Val1}|Acc]); -transform_opts([{Key, Val}|Rest], Acc) -> - transform_opts(Rest, [{Key, Val}|Acc]). - -to_atoms(List) -> - lists:map(fun(X) -> list_to_atom(X) end, List). - -split_string(String) -> - string:tokens(String, ","). - -parse_term(String) -> - String1 = "[" ++ String ++ "].", - {ok, Tokens, _} = erl_scan:string(String1), - case erl_parse:parse_term(Tokens) of - {ok, [Terms]} -> - Terms; - Term -> - Term - end. diff --git a/test/rebar_ct_SUITE.erl b/test/rebar_ct_SUITE.erl index cdd3774..ac016b1 100644 --- a/test/rebar_ct_SUITE.erl +++ b/test/rebar_ct_SUITE.erl @@ -17,15 +17,40 @@ multi_suite/1, all_suite/1, single_dir_and_single_suite/1, - symlinked_dir_overwritten_fix/1, - data_dir_correct/1]). + data_dir_correct/1, + cmd_label/1, + cmd_config/1, + cmd_allow_user_terms/1, + cmd_logdir/1, + cmd_logopts/1, + cmd_verbosity/1, + cmd_repeat/1, + cmd_duration/1, + cmd_until/1, + cmd_force_stop/1, + cmd_basic_html/1, + cmd_stylesheet/1, + cmd_decrypt_key/1, + cmd_decrypt_file/1, + cmd_abort_if_missing_suites/1, + cmd_multiply_timetraps/1, + cmd_scale_timetraps/1, + cmd_create_priv_dir/1, + cfg_opts/1, + cfg_arbitrary_opts/1, + cfg_test_spec_filtered/1, + cfg_atom_suites/1]). -include_lib("common_test/include/ct.hrl"). all() -> [{group, basic_app}, {group, multi_app}, {group, dirs_and_suites}, - {group, data_dirs}]. + {group, data_dirs}, + {group, ct_opts}, + cfg_opts, cfg_arbitrary_opts, + cfg_test_spec_filtered, + cfg_atom_suites]. groups() -> [{basic_app, [], [basic_app_default_dirs, basic_app_default_beams]}, @@ -39,9 +64,26 @@ groups() -> [{basic_app, [], [basic_app_default_dirs, single_unmanaged_suite, multi_suite, all_suite, - single_dir_and_single_suite, - symlinked_dir_overwritten_fix]}, - {data_dirs, [], [data_dir_correct]}]. + single_dir_and_single_suite]}, + {data_dirs, [], [data_dir_correct]}, + {ct_opts, [], [cmd_label, + cmd_config, + cmd_allow_user_terms, + cmd_logdir, + cmd_logopts, + cmd_verbosity, + cmd_repeat, + cmd_duration, + cmd_until, + cmd_force_stop, + cmd_basic_html, + cmd_stylesheet, + cmd_decrypt_key, + cmd_decrypt_file, + cmd_abort_if_missing_suites, + cmd_multiply_timetraps, + cmd_scale_timetraps, + cmd_create_priv_dir]}]. init_per_group(basic_app, Config) -> C = rebar_test_utils:init_rebar_state(Config, "ct_"), @@ -56,22 +98,14 @@ init_per_group(basic_app, Config) -> ok = filelib:ensure_dir(Suite), ok = file:write_file(Suite, test_suite(Name)), - {ok, State} = rebar_test_utils:run_and_check(C, [], ["as", "test", "compile"], return), + {ok, State} = rebar_test_utils:run_and_check(C, [], ["as", "test", "lock"], return), - LibDirs = rebar_dir:lib_dirs(State), - State1 = rebar_app_discover:do(State, LibDirs), - - Providers = rebar_state:providers(State1), - Namespace = rebar_state:namespace(State1), - CommandProvider = providers:get_provider(ct, Providers, Namespace), - GetOptSpec = providers:opts(CommandProvider), - {ok, GetOptResult} = getopt:parse(GetOptSpec, []), + Tests = rebar_prv_common_test:prepare_tests(State), + {ok, NewState} = rebar_prv_common_test:compile(State, Tests), + {ok, T} = Tests, + Opts = rebar_prv_common_test:translate_paths(NewState, T), - State2 = rebar_state:command_parsed_args(State1, GetOptResult), - - Result = rebar_prv_common_test:setup_ct(State2), - - [{result, Result}, {appnames, [Name]}|C]; + [{result, Opts}, {appnames, [Name]}|C]; init_per_group(multi_app, Config) -> C = rebar_test_utils:init_rebar_state(Config, "ct_"), @@ -99,22 +133,14 @@ init_per_group(multi_app, Config) -> ok = filelib:ensure_dir(Suite3), ok = file:write_file(Suite3, test_suite("extras")), - {ok, State} = rebar_test_utils:run_and_check(C, [], ["as", "test", "compile"], return), + {ok, State} = rebar_test_utils:run_and_check(C, [], ["as", "test", "lock"], return), - LibDirs = rebar_dir:lib_dirs(State), - State1 = rebar_app_discover:do(State, LibDirs), + Tests = rebar_prv_common_test:prepare_tests(State), + {ok, NewState} = rebar_prv_common_test:compile(State, Tests), + {ok, T} = Tests, + Opts = rebar_prv_common_test:translate_paths(NewState, T), - Providers = rebar_state:providers(State1), - Namespace = rebar_state:namespace(State1), - CommandProvider = providers:get_provider(ct, Providers, Namespace), - GetOptSpec = providers:opts(CommandProvider), - {ok, GetOptResult} = getopt:parse(GetOptSpec, []), - - State2 = rebar_state:command_parsed_args(State1, GetOptResult), - - Result = rebar_prv_common_test:setup_ct(State2), - - [{result, Result}, {appnames, [Name1, Name2]}|C]; + [{result, Opts}, {appnames, [Name1, Name2]}|C]; init_per_group(dirs_and_suites, Config) -> C = rebar_test_utils:init_rebar_state(Config, "ct_"), @@ -142,7 +168,21 @@ init_per_group(dirs_and_suites, Config) -> ok = filelib:ensure_dir(Suite3), ok = file:write_file(Suite3, test_suite("extras")), - [{appnames, [Name1, Name2]}|C]; + {ok, State} = rebar_test_utils:run_and_check(C, [], ["as", "test", "lock"], return), + + [{s, State}, {appnames, [Name1, Name2]}|C]; +init_per_group(ct_opts, Config) -> + C = rebar_test_utils:init_rebar_state(Config, "ct_opts"), + + AppDir = ?config(apps, C), + + Name = rebar_test_utils:create_random_name("ct_opts_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + {ok, State} = rebar_test_utils:run_and_check(C, [], ["as", "test", "lock"], return), + + [{result, State}|C]; init_per_group(_, Config) -> Config. end_per_group(_Group, _Config) -> ok. @@ -152,10 +192,10 @@ basic_app_default_dirs(Config) -> [Name] = ?config(appnames, Config), Result = ?config(result, Config), - Expect = filename:absname(filename:join([AppDir, "_build", "test", "lib", Name, "test"])), + Expect = filename:join([AppDir, "_build", "test", "lib", Name, "test"]), Dir = proplists:get_value(dir, Result), - Expect = Dir. + [Expect] = Dir. basic_app_default_beams(Config) -> AppDir = ?config(apps, Config), @@ -178,7 +218,7 @@ multi_app_default_dirs(Config) -> Expect1 = filename:absname(filename:join([AppDir, "_build", "test", "lib", Name1, "test"])), Expect2 = filename:absname(filename:join([AppDir, "_build", "test", "lib", Name2, "test"])), - Expect3 = filename:absname(filename:join([AppDir, "_build", "test", "test"])), + Expect3 = filename:absname(filename:join([AppDir, "_build", "test", "extras", "test"])), Dirs = proplists:get_value(dir, Result), true = (lists:sort([Expect1, Expect2, Expect3]) == lists:sort(Dirs)). @@ -215,8 +255,7 @@ multi_app_default_beams(Config) -> single_app_dir(Config) -> AppDir = ?config(apps, Config), [Name1, _Name2] = ?config(appnames, Config), - - {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return), + State = ?config(s, Config), LibDirs = rebar_dir:lib_dirs(State), State1 = rebar_app_discover:do(State, LibDirs), @@ -233,17 +272,19 @@ single_app_dir(Config) -> State2 = rebar_state:command_parsed_args(State1, GetOptResult), - Result = rebar_prv_common_test:setup_ct(State2), + Tests = rebar_prv_common_test:prepare_tests(State2), + {ok, NewState} = rebar_prv_common_test:compile(State2, Tests), + {ok, T} = Tests, + Opts = rebar_prv_common_test:translate_paths(NewState, T), - Expect = filename:absname(filename:join([AppDir, "_build", "test", "lib", Name1, "test"])), - Dir = proplists:get_value(dir, Result), + Expect = filename:join([AppDir, "_build", "test", "lib", Name1, "test"]), + Dir = proplists:get_value(dir, Opts), - Expect = Dir. + [Expect] = Dir. single_extra_dir(Config) -> AppDir = ?config(apps, Config), - - {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return), + State = ?config(s, Config), LibDirs = rebar_dir:lib_dirs(State), State1 = rebar_app_discover:do(State, LibDirs), @@ -257,22 +298,24 @@ single_extra_dir(Config) -> State2 = rebar_state:command_parsed_args(State1, GetOptResult), - Result = rebar_prv_common_test:setup_ct(State2), + Tests = rebar_prv_common_test:prepare_tests(State2), + {ok, NewState} = rebar_prv_common_test:compile(State2, Tests), + {ok, T} = Tests, + Opts = rebar_prv_common_test:translate_paths(NewState, T), - Expect = filename:absname(filename:join([AppDir, "_build", "test", "test"])), - Dir = proplists:get_value(dir, Result), + Expect = filename:join([AppDir, "_build", "test", "extras", "test"]), + Dir = proplists:get_value(dir, Opts), - Expect = Dir. + [Expect] = Dir. single_unmanaged_dir(Config) -> PrivDir = ?config(priv_dir, Config), + State = ?config(s, Config), Suite = filename:join([PrivDir, "unmanaged_dir", "unmanaged_dir_SUITE.erl"]), ok = filelib:ensure_dir(Suite), ok = file:write_file(Suite, test_suite("unmanaged_dir")), - {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return), - LibDirs = rebar_dir:lib_dirs(State), State1 = rebar_app_discover:do(State, LibDirs), @@ -285,18 +328,20 @@ single_unmanaged_dir(Config) -> State2 = rebar_state:command_parsed_args(State1, GetOptResult), - Result = rebar_prv_common_test:setup_ct(State2), + Tests = rebar_prv_common_test:prepare_tests(State2), + {ok, NewState} = rebar_prv_common_test:compile(State2, Tests), + {ok, T} = Tests, + Opts = rebar_prv_common_test:translate_paths(NewState, T), - Expect = filename:absname(filename:join([PrivDir, "unmanaged_dir"])), - Dir = proplists:get_value(dir, Result), + Expect = filename:join([PrivDir, "unmanaged_dir"]), + Dir = proplists:get_value(dir, Opts), - Expect = Dir. + [Expect] = Dir. single_suite(Config) -> AppDir = ?config(apps, Config), [Name1, _Name2] = ?config(appnames, Config), - - {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return), + State = ?config(s, Config), LibDirs = rebar_dir:lib_dirs(State), State1 = rebar_app_discover:do(State, LibDirs), @@ -314,24 +359,26 @@ single_suite(Config) -> State2 = rebar_state:command_parsed_args(State1, GetOptResult), - Result = rebar_prv_common_test:setup_ct(State2), + Tests = rebar_prv_common_test:prepare_tests(State2), + {ok, NewState} = rebar_prv_common_test:compile(State2, Tests), + {ok, T} = Tests, + Opts = rebar_prv_common_test:translate_paths(NewState, T), - Expect = [filename:absname(filename:join([AppDir, - "_build", - "test", - "lib", - Name1, - "test", - Name1 ++ "_SUITE"]))], - Suite = proplists:get_value(suite, Result), + Expect = filename:join([AppDir, + "_build", + "test", + "lib", + Name1, + "test", + Name1 ++ "_SUITE"]), + Suite = proplists:get_value(suite, Opts), - Expect = Suite. + [Expect] = Suite. single_extra_suite(Config) -> AppDir = ?config(apps, Config), [_Name1, _Name2] = ?config(appnames, Config), - - {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return), + State = ?config(s, Config), LibDirs = rebar_dir:lib_dirs(State), State1 = rebar_app_discover:do(State, LibDirs), @@ -347,27 +394,30 @@ single_extra_suite(Config) -> State2 = rebar_state:command_parsed_args(State1, GetOptResult), - Result = rebar_prv_common_test:setup_ct(State2), + Tests = rebar_prv_common_test:prepare_tests(State2), + {ok, NewState} = rebar_prv_common_test:compile(State2, Tests), + {ok, T} = Tests, + Opts = rebar_prv_common_test:translate_paths(NewState, T), - Expect = [filename:absname(filename:join([AppDir, - "_build", - "test", - "test", - "extra_SUITE"]))], - Suite = proplists:get_value(suite, Result), + Expect = filename:join([AppDir, + "_build", + "test", + "extras", + "test", + "extra_SUITE"]), + Suite = proplists:get_value(suite, Opts), - Expect = Suite. + [Expect] = Suite. single_unmanaged_suite(Config) -> PrivDir = ?config(priv_dir, Config), [_Name1, _Name2] = ?config(appnames, Config), + State = ?config(s, Config), Suite = filename:join([PrivDir, "unmanaged", "unmanaged_SUITE.erl"]), ok = filelib:ensure_dir(Suite), ok = file:write_file(Suite, test_suite("unmanaged")), - {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return), - LibDirs = rebar_dir:lib_dirs(State), State1 = rebar_app_discover:do(State, LibDirs), @@ -382,20 +432,22 @@ single_unmanaged_suite(Config) -> State2 = rebar_state:command_parsed_args(State1, GetOptResult), - Result = rebar_prv_common_test:setup_ct(State2), + Tests = rebar_prv_common_test:prepare_tests(State2), + {ok, NewState} = rebar_prv_common_test:compile(State2, Tests), + {ok, T} = Tests, + Opts = rebar_prv_common_test:translate_paths(NewState, T), - Expect = [filename:absname(filename:join([PrivDir, - "unmanaged", - "unmanaged_SUITE"]))], - SuitePath = proplists:get_value(suite, Result), + Expect = filename:join([PrivDir, + "unmanaged", + "unmanaged_SUITE"]), + SuitePath = proplists:get_value(suite, Opts), - Expect = SuitePath. + [Expect] = SuitePath. multi_suite(Config) -> AppDir = ?config(apps, Config), [Name1, Name2] = ?config(appnames, Config), - - {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return), + State = ?config(s, Config), LibDirs = rebar_dir:lib_dirs(State), State1 = rebar_app_discover:do(State, LibDirs), @@ -417,31 +469,33 @@ multi_suite(Config) -> State2 = rebar_state:command_parsed_args(State1, GetOptResult), - Result = rebar_prv_common_test:setup_ct(State2), - - Expect1 = filename:absname(filename:join([AppDir, - "_build", - "test", - "lib", - Name1, - "test", - Name1 ++ "_SUITE"])), - Expect2 = filename:absname(filename:join([AppDir, - "_build", - "test", - "lib", - Name2, - "test", - Name2 ++ "_SUITE"])), - Suites = proplists:get_value(suite, Result), + Tests = rebar_prv_common_test:prepare_tests(State2), + {ok, NewState} = rebar_prv_common_test:compile(State2, Tests), + {ok, T} = Tests, + Opts = rebar_prv_common_test:translate_paths(NewState, T), + + Expect1 = filename:join([AppDir, + "_build", + "test", + "lib", + Name1, + "test", + Name1 ++ "_SUITE"]), + Expect2 = filename:join([AppDir, + "_build", + "test", + "lib", + Name2, + "test", + Name2 ++ "_SUITE"]), + Suites = proplists:get_value(suite, Opts), true = (lists:sort([Expect1, Expect2]) == lists:sort(Suites)). all_suite(Config) -> AppDir = ?config(apps, Config), [Name1, Name2] = ?config(appnames, Config), - - {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return), + State = ?config(s, Config), LibDirs = rebar_dir:lib_dirs(State), State1 = rebar_app_discover:do(State, LibDirs), @@ -465,36 +519,39 @@ all_suite(Config) -> State2 = rebar_state:command_parsed_args(State1, GetOptResult), - Result = rebar_prv_common_test:setup_ct(State2), - - Expect1 = filename:absname(filename:join([AppDir, - "_build", - "test", - "lib", - Name1, - "test", - Name1 ++ "_SUITE"])), - Expect2 = filename:absname(filename:join([AppDir, - "_build", - "test", - "lib", - Name2, - "test", - Name2 ++ "_SUITE"])), - Expect3 = filename:absname(filename:join([AppDir, - "_build", - "test", - "test", - "extra_SUITE"])), - Suites = proplists:get_value(suite, Result), + Tests = rebar_prv_common_test:prepare_tests(State2), + {ok, NewState} = rebar_prv_common_test:compile(State2, Tests), + {ok, T} = Tests, + Opts = rebar_prv_common_test:translate_paths(NewState, T), + + Expect1 = filename:join([AppDir, + "_build", + "test", + "lib", + Name1, + "test", + Name1 ++ "_SUITE"]), + Expect2 = filename:join([AppDir, + "_build", + "test", + "lib", + Name2, + "test", + Name2 ++ "_SUITE"]), + Expect3 = filename:join([AppDir, + "_build", + "test", + "extras", + "test", + "extra_SUITE"]), + Suites = proplists:get_value(suite, Opts), true = (lists:sort([Expect1, Expect2, Expect3]) == lists:sort(Suites)). single_dir_and_single_suite(Config) -> AppDir = ?config(apps, Config), [_Name1, _Name2] = ?config(appnames, Config), - - {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return), + State = ?config(s, Config), LibDirs = rebar_dir:lib_dirs(State), State1 = rebar_app_discover:do(State, LibDirs), @@ -509,52 +566,369 @@ single_dir_and_single_suite(Config) -> State2 = rebar_state:command_parsed_args(State1, GetOptResult), - Result = rebar_prv_common_test:setup_ct(State2), + Tests = rebar_prv_common_test:prepare_tests(State2), + {ok, NewState} = rebar_prv_common_test:compile(State2, Tests), + {ok, T} = Tests, + Opts = rebar_prv_common_test:translate_paths(NewState, T), - Expect = [filename:absname(filename:join([AppDir, - "_build", - "test", - "test", - "extra_SUITE"]))], - Suite = proplists:get_value(suite, Result), + Expect = filename:join([AppDir, + "_build", + "test", + "extras", + "test"]), + Dir = proplists:get_value(dir, Opts), + [Expect] = Dir, - Expect = Suite. + Suite = proplists:get_value(suite, Opts), + ["extra_SUITE"] = Suite. -symlinked_dir_overwritten_fix(Config) -> - AppDir = ?config(apps, Config), - [Name1, _Name2] = ?config(appnames, Config), +%% this test probably only fails when this suite is run via rebar3 with the --cover flag +data_dir_correct(Config) -> + DataDir = ?config(data_dir, Config), + Parts = filename:split(DataDir), + ["rebar_ct_SUITE_data","test","rebar","lib","test","_build"|_] = lists:reverse(Parts). - {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return), +cmd_label(Config) -> + State = ?config(result, Config), - LibDirs = rebar_dir:lib_dirs(State), - State1 = rebar_app_discover:do(State, LibDirs), + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--label=this_is_a_label"]), - Providers = rebar_state:providers(State1), - Namespace = rebar_state:namespace(State1), + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({label, "this_is_a_label"}, TestOpts). + +cmd_config(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), CommandProvider = providers:get_provider(ct, Providers, Namespace), GetOptSpec = providers:opts(CommandProvider), - {ok, GetOptResult} = getopt:parse(GetOptSpec, - ["--dir=" ++ filename:join([AppDir, - "apps", - Name1])]), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--config=config/foo,config/bar,config/baz"]), - State2 = rebar_state:command_parsed_args(State1, GetOptResult), + NewState = rebar_state:command_parsed_args(State, GetOptResult), - Result = rebar_prv_common_test:setup_ct(State2), + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), - Expect = filename:absname(filename:join([AppDir, "_build", "test", "lib", Name1])), - Dir = proplists:get_value(dir, Result), + true = lists:member({config, ["config/foo", "config/bar", "config/baz"]}, TestOpts). - Expect = Dir, +cmd_allow_user_terms(Config) -> + State = ?config(result, Config), - {ok, _} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return). + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--allow_user_terms=true"]), -%% this test probably only fails when this suite is run via rebar3 with the --cover flag -data_dir_correct(Config) -> - DataDir = ?config(data_dir, Config), - Parts = filename:split(DataDir), - ["rebar_ct_SUITE_data","test","rebar","lib","test","_build"|_] = lists:reverse(Parts). + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({allow_user_terms, true}, TestOpts). + +cmd_logdir(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--logdir=/tmp/ct_logs"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({logdir, "/tmp/ct_logs"}, TestOpts). + +cmd_logopts(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--logopts=no_src,no_nl"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({logopts, [no_src, no_nl]}, TestOpts). + +cmd_verbosity(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--verbosity=43"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({verbosity, 43}, TestOpts). + +cmd_repeat(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--repeat=3"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({repeat, 3}, TestOpts). + +cmd_duration(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--duration=001500"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({duration, "001500"}, TestOpts). + +cmd_until(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--until=001500"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({until, "001500"}, TestOpts). + +cmd_force_stop(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--force_stop=skip_rest"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({force_stop, skip_rest}, TestOpts). + +cmd_basic_html(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--basic_html"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({basic_html, true}, TestOpts). + +cmd_stylesheet(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--stylesheet=resources/tests.css"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({stylesheet, "resources/tests.css"}, TestOpts). + +cmd_decrypt_key(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--decrypt_key==ac467e30"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({decrypt_key, "=ac467e30"}, TestOpts). + +cmd_decrypt_file(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--decrypt_file=../keyfile.pem"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({decrypt_file, "../keyfile.pem"}, TestOpts). + +cmd_abort_if_missing_suites(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--abort_if_missing_suites"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({abort_if_missing_suites, true}, TestOpts). + +cmd_multiply_timetraps(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--multiply_timetraps=3"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({multiply_timetraps, 3}, TestOpts). + +cmd_scale_timetraps(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--scale_timetraps"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({scale_timetraps, true}, TestOpts). + +cmd_create_priv_dir(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--create_priv_dir=manual_per_tc"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({create_priv_dir, manual_per_tc}, TestOpts). + +cfg_opts(Config) -> + C = rebar_test_utils:init_rebar_state(Config, "ct_cfg_opts_"), + + AppDir = ?config(apps, C), + + Name = rebar_test_utils:create_random_name("ct_cfg_opts_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + RebarConfig = [{ct_opts, [{label, "this_is_a_label"}, {decrypt_file, "../keyfile.pem"}]}], + + {ok, State} = rebar_test_utils:run_and_check(C, RebarConfig, ["as", "test", "lock"], return), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(State), + + true = lists:member({label, "this_is_a_label"}, TestOpts), + true = lists:member({decrypt_file, "../keyfile.pem"}, TestOpts). + +%% allow even nonsensical opts to be passed to ct_run for futureproofing +cfg_arbitrary_opts(Config) -> + C = rebar_test_utils:init_rebar_state(Config, "ct_cfg_arbitrary_opts_"), + + AppDir = ?config(apps, C), + + Name = rebar_test_utils:create_random_name("ct_cfg_arbitrary_opts_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + RebarConfig = [{ct_opts, [{foo, 1}, {bar, 2}, {baz, 3}]}], + + {ok, State} = rebar_test_utils:run_and_check(C, RebarConfig, ["as", "test", "lock"], return), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(State), + + true = lists:member({foo, 1}, TestOpts), + true = lists:member({bar, 2}, TestOpts), + true = lists:member({baz, 3}, TestOpts). + +cfg_test_spec_filtered(Config) -> + C = rebar_test_utils:init_rebar_state(Config, "ct_cfg_test_spec_filtered_opts_"), + + AppDir = ?config(apps, C), + + Name = rebar_test_utils:create_random_name("ct_cfg_test_spec_filtered_opts_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + RebarConfig = [{ct_opts, [{test_spec, "spec/foo.spec"}]}], + + {ok, State} = rebar_test_utils:run_and_check(C, RebarConfig, ["as", "test", "lock"], return), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(State), + + false = lists:keysearch(test_spec, 1, TestOpts). + +cfg_atom_suites(Config) -> + C = rebar_test_utils:init_rebar_state(Config, "ct_cfg_atom_suites_"), + + AppDir = ?config(apps, C), + + Name = rebar_test_utils:create_random_name("ct_cfg_atom_suites_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + RebarConfig = [{ct_opts, [{suite, [foo, bar, baz]}]}], + + {ok, State} = rebar_test_utils:run_and_check(C, RebarConfig, ["as", "test", "lock"], return), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(State), + true = lists:member({suite, ["foo", "bar", "baz"]}, TestOpts). %% helper for generating test data test_suite(Name) -> |