diff options
| -rw-r--r-- | src/rebar_prv_common_test.erl | 43 | ||||
| -rw-r--r-- | test/rebar_ct_SUITE.erl | 403 |
2 files changed, 435 insertions, 11 deletions
diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index ede88dc..7af2819 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -125,14 +125,16 @@ transform_opts([{testcase, Cases}|Rest], Acc) -> transform_opts(Rest, [{testcase, split_string(Cases)}|Acc]); transform_opts([{config, Configs}|Rest], Acc) -> transform_opts(Rest, [{config, split_string(Configs)}|Acc]); -transform_opts([{include, Includes}|Rest], Acc) -> - transform_opts(Rest, [{include, split_string(Includes)}|Acc]); +transform_opts([{logopts, LogOpts}|Rest], Acc) -> + transform_opts(Rest, [{logopts, lists:map(fun(P) -> list_to_atom(P) end, split_string(LogOpts))}|Acc]); transform_opts([{force_stop, "true"}|Rest], Acc) -> transform_opts(Rest, [{force_stop, true}|Acc]); transform_opts([{force_stop, "false"}|Rest], Acc) -> transform_opts(Rest, [{force_stop, false}|Acc]); transform_opts([{force_stop, "skip_rest"}|Rest], Acc) -> transform_opts(Rest, [{force_stop, skip_rest}|Acc]); +transform_opts([{create_priv_dir, CreatePrivDir}|Rest], Acc) -> + transform_opts(Rest, [{create_priv_dir, list_to_atom(CreatePrivDir)}|Acc]); %% drop cover from opts, ct doesn't care about it transform_opts([{cover, _}|Rest], Acc) -> transform_opts(Rest, Acc); @@ -147,7 +149,7 @@ split_string(String) -> string:tokens(String, [$,]). cfgopts(State) -> - Opts = rebar_state:get(State, ct_tests, []), + Opts = rebar_state:get(State, ct_opts, []), rebar_utils:filtermap(fun filter_opts/1, Opts). filter_opts({test_spec, _}) -> @@ -164,9 +166,10 @@ filter_opts({suite, Suites}) -> filter_opts(_) -> true. select_tests(State, ProjectApps, CmdOpts, CfgOpts) -> + FixedOpts = lists:filter(fun({_, _}) -> true; (V) -> ?WARN("`~p` is not a valid option for `ct_opts`", [V]) end, CfgOpts), Merged = lists:ukeymerge(1, lists:ukeysort(1, CmdOpts), - lists:ukeysort(1, CfgOpts)), + lists:ukeysort(1, FixedOpts)), %% make sure `dir` and/or `suite` from command line go in as %% a pair overriding both `dir` and `suite` from config if %% they exist @@ -464,13 +467,21 @@ ct_opts(_State) -> {config, undefined, "config", string, help(config)}, %% comma-seperated list {allow_user_terms, undefined, "allow_user_terms", boolean, help(allow_user_terms)}, %% Bool {logdir, undefined, "logdir", string, help(logdir)}, %% dir + {logopts, undefined, "logopts", string, help(logopts)}, %% comma seperated list {verbosity, undefined, "verbosity", integer, help(verbosity)}, %% Integer {cover, $c, "cover", {boolean, false}, help(cover)}, - {include, undefined, "include", string, help(include)}, % comma-seperated list {repeat, undefined, "repeat", integer, help(repeat)}, %% integer {duration, undefined, "duration", string, help(duration)}, % format: HHMMSS {until, undefined, "until", string, help(until)}, %% format: YYMoMoDD[HHMMSS] - {basic_html, undefined, "basic_html", boolean, help(basic_html)}, %% Booloean + {force_stop, undefined, "force_stop", string, help(force_stop)}, %% String + {basic_html, undefined, "basic_html", boolean, help(basic_html)}, %% Boolean + {stylesheet, undefined, "stylesheet", string, help(stylesheet)}, %% String + {decrypt_key, undefined, "decrypt_key", string, help(decrypt_key)}, %% String + {decrypt_file, undefined, "decrypt_file", string, help(decrypt_file)}, %% String + {abort_if_missing_suites, undefined, "abort_if_missing_suites", {boolean, true}, help(abort_if_missing_suites)}, %% Boolean + {multiply_timetraps, undefined, "multiply_timetraps", integer, help(multiple_timetraps)}, %% Integer + {scale_timetraps, undefined, "scale_timetraps", boolean, help(scale_timetraps)}, + {create_priv_dir, undefined, "create_priv_dir", string, help(create_priv_dir)}, {verbose, $v, "verbose", boolean, help(verbose)} ]. @@ -490,20 +501,36 @@ help(allow_user_terms) -> "Allow user defined config values in config files"; help(logdir) -> "Log folder"; +help(logopts) -> + "Options for common test logging"; help(verbosity) -> "Verbosity"; help(cover) -> "Generate cover data"; -help(include) -> - "Include folders"; help(repeat) -> "How often to repeat tests"; help(duration) -> "Max runtime (format: HHMMSS)"; help(until) -> "Run until (format: HHMMSS)"; +help(force_stop) -> + "Force stop on test timeout (true | false | skip_rest)"; help(basic_html) -> "Show basic HTML"; +help(stylesheet) -> + "CSS stylesheet to apply to html output"; +help(decrypt_key) -> + "Path to key for decrypting config"; +help(decrypt_file) -> + "Path to file containing key for decrypting config"; +help(abort_if_missing_suites) -> + "Abort if suites are missing"; +help(multiply_timetraps) -> + "Multiply timetraps"; +help(scale_timetraps) -> + "Scale timetraps"; +help(create_priv_dir) -> + "Create priv dir (auto_per_run | auto_per_tc | manual_per_tc)"; help(verbose) -> "Verbose output"; help(_) -> diff --git a/test/rebar_ct_SUITE.erl b/test/rebar_ct_SUITE.erl index 821a16e..ac016b1 100644 --- a/test/rebar_ct_SUITE.erl +++ b/test/rebar_ct_SUITE.erl @@ -17,14 +17,40 @@ multi_suite/1, all_suite/1, single_dir_and_single_suite/1, - data_dir_correct/1]). + data_dir_correct/1, + cmd_label/1, + cmd_config/1, + cmd_allow_user_terms/1, + cmd_logdir/1, + cmd_logopts/1, + cmd_verbosity/1, + cmd_repeat/1, + cmd_duration/1, + cmd_until/1, + cmd_force_stop/1, + cmd_basic_html/1, + cmd_stylesheet/1, + cmd_decrypt_key/1, + cmd_decrypt_file/1, + cmd_abort_if_missing_suites/1, + cmd_multiply_timetraps/1, + cmd_scale_timetraps/1, + cmd_create_priv_dir/1, + cfg_opts/1, + cfg_arbitrary_opts/1, + cfg_test_spec_filtered/1, + cfg_atom_suites/1]). -include_lib("common_test/include/ct.hrl"). all() -> [{group, basic_app}, {group, multi_app}, {group, dirs_and_suites}, - {group, data_dirs}]. + {group, data_dirs}, + {group, ct_opts}, + cfg_opts, cfg_arbitrary_opts, + cfg_test_spec_filtered, + cfg_atom_suites]. groups() -> [{basic_app, [], [basic_app_default_dirs, basic_app_default_beams]}, @@ -39,7 +65,25 @@ groups() -> [{basic_app, [], [basic_app_default_dirs, multi_suite, all_suite, single_dir_and_single_suite]}, - {data_dirs, [], [data_dir_correct]}]. + {data_dirs, [], [data_dir_correct]}, + {ct_opts, [], [cmd_label, + cmd_config, + cmd_allow_user_terms, + cmd_logdir, + cmd_logopts, + cmd_verbosity, + cmd_repeat, + cmd_duration, + cmd_until, + cmd_force_stop, + cmd_basic_html, + cmd_stylesheet, + cmd_decrypt_key, + cmd_decrypt_file, + cmd_abort_if_missing_suites, + cmd_multiply_timetraps, + cmd_scale_timetraps, + cmd_create_priv_dir]}]. init_per_group(basic_app, Config) -> C = rebar_test_utils:init_rebar_state(Config, "ct_"), @@ -127,6 +171,18 @@ init_per_group(dirs_and_suites, Config) -> {ok, State} = rebar_test_utils:run_and_check(C, [], ["as", "test", "lock"], return), [{s, State}, {appnames, [Name1, Name2]}|C]; +init_per_group(ct_opts, Config) -> + C = rebar_test_utils:init_rebar_state(Config, "ct_opts"), + + AppDir = ?config(apps, C), + + Name = rebar_test_utils:create_random_name("ct_opts_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + {ok, State} = rebar_test_utils:run_and_check(C, [], ["as", "test", "lock"], return), + + [{result, State}|C]; init_per_group(_, Config) -> Config. end_per_group(_Group, _Config) -> ok. @@ -532,6 +588,347 @@ data_dir_correct(Config) -> Parts = filename:split(DataDir), ["rebar_ct_SUITE_data","test","rebar","lib","test","_build"|_] = lists:reverse(Parts). +cmd_label(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--label=this_is_a_label"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({label, "this_is_a_label"}, TestOpts). + +cmd_config(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--config=config/foo,config/bar,config/baz"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({config, ["config/foo", "config/bar", "config/baz"]}, TestOpts). + +cmd_allow_user_terms(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--allow_user_terms=true"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({allow_user_terms, true}, TestOpts). + +cmd_logdir(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--logdir=/tmp/ct_logs"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({logdir, "/tmp/ct_logs"}, TestOpts). + +cmd_logopts(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--logopts=no_src,no_nl"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({logopts, [no_src, no_nl]}, TestOpts). + +cmd_verbosity(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--verbosity=43"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({verbosity, 43}, TestOpts). + +cmd_repeat(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--repeat=3"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({repeat, 3}, TestOpts). + +cmd_duration(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--duration=001500"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({duration, "001500"}, TestOpts). + +cmd_until(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--until=001500"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({until, "001500"}, TestOpts). + +cmd_force_stop(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--force_stop=skip_rest"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({force_stop, skip_rest}, TestOpts). + +cmd_basic_html(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--basic_html"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({basic_html, true}, TestOpts). + +cmd_stylesheet(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--stylesheet=resources/tests.css"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({stylesheet, "resources/tests.css"}, TestOpts). + +cmd_decrypt_key(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--decrypt_key==ac467e30"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({decrypt_key, "=ac467e30"}, TestOpts). + +cmd_decrypt_file(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--decrypt_file=../keyfile.pem"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({decrypt_file, "../keyfile.pem"}, TestOpts). + +cmd_abort_if_missing_suites(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--abort_if_missing_suites"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({abort_if_missing_suites, true}, TestOpts). + +cmd_multiply_timetraps(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--multiply_timetraps=3"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({multiply_timetraps, 3}, TestOpts). + +cmd_scale_timetraps(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--scale_timetraps"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({scale_timetraps, true}, TestOpts). + +cmd_create_priv_dir(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--create_priv_dir=manual_per_tc"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({create_priv_dir, manual_per_tc}, TestOpts). + +cfg_opts(Config) -> + C = rebar_test_utils:init_rebar_state(Config, "ct_cfg_opts_"), + + AppDir = ?config(apps, C), + + Name = rebar_test_utils:create_random_name("ct_cfg_opts_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + RebarConfig = [{ct_opts, [{label, "this_is_a_label"}, {decrypt_file, "../keyfile.pem"}]}], + + {ok, State} = rebar_test_utils:run_and_check(C, RebarConfig, ["as", "test", "lock"], return), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(State), + + true = lists:member({label, "this_is_a_label"}, TestOpts), + true = lists:member({decrypt_file, "../keyfile.pem"}, TestOpts). + +%% allow even nonsensical opts to be passed to ct_run for futureproofing +cfg_arbitrary_opts(Config) -> + C = rebar_test_utils:init_rebar_state(Config, "ct_cfg_arbitrary_opts_"), + + AppDir = ?config(apps, C), + + Name = rebar_test_utils:create_random_name("ct_cfg_arbitrary_opts_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + RebarConfig = [{ct_opts, [{foo, 1}, {bar, 2}, {baz, 3}]}], + + {ok, State} = rebar_test_utils:run_and_check(C, RebarConfig, ["as", "test", "lock"], return), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(State), + + true = lists:member({foo, 1}, TestOpts), + true = lists:member({bar, 2}, TestOpts), + true = lists:member({baz, 3}, TestOpts). + +cfg_test_spec_filtered(Config) -> + C = rebar_test_utils:init_rebar_state(Config, "ct_cfg_test_spec_filtered_opts_"), + + AppDir = ?config(apps, C), + + Name = rebar_test_utils:create_random_name("ct_cfg_test_spec_filtered_opts_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + RebarConfig = [{ct_opts, [{test_spec, "spec/foo.spec"}]}], + + {ok, State} = rebar_test_utils:run_and_check(C, RebarConfig, ["as", "test", "lock"], return), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(State), + + false = lists:keysearch(test_spec, 1, TestOpts). + +cfg_atom_suites(Config) -> + C = rebar_test_utils:init_rebar_state(Config, "ct_cfg_atom_suites_"), + + AppDir = ?config(apps, C), + + Name = rebar_test_utils:create_random_name("ct_cfg_atom_suites_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + RebarConfig = [{ct_opts, [{suite, [foo, bar, baz]}]}], + + {ok, State} = rebar_test_utils:run_and_check(C, RebarConfig, ["as", "test", "lock"], return), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(State), + + true = lists:member({suite, ["foo", "bar", "baz"]}, TestOpts). %% helper for generating test data test_suite(Name) -> |