Merge pull request #894 from talentdeficit/ct_test_specs

rebar_prv_common_test refactor

1 files changed, 359 insertions, 375 deletions

diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl
index 32d4433..7a088e5 100644
--- a/src/rebar_prv_common_test.erl
+++ b/src/rebar_prv_common_test.erl
@@ -2,19 +2,21 @@
 %% ex: ts=4 sw=4 et
 
 -module(rebar_prv_common_test).
+
 -behaviour(provider).
 
 -export([init/1,
          do/1,
          format_error/1]).
 %% exported for test purposes, consider private
--export([setup_ct/1]).
+-export([compile/2, prepare_tests/1, translate_paths/2]).
 
 -include("rebar.hrl").
 -include_lib("providers/include/providers.hrl").
 
 -define(PROVIDER, ct).
--define(DEPS, [compile]).
+%% we need to modify app_info state before compile
+-define(DEPS, [lock]).
 
 %% ===================================================================
 %% Public API
@@ -31,12 +33,19 @@ init(State) ->
                                  {desc, "Run Common Tests."},
                                  {opts, ct_opts(State)},
                                  {profiles, [test]}]),
-    State1 = rebar_state:add_provider(State, Provider),
-    State2 = rebar_state:add_to_profile(State1, test, test_state(State1)),
-    {ok, State2}.
+    {ok, rebar_state:add_provider(State, Provider)}.
 
 -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
 do(State) ->
+    Tests = prepare_tests(State),
+    case compile(State, Tests) of
+        %% successfully compiled apps
+        {ok, S} -> do(S, Tests);
+        %% this should look like a compiler error, not a ct error
+        Error   -> Error
+    end.
+
+do(State, Tests) ->
     ?INFO("Running Common Test suites...", []),
     rebar_utils:update_code(rebar_state:code_paths(State, all_deps), [soft_purge]),
 
@@ -45,63 +54,332 @@ do(State) ->
     Cwd = rebar_dir:get_cwd(),
     rebar_hooks:run_all_hooks(Cwd, pre, ?PROVIDER, Providers, State),
 
-    try run_test(State) of
-        {ok, State1} = Result ->
-            %% Run ct provider posthooks
-            rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State1),
-            rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)),
-            Result;
-        ?PRV_ERROR(_) = Error ->
+    case Tests of
+        {ok, T} ->
+            case run_tests(State, T) of
+                ok    ->
+                    %% Run ct provider posthooks
+                    rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State),
+                    rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)),
+                    {ok, State};
+                Error ->
+                    rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)),
+                    Error
+            end;
+        Error ->
             rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)),
             Error
-    catch
-        throw:{error, Reason} ->
-            rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)),
-            ?PRV_ERROR(Reason)
+    end.
+
+run_tests(State, Opts) ->
+    T = translate_paths(State, Opts),
+    Opts1 = setup_logdir(State, T),
+    Opts2 = turn_off_auto_compile(Opts1),
+    ?DEBUG("ct_opts ~p", [Opts2]),
+    {RawOpts, _} = rebar_state:command_parsed_args(State),
+    ok = maybe_write_coverdata(State),
+    case proplists:get_value(verbose, RawOpts, false) of
+        true  -> run_test_verbose(Opts2);
+        false -> run_test_quiet(Opts2)
     end.
 
 -spec format_error(any()) -> iolist().
-format_error({multiple_dirs_and_suites, Opts}) ->
-    io_lib:format("Multiple dirs declared alongside suite in opts: ~p", [Opts]);
-format_error({bad_dir_or_suite, Opts}) ->
-    io_lib:format("Bad value for dir or suite in opts: ~p", [Opts]);
+format_error({error, Reason}) ->
+    io_lib:format("Error running tests:~n  ~p", [Reason]);
+format_error({error_running_tests, Reason}) ->
+    format_error({error, Reason});
 format_error({failures_running_tests, {Failed, AutoSkipped}}) ->
     io_lib:format("Failures occured running tests: ~b", [Failed+AutoSkipped]);
-format_error({error_running_tests, Reason}) ->
-    io_lib:format("Error running tests: ~p", [Reason]);
-format_error(suite_at_project_root) ->
-    io_lib:format("Test suites can't be located in project root", []);
-format_error({error, Reason}) ->
-    io_lib:format("Unknown error: ~p", [Reason]).
+format_error({multiple_errors, Errors}) ->
+    io_lib:format(lists:concat(["Error running tests:"] ++
+                               lists:map(fun(Error) -> "~n  " ++ Error end, Errors)), []).
 
 %% ===================================================================
 %% Internal functions
 %% ===================================================================
 
-run_test(State) ->
-    case setup_ct(State) of
-        {error, {no_tests_specified, Opts}} ->
-            ?WARN("No tests specified in opts: ~p", [Opts]),
-            {ok, State};
-        Opts ->
-            Opts1 = setup_logdir(State, Opts),
-            ?DEBUG("common test opts: ~p", [Opts1]),
-            run_test(State, Opts1)
-    end.
+prepare_tests(State) ->
+    %% command line test options
+    CmdOpts = cmdopts(State),
+    %% rebar.config test options
+    CfgOpts = cfgopts(State),
+    ProjectApps = rebar_state:project_apps(State),
+    %% prioritize tests to run first trying any command line specified
+    %% tests falling back to tests specified in the config file finally
+    %% running a default set if no other tests are present
+    select_tests(State, ProjectApps, CmdOpts, CfgOpts).
 
-run_test(State, Opts) ->
+cmdopts(State) ->
     {RawOpts, _} = rebar_state:command_parsed_args(State),
-    ok = rebar_prv_cover:maybe_cover_compile(State, apps),
-    Result = case proplists:get_value(verbose, RawOpts, false) of
-        true  -> run_test_verbose(Opts);
-        false -> run_test_quiet(Opts)
+    %% filter out opts common_test doesn't know about and convert
+    %% to ct acceptable forms
+    transform_opts(RawOpts, []).
+
+transform_opts([], Acc) -> lists:reverse(Acc);
+transform_opts([{dir, Dirs}|Rest], Acc) ->
+    transform_opts(Rest, [{dir, split_string(Dirs)}|Acc]);
+transform_opts([{suite, Suites}|Rest], Acc) ->
+    transform_opts(Rest, [{suite, split_string(Suites)}|Acc]);
+transform_opts([{group, Groups}|Rest], Acc) ->
+    transform_opts(Rest, [{group, split_string(Groups)}|Acc]);
+transform_opts([{testcase, Cases}|Rest], Acc) ->
+    transform_opts(Rest, [{testcase, split_string(Cases)}|Acc]);
+transform_opts([{config, Configs}|Rest], Acc) ->
+    transform_opts(Rest, [{config, split_string(Configs)}|Acc]);
+transform_opts([{logopts, LogOpts}|Rest], Acc) ->
+    transform_opts(Rest, [{logopts, lists:map(fun(P) -> list_to_atom(P) end, split_string(LogOpts))}|Acc]);
+transform_opts([{force_stop, "true"}|Rest], Acc) ->
+    transform_opts(Rest, [{force_stop, true}|Acc]);
+transform_opts([{force_stop, "false"}|Rest], Acc) ->
+    transform_opts(Rest, [{force_stop, false}|Acc]);
+transform_opts([{force_stop, "skip_rest"}|Rest], Acc) ->
+    transform_opts(Rest, [{force_stop, skip_rest}|Acc]);
+transform_opts([{create_priv_dir, CreatePrivDir}|Rest], Acc) ->
+    transform_opts(Rest, [{create_priv_dir, list_to_atom(CreatePrivDir)}|Acc]);
+%% drop cover from opts, ct doesn't care about it
+transform_opts([{cover, _}|Rest], Acc) ->
+    transform_opts(Rest, Acc);
+%% drop verbose from opts, ct doesn't care about it
+transform_opts([{verbose, _}|Rest], Acc) ->
+    transform_opts(Rest, Acc);
+%% getopt should handle anything else
+transform_opts([Opt|Rest], Acc) ->
+    transform_opts(Rest, [Opt|Acc]).
+
+split_string(String) ->
+    string:tokens(String, [$,]).
+
+cfgopts(State) ->
+    Opts = rebar_state:get(State, ct_opts, []),
+    rebar_utils:filtermap(fun filter_opts/1, Opts).
+
+filter_opts({test_spec, _}) ->
+    ?WARN("Test specs not supported", []),
+    false;
+filter_opts({auto_compile, _}) ->
+    ?WARN("Auto compile not supported", []),
+    false;
+filter_opts({suite, Suite}) when is_integer(hd(Suite)) -> true;
+filter_opts({suite, Suite}) when is_atom(Suite) ->
+    {true, {suite, atom_to_list(Suite)}};
+filter_opts({suite, Suites}) ->
+    {true, {suite, lists:map(fun(S) when is_atom(S) -> atom_to_list(S);
+                                (S) when is_list(S) -> S
+                             end,
+                             Suites)}};
+filter_opts(_) -> true.
+
+select_tests(State, ProjectApps, CmdOpts, CfgOpts) ->
+    FixedOpts = lists:filter(fun({_, _}) -> true; (V) -> ?WARN("`~p` is not a valid option for `ct_opts`", [V]) end, CfgOpts),
+    Merged = lists:ukeymerge(1,
+                             lists:ukeysort(1, CmdOpts),
+                             lists:ukeysort(1, FixedOpts)),
+    %% make sure `dir` and/or `suite` from command line go in as
+    %% a pair overriding both `dir` and `suite` from config if
+    %% they exist
+    Opts = case {proplists:get_value(suite, CmdOpts), proplists:get_value(dir, CmdOpts)} of
+        {undefined, undefined} -> Merged;
+        {_Suite, undefined}    -> lists:keydelete(dir, 1, Merged);
+        {undefined, _Dir}      -> lists:keydelete(suite, 1, Merged);
+        {_Suite, _Dir}         -> Merged
     end,
-    ok = rebar_prv_cover:maybe_write_coverdata(State, ?PROVIDER),
-    case Result of
-        ok    -> {ok, State};
-        Error -> Error
+    discover_tests(State, ProjectApps, Opts).
+
+discover_tests(State, ProjectApps, Opts) ->
+    case {proplists:get_value(suite, Opts), proplists:get_value(dir, Opts)} of
+        %% no dirs or suites defined, try using `$APP/test` and `$ROOT/test`
+        %%  as suites
+        {undefined, undefined} -> {ok, [default_tests(State, ProjectApps)|Opts]};
+        {_, _}                 -> {ok, Opts}
+    end.
+
+default_tests(State, ProjectApps) ->
+    BareTest = filename:join([rebar_state:dir(State), "test"]),
+    F = fun(App) -> rebar_app_info:dir(App) == rebar_state:dir(State) end,
+    AppTests = application_dirs(ProjectApps, []),
+    case filelib:is_dir(BareTest) andalso not lists:any(F, ProjectApps) of
+        %% `test` dir at root of project is already scheduled to be
+        %%  included or `test` does not exist
+        false -> {dir, AppTests};
+        %% need to add `test` dir at root to dirs to be included
+        true  -> {dir, AppTests ++ [BareTest]}
+    end.
+
+application_dirs([], []) -> [];
+application_dirs([], Acc) -> lists:reverse(Acc);
+application_dirs([App|Rest], Acc) ->
+    TestDir = filename:join([rebar_app_info:dir(App), "test"]),
+    case filelib:is_dir(TestDir) of
+        true  -> application_dirs(Rest, [TestDir|Acc]);
+        false -> application_dirs(Rest, Acc)
+    end.
+
+compile(State, {ok, Tests}) ->
+    %% inject `ct_first_files` and `ct_compile_opts` into the applications
+    %% to be compiled
+    case inject_ct_state(State, Tests) of
+        {ok, NewState} -> do_compile(NewState);
+        Error          -> Error
+    end;
+%% maybe compile even in the face of errors?
+compile(_State, Error) -> Error.
+
+do_compile(State) ->
+    case rebar_prv_compile:do(State) of
+        %% successfully compiled apps
+        {ok, S} ->
+            ok = maybe_cover_compile(S),
+            {ok, S};
+        %% this should look like a compiler error, not an eunit error
+        Error   -> Error
+    end.
+
+inject_ct_state(State, Tests) ->
+    Apps = rebar_state:project_apps(State),
+    ModdedApps = lists:map(fun(App) ->
+        NewOpts = inject(rebar_app_info:opts(App), State),
+        rebar_app_info:opts(App, NewOpts)
+    end, Apps),
+    NewOpts = inject(rebar_state:opts(State), State),
+    NewState = rebar_state:opts(State, NewOpts),
+    test_dirs(NewState, ModdedApps, Tests).
+
+inject(Opts, State) ->
+    %% append `ct_compile_opts` to app defined `erl_opts`
+    ErlOpts = rebar_opts:get(Opts, erl_opts, []),
+    CTOpts = rebar_state:get(State, ct_compile_opts, []),
+    NewErlOpts = CTOpts ++ ErlOpts,
+    %% append `ct_first_files` to app defined `erl_first_files`
+    FirstFiles = rebar_opts:get(Opts, erl_first_files, []),
+    CTFirstFiles = rebar_state:get(State, ct_first_files, []),
+    NewFirstFiles = CTFirstFiles ++ FirstFiles,
+    %% insert the new keys into the opts
+    lists:foldl(fun({K, V}, NewOpts) -> rebar_opts:set(NewOpts, K, V) end,
+                Opts,
+                [{erl_opts, NewErlOpts}, {erl_first_files, NewFirstFiles}]).
+
+test_dirs(State, Apps, Opts) ->
+    case {proplists:get_value(suite, Opts), proplists:get_value(dir, Opts)} of
+        {Suites, undefined} -> set_compile_dirs(State, Apps, {suite, Suites});
+        {undefined, Dirs}   -> set_compile_dirs(State, Apps, {dir, Dirs});
+        {Suites, Dir} when is_integer(hd(Dir)) ->
+            set_compile_dirs(State, Apps, join(Suites, Dir));
+        {Suites, [Dir]} when is_integer(hd(Dir)) ->
+            set_compile_dirs(State, Apps, join(Suites, Dir));          
+        {_Suites, _Dirs}    -> {error, "Only a single directory may be specified when specifying suites"}
+    end.
+
+join(Suite, Dir) when is_integer(hd(Suite)) ->
+    {suite, [filename:join([Dir, Suite])]};
+join(Suites, Dir) ->
+    {suite, lists:map(fun(S) -> filename:join([Dir, S]) end, Suites)}.
+
+set_compile_dirs(State, Apps, {dir, Dir}) when is_integer(hd(Dir)) ->
+    %% single directory
+    %% insert `Dir` into an app if relative, or the base state if not
+    %% app relative but relative to the root or not at all if outside
+    %% project scope
+    {NewState, NewApps} = maybe_inject_test_dir(State, [], Apps, Dir),
+    {ok, rebar_state:project_apps(NewState, NewApps)};
+set_compile_dirs(State, Apps, {dir, Dirs}) ->
+    %% multiple directories
+    F = fun(Dir, {S, A}) -> maybe_inject_test_dir(S, [], A, Dir) end,
+    {NewState, NewApps} = lists:foldl(F, {State, Apps}, Dirs),
+    {ok, rebar_state:project_apps(NewState, NewApps)};
+set_compile_dirs(State, Apps, {suite, Suites}) ->
+    %% suites with dir component
+    Dirs = find_suite_dirs(Suites),
+    F = fun(Dir, {S, A}) -> maybe_inject_test_dir(S, [], A, Dir) end,
+    {NewState, NewApps} = lists:foldl(F, {State, Apps}, Dirs),
+    {ok, rebar_state:project_apps(NewState, NewApps)}.
+
+find_suite_dirs(Suites) ->
+    AllDirs = lists:map(fun(S) -> filename:dirname(filename:absname(S)) end, Suites),
+    %% eliminate duplicates
+    lists:usort(AllDirs).
+
+maybe_inject_test_dir(State, AppAcc, [App|Rest], Dir) ->
+    case rebar_file_utils:path_from_ancestor(Dir, rebar_app_info:dir(App)) of
+        {ok, Path} ->
+            Opts = inject_test_dir(rebar_app_info:opts(App), Path),
+            {State, AppAcc ++ [rebar_app_info:opts(App, Opts)] ++ Rest};
+        {error, badparent} ->
+            maybe_inject_test_dir(State, AppAcc ++ [App], Rest, Dir)
+    end;
+maybe_inject_test_dir(State, AppAcc, [], Dir) ->
+    case rebar_file_utils:path_from_ancestor(Dir, rebar_state:dir(State)) of
+        {ok, []}   ->
+            ?WARN("Can't have suites in root of project dir, dropping from tests", []),
+            {State, AppAcc};
+        {ok, Path} ->
+            Opts = inject_test_dir(rebar_state:opts(State), Path),
+            {rebar_state:opts(State, Opts), AppAcc};
+        {error, badparent} ->
+            {State, AppAcc}
     end.
 
+inject_test_dir(Opts, Dir) ->
+    %% append specified test targets to app defined `extra_src_dirs`
+    ExtraSrcDirs = rebar_opts:get(Opts, extra_src_dirs, []),
+    rebar_opts:set(Opts, extra_src_dirs, ExtraSrcDirs ++ [Dir]).
+
+translate_paths(State, Opts) ->
+    case {proplists:get_value(suite, Opts), proplists:get_value(dir, Opts)} of
+        {_Suites, undefined} -> translate_suites(State, Opts, []);
+        {undefined, _Dirs}   -> translate_dirs(State, Opts, []);
+        %% both dirs and suites are defined, only translate dir paths
+        _                    -> translate_dirs(State, Opts, [])
+    end.
+
+translate_dirs(_State, [], Acc) -> lists:reverse(Acc);
+translate_dirs(State, [{dir, Dir}|Rest], Acc) when is_integer(hd(Dir)) ->
+    %% single dir
+    Apps = rebar_state:project_apps(State),
+    translate_dirs(State, Rest, [{dir, translate(State, Apps, Dir)}|Acc]);
+translate_dirs(State, [{dir, Dirs}|Rest], Acc) ->
+    %% multiple dirs
+    Apps = rebar_state:project_apps(State),
+    NewDirs = {dir, lists:map(fun(Dir) -> translate(State, Apps, Dir) end, Dirs)},
+    translate_dirs(State, Rest, [NewDirs|Acc]);
+translate_dirs(State, [Test|Rest], Acc) ->
+    translate_dirs(State, Rest, [Test|Acc]).
+
+translate_suites(_State, [], Acc) -> lists:reverse(Acc);
+translate_suites(State, [{suite, Suite}|Rest], Acc) when is_integer(hd(Suite)) ->
+    %% single suite
+    Apps = rebar_state:project_apps(State),
+    translate_suites(State, Rest, [{suite, translate(State, Apps, Suite)}|Acc]);
+translate_suites(State, [{suite, Suites}|Rest], Acc) ->
+    %% multiple suites
+    Apps = rebar_state:project_apps(State),
+    NewSuites = {suite, lists:map(fun(Suite) -> translate(State, Apps, Suite) end, Suites)},
+    translate_suites(State, Rest, [NewSuites|Acc]);
+translate_suites(State, [Test|Rest], Acc) ->
+    translate_suites(State, Rest, [Test|Acc]).
+
+translate(State, [App|Rest], Path) ->
+    case rebar_file_utils:path_from_ancestor(Path, rebar_app_info:dir(App)) of
+        {ok, P}            -> filename:join([rebar_app_info:out_dir(App), P]);
+        {error, badparent} -> translate(State, Rest, Path)
+    end;
+translate(State, [], Path) ->
+    case rebar_file_utils:path_from_ancestor(Path, rebar_state:dir(State)) of
+        {ok, P}            -> filename:join([rebar_dir:base_dir(State), "extras", P]);
+        %% not relative, leave as is
+        {error, badparent} -> Path
+    end.
+
+setup_logdir(State, Opts) ->
+    Logdir = case proplists:get_value(logdir, Opts) of
+        undefined -> filename:join([rebar_dir:base_dir(State), "logs"]);
+        Dir       -> Dir
+    end,
+    filelib:ensure_dir(filename:join([Logdir, "dummy.beam"])),
+    [{logdir, Logdir}|lists:keydelete(logdir, 1, Opts)].
+
+turn_off_auto_compile(Opts) ->
+    [{auto_compile, false}|lists:keydelete(auto_compile, 1, Opts)].
+
 run_test_verbose(Opts) -> handle_results(ct:run_test(Opts)).
 
 run_test_quiet(Opts) ->
@@ -171,272 +449,46 @@ format_skipped({0, 0}) ->
 format_skipped({User, Auto}) ->
     io_lib:format("Skipped ~p (~p, ~p) tests. ", [User+Auto, User, Auto]).
 
-test_state(State) ->
-    TestOpts = case rebar_state:get(State, ct_compile_opts, []) of
-        []    -> [];
-        Opts  -> [{erl_opts, Opts}]
-    end,
-    [first_files(State)|TestOpts].
-
-first_files(State) ->
-    CTFirst = rebar_state:get(State, ct_first_files, []),
-    {erl_first_files, CTFirst}.
-
-setup_ct(State) ->
-    Opts = resolve_ct_opts(State),
-    Opts1 = discover_tests(State, Opts),
-    copy_and_compile_tests(State, Opts1).
-
-resolve_ct_opts(State) ->
-    {RawOpts, _} = rebar_state:command_parsed_args(State),
-    CmdOpts = transform_opts(RawOpts),
-    CfgOpts = rebar_state:get(State, ct_opts, []),
-    Merged = lists:ukeymerge(1,
-                             lists:ukeysort(1, CmdOpts),
-                             lists:ukeysort(1, CfgOpts)),
-    %% make sure `dir` and/or `suite` from command line go in as
-    %%  a pair overriding both `dir` and `suite` from config if
-    %%  they exist
-    case {proplists:get_value(suite, CmdOpts), proplists:get_value(dir, CmdOpts)} of
-        {undefined, undefined} -> Merged;
-        {_Suite, undefined}    -> lists:keydelete(dir, 1, Merged);
-        {undefined, _Dir}      -> lists:keydelete(suite, 1, Merged);
-        {_Suite, _Dir}         -> Merged
-    end.
-
-discover_tests(State, Opts) ->
-    case proplists:get_value(spec, Opts) of
-        undefined -> discover_dirs_and_suites(State, Opts);
-        TestSpec  -> discover_testspec(TestSpec, Opts)
-    end.
-
-discover_dirs_and_suites(State, Opts) ->
-    case {proplists:get_value(dir, Opts), proplists:get_value(suite, Opts)} of
-        %% no dirs or suites defined, try using `$APP/test` and `$ROOT/test`
-        %%  as suites
-        {undefined, undefined} -> test_dirs(State, Opts);
-        %% no dirs defined
-        {undefined, _} -> Opts;
-        %% no suites defined
-        {_, undefined} -> Opts;
-        %% a single dir defined, this is ok
-        {Dirs, Suites} when is_integer(hd(Dirs)), is_list(Suites) -> Opts;
-        %% still a single dir defined, adjust to make acceptable to ct
-        {[Dir], Suites} when is_integer(hd(Dir)), is_list(Suites) ->
-            [{dir, Dir}|lists:keydelete(dir, 1, Opts)];
-        %% multiple dirs and suites, error now to simplify later steps
-        {_, _} -> throw({error, {multiple_dirs_and_suites, Opts}})
-    end.
-
-discover_testspec(_TestSpec, Opts) ->
-    lists:keydelete(auto_compile, 1, Opts).
-
-copy_and_compile_tests(State, Opts) ->
-    %% possibly enable cover
+maybe_cover_compile(State) ->
     {RawOpts, _} = rebar_state:command_parsed_args(State),
     State1 = case proplists:get_value(cover, RawOpts, false) of
         true  -> rebar_state:set(State, cover_enabled, true);
         false -> State
     end,
-    copy_and_compile_test_suites(State1, Opts).
-
-copy_and_compile_test_suites(State, Opts) ->
-    case proplists:get_value(suite, Opts) of
-        %% no suites, try dirs
-        undefined -> copy_and_compile_test_dirs(State, Opts);
-        Suites    ->
-            Dir = proplists:get_value(dir, Opts, undefined),
-            AllSuites = join(Dir, Suites),
-            Dirs = find_suite_dirs(AllSuites),
-            lists:foreach(fun(S) ->
-                NewPath = copy(State, S),
-                compile_dir(State, NewPath)
-            end, Dirs),
-            NewSuites = lists:map(fun(S) -> retarget_path(State, S) end, AllSuites),
-            [{suite, NewSuites}|lists:keydelete(suite, 1, Opts)]
-    end.
-
-copy_and_compile_test_dirs(State, Opts) ->
-    case proplists:get_value(dir, Opts) of
-        undefined -> {error, {no_tests_specified, Opts}};
-        %% dir is a single directory
-        Dir when is_list(Dir), is_integer(hd(Dir)) ->
-            NewPath = copy(State, Dir),
-            [{dir, compile_dir(State, NewPath)}|lists:keydelete(dir, 1, Opts)];
-        %% dir is a list of directories
-        Dirs when is_list(Dirs) ->
-            NewDirs = lists:map(fun(Dir) ->
-                NewPath = copy(State, Dir),
-                compile_dir(State, NewPath)
-            end, Dirs),
-            [{dir, NewDirs}|lists:keydelete(dir, 1, Opts)]
-    end.
-
-join(undefined, Suites) -> Suites;
-join(Dir, Suites) when is_list(Dir), is_integer(hd(Dir)) ->
-    lists:map(fun(S) -> filename:join([Dir, S]) end, Suites);
-%% multiple dirs or a bad dir argument, try to continue anyways
-join(_, Suites) -> Suites.
-
-find_suite_dirs(Suites) ->
-    AllDirs = lists:map(fun(S) -> filename:dirname(filename:absname(S)) end, Suites),
-    %% eliminate duplicates
-    lists:usort(AllDirs).
-
-copy(State, Dir) ->
-    From = reduce_path(Dir),
-    retarget_path(State, From).
-
-compile_dir(State, Dir) ->
-    NewState = replace_src_dirs(State, [filename:absname(Dir)]),
-    ok = rebar_erlc_compiler:compile(rebar_state:opts(NewState), rebar_dir:base_dir(State), Dir),
-    ok = maybe_cover_compile(State, Dir),
-    Dir.
-
-retarget_path(State, Path) ->
-    ProjectApps = rebar_state:project_apps(State),
-    retarget_path(State, Path, ProjectApps).
-
-%% not relative to any apps in project, check to see it's relative to
-%%  project root
-retarget_path(State, Path, []) ->
-    case relative_path(reduce_path(Path), rebar_state:dir(State)) of
-        {ok, NewPath}         -> filename:join([rebar_dir:base_dir(State), NewPath]);
-        %% not relative to project root, don't modify
-        {error, not_relative} -> Path
-    end;
-%% relative to current app, retarget to the same dir relative to
-%%  the app's out_dir
-retarget_path(State, Path, [App|Rest]) ->
-    case relative_path(reduce_path(Path), rebar_app_info:dir(App)) of
-        {ok, NewPath}         -> filename:join([rebar_app_info:out_dir(App), NewPath]);
-        {error, not_relative} -> retarget_path(State, Path, Rest)
-    end.
-
-relative_path(Target, To) ->
-    relative_path1(filename:split(filename:absname(Target)),
-                   filename:split(filename:absname(To))).
-
-relative_path1([Part|Target], [Part|To]) -> relative_path1(Target, To);
-relative_path1([], [])                   -> {ok, ""};
-relative_path1(Target, [])               -> {ok, filename:join(Target)};
-relative_path1(_, _)                     -> {error, not_relative}.
-
-reduce_path(Dir) -> reduce_path([], filename:split(filename:absname(Dir))).
-
-reduce_path([], [])                -> filename:nativename("/");
-reduce_path(Acc, [])               -> filename:join(lists:reverse(Acc));
-reduce_path(Acc, ["."|Rest])       -> reduce_path(Acc, Rest);
-reduce_path([_|Acc], [".."|Rest])  -> reduce_path(Acc, Rest);
-reduce_path([], [".."|Rest])       -> reduce_path([], Rest);
-reduce_path(Acc, [Component|Rest]) -> reduce_path([Component|Acc], Rest).
-
-replace_src_dirs(State, Dirs) ->
-    %% replace any `src_dirs` with the test dirs
-    ErlOpts = rebar_state:get(State, erl_opts, []),
-    StrippedErlOpts = filter_src_dirs(ErlOpts),
-    State1 = rebar_state:set(State, erl_opts, StrippedErlOpts),
-    State2 = rebar_state:set(State1, src_dirs, []),
-    rebar_state:set(State2, extra_src_dirs, Dirs).
-
-filter_src_dirs(ErlOpts) ->
-    lists:filter(fun({src_dirs, _}) -> false; ({extra_src_dirs, _}) -> false; (_) -> true end, ErlOpts).
-
-test_dirs(State, Opts) ->
-    BareTest = filename:join([rebar_state:dir(State), "test"]),
-    F = fun(App) -> rebar_app_info:dir(App) == rebar_state:dir(State) end,
-    TestApps = project_apps(State),
-    case filelib:is_dir(BareTest) andalso not lists:any(F, TestApps) of
-        %% `test` dir at root of project is already scheduled to be
-        %%  included or `test` does not exist
-        false -> application_dirs(TestApps, Opts, []);
-        %% need to add `test` dir at root to dirs to be included
-        true  -> application_dirs(TestApps, Opts, [BareTest])
-    end.
-
-project_apps(State) ->
-    filter_checkouts(rebar_state:project_apps(State)).
-
-filter_checkouts(Apps) -> filter_checkouts(Apps, []).
-
-filter_checkouts([], Acc) -> lists:reverse(Acc);
-filter_checkouts([App|Rest], Acc) ->
-    case rebar_app_info:is_checkout(App) of
-        true  -> filter_checkouts(Rest, Acc);
-        false -> filter_checkouts(Rest, [App|Acc])
-    end.
-
-application_dirs([], Opts, []) -> Opts;
-application_dirs([], Opts, [Acc]) -> [{dir, Acc}|Opts];
-application_dirs([], Opts, Acc) -> [{dir, lists:reverse(Acc)}|Opts];
-application_dirs([App|Rest], Opts, Acc) ->
-    TestDir = filename:join([rebar_app_info:dir(App), "test"]),
-    case filelib:is_dir(TestDir) of
-        true  -> application_dirs(Rest, Opts, [TestDir|Acc]);
-        false -> application_dirs(Rest, Opts, Acc)
-    end.
-
-setup_logdir(State, Opts) ->
-    Logdir = case proplists:get_value(logdir, Opts) of
-        undefined -> filename:join([rebar_dir:base_dir(State), "logs"]);
-        Dir       -> Dir
-    end,
-    ensure_dir([Logdir]),
-    [{logdir, Logdir}|lists:keydelete(logdir, 1, Opts)].
+    rebar_prv_cover:maybe_cover_compile(State1).
 
-ensure_dir([]) -> ok;
-ensure_dir([Dir|Rest]) ->
-    case ec_file:is_dir(Dir) of
-        true ->
-            ok;
-        false ->
-            ec_file:mkdir_path(Dir)
-    end,
-    ensure_dir(Rest).
-
-maybe_cover_compile(State, Dir) ->
-    {Opts, _} = rebar_state:command_parsed_args(State),
-    State1 = case proplists:get_value(cover, Opts, false) of
+maybe_write_coverdata(State) ->
+    {RawOpts, _} = rebar_state:command_parsed_args(State),
+    State1 = case proplists:get_value(cover, RawOpts, false) of
         true  -> rebar_state:set(State, cover_enabled, true);
         false -> State
     end,
-    rebar_prv_cover:maybe_cover_compile(State1, [Dir]).
+    rebar_prv_cover:maybe_write_coverdata(State1, ?PROVIDER).
 
 ct_opts(_State) ->
     [{dir, undefined, "dir", string, help(dir)}, %% comma-seperated list
      {suite, undefined, "suite", string, help(suite)}, %% comma-seperated list
      {group, undefined, "group", string, help(group)}, %% comma-seperated list
      {testcase, undefined, "case", string, help(testcase)}, %% comma-seperated list
-     {spec, undefined, "spec", string, help(spec)}, %% comma-seperated list
-     {join_specs, undefined, "join_specs", boolean, help(join_specs)}, %% Boolean
      {label, undefined, "label", string, help(label)}, %% String
      {config, undefined, "config", string, help(config)}, %% comma-seperated list
-     {userconfig, undefined, "userconfig", string, help(userconfig)}, %% [{CallbackMod, CfgStrings}] | {CallbackMod, CfgStrings}
      {allow_user_terms, undefined, "allow_user_terms", boolean, help(allow_user_terms)}, %% Bool
      {logdir, undefined, "logdir", string, help(logdir)}, %% dir
-     {logopts, undefined, "logopts", string, help(logopts)}, %% enum, no_nl | no_src
-     {verbosity, undefined, "verbosity", string, help(verbosity)}, %% Integer OR [{Category, VLevel}]
-     {silent_connections, undefined, "silent_connections", string,
-      help(silent_connections)}, % all OR %% comma-seperated list
-     {stylesheet, undefined, "stylesheet", string, help(stylesheet)}, %% file
+     {logopts, undefined, "logopts", string, help(logopts)}, %% comma seperated list
+     {verbosity, undefined, "verbosity", integer, help(verbosity)}, %% Integer
      {cover, $c, "cover", {boolean, false}, help(cover)},
-     {cover_spec, undefined, "cover_spec", string, help(cover_spec)}, %% file
-     {cover_stop, undefined, "cover_stop", boolean, help(cover_stop)}, %% Boolean
-     {event_handler, undefined, "event_handler", string, help(event_handler)}, %% EH | [EH] WHERE EH atom() | {atom(), InitArgs} | {[atom()], InitArgs}
-     {include, undefined, "include", string, help(include)}, % comma-seperated list
-     {abort_if_missing_suites, undefined, "abort_if_missing_suites", {boolean, true},
-      help(abort_if_missing_suites)}, %% boolean
-     {multiply_timetraps, undefined, "multiply_timetraps", integer,
-      help(multiply_timetraps)}, %% integer
-     {scale_timetraps, undefined, "scale_timetraps", boolean, help(scale_timetraps)}, %% Boolean
-     {create_priv_dir, undefined, "create_priv_dir", string, help(create_priv_dir)}, %% enum: auto_per_run | auto_per_tc | manual_per_tc
      {repeat, undefined, "repeat", integer, help(repeat)}, %% integer
      {duration, undefined, "duration", string, help(duration)}, % format: HHMMSS
      {until, undefined, "until", string, help(until)}, %% format: YYMoMoDD[HHMMSS]
-     {force_stop, undefined, "force_stop", string, help(force_stop)}, % enum: skip_rest, bool
-     {basic_html, undefined, "basic_html", boolean, help(basic_html)}, %% Booloean
-     {ct_hooks, undefined, "ct_hooks", string, help(ct_hooks)}, %% List: [CTHModule | {CTHModule, CTHInitArgs}] where CTHModule is atom CthInitArgs is term
-     {auto_compile, undefined, "auto_compile", {boolean, false}, help(auto_compile)},
+     {force_stop, undefined, "force_stop", string, help(force_stop)}, %% String
+     {basic_html, undefined, "basic_html", boolean, help(basic_html)}, %% Boolean
+     {stylesheet, undefined, "stylesheet", string, help(stylesheet)}, %% String
+     {decrypt_key, undefined, "decrypt_key", string, help(decrypt_key)}, %% String
+     {decrypt_file, undefined, "decrypt_file", string, help(decrypt_file)}, %% String
+     {abort_if_missing_suites, undefined, "abort_if_missing_suites", {boolean, true}, help(abort_if_missing_suites)}, %% Boolean
+     {multiply_timetraps, undefined, "multiply_timetraps", integer, help(multiple_timetraps)}, %% Integer
+     {scale_timetraps, undefined, "scale_timetraps", boolean, help(scale_timetraps)},
+     {create_priv_dir, undefined, "create_priv_dir", string, help(create_priv_dir)},
      {verbose, $v, "verbose", boolean, help(verbose)}
     ].
 
@@ -448,28 +500,20 @@ help(group) ->
     "List of test groups to run";
 help(testcase) ->
     "List of test cases to run";
-help(spec) ->
-    "List of test specs to run";
 help(label) ->
     "Test label";
 help(config) ->
     "List of config files";
+help(allow_user_terms) ->
+    "Allow user defined config values in config files";
 help(logdir) ->
     "Log folder";
+help(logopts) ->
+    "Options for common test logging";
 help(verbosity) ->
     "Verbosity";
-help(stylesheet) ->
-    "Stylesheet to use for test results";
 help(cover) ->
     "Generate cover data";
-help(cover_spec) ->
-    "Cover file to use";
-help(event_handler) ->
-    "Event handlers to attach to the runner";
-help(include) ->
-    "Include folder";
-help(abort_if_missing_suites) ->
-    "Abort if suites are missing";
 help(repeat) ->
     "How often to repeat tests";
 help(duration) ->
@@ -477,85 +521,25 @@ help(duration) ->
 help(until) ->
     "Run until (format: HHMMSS)";
 help(force_stop) ->
-    "Force stop after time";
+    "Force stop on test timeout (true | false | skip_rest)";
 help(basic_html) ->
     "Show basic HTML";
+help(stylesheet) ->
+    "CSS stylesheet to apply to html output";
+help(decrypt_key) ->
+    "Path to key for decrypting config";
+help(decrypt_file) ->
+    "Path to file containing key for decrypting config";
+help(abort_if_missing_suites) ->
+    "Abort if suites are missing";
+help(multiply_timetraps) ->
+    "Multiply timetraps";
+help(scale_timetraps) ->
+    "Scale timetraps";
+help(create_priv_dir) ->
+    "Create priv dir (auto_per_run | auto_per_tc | manual_per_tc)";
 help(verbose) ->
     "Verbose output";
 help(_) ->
     "".
 
-transform_opts(Opts) ->
-    transform_opts(Opts, []).
-
-transform_opts([], Acc) -> Acc;
-%% drop `cover` and `verbose` so they're not passed as an option to common_test
-transform_opts([{cover, _}|Rest], Acc) ->
-    transform_opts(Rest, Acc);
-transform_opts([{cover_spec, CoverSpec}|Rest], Acc) ->
-    transform_opts(Rest, [{cover, CoverSpec}|Acc]);
-transform_opts([{verbose, _}|Rest], Acc) ->
-    transform_opts(Rest, Acc);
-transform_opts([{ct_hooks, CtHooks}|Rest], Acc) ->
-    transform_opts(Rest, [{ct_hooks, parse_term(CtHooks)}|Acc]);
-transform_opts([{force_stop, "skip_rest"}|Rest], Acc) ->
-    transform_opts(Rest, [{force_stop, skip_rest}|Acc]);
-transform_opts([{force_stop, _}|Rest], Acc) ->
-    transform_opts(Rest, [{force_stop, true}|Acc]);
-transform_opts([{repeat, Repeat}|Rest], Acc) ->
-    transform_opts(Rest, [{repeat,
-                           ec_cnv:to_integer(Repeat)}|Acc]);
-transform_opts([{create_priv_dir, CreatePrivDir}|Rest], Acc) ->
-    transform_opts(Rest, [{create_priv_dir,
-                           to_atoms(CreatePrivDir)}|Acc]);
-transform_opts([{multiply_timetraps, MultiplyTimetraps}|Rest], Acc) ->
-    transform_opts(Rest, [{multiply_timetraps,
-                           ec_cnv:to_integer(MultiplyTimetraps)}|Acc]);
-transform_opts([{event_handler, EventHandler}|Rest], Acc) ->
-    transform_opts(Rest, [{event_handler, parse_term(EventHandler)}|Acc]);
-transform_opts([{silent_connections, "all"}|Rest], Acc) ->
-    transform_opts(Rest, [{silent_connections, all}|Acc]);
-transform_opts([{silent_connections, SilentConnections}|Rest], Acc) ->
-    transform_opts(Rest, [{silent_connections,
-                           to_atoms(split_string(SilentConnections))}|Acc]);
-transform_opts([{verbosity, Verbosity}|Rest], Acc) ->
-    transform_opts(Rest, [{verbosity, parse_term(Verbosity)}|Acc]);
-transform_opts([{logopts, LogOpts}|Rest], Acc) ->
-    transform_opts(Rest, [{logopts, to_atoms(split_string(LogOpts))}|Acc]);
-transform_opts([{userconfig, UserConfig}|Rest], Acc) ->
-    transform_opts(Rest, [{userconfig, parse_term(UserConfig)}|Acc]);
-transform_opts([{testcase, Testcase}|Rest], Acc) ->
-    transform_opts(Rest, [{testcase, to_atoms(split_string(Testcase))}|Acc]);
-transform_opts([{group, Group}|Rest], Acc) -> % @TODO handle ""
-    % Input is a list or an atom. It can also be a nested list.
-    transform_opts(Rest, [{group, parse_term(Group)}|Acc]);
-transform_opts([{suite, Suite}|Rest], Acc) ->
-    transform_opts(Rest, [{suite, split_string(Suite)}|Acc]);
-transform_opts([{Key, Val}|Rest], Acc) when is_list(Val) ->
-    % Default to splitting a string on comma, that works fine for both flat
-    % lists of which there are many and single-items.
-    Val1 = case split_string(Val) of
-               [Val2] ->
-                   Val2;
-               Val2 ->
-                   Val2
-           end,
-    transform_opts(Rest, [{Key, Val1}|Acc]);
-transform_opts([{Key, Val}|Rest], Acc) ->
-    transform_opts(Rest, [{Key, Val}|Acc]).
-
-to_atoms(List) ->
-    lists:map(fun(X) -> list_to_atom(X) end, List).
-
-split_string(String) ->
-    string:tokens(String, ",").
-
-parse_term(String) ->
-    String1 = "[" ++ String ++ "].",
-    {ok, Tokens, _} = erl_scan:string(String1),
-    case erl_parse:parse_term(Tokens) of
-        {ok, [Terms]} ->
-            Terms;
-        Term ->
-            Term
-    end.