Merge pull request #88 from talentdeficit/test_providers

test providers

5 files changed, 375 insertions, 575 deletions

diff --git a/src/rebar.app.src b/src/rebar.app.src
index 1a1e528..73c1eff 100644
--- a/src/rebar.app.src
+++ b/src/rebar.app.src
@@ -13,6 +13,8 @@
                   crypto,
                   syntax_tools,
                   tools,
+                  eunit,
+                  common_test,
                   erlware_commons,
                   providers,
                   relx,
diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl
index 4576d05..7a52c92 100644
--- a/src/rebar_prv_common_test.erl
+++ b/src/rebar_prv_common_test.erl
@@ -25,19 +25,39 @@ init(State) ->
                                  {deps, ?DEPS},
                                  {bare, false},
                                  {example, "rebar ct"},
-                                 {short_desc, "Run Common Tests"},
+                                 {short_desc, "Run Common Tests."},
                                  {desc, ""},
                                  {opts, ct_opts(State)},
                                  {profile, test}]),
     State1 = rebar_state:add_provider(State, Provider),
     {ok, State1}.
 
--spec do(rebar_state:t()) -> {ok, rebar_state:t()}.
+-spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
 do(State) ->
-    {Opts, _} = rebar_state:command_parsed_args(State),
-    Opts1 = transform_opts(Opts),
-    ok = create_dirs(Opts1),
-    case handle_results(ct:run_test(Opts1)) of
+    ?INFO("Running Common Test suites...", []),
+    {RawOpts, _} = rebar_state:command_parsed_args(State),
+    {InDirs, OutDir} = split_ct_dirs(State, RawOpts),
+    Opts = transform_opts(RawOpts),
+    TestApps = filter_checkouts(rebar_state:project_apps(State)),
+    ok = create_dirs(Opts),
+    ?DEBUG("Compiling Common Test suites in: ~p", [OutDir]),
+    lists:foreach(fun(App) ->
+                      AppDir = rebar_app_info:dir(App),
+                      C = rebar_config:consult(AppDir),
+                      S = rebar_state:new(State, C, AppDir),
+                      %% combine `erl_first_files` and `common_test_first_files` and
+                      %% adjust compile opts to include `common_test_compile_opts`
+                      %% and `{src_dirs, "test"}`
+                      TestState = test_state(S, InDirs, OutDir),
+                      ok = rebar_erlc_compiler:compile(TestState, AppDir)
+                  end, TestApps),
+    ok = maybe_compile_extra_tests(TestApps, State, InDirs, OutDir),
+    Path = code:get_path(),
+    true = code:add_patha(OutDir),
+    CTOpts = resolve_ct_opts(State, Opts, OutDir),
+    Result = handle_results(ct:run_test(CTOpts)),
+    true = code:set_path(Path),
+    case Result of
         {error, Reason} ->
             {error, {?MODULE, Reason}};
         ok ->
@@ -51,9 +71,9 @@ format_error({error_running_tests, Reason}) ->
     io_lib:format("Error running tests: ~p", [Reason]).
 
 ct_opts(State) ->
-    DefaultTestDir = filename:join([rebar_state:dir(State), "test"]),
     DefaultLogsDir = filename:join([rebar_state:dir(State), "logs"]),
-    [{dir, undefined, "dir", {string, DefaultTestDir}, help(dir)}, %% dir
+    [{dir, undefined, "dir", string, help(dir)}, %% comma-seperated list
+     {outdir, undefined, "outdir", string, help(outdir)}, %% string
      {suite, undefined, "suite", string, help(suite)}, %% comma-seperated list
      {group, undefined, "group", string, help(group)}, %% comma-seperated list
      {testcase, undefined, "case", string, help(testcase)}, %% comma-seperated list
@@ -63,7 +83,7 @@ ct_opts(State) ->
      {config, undefined, "config", string, help(config)}, %% comma-seperated list
      {userconfig, undefined, "userconfig", string, help(userconfig)}, %% [{CallbackMod, CfgStrings}] | {CallbackMod, CfgStrings}
      {allow_user_terms, undefined, "allow_user_terms", boolean, help(allow_user_terms)}, %% Bool
-     {logdir, undefined, "logdir", {string, DefaultLogsDir}, help(logdir)}, %% string
+     {logdir, undefined, "logdir", {string, DefaultLogsDir}, help(logdir)}, %% dir
      {logopts, undefined, "logopts", string, help(logopts)}, %% enum, no_nl | no_src
      {verbosity, undefined, "verbosity", string, help(verbosity)}, %% Integer OR [{Category, VLevel}]
      {silent_connections, undefined, "silent_connections", string,
@@ -87,10 +107,91 @@ ct_opts(State) ->
      {ct_hooks, undefined, "ct_hooks", string, help(ct_hooks)} %% List: [CTHModule | {CTHModule, CTHInitArgs}] where CTHModule is atom CthInitArgs is term
     ].
 
+help(outdir) ->
+    "Output directory for compiled modules";
+help(dir) ->
+    "List of additional directories containing test suites";
+help(suite) ->
+    "List of test suites to run";
+help(group) ->
+    "List of test groups to run";
+help(testcase) ->
+    "List of test cases to run";
+help(spec) ->
+    "List of test specs to run";
+help(join_specs) ->
+    ""; %% ??
+help(label) ->
+    "Test label";
+help(config) ->
+    "List of config files";
+help(allow_user_terms) ->
+    ""; %% ??
+help(logdir) ->
+    "Log folder";
+help(logopts) ->
+    ""; %% ??
+help(verbosity) ->
+    "Verbosity";
+help(silent_connections) ->
+    ""; %% ??
+help(stylesheet) ->
+    "Stylesheet to use for test results";
+help(cover) ->
+    "Cover file to use";
+help(cover_stop) ->
+    ""; %% ??
+help(event_handler) ->
+    "Event handlers to attach to the runner";
+help(include) ->
+    "Include folder";
+help(abort_if_missing_suites) ->
+    "Abort if suites are missing";
+help(multiply_timetraps) ->
+    ""; %% ??
+help(scale_timetraps) ->
+    ""; %% ??
+help(create_priv_dir) ->
+    ""; %% ??
+help(repeat) ->
+    "How often to repeat tests";
+help(duration) ->
+    "Max runtime (format: HHMMSS)";
+help(until) ->
+    "Run until (format: HHMMSS)";
+help(force_stop) ->
+    "Force stop after time";
+help(basic_html) ->
+    "Show basic HTML";
+help(ct_hooks) ->
+    "";
+help(userconfig) ->
+    "".
+
+split_ct_dirs(State, RawOpts) ->
+    %% preserve the override nature of command line opts by only checking
+    %% `rebar.config` defined additional test dirs if none are defined via
+    %% command line flag
+    InDirs = case proplists:get_value(dir, RawOpts) of
+        undefined ->
+            CTOpts = rebar_state:get(State, common_test_opts, []),
+            proplists:get_value(dir, CTOpts, []);
+        Dirs -> split_string(Dirs)
+    end,
+    OutDir = case proplists:get_value(outdir, RawOpts) of
+        undefined -> filename:join([rebar_state:dir(State),
+                                    ec_file:insecure_mkdtemp()]);
+        Out -> Out
+    end,
+    {InDirs, OutDir}.
+
 transform_opts(Opts) ->
     transform_opts(Opts, []).
 
 transform_opts([], Acc) -> Acc;
+%% drop `outdir` so it's not passed to common_test
+transform_opts([{outdir, _}|Rest], Acc) ->
+    transform_opts(Rest, Acc);
 transform_opts([{ct_hooks, CtHooks}|Rest], Acc) ->
     transform_opts(Rest, [{ct_hooks, parse_term(CtHooks)}|Acc]);
 transform_opts([{force_stop, "skip_rest"}|Rest], Acc) ->
@@ -153,89 +254,78 @@ parse_term(String) ->
             Term
     end.
 
+filter_checkouts(Apps) -> filter_checkouts(Apps, []).
+
+filter_checkouts([], Acc) -> lists:reverse(Acc);
+filter_checkouts([App|Rest], Acc) ->
+    AppDir = filename:absname(rebar_app_info:dir(App)),
+    CheckoutsDir = filename:absname("_checkouts"),
+    case lists:prefix(CheckoutsDir, AppDir) of
+        true -> filter_checkouts(Rest, Acc);
+        false -> filter_checkouts(Rest, [App|Acc])
+    end.
+
 create_dirs(Opts) ->
     LogDir = proplists:get_value(logdir, Opts),
-    TestDir = proplists:get_value(dir, Opts),
-    ensure_logdir(LogDir),
-    ensure_testdir(TestDir),
+    ensure_dir([LogDir]),
     ok.
 
-ensure_logdir(Logdir) ->
-    case ec_file:is_dir(Logdir) of
+ensure_dir([]) -> ok;
+ensure_dir([Dir|Rest]) ->
+    case ec_file:is_dir(Dir) of
         true ->
             ok;
         false ->
-            ec_file:mkdir_path(Logdir)
+            ec_file:mkdir_path(Dir)
+    end,
+    ensure_dir(Rest).
+
+test_state(State, InDirs, OutDir) ->
+    ErlOpts = rebar_state:get(State, common_test_compile_opts, []) ++
+              rebar_utils:erl_opts(State),
+    TestOpts = [{outdir, OutDir}] ++
+               add_test_dir(ErlOpts, InDirs),
+    first_files(rebar_state:set(State, erl_opts, TestOpts)).
+
+add_test_dir(Opts, InDirs) ->
+    %% if no src_dirs are set we have to specify `src` or it won't
+    %% be built
+    case proplists:append_values(src_dirs, Opts) of
+        [] -> [{src_dirs, ["src", "test"|InDirs]}];
+        _ -> [{src_dirs, ["test"|InDirs]}]
     end.
 
-ensure_testdir(Testdir) ->
-    case ec_file:is_dir(Testdir) of
-        false ->
-            ?INFO("Test directory ~s does not exist:\n",
-                  [Testdir]),
-            ?FAIL;
+first_files(State) ->
+    BaseFirst = rebar_state:get(State, erl_first_files, []),
+    CTFirst = rebar_state:get(State, common_test_first_files, []),
+    rebar_state:set(State, erl_first_modules, BaseFirst ++ CTFirst).
+
+resolve_ct_opts(State, CmdLineOpts, OutDir) ->
+    CTOpts = rebar_state:get(State, common_test_opts, []),
+    Opts = lists:ukeymerge(1,
+                    lists:ukeysort(1, CmdLineOpts),
+                    lists:ukeysort(1, CTOpts)),
+    %% rebar has seperate input and output directories whereas `common_test`
+    %% uses only a single directory so set `dir` to our precompiled `OutDir`
+    %% and disable `auto_compile`
+    [{auto_compile, false}, {dir, OutDir}] ++ lists:keydelete(dir, 1, Opts).
+
+maybe_compile_extra_tests(TestApps, State, InDirs, OutDir) ->
+    F = fun(App) -> rebar_app_info:dir(App) == rebar_dir:get_cwd() end,
+    case lists:filter(F, TestApps) of
+        %% compile just the `test` and extra test directories of the base dir
+        [] ->
+            ErlOpts = rebar_state:get(State, common_test_compile_opts, []) ++
+                      rebar_utils:erl_opts(State),
+            TestOpts = [{outdir, OutDir}] ++
+                       [{src_dirs, ["test"|InDirs]}] ++
+                       lists:keydelete(src_dirs, 1, ErlOpts),
+            TestState = first_files(rebar_state:set(State, erl_opts, TestOpts)),
+            rebar_erlc_compiler:compile(TestState, rebar_dir:get_cwd());
+        %% already compiled `./test` so do nothing
         _ -> ok
     end.
 
-help(dir) ->
-    "Test folder (default: test/)";
-help(suite) ->
-    "List of test suites to run";
-help(group) ->
-    "List of test groups to run";
-help(testcase) ->
-    "List of test cases to run";
-help(spec) ->
-    "List of test specs to run";
-help(join_specs) ->
-    ""; %% ??
-help(label) ->
-    "Test label";
-help(config) ->
-    "List of config files";
-help(allow_user_terms) ->
-    ""; %% ??
-help(logdir) ->
-    "Log folder";
-help(logopts) ->
-    ""; %% ??
-help(verbosity) ->
-    "Verbosity";
-help(silent_connections) ->
-    ""; %% ??
-help(stylesheet) ->
-    "Stylesheet to use for test results";
-help(cover) ->
-    "Cover file to use";
-help(cover_stop) ->
-    ""; %% ??
-help(event_handler) ->
-    "Event handlers to attach to the runner";
-help(include) ->
-    "Include folder";
-help(abort_if_missing_suites) ->
-    "Abort if suites are missing";
-help(multiply_timetraps) ->
-    ""; %% ??
-help(scale_timetraps) ->
-    ""; %% ??
-help(create_priv_dir) ->
-    ""; %% ??
-help(repeat) ->
-    "How often to repeat tests";
-help(duration) ->
-    "Max runtime (format: HHMMSS)";
-help(until) ->
-    "Run until (format: HHMMSS)";
-help(force_stop) ->
-    "Force stop after time";
-help(basic_html) ->
-    "Show basic HTML";
-help(ct_hooks) ->
-    "";
-help(userconfig) ->
-    "".
-
 handle_results([Result]) ->
     handle_results(Result);
 handle_results([Result|Results]) when is_list(Results) ->
diff --git a/src/rebar_prv_eunit.erl b/src/rebar_prv_eunit.erl
index c14a15d..ee8a235 100644
--- a/src/rebar_prv_eunit.erl
+++ b/src/rebar_prv_eunit.erl
@@ -12,9 +12,7 @@
 -include("rebar.hrl").
 
 -define(PROVIDER, eunit).
-
 -define(DEPS, [compile]).
--define(EUNIT_DIR, ".eunit").
 
 %% ===================================================================
 %% Public API
@@ -22,507 +20,157 @@
 
 -spec init(rebar_state:t()) -> {ok, rebar_state:t()}.
 init(State) ->
-    State1 = rebar_state:add_provider(State, providers:create([{name, ?PROVIDER},
-                                                               {module, ?MODULE},
-                                                               {bare, false},
-                                                               {deps, ?DEPS},
-                                                               {example, "rebar eunit"},
-                                                               {short_desc, "Run eunit tests on project apps."},
-                                                               {desc, ""},
-                                                               {opts, []}])),
-
+    Provider = providers:create([{name, ?PROVIDER},
+                                 {module, ?MODULE},
+                                 {deps, ?DEPS},
+                                 {bare, false},
+                                 {example, "rebar eunit"},
+                                 {short_desc, "Run EUnit Tests."},
+                                 {desc, ""},
+                                 {opts, eunit_opts(State)},
+                                 {profile, test}]),
+    State1 = rebar_state:add_provider(State, Provider),
     {ok, State1}.
 
 -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
 do(State) ->
-    ok = ensure_dirs(),
-    %% Save code path
-    CodePath = setup_code_path(),
-    CompileOnly = rebar_state:get(State, compile_only, false),
-    {ok, SrcErls} = rebar_erlc_compiler:test_compile(State, "eunit",
-                                                     ?EUNIT_DIR),
-    case CompileOnly of
-        "true" ->
-            true = rebar_utils:cleanup_code_path(CodePath),
-            ?CONSOLE("Compiled modules for eunit~n", []);
-        false ->
-            run_eunit(State, CodePath, SrcErls)
+    ?INFO("Performing EUnit tests...", []),
+    {RawOpts, _} = rebar_state:command_parsed_args(State),
+    Opts = transform_opts(RawOpts, State),
+    TestApps = filter_checkouts(rebar_state:project_apps(State)),
+    OutDir = case proplists:get_value(outdir, Opts, undefined) of
+        undefined -> filename:join([rebar_state:dir(State),
+                     ec_file:insecure_mkdtemp()]);
+        Out -> Out
     end,
-    {ok, State}.
+    ?DEBUG("Compiling EUnit instrumented modules in: ~p", [OutDir]),
+    lists:foreach(fun(App) ->
+                      AppDir = rebar_app_info:dir(App),
+                      C = rebar_config:consult(AppDir),
+                      S = rebar_state:new(State, C, AppDir),
+                      %% combine `erl_first_files` and `eunit_first_files` and adjust
+                      %% compile opts to include `eunit_compile_opts`, `{d, 'TEST'}`
+                      %% and `{src_dirs, "test"}`
+                      TestState = first_files(test_state(S, OutDir)),
+                      ok = rebar_erlc_compiler:compile(TestState, AppDir)
+                  end, TestApps),
+    ok = maybe_compile_extra_tests(TestApps, State, OutDir),
+    Path = code:get_path(),
+    true = code:add_patha(OutDir),
+    EUnitOpts = resolve_eunit_opts(State, Opts),
+    AppsToTest = [{application, erlang:binary_to_atom(rebar_app_info:name(App), unicode)}
+                  || App <- TestApps],
+    Result = eunit:test(AppsToTest, EUnitOpts),
+    true = code:set_path(Path),
+    case handle_results(Result) of
+        {error, Reason} ->
+            {error, {?MODULE, Reason}};
+        ok ->
+            {ok, State}
+    end.
 
 -spec format_error(any()) -> iolist().
-format_error(Reason) ->
-    io_lib:format("~p", [Reason]).
+format_error(unknown_error) ->
+    io_lib:format("Error running tests", []);
+format_error({error_running_tests, Reason}) ->
+    io_lib:format("Error running tests: ~p", [Reason]).
 
-run_eunit(Config, CodePath, SrcErls) ->
-    %% Build a list of all the .beams in ?EUNIT_DIR -- use this for
-    %% cover and eunit testing. Normally you can just tell cover
-    %% and/or eunit to scan the directory for you, but eunit does a
-    %% code:purge in conjunction with that scan and causes any cover
-    %% compilation info to be lost.
-    AllBeamFiles = rebar_utils:beams(?EUNIT_DIR),
-    {BeamFiles, TestBeamFiles} =
-        lists:partition(fun(N) -> string:str(N, "_tests.beam") =:= 0 end,
-                        AllBeamFiles),
-    OtherBeamFiles = TestBeamFiles --
-        [filename:rootname(N) ++ "_tests.beam" || N <- AllBeamFiles],
-    ModuleBeamFiles = randomize_suites(Config, BeamFiles ++ OtherBeamFiles),
-    %% Get matching tests and modules
-    AllModules = [rebar_utils:beam_to_mod(?EUNIT_DIR, N) || N <- AllBeamFiles],
-    {Tests, FilteredModules} =
-        get_tests_and_modules(Config, ModuleBeamFiles, AllModules),
-    SrcModules = [rebar_utils:erl_to_mod(M) || M <- SrcErls],
-    {ok, CoverLog} = rebar_cover_utils:init(Config, ModuleBeamFiles,
-                                            eunit_dir()),
-    StatusBefore = status_before_eunit(),
-    EunitResult = perform_eunit(Config, Tests),
-    rebar_cover_utils:perform_cover(Config, FilteredModules, SrcModules,
-                                    eunit_dir()),
-    rebar_cover_utils:close(CoverLog),
-    case proplists:get_value(reset_after_eunit, get_eunit_opts(Config),
-                             true) of
-        true ->
-            reset_after_eunit(StatusBefore);
-        false ->
-            ok
-    end,
-    %% Stop cover to clean the cover_server state. This is important if we want
-    %% eunit+cover to not slow down when analyzing many Erlang modules.
-    ok = rebar_cover_utils:exit(),
-    case EunitResult of
-        ok ->
-            ok;
-        _ ->
-            ?ABORT("One or more eunit tests failed.", [])
+eunit_opts(_State) ->
+    [{outdir, $o, "outdir", string, help(outdir)},
+     {verbose, $v, "verbose", boolean, help(verbose)}].
+
+help(outdir) -> "Output directory for EUnit compiled modules";
+help(verbose) -> "Verbose output".
+
+transform_opts(Opts, State) -> transform_opts(Opts, State, []).
+
+transform_opts([], _State, Acc) -> Acc;
+transform_opts([{outdir, Path}|Rest], State, Acc) ->
+    NewAcc = case filename:pathtype(Path) of
+        absolute -> [{outdir, Path}] ++ Acc;
+        _ -> [{outdir, filename:join([rebar_state:dir(State), Path])}] ++ Acc
     end,
-    %% Restore code path
-    true = rebar_utils:cleanup_code_path(CodePath),
-    ok.
-ensure_dirs() ->
-    %% Make sure ?EUNIT_DIR/ and ebin/ directory exists (append dummy module)
-    ok = filelib:ensure_dir(filename:join(eunit_dir(), "dummy")),
-    ok = filelib:ensure_dir(filename:join(rebar_dir:ebin_dir(), "dummy")).
-eunit_dir() ->
-    filename:join(rebar_dir:get_cwd(), ?EUNIT_DIR).
-setup_code_path() ->
-    %% Setup code path prior to compilation so that parse_transforms
-    %% and the like work properly. Also, be sure to add ebin_dir()
-    %% to the END of the code path so that we don't have to jump
-    %% through hoops to access the .app file
-    CodePath = code:get_path(),
-    true = code:add_patha(eunit_dir()),
-    true = code:add_pathz(rebar_dir:ebin_dir()),
-    CodePath.
-%%
-%% == get matching tests ==
-%%
-get_tests_and_modules(Config, ModuleBeamFiles, AllModules) ->
-    SelectedSuites = get_selected_suites(Config, AllModules),
-    {Tests, QualifiedTests} = get_qualified_and_unqualified_tests(Config),
-    Modules = get_test_modules(SelectedSuites, Tests,
-                               QualifiedTests, ModuleBeamFiles),
-    FilteredModules = get_matching_modules(AllModules, Modules, QualifiedTests),
-    MatchedTests = get_matching_tests(Modules, Tests, QualifiedTests),
-    {MatchedTests, FilteredModules}.
-%%
-%% == get suites specified via 'suites' option ==
-%%
-get_selected_suites(Config, Modules) ->
-    RawSuites = get_suites(Config),
-    Suites = [list_to_atom(Suite) || Suite <- string:tokens(RawSuites, ",")],
-    [M || M <- Suites, lists:member(M, Modules)].
-get_suites(Config) ->
-    case rebar_state:get(Config, suites, "") of
-        "" ->
-            rebar_state:get(Config, suite, "");
-        Suites ->
-            Suites
-    end.
-get_qualified_and_unqualified_tests(Config) ->
-    RawFunctions = rebar_state:get(Config, tests, ""),
-    FunctionNames = [FunctionName ||
-                        FunctionName <- string:tokens(RawFunctions, ",")],
-    get_qualified_and_unqualified_tests1(FunctionNames, [], []).
-get_qualified_and_unqualified_tests1([], Functions, QualifiedFunctions) ->
-    {Functions, QualifiedFunctions};
-get_qualified_and_unqualified_tests1([TestName|TestNames], Functions,
-                                     QualifiedFunctions) ->
-    case string:tokens(TestName, ":") of
-        [TestName] ->
-            Function = list_to_atom(TestName),
-            get_qualified_and_unqualified_tests1(
-              TestNames, [Function|Functions], QualifiedFunctions);
-        [ModuleName, FunctionName] ->
-            M = list_to_atom(ModuleName),
-            F = list_to_atom(FunctionName),
-            get_qualified_and_unqualified_tests1(TestNames, Functions,
-                                                 [{M, F}|QualifiedFunctions]);
-        _ ->
-            ?ABORT("Unsupported test function specification: ~s", [TestName])
-    end.
-%% Provide modules which are to be searched for tests.
-%% Several scenarios are possible:
-%%
-%% == randomize suites ==
-%%
-randomize_suites(Config, Modules) ->
-    case rebar_state:get(Config, random_suite_order, undefined) of
-        undefined ->
-            Modules;
-        "true" ->
-            Seed = crypto:rand_uniform(1, 65535),
-            randomize_suites1(Modules, Seed);
-        String ->
-            try list_to_integer(String) of
-                Seed ->
-                    randomize_suites1(Modules, Seed)
-            catch
-                error:badarg ->
-                    ?ERROR("Bad random seed provided: ~p", [String]),
-                    ?FAIL
-            end
-    end.
-randomize_suites1(Modules, Seed) ->
-    _ = random:seed(35, Seed, 1337),
-    ?CONSOLE("Randomizing suite order with seed ~b~n", [Seed]),
-    [X||{_,X} <- lists:sort([{random:uniform(), M} || M <- Modules])].
-%%
-%% == get matching tests ==
-%% 1) Specific tests have been provided and/or
-%% no unqualified tests have been specified and
-%% there were some qualified tests, then we can search for
-%% functions in specified suites (or in empty set of suites).
-%%
-%% 2) Neither specific suites nor qualified test names have been
-%% provided use ModuleBeamFiles which filters out "*_tests"
-%% modules so EUnit won't doubly run them and cover only
-%% calculates coverage on production code. However,
-%% keep "*_tests" modules that are not automatically
-%% included by EUnit.
-%%
-%% From 'Primitives' in the EUnit User's Guide
-%% http://www.erlang.org/doc/apps/eunit/chapter.html
-%% "In addition, EUnit will also look for another
-%% module whose name is ModuleName plus the suffix
-%% _tests, and if it exists, all the tests from that
-%% module will also be added. (If ModuleName already
-%% contains the suffix _tests, this is not done.) E.g.,
-%% the specification {module, mymodule} will run all
-%% tests in the modules mymodule and mymodule_tests.
-%% Typically, the _tests module should only contain
-%% test cases that use the public interface of the main
-%% module (and no other code)."
-get_test_modules(SelectedSuites, Tests, QualifiedTests, ModuleBeamFiles) ->
-    SuitesProvided = SelectedSuites =/= [],
-    OnlyQualifiedTestsProvided = QualifiedTests =/= [] andalso Tests =:= [],
-    if
-        SuitesProvided orelse OnlyQualifiedTestsProvided ->
-            SelectedSuites;
-        true ->
-            [rebar_utils:beam_to_mod(?EUNIT_DIR, N) ||
-                N <- ModuleBeamFiles]
+    transform_opts(Rest, State, NewAcc);
+transform_opts([{Key, Val}|Rest], State, Acc) ->
+    transform_opts(Rest, State, [{Key, Val}|Acc]).
+
+filter_checkouts(Apps) -> filter_checkouts(Apps, []).
+
+filter_checkouts([], Acc) -> lists:reverse(Acc);
+filter_checkouts([App|Rest], Acc) ->
+    AppDir = filename:absname(rebar_app_info:dir(App)),
+    CheckoutsDir = filename:absname("_checkouts"),
+    case lists:prefix(CheckoutsDir, AppDir) of
+        true -> filter_checkouts(Rest, Acc);
+        false -> filter_checkouts(Rest, [App|Acc])
     end.
-get_matching_modules(AllModules, Modules, QualifiedTests) ->
-    ModuleFilterMapper =
-        fun({M, _}) ->
-                case lists:member(M, AllModules) of
-                    true -> {true, M};
-                    _-> false
-                end
-        end,
-    ModulesFromQualifiedTests = lists:zf(ModuleFilterMapper, QualifiedTests),
-    lists:usort(Modules ++ ModulesFromQualifiedTests).
-get_matching_tests(Modules, [], []) ->
-    Modules;
-get_matching_tests(Modules, [], QualifiedTests) ->
-    FilteredQualifiedTests = filter_qualified_tests(Modules, QualifiedTests),
-    lists:merge(Modules, make_test_primitives(FilteredQualifiedTests));
-get_matching_tests(Modules, Tests, QualifiedTests) ->
-    AllTests = lists:merge(QualifiedTests,
-                           get_matching_tests1(Modules, Tests, [])),
-    make_test_primitives(AllTests).
-filter_qualified_tests(Modules, QualifiedTests) ->
-    TestsFilter = fun({Module, _Function}) ->
-                          lists:all(fun(M) -> M =/= Module end, Modules) end,
-    lists:filter(TestsFilter, QualifiedTests).
-get_matching_tests1([], _Functions, TestFunctions) ->
-    TestFunctions;
-get_matching_tests1([Module|TModules], Functions, TestFunctions) ->
-    %% Get module exports
-    ModuleStr = atom_to_list(Module),
-    ModuleExports = get_beam_test_exports(ModuleStr),
-    %% Get module _tests exports
-    TestModuleStr = string:concat(ModuleStr, "_tests"),
-    TestModuleExports = get_beam_test_exports(TestModuleStr),
-    %% Build tests {M, F} list
-    Tests = get_matching_tests2(Functions, {Module, ModuleExports},
-                                {list_to_atom(TestModuleStr),
-                                 TestModuleExports}),
-    get_matching_tests1(TModules, Functions,
-                        lists:merge([TestFunctions, Tests])).
-get_matching_tests2(Functions, {Mod, ModExports}, {TestMod, TestModExports}) ->
-    %% Look for matching functions into ModExports
-    ModExportsStr = [atom_to_list(E1) || E1 <- ModExports],
-    TestModExportsStr = [atom_to_list(E2) || E2 <- TestModExports],
-    get_matching_exports(Functions, {Mod, ModExportsStr},
-                         {TestMod, TestModExportsStr}, []).
-get_matching_exports([], _, _, Matched) ->
-    Matched;
-get_matching_exports([Function|TFunctions], {Mod, ModExportsStr},
-                     {TestMod, TestModExportsStr}, Matched) ->
-    FunctionStr = atom_to_list(Function),
-    %% Get matching Function in module, otherwise look in _tests module
-    NewMatch = case get_matching_export(FunctionStr, ModExportsStr) of
-                   [] ->
-                       {TestMod, get_matching_export(FunctionStr,
-                                                     TestModExportsStr)};
-                   MatchingExport ->
-                       {Mod, MatchingExport}
-               end,
-    case NewMatch of
-        {_, []} ->
-            get_matching_exports(TFunctions, {Mod, ModExportsStr},
-                                 {TestMod, TestModExportsStr}, Matched);
-        _ ->
-            get_matching_exports(TFunctions, {Mod, ModExportsStr},
-                                 {TestMod, TestModExportsStr},
-                                 [NewMatch|Matched])
+
+test_state(State, TmpDir) ->
+    ErlOpts = rebar_state:get(State, eunit_compile_opts, []) ++
+              rebar_utils:erl_opts(State),
+    ErlOpts1 = [{outdir, TmpDir}] ++
+               add_test_dir(ErlOpts),
+    TestOpts = safe_define_test_macro(ErlOpts1),
+    rebar_state:set(State, erl_opts, TestOpts).
+
+add_test_dir(Opts) ->
+    %% if no src_dirs are set we have to specify `src` or it won't
+    %% be built
+    case proplists:append_values(src_dirs, Opts) of
+        [] -> [{src_dirs, ["src", "test"]}];
+        Srcs -> [{src_dirs, ["test"|Srcs]}]
+    end ++ lists:keydelete(src_dirs, 1, Opts).
+
+safe_define_test_macro(Opts) ->
+    %% defining a compile macro twice results in an exception so
+    %% make sure 'TEST' is only defined once
+    case test_defined(Opts) of
+       true -> Opts;
+       false -> [{d, 'TEST'}] ++ Opts
     end.
-get_matching_export(_FunctionStr, []) ->
-    [];
-get_matching_export(FunctionStr, [ExportStr|TExportsStr]) ->
-    case string:str(ExportStr, FunctionStr) of
-        1 ->
-            list_to_atom(ExportStr);
-        _ ->
-            get_matching_export(FunctionStr, TExportsStr)
+
+test_defined([{d, 'TEST'}|_]) -> true;
+test_defined([{d, 'TEST', true}|_]) -> true;
+test_defined([_|Rest]) -> test_defined(Rest);
+test_defined([]) -> false.
+
+first_files(State) ->
+    BaseFirst = rebar_state:get(State, erl_first_files, []),
+    EUnitFirst = rebar_state:get(State, eunit_first_files, []),
+    rebar_state:set(State, erl_first_modules, BaseFirst ++ EUnitFirst).
+
+resolve_eunit_opts(State, Opts) ->
+    EUnitOpts = rebar_state:get(State, eunit_opts, []),
+    case lists:member({verbose, true}, Opts) of
+        true -> set_verbose(EUnitOpts);
+        false -> EUnitOpts
     end.
-get_beam_test_exports(ModuleStr) ->
-    FilePath = filename:join(eunit_dir(),
-                             string:concat(ModuleStr, ".beam")),
-    case filelib:is_regular(FilePath) of
-        true ->
-            {beam_file, _, Exports0, _, _, _} = beam_disasm:file(FilePath),
-            Exports1 = [FunName || {FunName, FunArity, _} <- Exports0,
-                                   FunArity =:= 0],
-            F = fun(FName) ->
-                        FNameStr = atom_to_list(FName),
-                        re:run(FNameStr, "_test(_)?") =/= nomatch
-                end,
-            lists:filter(F, Exports1);
-        _ ->
-            []
+
+set_verbose(Opts) ->
+    case lists:member(verbose, Opts) of
+        true -> Opts;
+        false -> [verbose] ++ Opts
     end.
-make_test_primitives(RawTests) ->
-    %% Use {test,M,F} and {generator,M,F} if at least R15B02. Otherwise,
-    %% use eunit_test:function_wrapper/2 fallback.
-    %% eunit_test:function_wrapper/2 was renamed to eunit_test:mf_wrapper/2
-    %% in R15B02; use that as >= R15B02 check.
-    %% TODO: remove fallback and use only {test,M,F} and {generator,M,F}
-    %% primitives once at least R15B02 is required.
-    {module, eunit_test} = code:ensure_loaded(eunit_test),
-    MakePrimitive = case erlang:function_exported(eunit_test, mf_wrapper, 2) of
-                        true -> fun eunit_primitive/3;
-                        false -> fun pre15b02_eunit_primitive/3
-                    end,
-    ?CONSOLE(" Running test function(s):~n", []),
-    F = fun({M, F2}, Acc) ->
-                ?CONSOLE(" ~p:~p/0~n", [M, F2]),
-                FNameStr = atom_to_list(F2),
-                NewFunction =
-                    case re:run(FNameStr, "_test_") of
-                        nomatch ->
-                            %% Normal test
-                            MakePrimitive(test, M, F2);
-                        _ ->
-                            %% Generator
-                            MakePrimitive(generator, M, F2)
-                    end,
-                [NewFunction|Acc]
-        end,
-    lists:foldl(F, [], RawTests).
-eunit_primitive(Type, M, F) ->
-    {Type, M, F}.
-pre15b02_eunit_primitive(test, M, F) ->
-    eunit_test:function_wrapper(M, F);
-pre15b02_eunit_primitive(generator, M, F) ->
-    {generator, eunit_test:function_wrapper(M, F)}.
-%%
-%% == run tests ==
-%%
-perform_eunit(Config, Tests) ->
-    EunitOpts = get_eunit_opts(Config),
-    %% Move down into ?EUNIT_DIR while we run tests so any generated files
-    %% are created there (versus in the source dir)
-    Cwd = rebar_dir:get_cwd(),
-    ok = file:set_cwd(?EUNIT_DIR),
-    EunitResult = (catch eunit:test(Tests, EunitOpts)),
-    %% Return to original working dir
-    ok = file:set_cwd(Cwd),
-    EunitResult.
-get_eunit_opts(Config) ->
-    %% Enable verbose in eunit if so requested..
-    BaseOpts = case rebar_log:is_verbose(Config) of
-                   true ->
-                       [verbose];
-                   false ->
-                       []
-               end,
-    BaseOpts ++ rebar_state:get(Config, eunit_opts, []).
-%%
-%% == reset_after_eunit ==
-%%
-status_before_eunit() ->
-    Apps = get_app_names(),
-    AppEnvs = [{App, application:get_all_env(App)} || App <- Apps],
-    {erlang:processes(), erlang:is_alive(), AppEnvs, ets:tab2list(ac_tab)}.
-get_app_names() ->
-    [AppName || {AppName, _, _} <- application:loaded_applications()].
-reset_after_eunit({OldProcesses, WasAlive, OldAppEnvs, _OldACs}) ->
-    IsAlive = erlang:is_alive(),
-    if not WasAlive andalso IsAlive ->
-            ?DEBUG("Stopping net kernel....\n", []),
-            erl_epmd:stop(),
-            _ = net_kernel:stop(),
-            pause_until_net_kernel_stopped();
-       true ->
-            ok
-    end,
-    OldApps = [App || {App, _} <- OldAppEnvs],
-    Apps = get_app_names(),
-    _ = [begin
-             _ = case lists:member(App, OldApps) of
-                     true -> ok;
-                     false -> application:stop(App)
-                 end,
-             ok = application:unset_env(App, K)
-         end || App <- Apps, App /= rebar,
-                {K, _V} <- application:get_all_env(App),
-                K =/= included_applications],
-    reconstruct_app_env_vars(Apps),
-    Processes = erlang:processes(),
-    _ = kill_extras(Processes -- OldProcesses),
-    ok.
-kill_extras(Pids) ->
-    %% Killing any of the procs below will either:
-    %% 1. Interfere with stuff that we don't want interfered with, or
-    %% 2. May/will force the 'kernel' app to shutdown, which *will*
-    %% interfere with rebar's ability To Do Useful Stuff(tm).
-    %% This list may require changes as OTP versions and/or
-    %% rebar use cases change.
-    KeepProcs = [cover_server, eunit_server,
-                 eqc, eqc_license, eqc_locked,
-                 %% inet_gethost_native is started on demand, when
-                 %% doing name lookups. It is under kernel_sup, under
-                 %% a supervisor_bridge.
-                 inet_gethost_native],
-    Killed = [begin
-                  Info = case erlang:process_info(Pid) of
-                             undefined -> [];
-                             Else -> Else
-                         end,
-                  Keep1 = case proplists:get_value(registered_name, Info) of
-                              undefined ->
-                                  false;
-                              Name ->
-                                  lists:member(Name, KeepProcs)
-                          end,
-                  Keep2 = case proplists:get_value(dictionary, Info) of
-                              undefined ->
-                                  false;
-                              Ds ->
-                                  case proplists:get_value('$ancestors', Ds) of
-                                      undefined ->
-                                          false;
-                                      As ->
-                                          lists:member(kernel_sup, As)
-                                  end
-                          end,
-                  if Keep1 orelse Keep2 ->
-                          ok;
-                     true ->
-                          ?DEBUG("Kill ~p ~p\n", [Pid, Info]),
-                          exit(Pid, kill),
-                          Pid
-                  end
-              end || Pid <- Pids],
-    case lists:usort(Killed) -- [ok] of
+
+maybe_compile_extra_tests(TestApps, State, OutDir) ->
+    F = fun(App) -> rebar_app_info:dir(App) == rebar_dir:get_cwd() end,
+    case lists:filter(F, TestApps) of
+        %% compile just the `test` and extra test directories of the base dir
         [] ->
-            ?DEBUG("No processes to kill\n", []),
-            [];
-        Else ->
-            lists:foreach(fun(Pid) -> wait_until_dead(Pid) end, Else),
-            Else
-    end.
-reconstruct_app_env_vars([App|Apps]) ->
-    CmdLine0 = proplists:get_value(App, init:get_arguments(), []),
-    CmdVars = [{list_to_atom(K), list_to_atom(V)} || {K, V} <- CmdLine0],
-    AppFile = (catch filename:join([code:lib_dir(App),
-                                    "ebin",
-                                    atom_to_list(App) ++ ".app"])),
-    AppVars = case file:consult(AppFile) of
-                  {ok, [{application, App, Ps}]} ->
-                      proplists:get_value(env, Ps, []);
-                  _ ->
-                      []
-              end,
-    %% App vars specified in config files override those in the .app file.
-    %% Config files later in the args list override earlier ones.
-    AppVars1 = case init:get_argument(config) of
-                   {ok, ConfigFiles} ->
-                       {App, MergedAppVars} = lists:foldl(fun merge_app_vars/2,
-                                                          {App, AppVars},
-                                                          ConfigFiles),
-                       MergedAppVars;
-                   error ->
-                       AppVars
-               end,
-    AllVars = CmdVars ++ AppVars1,
-    ?DEBUG("Reconstruct ~p ~p\n", [App, AllVars]),
-    lists:foreach(fun({K, V}) -> application:set_env(App, K, V) end, AllVars),
-    reconstruct_app_env_vars(Apps);
-reconstruct_app_env_vars([]) ->
-    ok.
-merge_app_vars(ConfigFile, {App, AppVars}) ->
-    File = ensure_config_extension(ConfigFile),
-    FileAppVars = app_vars_from_config_file(File, App),
-    Dict1 = dict:from_list(AppVars),
-    Dict2 = dict:from_list(FileAppVars),
-    Dict3 = dict:merge(fun(_Key, _Value1, Value2) -> Value2 end, Dict1, Dict2),
-    {App, dict:to_list(Dict3)}.
-ensure_config_extension(File) ->
-    %% config files must end with .config on disk but when specifying them
-    %% via the -config option the extension is optional
-    BaseFileName = filename:basename(File, ".config"),
-    DirName = filename:dirname(File),
-    filename:join(DirName, BaseFileName ++ ".config").
-app_vars_from_config_file(File, App) ->
-    case file:consult(File) of
-        {ok, [Env]} ->
-            proplists:get_value(App, Env, []);
-        _ ->
-            []
-    end.
-wait_until_dead(Pid) when is_pid(Pid) ->
-    Ref = erlang:monitor(process, Pid),
-    receive
-        {'DOWN', Ref, process, _Obj, Info} ->
-            Info
-    after 10*1000 ->
-            exit({timeout_waiting_for, Pid})
-    end;
-wait_until_dead(_) ->
-    ok.
-pause_until_net_kernel_stopped() ->
-    pause_until_net_kernel_stopped(10).
-pause_until_net_kernel_stopped(0) ->
-    exit(net_kernel_stop_failed);
-pause_until_net_kernel_stopped(N) ->
-    case node() of
-        'nonode@nohost' ->
-            ?DEBUG("Stopped net kernel.\n", []),
-            ok;
-        _ ->
-            timer:sleep(100),
-            pause_until_net_kernel_stopped(N - 1)
+            ErlOpts = rebar_state:get(State, common_test_compile_opts, []) ++
+                      rebar_utils:erl_opts(State),
+            TestOpts = [{outdir, OutDir}] ++
+                       [{src_dirs, ["test"]}] ++
+                       safe_define_test_macro(lists:keydelete(src_dirs, 1, ErlOpts)),
+            TestState = first_files(rebar_state:set(State, erl_opts, TestOpts)),
+            rebar_erlc_compiler:compile(TestState, rebar_dir:get_cwd());
+        %% already compiled `./test` so do nothing
+        _ -> ok
     end.
+
+handle_results(ok) -> ok;
+handle_results(error) ->
+    {error, unknown_error};
+handle_results({error, Reason}) ->
+    {error, {error_running_tests, Reason}}.
+
diff --git a/test/rebar_eunit_SUITE.erl b/test/rebar_eunit_SUITE.erl
new file mode 100644
index 0000000..190fbfa
--- /dev/null
+++ b/test/rebar_eunit_SUITE.erl
@@ -0,0 +1,37 @@
+-module(rebar_eunit_SUITE).
+
+-export([suite/0,
+         init_per_suite/1,
+         end_per_suite/1,
+         init_per_testcase/2,
+         all/0,
+         test_basic_app/1]).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("kernel/include/file.hrl").
+
+suite() ->
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_testcase(_, Config) ->
+    rebar_test_utils:init_rebar_state(Config).
+
+all() ->
+    [test_basic_app].
+
+test_basic_app(Config) ->
+    AppDir = ?config(apps, Config),
+
+    Name = rebar_test_utils:create_random_name("eunit_"),
+    Vsn = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]),
+    
+    RebarConfig = [{erl_opts, [{d, some_define}]}],
+    rebar_test_utils:run_and_check(Config, RebarConfig, ["eunit"], {ok, [{app, Name}]}).
diff --git a/test/rebar_test_utils.erl b/test/rebar_test_utils.erl
index 4982929..0a74a5f 100644
--- a/test/rebar_test_utils.erl
+++ b/test/rebar_test_utils.erl
@@ -62,6 +62,7 @@ run_and_check(Config, RebarConfig, Command, Expect) ->
 %% And returns a `rebar_app_info' object.
 create_app(AppDir, Name, Vsn, Deps) ->
     write_src_file(AppDir, Name),
+    write_test_file(AppDir, Name),
     write_app_src_file(AppDir, Name, Vsn, Deps),
     rebar_app_info:new(Name, Vsn, AppDir, Deps).
 
@@ -133,9 +134,14 @@ check_results(AppDir, Expected) ->
         end, Expected).
 
 write_src_file(Dir, Name) ->
-    Erl = filename:join([Dir, "src", "not_a_real_src" ++ Name ++ ".erl"]),
+    Erl = filename:join([Dir, "src", "not_a_real_src_" ++ Name ++ ".erl"]),
     ok = filelib:ensure_dir(Erl),
-    ok = ec_file:write(Erl, erl_src_file("not_a_real_src" ++ Name ++ ".erl")).
+    ok = ec_file:write(Erl, erl_src_file("not_a_real_src_" ++ Name ++ ".erl")).
+
+write_test_file(Dir, Name) ->
+    Erl = filename:join([Dir, "test", "not_a_real_src_" ++ Name ++ "_tests.erl"]),
+    ok = filelib:ensure_dir(Erl),
+    ok = ec_file:write(Erl, erl_test_file("not_a_real_src_" ++ Name ++ ".erl")).
 
 write_app_file(Dir, Name, Version, Deps) ->
     Filename = filename:join([Dir, "ebin", Name ++ ".app"]),
@@ -149,8 +155,25 @@ write_app_src_file(Dir, Name, Version, Deps) ->
 
 erl_src_file(Name) ->
     io_lib:format("-module(~s).\n"
-                 "-export([main/0]).\n"
-                 "main() -> ok.\n", [filename:basename(Name, ".erl")]).
+                  "-export([main/0]).\n"
+                  "main() -> ok.\n"
+                  "-ifdef(TEST).\n"
+                  "-include_lib(\"eunit/include/eunit.hrl\").\n"
+                  "some_test_() -> ?_assertEqual(ok, main()).\n"
+                  "-endif.\n", [filename:basename(Name, ".erl")]).
+
+erl_test_file(Name) ->
+    BaseName = filename:basename(Name, ".erl"),
+    io_lib:format("-module(~s_tests).\n"
+                  "-compile(export_all).\n"
+                  "-ifndef(some_define).\n"
+                  "-define(some_define, false).\n"
+                  "-endif.\n"
+                  "-ifdef(TEST).\n"
+                  "-include_lib(\"eunit/include/eunit.hrl\").\n"
+                  "some_test_() -> ?_assertEqual(ok, ~s:main()).\n"
+                  "define_test_() -> ?_assertEqual(true, ?some_define).\n"
+                  "-endif.\n", [BaseName, BaseName]).
 
 get_app_metadata(Name, Vsn, Deps) ->
     {application, erlang:list_to_atom(Name),