Merge pull request #805 from talentdeficit/eunit_test_specs

eunit test provider

7 files changed, 447 insertions, 409 deletions

diff --git a/test/rebar_eunit_SUITE.erl b/test/rebar_eunit_SUITE.erl
index d2dac1d..903bd96 100644
--- a/test/rebar_eunit_SUITE.erl
+++ b/test/rebar_eunit_SUITE.erl
@@ -1,464 +1,472 @@
 -module(rebar_eunit_SUITE).
 
--export([suite/0,
-         init_per_suite/1,
-         end_per_suite/1,
-         init_per_testcase/2,
-         all/0]).
--export([test_basic_app/1,
-         test_multi_app/1,
-         test_profile/1,
-         test_basic_exports/1,
-         test_multi_exports/1,
-         test_basic_defines/1,
-         test_multi_defines/1,
-         test_single_app_flag/1,
-         test_multiple_app_flag/1,
-         test_nonexistent_app_flag/1,
-         test_single_suite_flag/1,
-         test_suite_in_app_flag/1,
-         test_suite_in_wrong_app_flag/1,
-         test_nonexistent_suite_flag/1,
-         test_single_file_flag/1,
-         test_multiple_file_flag/1,
-         test_nonexistent_file_flag/1]).
+-export([all/0, groups/0]).
+-export([init_per_suite/1, init_per_group/2, end_per_group/2]).
+-export([basic_app_compiles/1, basic_app_files/1, basic_app_exports/1, basic_app_testset/1]).
+-export([multi_app_compiles/1, multi_app_files/1, multi_app_exports/1, multi_app_testset/1]).
+-export([eunit_tests/1, eunit_opts/1, eunit_first_files/1]).
+-export([single_application_arg/1, multi_application_arg/1, missing_application_arg/1]).
+-export([single_module_arg/1, multi_module_arg/1, missing_module_arg/1]).
+-export([single_suite_arg/1, multi_suite_arg/1, missing_suite_arg/1]).
+-export([single_file_arg/1, multi_file_arg/1, missing_file_arg/1]).
+-export([single_dir_arg/1, multi_dir_arg/1, missing_dir_arg/1]).
+-export([multiple_arg_composition/1, multiple_arg_errors/1]).
 
 -include_lib("common_test/include/ct.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("kernel/include/file.hrl").
 
-suite() ->
-    [].
-
+all() ->
+    [{group, basic_app}, {group, multi_app}, {group, cmd_line_args}].
+
+groups() ->
+    [{basic_app, [sequence], [basic_app_compiles, {group, basic_app_results}]},
+     {basic_app_results, [], [basic_app_files, basic_app_exports, basic_app_testset]},
+     {multi_app, [sequence], [multi_app_compiles, {group, multi_app_results}]},
+     {multi_app_results, [], [multi_app_files, multi_app_exports, multi_app_testset]},
+     {cmd_line_args, [], [eunit_tests, eunit_opts, eunit_first_files,
+                          single_application_arg, multi_application_arg, missing_application_arg,
+                          single_module_arg, multi_module_arg, missing_module_arg,
+                          single_suite_arg, multi_suite_arg, missing_suite_arg,
+                          single_file_arg, multi_file_arg, missing_file_arg,
+                          single_dir_arg, multi_dir_arg, missing_dir_arg,
+                          multiple_arg_composition, multiple_arg_errors]}].
+
+%% this just unzips the example apps used by tests to the priv dir for later use
 init_per_suite(Config) ->
+    PrivDir = ?config(priv_dir, Config),
+    DataDir = ?config(data_dir, Config),
+    {ok, Cwd} = file:get_cwd(),
+    file:set_cwd(PrivDir),
+    ok = ec_file:copy(filename:join([DataDir, "basic_app.zip"]), filename:join([PrivDir, "basic_app.zip"])),
+    {ok, _} = zip:extract(filename:join([PrivDir, "basic_app.zip"])),
+    ok = ec_file:copy(filename:join([DataDir, "multi_app.zip"]), filename:join([PrivDir, "multi_app.zip"])),
+    {ok, _} = zip:extract(filename:join([PrivDir, "multi_app.zip"])),
+    file:set_cwd(Cwd),
     Config.
 
-end_per_suite(_Config) ->
-    ok.
+init_per_group(basic_app, Config) ->
+    GroupState = rebar_test_utils:init_rebar_state(Config, "basic_app_"),
 
-init_per_testcase(_, Config) ->
-    rebar_test_utils:init_rebar_state(Config, "eunit_").
+    AppDir = ?config(apps, GroupState),
+    PrivDir = ?config(priv_dir, GroupState),
 
-all() ->
-    [test_basic_app, test_multi_app, test_profile,
-     test_basic_exports, test_multi_exports,
-     test_basic_defines, test_multi_defines,
-     test_single_app_flag, test_multiple_app_flag, test_nonexistent_app_flag,
-     test_single_suite_flag, test_suite_in_app_flag,
-     test_suite_in_wrong_app_flag, test_nonexistent_suite_flag,
-     test_single_file_flag, test_multiple_file_flag, test_nonexistent_file_flag].
-
-test_basic_app(Config) ->
+    AppDirs = ["src", "include", "test"],
+
+    lists:foreach(fun(F) -> ec_file:copy(filename:join([PrivDir, "basic_app", F]),
+                                         filename:join([AppDir, F]),
+                                         [recursive]) end, AppDirs),
+
+    RebarConfig = [{erl_opts, [{d, config_define}]}, {eunit_compile_opts, [{d, eunit_compile_define}]}],
+
+    {ok, State} = rebar_test_utils:run_and_check(GroupState, RebarConfig, ["as", "test", "lock"], return),
+
+    [{result, State}|GroupState];
+init_per_group(multi_app, Config) ->
+    GroupState = rebar_test_utils:init_rebar_state(Config, "multi_app_"),
+
+    AppDir = ?config(apps, GroupState),
+    PrivDir = ?config(priv_dir, GroupState),
+
+    AppDirs = ["apps", "test"],
+
+    lists:foreach(fun(F) -> ec_file:copy(filename:join([PrivDir, "multi_app", F]),
+                                         filename:join([AppDir, F]),
+                                         [recursive]) end, AppDirs),
+
+    RebarConfig = [{erl_opts, [{d, config_define}]}, {eunit_compile_opts, [{d, eunit_compile_define}]}],
+
+    {ok, State} = rebar_test_utils:run_and_check(GroupState, RebarConfig, ["as", "test", "lock"], return),
+
+    [{result, State}|GroupState];
+init_per_group(cmd_line_args, Config) ->
+    GroupState = rebar_test_utils:init_rebar_state(Config, "cmd_line_args_"),
+
+    AppDir = ?config(apps, GroupState),
+    PrivDir = ?config(priv_dir, GroupState),
+
+    AppDirs = ["apps", "test"],
+
+    lists:foreach(fun(F) -> ec_file:copy(filename:join([PrivDir, "multi_app", F]),
+                                         filename:join([AppDir, F]),
+                                         [recursive]) end, AppDirs),
+
+    RebarConfig = [{erl_opts, [{d, config_define}]},
+                   {eunit_compile_opts, [{d, eunit_compile_define}]},
+                   %% test set not supported by cmd line args
+                   {eunit_tests, [{test, multi_app_bar, sanity_test},
+                                  {test, multi_app_baz, sanity_test}]},
+                   {eunit_opts, [verbose]},
+                   {eunit_first_files, [filename:join(["apps", "multi_app_bar", "test", "multi_app_bar_tests_helper.erl"]),
+                                        filename:join(["apps", "multi_app_baz", "test", "multi_app_baz_tests_helper.erl"])]}],
+
+    {ok, State} = rebar_test_utils:run_and_check(GroupState, RebarConfig, ["eunit"], return),
+
+    [{result, State}|GroupState];
+init_per_group(_, Config) -> Config.
+
+end_per_group(_, Config) -> Config.
+
+
+
+%% === tests for a single application at the root of a project ===
+
+%% check that project compiles properly
+basic_app_compiles(Config) ->
     AppDir = ?config(apps, Config),
+    State = ?config(result, Config),
 
-    Name = rebar_test_utils:create_random_name("basic_"),
-    Vsn = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(AppDir, Name, Vsn, [kernel, stdlib]),
+    {ok, _} = rebar_prv_eunit:do(State),
 
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    rebar_test_utils:run_and_check(Config, RebarConfig, ["eunit"], {ok, [{app, Name}]}).
+    rebar_test_utils:check_results(AppDir, [{app, "basic_app"}], "*").
 
-test_multi_app(Config) ->
+%% check that all files expected to be present are present
+basic_app_files(Config) ->
     AppDir = ?config(apps, Config),
 
-    Name1 = rebar_test_utils:create_random_name("multi_app1_"),
-    Vsn1 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name1]),
-                                      Name1,
-                                      Vsn1,
-                                      [kernel, stdlib]),
-    Name2 = rebar_test_utils:create_random_name("multi_app2_"),
-    Vsn2 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name2]),
-                                      Name2,
-                                      Vsn2,
-                                      [kernel, stdlib]),
-
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    rebar_test_utils:run_and_check(Config,
-                                   RebarConfig,
-                                   ["eunit"],
-                                   {ok, [{app, Name1}, {app, Name2}]}).
-
-test_profile(Config) ->
+    lists:foreach(fun(F) -> true = ec_file:exists(filename:join([AppDir, "_build", "test", "lib", "basic_app", "ebin", F])) end,
+                  ["basic_app.app", "basic_app.beam", "basic_app_tests.beam", "basic_app_tests_helper.beam"]).
+
+%% check that the correct tests are exported from modules for project
+%% note that this implies `TEST` is set correctly
+basic_app_exports(_Config) ->
+    Tests = fun(Mod) ->
+        begin
+            Path = code:which(Mod),
+            {ok, {Mod, [{exports, Ex}]}} = beam_lib:chunks(Path, [exports]),
+            true = lists:member({sanity_test, 0}, Ex)
+        end
+    end,
+    Helpers = fun(Mod) ->
+        begin
+            Path = code:which(Mod),
+            {ok, {Mod, [{exports, Ex}]}} = beam_lib:chunks(Path, [exports]),
+            true = lists:member({help, 0}, Ex)
+        end
+    end,
+    lists:foreach(Tests, [basic_app, basic_app_tests]),
+    lists:foreach(Helpers, [basic_app_tests_helper]).
+
+%% check that the correct tests are schedule to run for project
+basic_app_testset(Config) ->
+    Result = ?config(result, Config),
+
+    {ok, [{application, basic_app}]} = rebar_prv_eunit:prepare_tests(Result).
+
+
+
+%% === tests for multiple applications in the `apps' directory of a project ===
+
+%% check that project compiles properly
+multi_app_compiles(Config) ->
     AppDir = ?config(apps, Config),
+    State = ?config(result, Config),
 
-    Name = rebar_test_utils:create_random_name("profile_"),
-    Vsn = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]),
+    {ok, _} = rebar_prv_eunit:do(State),
 
-    RebarConfig = [{erl_opts, [{d, some_define}]},
-                  {profiles, [{test, [{erl_opts, [debug_info]}]}]}],
-    rebar_test_utils:run_and_check(Config,
-                                   RebarConfig,
-                                   ["as", "test", "eunit"],
-                                   {ok, [{app, Name}]}).
+    rebar_test_utils:check_results(AppDir, [{app, "multi_app_bar"}, {app, "multi_app_baz"}], "*").
 
-test_basic_exports(Config) ->
+%% check that all files expected to be present are present
+multi_app_files(Config) ->
     AppDir = ?config(apps, Config),
 
-    Name = rebar_test_utils:create_random_name("basic_exports_"),
-    Vsn = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(AppDir, Name, Vsn, [kernel, stdlib]),
-
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    rebar_test_utils:run_and_check(Config,
-                                   RebarConfig,
-                                   ["eunit"],
-                                   {ok, [{app, Name}]}),
-
-    App = list_to_atom("not_a_real_src_" ++ Name),
-    Suite = list_to_atom("not_a_real_src_" ++ Name ++ "_tests"),
-    AppExports = App:module_info(exports),
-    SuiteExports = Suite:module_info(exports),
-    AppExpect = [{some_test_, 0}],
-    SuiteExpect = [{some_test_, 0}, {define_test_, 0}],
-    lists:foreach(fun(Expect) -> true = lists:member(Expect, AppExports) end, AppExpect),
-    lists:foreach(fun(Expect) -> true = lists:member(Expect, SuiteExports) end, SuiteExpect).
-
-test_multi_exports(Config) ->
+    lists:foreach(fun(F) -> true = ec_file:exists(filename:join([AppDir, "_build", "test", "lib", "multi_app_bar", "ebin", F])) end,
+                  ["multi_app_bar.app", "multi_app_bar.beam", "multi_app_bar_tests.beam", "multi_app_bar_tests_helper.beam"]),
+    lists:foreach(fun(F) -> true = ec_file:exists(filename:join([AppDir, "_build", "test", "lib", "multi_app_baz", "ebin", F])) end,
+                  ["multi_app_baz.app", "multi_app_baz.beam", "multi_app_baz_tests.beam", "multi_app_baz_tests_helper.beam"]),
+    lists:foreach(fun(F) -> true = ec_file:exists(filename:join([AppDir, "_build", "test", "test", F])) end,
+                  ["multi_app_tests.beam", "multi_app_tests_helper.beam"]).
+
+%% check that the correct tests are exported from modules for project
+%% note that this implies `TEST` is set correctly
+multi_app_exports(_Config) ->
+    Tests = fun(Mod) ->
+        begin
+            Ex = Mod:module_info(exports),
+            true = lists:member({sanity_test, 0}, Ex)
+        end
+    end,
+    Helpers = fun(Mod) ->
+        begin
+            Ex = Mod:module_info(exports),
+            true = lists:member({help, 0}, Ex)
+        end
+    end,
+    lists:foreach(Tests, [multi_app_bar, multi_app_bar_tests,
+                          multi_app_baz, multi_app_baz_tests,
+                          multi_app_tests]),
+    lists:foreach(Helpers, [multi_app_bar_tests_helper, multi_app_baz_tests_helper, multi_app_tests_helper]).
+
+%% check that the correct tests are schedule to run for project
+multi_app_testset(Config) ->
     AppDir = ?config(apps, Config),
+    Result = ?config(result, Config),
 
-    Name1 = rebar_test_utils:create_random_name("multi_exports_app1_"),
-    Vsn1 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name1]),
-                                      Name1,
-                                      Vsn1,
-                                      [kernel, stdlib]),
-    Name2 = rebar_test_utils:create_random_name("multi_exports_app2_"),
-    Vsn2 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name2]),
-                                      Name2,
-                                      Vsn2,
-                                      [kernel, stdlib]),
-
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    rebar_test_utils:run_and_check(Config,
-                                   RebarConfig,
-                                   ["eunit"],
-                                   {ok, [{app, Name1}, {app, Name2}]}),
-
-    App1 = list_to_atom("not_a_real_src_" ++ Name1),
-    Suite1 = list_to_atom("not_a_real_src_" ++ Name1 ++ "_tests"),
-    AppExports1 = App1:module_info(exports),
-    SuiteExports1 = Suite1:module_info(exports),
-    App2 = list_to_atom("not_a_real_src_" ++ Name2),
-    Suite2 = list_to_atom("not_a_real_src_" ++ Name2 ++ "_tests"),
-    AppExports2 = App2:module_info(exports),
-    SuiteExports2 = Suite2:module_info(exports),
-    AppExpect = [{some_test_, 0}],
-    SuiteExpect = [{some_test_, 0}, {define_test_, 0}],
-    lists:foreach(fun(Expect) -> true = lists:member(Expect, AppExports1) end, AppExpect),
-    lists:foreach(fun(Expect) -> true = lists:member(Expect, SuiteExports1) end, SuiteExpect),
-    lists:foreach(fun(Expect) -> true = lists:member(Expect, AppExports2) end, AppExpect),
-    lists:foreach(fun(Expect) -> true = lists:member(Expect, SuiteExports2) end, SuiteExpect).
-
-test_basic_defines(Config) ->
-    AppDir = ?config(apps, Config),
+    Set = {ok, [{application, multi_app_bar}, {application, multi_app_baz}, {dir, filename:join([AppDir, "_build", "test", "test"])}]},
+    Set = rebar_prv_eunit:prepare_tests(Result).
 
-    Name = rebar_test_utils:create_random_name("basic_"),
-    Vsn = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(AppDir, Name, Vsn, [kernel, stdlib]),
 
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    rebar_test_utils:run_and_check(Config, RebarConfig, ["eunit"], {ok, [{app, Name}]}),
 
-    App = list_to_atom("not_a_real_src_" ++ Name),
-    Suite = list_to_atom("not_a_real_src_" ++ Name ++ "_tests"),
-    AppOpts = proplists:get_value(options, App:module_info(compile), []),
-    SuiteOpts = proplists:get_value(options, Suite:module_info(compile), []),
-    Expect = [{d, some_define}],
-    lists:foreach(fun(E) -> true = lists:member(E, AppOpts) end, Expect),
-    lists:foreach(fun(E) -> true = lists:member(E, SuiteOpts) end, Expect).
+%% === tests for command line arguments ===
 
-test_multi_defines(Config) ->
-    AppDir = ?config(apps, Config),
+%% no explicit test for cmd line args taking precedence over the rebar.config since
+%% almost every single test implies it
 
-    Name1 = rebar_test_utils:create_random_name("multi_app1_"),
-    Vsn1 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name1]),
-                                      Name1,
-                                      Vsn1,
-                                      [kernel, stdlib]),
-    Name2 = rebar_test_utils:create_random_name("multi_app2_"),
-    Vsn2 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name2]),
-                                      Name2,
-                                      Vsn2,
-                                      [kernel, stdlib]),
-
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    rebar_test_utils:run_and_check(Config,
-                                   RebarConfig,
-                                   ["eunit"],
-                                   {ok, [{app, Name1}, {app, Name2}]}),
-
-    App1 = list_to_atom("not_a_real_src_" ++ Name1),
-    Suite1 = list_to_atom("not_a_real_src_" ++ Name1 ++ "_tests"),
-    AppOpts1 = proplists:get_value(options, App1:module_info(compile), []),
-    SuiteOpts1 = proplists:get_value(options, Suite1:module_info(compile), []),
-    App2 = list_to_atom("not_a_real_src_" ++ Name2),
-    Suite2 = list_to_atom("not_a_real_src_" ++ Name2 ++ "_tests"),
-    AppOpts2 = proplists:get_value(options, App2:module_info(compile), []),
-    SuiteOpts2 = proplists:get_value(options, Suite2:module_info(compile), []),
-    Expect = [{d, some_define}],
-    lists:foreach(fun(E) -> true = lists:member(E, AppOpts1) end, Expect),
-    lists:foreach(fun(E) -> true = lists:member(E, SuiteOpts1) end, Expect),
-    lists:foreach(fun(E) -> true = lists:member(E, AppOpts2) end, Expect),
-    lists:foreach(fun(E) -> true = lists:member(E, SuiteOpts2) end, Expect).
-
-test_single_app_flag(Config) ->
-    AppDir = ?config(apps, Config),
+%% check tests in the rebar.config are run if no cmd line opts are specified
+eunit_tests(Config) ->
+    State = ?config(result, Config),
+
+    Expect = {ok, [{test, multi_app_bar, sanity_test}, {test, multi_app_baz, sanity_test}]},
+    Expect = rebar_prv_eunit:prepare_tests(State).
+
+%% check eunit_opts from the rebar.config are respected
+eunit_opts(Config) ->
+    State = ?config(result, Config),
+
+    Apps = rebar_state:project_apps(State),
+    lists:foreach(fun(App) -> [verbose] = rebar_app_info:get(App, eunit_opts) end,
+                  Apps).
+
+%% check eunit_first_files from the rebar.config are respected
+eunit_first_files(Config) ->
+    State = ?config(result, Config),
+
+    FirstFiles = [filename:join(["apps", "multi_app_bar", "test", "multi_app_bar_tests_helper.erl"]),
+                  filename:join(["apps", "multi_app_baz", "test", "multi_app_baz_tests_helper.erl"])],
+
+    Apps = rebar_state:project_apps(State),
+    lists:foreach(fun(App) -> FirstFiles = rebar_app_info:get(App, eunit_first_files) end,
+                  Apps).
+
+%% check that the --application cmd line opt generates the correct test set
+single_application_arg(Config) ->
+    S = ?config(result, Config),
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--application=multi_app_bar"]),
+    State = rebar_state:command_parsed_args(S, Args),
+
+    {ok, [{application, multi_app_bar}]} = rebar_prv_eunit:prepare_tests(State).
+
+multi_application_arg(Config) ->
+    S = ?config(result, Config),
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--application=multi_app_bar,multi_app_baz"]),
+    State = rebar_state:command_parsed_args(S, Args),
+
+    {ok, [{application, multi_app_bar}, {application, multi_app_baz}]} = rebar_prv_eunit:prepare_tests(State).
+
+%% check that an invalid --application cmd line opt generates an error
+missing_application_arg(Config) ->
+    S = ?config(result, Config),
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--application=missing_app"]),
+    State = rebar_state:command_parsed_args(S, Args),
 
-    Name1 = rebar_test_utils:create_random_name("multi_exports_app1_"),
-    Vsn1 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name1]),
-                                      Name1,
-                                      Vsn1,
-                                      [kernel, stdlib]),
-    Name2 = rebar_test_utils:create_random_name("multi_exports_app2_"),
-    Vsn2 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name2]),
-                                      Name2,
-                                      Vsn2,
-                                      [kernel, stdlib]),
-
-    BareSuite = io_lib:format("-module(all_tests).\n"
-                              "-compile(export_all).\n"
-                              "-include_lib(\"eunit/include/eunit.hrl\").\n"
-                              "some_test_() -> ?_assert(true).\n"
-                              "define_test_() -> ?_assertEqual(true, ?some_define).\n", []),
-    FileName = filename:join([AppDir, "test", "all_tests.erl"]),
-    ok = filelib:ensure_dir(FileName),
-    ok = ec_file:write(FileName, BareSuite),
-
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    rebar_test_utils:run_and_check(Config,
-                                   RebarConfig,
-                                   ["eunit", "--app=" ++ Name1],
-                                   {ok, [{app, Name1}, {app, Name2}]}),
-
-    Suite1 = list_to_atom("not_a_real_src_" ++ Name1 ++ "_tests"),
-    {module, Suite1} = code:ensure_loaded(Suite1),
-    Suite2 = list_to_atom("not_a_real_src_" ++ Name2 ++ "_tests"),
-    {error, nofile} = code:ensure_loaded(Suite2),
-    {error, nofile} = code:ensure_loaded(all_tests).
-
-test_multiple_app_flag(Config) ->
+    Error = {error, {rebar_prv_eunit, {eunit_test_errors, ["Application `missing_app' not found in project."]}}},
+    Error = rebar_prv_eunit:prepare_tests(State).
+
+%% check that the --module cmd line opt generates the correct test set
+single_module_arg(Config) ->
     AppDir = ?config(apps, Config),
+    S = ?config(result, Config),
+
+    %% necessary to fix paths
+    Path = code:get_path(),
+    code:add_paths([filename:join([AppDir, "_build", "test", "lib", "multi_app_bar", "ebin"])]),
+
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--module=multi_app_bar"]),
+    State = rebar_state:command_parsed_args(S, Args),
+
+    {ok, [{module, multi_app_bar}]} = rebar_prv_eunit:prepare_tests(State),
+
+    %% restore path
+    code:set_path(Path).
 
-    Name1 = rebar_test_utils:create_random_name("multi_exports_app1_"),
-    Vsn1 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name1]),
-                                      Name1,
-                                      Vsn1,
-                                      [kernel, stdlib]),
-    Name2 = rebar_test_utils:create_random_name("multi_exports_app2_"),
-    Vsn2 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name2]),
-                                      Name2,
-                                      Vsn2,
-                                      [kernel, stdlib]),
-
-    BareSuite = io_lib:format("-module(all_tests).\n"
-                              "-compile(export_all).\n"
-                              "-include_lib(\"eunit/include/eunit.hrl\").\n"
-                              "some_test_() -> ?_assert(true).\n"
-                              "define_test_() -> ?_assertEqual(true, ?some_define).\n", []),
-    FileName = filename:join([AppDir, "test", "all_tests.erl"]),
-    ok = filelib:ensure_dir(FileName),
-    ok = ec_file:write(FileName, BareSuite),
-
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    rebar_test_utils:run_and_check(Config,
-                                   RebarConfig,
-                                   ["eunit", "--app=" ++ Name1 ++ "," ++ Name2],
-                                   {ok, [{app, Name1}, {app, Name2}]}),
-
-    Suite1 = list_to_atom("not_a_real_src_" ++ Name1 ++ "_tests"),
-    {module, Suite1} = code:ensure_loaded(Suite1),
-    Suite2 = list_to_atom("not_a_real_src_" ++ Name2 ++ "_tests"),
-    {module, Suite2} = code:ensure_loaded(Suite2),
-    {error, nofile} = code:ensure_loaded(all_tests).
-
-test_nonexistent_app_flag(Config) ->
+multi_module_arg(Config) ->
     AppDir = ?config(apps, Config),
+    S = ?config(result, Config),
+
+    %% necessary to fix paths
+    Path = code:get_path(),
+    code:add_paths([filename:join([AppDir, "_build", "test", "lib", "multi_app_bar", "ebin"])]),
+    code:add_paths([filename:join([AppDir, "_build", "test", "lib", "multi_app_baz", "ebin"])]),
+
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--module=multi_app_bar,multi_app_baz"]),
+    State = rebar_state:command_parsed_args(S, Args),
+
+    {ok, [{module, multi_app_bar}, {module, multi_app_baz}]} = rebar_prv_eunit:prepare_tests(State),
+
+    %% restore path
+    code:set_path(Path).
+
+%% check that an invalid --module cmd line opt generates an error
+missing_module_arg(Config) ->
+    S = ?config(result, Config),
 
-    Name1 = rebar_test_utils:create_random_name("multi_exports_app1_"),
-    Vsn1 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,Name1]),
-                                      Name1,
-                                      Vsn1,
-                                      [kernel, stdlib]),
-    Name2 = rebar_test_utils:create_random_name("multi_exports_app2_"),
-    Vsn2 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name2]),
-                                      Name2,
-                                      Vsn2,
-                                      [kernel, stdlib]),
-
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    {error, {_, Error}} = rebar_test_utils:run_and_check(Config,
-                                                         RebarConfig,
-                                                         ["eunit", "--app=not_a_real_app"],
-                                                         return),
-
-    Error = {error_running_tests, "Application `not_a_real_app' not found in project."}.
-
-test_single_suite_flag(Config) ->
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--module=missing_app"]),
+    State = rebar_state:command_parsed_args(S, Args),
+
+    Error = {error, {rebar_prv_eunit, {eunit_test_errors, ["Module `missing_app' not found in project."]}}},
+    Error = rebar_prv_eunit:prepare_tests(State).
+
+%% check that the --suite cmd line opt generates the correct test set
+single_suite_arg(Config) ->
     AppDir = ?config(apps, Config),
+    S = ?config(result, Config),
+
+    %% necessary to fix paths
+    Path = code:get_path(),
+    code:add_paths([filename:join([AppDir, "_build", "test", "lib", "multi_app_bar", "ebin"])]),
+
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--suite=multi_app_bar"]),
+    State = rebar_state:command_parsed_args(S, Args),
+
+    {ok, [{module, multi_app_bar}]} = rebar_prv_eunit:prepare_tests(State),
 
-    Name1 = rebar_test_utils:create_random_name("multi_exports_app1_"),
-    Vsn1 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name1]),
-                                      Name1,
-                                      Vsn1,
-                                      [kernel, stdlib]),
-    Name2 = rebar_test_utils:create_random_name("multi_exports_app2_"),
-    Vsn2 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name2]),
-                                      Name2,
-                                      Vsn2,
-                                      [kernel, stdlib]),
-
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    rebar_test_utils:run_and_check(Config,
-                                   RebarConfig,
-                                   ["eunit", "--suite=not_a_real_src_" ++ Name1],
-                                   {ok, [{app, Name1}, {app, Name2}]}),
-
-    Suite1 = list_to_atom("not_a_real_src_" ++ Name1 ++ "_tests"),
-    {module, Suite1} = code:ensure_loaded(Suite1).
-
-test_suite_in_app_flag(Config) ->
+    %% restore path
+    code:set_path(Path).
+
+multi_suite_arg(Config) ->
     AppDir = ?config(apps, Config),
+    S = ?config(result, Config),
+
+    %% necessary to fix paths
+    Path = code:get_path(),
+    code:add_paths([filename:join([AppDir, "_build", "test", "lib", "multi_app_bar", "ebin"])]),
+    code:add_paths([filename:join([AppDir, "_build", "test", "lib", "multi_app_baz", "ebin"])]),
+
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--suite=multi_app_bar,multi_app_baz"]),
+    State = rebar_state:command_parsed_args(S, Args),
+
+    {ok, [{module, multi_app_bar}, {module, multi_app_baz}]} = rebar_prv_eunit:prepare_tests(State),
 
-    Name1 = rebar_test_utils:create_random_name("multi_exports_app1_"),
-    Vsn1 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name1]),
-                                      Name1,
-                                      Vsn1,
-                                      [kernel, stdlib]),
-    Name2 = rebar_test_utils:create_random_name("multi_exports_app2_"),
-    Vsn2 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name2]),
-                                      Name2,
-                                      Vsn2,
-                                      [kernel, stdlib]),
-
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    rebar_test_utils:run_and_check(Config,
-                                   RebarConfig,
-                                   ["eunit",
-                                    "--app=" ++ Name1,
-                                    "--suite=not_a_real_src_" ++ Name1],
-                                   {ok, [{app, Name1}, {app, Name2}]}),
-
-    Suite1 = list_to_atom("not_a_real_src_" ++ Name1 ++ "_tests"),
-    {module, Suite1} = code:ensure_loaded(Suite1),
-    Suite2 = list_to_atom("not_a_real_src_" ++ Name2 ++ "_tests"),
-    {error, nofile} = code:ensure_loaded(Suite2).
-
-test_suite_in_wrong_app_flag(Config) ->
+    %% restore path
+    code:set_path(Path).
+
+%% check that an invalid --suite cmd line opt generates an error
+missing_suite_arg(Config) ->
+    S = ?config(result, Config),
+
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--suite=missing_app"]),
+    State = rebar_state:command_parsed_args(S, Args),
+
+    Error = {error, {rebar_prv_eunit, {eunit_test_errors, ["Module `missing_app' not found in project."]}}},
+    Error = rebar_prv_eunit:prepare_tests(State).
+
+%% check that the --file cmd line opt generates the correct test set
+single_file_arg(Config) ->
+    S = ?config(result, Config),
     AppDir = ?config(apps, Config),
 
-    Name1 = rebar_test_utils:create_random_name("multi_exports_app1_"),
-    Vsn1 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name1]),
-                                      Name1,
-                                      Vsn1,
-                                      [kernel, stdlib]),
-    Name2 = rebar_test_utils:create_random_name("multi_exports_app2_"),
-    Vsn2 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name2]),
-                                      Name2,
-                                      Vsn2,
-                                      [kernel, stdlib]),
-
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    {error, {_, Error}} = rebar_test_utils:run_and_check(Config,
-                                                         RebarConfig,
-                                                         ["eunit",
-                                                         "--app=" ++ Name1,
-                                                         "--suite=not_a_real_src_" ++ Name2],
-                                                         return),
-
-    Error = {error_running_tests, "Module `not_a_real_src_" ++
-                                  Name2 ++
-                                  "' not found in applications."}.
-
-test_nonexistent_suite_flag(Config) ->
+    Path = filename:join([AppDir, "_build", "test", "lib", "multi_app_bar", "ebin", "multi_app_bar.beam"]),
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--file=" ++ Path]),
+    State = rebar_state:command_parsed_args(S, Args),
+
+    {ok, [{file, Path}]} = rebar_prv_eunit:prepare_tests(State).
+
+multi_file_arg(Config) ->
+    S = ?config(result, Config),
     AppDir = ?config(apps, Config),
 
-    Name1 = rebar_test_utils:create_random_name("multi_exports_app1_"),
-    Vsn1 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name1]),
-                                      Name1,
-                                      Vsn1,
-                                      [kernel, stdlib]),
-    Name2 = rebar_test_utils:create_random_name("multi_exports_app2_"),
-    Vsn2 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name2]),
-                                      Name2,
-                                      Vsn2,
-                                      [kernel, stdlib]),
-
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    {error, {_, Error}} = rebar_test_utils:run_and_check(Config,
-                                                         RebarConfig,
-                                                         ["eunit", "--suite=not_a_real_module"],
-                                                         return),
-
-    Error = {error_running_tests, "Module `not_a_real_module' not found in applications."}.
-
-test_single_file_flag(Config) ->
+    BarPath = filename:join([AppDir, "_build", "test", "lib", "multi_app_bar", "ebin", "multi_app_bar.beam"]),
+    BazPath = filename:join([AppDir, "_build", "test", "lib", "multi_app_baz", "ebin", "multi_app_baz.beam"]),
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--file=" ++ BarPath ++ "," ++ BazPath]),
+    State = rebar_state:command_parsed_args(S, Args),
+
+    {ok, [{file, BarPath}, {file, BazPath}]} = rebar_prv_eunit:prepare_tests(State).
+
+%% check that an invalid --file cmd line opt generates an error
+missing_file_arg(Config) ->
+    S = ?config(result, Config),
     AppDir = ?config(apps, Config),
 
-    Name = rebar_test_utils:create_random_name("single_file_flag_app_"),
-    Vsn = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(AppDir, Name, Vsn, [kernel, stdlib]),
+    Path = filename:join([AppDir, "_build", "test", "lib", "missing_app", "ebin", "missing_app.beam"]),
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--file=" ++ Path]),
+    State = rebar_state:command_parsed_args(S, Args),
+
+    Error = {error, {rebar_prv_eunit, {eunit_test_errors, ["File `" ++ Path ++"' not found."]}}},
+    Error = rebar_prv_eunit:prepare_tests(State).
 
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    rebar_test_utils:run_and_check(Config,
-                                   RebarConfig,
-                                   ["eunit", "--file=not_a_real_src_" ++ Name ++ "_tests.beam"],
-                                   {ok, [{app, Name}]}),
+%% check that the --dir cmd line opt generates the correct test set
+single_dir_arg(Config) ->
+    S = ?config(result, Config),
+    AppDir = ?config(apps, Config),
+
+    Path = filename:join([AppDir, "_build", "test", "lib", "multi_app_bar", "ebin"]),
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--dir=" ++ Path]),
+    State = rebar_state:command_parsed_args(S, Args),
 
-    File = list_to_atom("not_a_real_src_" ++ Name ++ "_tests"),
-    {module, File} = code:ensure_loaded(File).
+    {ok, [{dir, Path}]} = rebar_prv_eunit:prepare_tests(State).
 
-test_multiple_file_flag(Config) ->
+multi_dir_arg(Config) ->
+    S = ?config(result, Config),
     AppDir = ?config(apps, Config),
 
-    Name = rebar_test_utils:create_random_name("multiple_file_flag_app_"),
-    Vsn = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(AppDir, Name, Vsn, [kernel, stdlib]),
+    BarPath = filename:join([AppDir, "_build", "test", "lib", "multi_app_bar", "ebin"]),
+    BazPath = filename:join([AppDir, "_build", "test", "lib", "multi_app_baz", "ebin"]),
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--dir=" ++ BarPath ++ "," ++ BazPath]),
+    State = rebar_state:command_parsed_args(S, Args),
 
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    rebar_test_utils:run_and_check(Config,
-                                   RebarConfig,
-                                   ["eunit", "--file=not_a_real_src_" ++ Name ++ "_tests.beam,not_a_real_src_" ++ Name ++ ".beam"],
-                                   {ok, [{app, Name}]}),
+    {ok, [{dir, BarPath}, {dir, BazPath}]} = rebar_prv_eunit:prepare_tests(State).
 
-    File1 = list_to_atom("not_a_real_src_" ++ Name ++ "_tests"),
-    {module, File1} = code:ensure_loaded(File1),
+%% check that an invalid --dir cmd line opt generates an error
+missing_dir_arg(Config) ->
+    S = ?config(result, Config),
+    AppDir = ?config(apps, Config),
 
-    File2 = list_to_atom("not_a_real_src_" ++ Name),
-    {module, File2} = code:ensure_loaded(File2).
+    Path = filename:join([AppDir, "_build", "test", "lib", "missing_app", "ebin"]),
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--dir=" ++ Path]),
+    State = rebar_state:command_parsed_args(S, Args),
 
-test_nonexistent_file_flag(Config) ->
+    Error = {error, {rebar_prv_eunit, {eunit_test_errors, ["Directory `" ++ Path ++"' not found."]}}},
+    Error = rebar_prv_eunit:prepare_tests(State).
+
+%% check that multiple args are composed
+multiple_arg_composition(Config) ->
+    S = ?config(result, Config),
     AppDir = ?config(apps, Config),
 
-    Name = rebar_test_utils:create_random_name("nonexistent_file_flag_app_"),
-    Vsn = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(AppDir,
-                                      Name,
-                                      Vsn,
-                                      [kernel, stdlib]),
-
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    {error, {rebar_prv_eunit, _Error}} = rebar_test_utils:run_and_check(Config,
-                                                         RebarConfig,
-                                                         ["eunit", "--file=" ++ filename:join(["some_path", "not_a_real_file.erl"])],
-                                                         return).
+    %% necessary to fix paths
+    Path = code:get_path(),
+    code:add_paths([filename:join([AppDir, "_build", "test", "lib", "multi_app_bar", "ebin"])]),
+    FilePath = filename:join([AppDir, "_build", "test", "lib", "multi_app_bar", "ebin", "multi_app_bar.beam"]),
+    DirPath = filename:join([AppDir, "_build", "test", "lib", "multi_app_bar", "ebin"]),
+
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--application=multi_app_bar",
+                                                              "--module=multi_app_bar",
+                                                              "--suite=multi_app_bar",
+                                                              "--file=" ++ FilePath,
+                                                              "--dir=" ++ DirPath]),
+    State = rebar_state:command_parsed_args(S, Args),
+
+    Expect = [{application, multi_app_bar},
+              {dir, DirPath},
+              {file, FilePath},
+              {module, multi_app_bar},
+              {module, multi_app_bar}],
+
+    {ok, Expect} = rebar_prv_eunit:prepare_tests(State),
+
+    %% restore path
+    code:set_path(Path).
+
+%% check that multiple errors are reported
+multiple_arg_errors(Config) ->
+    S = ?config(result, Config),
+    AppDir = ?config(apps, Config),
+
+    FilePath = filename:join([AppDir, "_build", "test", "lib", "missing_app", "ebin", "missing_app.beam"]),
+    DirPath = filename:join([AppDir, "_build", "test", "lib", "missing_app", "ebin"]),
+
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--application=missing_app",
+                                                              "--module=missing_app",
+                                                              "--suite=missing_app",
+                                                              "--file=" ++ FilePath,
+                                                              "--dir=" ++ DirPath]),
+    State = rebar_state:command_parsed_args(S, Args),
+
+    Expect = ["Application `missing_app' not found in project.",
+              "Directory `" ++ DirPath ++ "' not found.",
+              "File `" ++ FilePath ++ "' not found.",
+              "Module `missing_app' not found in project.",
+              "Module `missing_app' not found in project."],
+
+    {error, {rebar_prv_eunit, {eunit_test_errors, Expect}}} = rebar_prv_eunit:prepare_tests(State).
 
diff --git a/test/rebar_eunit_SUITE_data/basic_app.zip b/test/rebar_eunit_SUITE_data/basic_app.zip
new file mode 100644
index 0000000..0eaa0af
--- /dev/null
+++ b/test/rebar_eunit_SUITE_data/basic_app.zip
diff --git a/test/rebar_eunit_SUITE_data/deflate b/test/rebar_eunit_SUITE_data/deflate
new file mode 100755
index 0000000..368228f
--- /dev/null
+++ b/test/rebar_eunit_SUITE_data/deflate
@@ -0,0 +1,15 @@
+#!/usr/bin/env escript
+
+main(["basic_app"]) ->
+  case filelib:is_dir("basic_app") of
+    true  -> zip:create("basic_app.zip", ["basic_app"]), halt(0);
+    false -> io:format("unable to locate basic_app directory~n", []), halt(1)
+  end;
+main(["multi_app"]) ->
+  case filelib:is_dir("multi_app") of
+    true  -> zip:create("multi_app.zip", ["multi_app"]), halt(0);
+    false -> io:format("unable to locate multi_app directory~n", []), halt(1)
+  end,
+  halt(0);
+main(_) ->
+  io:format("usage: deflate basic_app | multi_app~n", []), halt(1).
\ No newline at end of file
diff --git a/test/rebar_eunit_SUITE_data/inflate b/test/rebar_eunit_SUITE_data/inflate
new file mode 100755
index 0000000..d402f80
--- /dev/null
+++ b/test/rebar_eunit_SUITE_data/inflate
@@ -0,0 +1,15 @@
+#!/usr/bin/env escript
+
+main(["basic_app"]) ->
+  case filelib:is_file("basic_app.zip") of
+    true  -> zip:unzip("basic_app.zip"), halt(0);
+    false -> io:format("unable to locate basic_app.zip~n", []), halt(1)
+  end;
+main(["multi_app"]) ->
+  case filelib:is_file("multi_app.zip") of
+    true  -> zip:unzip("multi_app.zip"), halt(0);
+    false -> io:format("unable to locate multi_app.zip~n", []), halt(1)
+  end,
+  halt(0);
+main(_) ->
+  io:format("usage: inflate basic_app | multi_app~n", []), halt(1).
\ No newline at end of file
diff --git a/test/rebar_eunit_SUITE_data/multi_app.zip b/test/rebar_eunit_SUITE_data/multi_app.zip
new file mode 100644
index 0000000..36bf9a6
--- /dev/null
+++ b/test/rebar_eunit_SUITE_data/multi_app.zip
diff --git a/test/rebar_profiles_SUITE.erl b/test/rebar_profiles_SUITE.erl
index d4c10c5..a31a4c9 100644
--- a/test/rebar_profiles_SUITE.erl
+++ b/test/rebar_profiles_SUITE.erl
@@ -375,8 +375,8 @@ test_profile_applied_at_completion(Config) ->
                                                  ["eunit"],
                                                  return),
 
-    Opts = rebar_state:opts(State),
-    ErlOpts = dict:fetch(erl_opts, Opts),
+    [App] = rebar_state:project_apps(State),
+    ErlOpts = rebar_app_info:get(App, erl_opts),
     true = lists:member({d, 'TEST'}, ErlOpts).
 
 test_profile_applied_before_compile(Config) ->
diff --git a/test/rebar_test_utils.erl b/test/rebar_test_utils.erl
index 8d1d408..ca5e91a 100644
--- a/test/rebar_test_utils.erl
+++ b/test/rebar_test_utils.erl
@@ -1,7 +1,7 @@
 -module(rebar_test_utils).
 -include_lib("common_test/include/ct.hrl").
 -include_lib("eunit/include/eunit.hrl").
--export([init_rebar_state/1, init_rebar_state/2, run_and_check/4]).
+-export([init_rebar_state/1, init_rebar_state/2, run_and_check/4, check_results/3]).
 -export([expand_deps/2, flat_deps/1, top_level_deps/1]).
 -export([create_app/4, create_eunit_app/4, create_empty_app/4, create_config/2,
          package_app/3]).