Merge pull request #883 from talentdeficit/erlc_compiler_refactor

compiler refactor/eunit & ct fixes

7 files changed, 430 insertions, 229 deletions

diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl
index b9192c3..57b7387 100644
--- a/src/rebar_erlc_compiler.erl
+++ b/src/rebar_erlc_compiler.erl
@@ -26,9 +26,9 @@
 %% -------------------------------------------------------------------
 -module(rebar_erlc_compiler).
 
--export([compile/1,
-         compile/3,
-         compile/4,
+-export([compile/1, compile/2, compile/3,
+         compile_dir/3, compile_dir/4,
+         compile_dirs/5,
          clean/1]).
 
 -include("rebar.hrl").
@@ -39,14 +39,22 @@
 -type erlc_info_v() :: {digraph:vertex(), term()} | 'false'.
 -type erlc_info_e() :: {digraph:vertex(), digraph:vertex()}.
 -type erlc_info() :: {list(erlc_info_v()), list(erlc_info_e()), list(string())}.
--record(erlcinfo,
-        {
-          vsn = ?ERLCINFO_VSN :: pos_integer(),
-          info = {[], [], []} :: erlc_info()
-        }).
+-record(erlcinfo, {
+    vsn = ?ERLCINFO_VSN :: pos_integer(),
+    info = {[], [], []} :: erlc_info()
+}).
 
+-type compile_opts() :: [compile_opt()].
+-type compile_opt() :: {recursive, boolean()}.
+
+-record(compile_opts, {
+    recursive = true
+}).
+
+-define(DEFAULT_OUTDIR, "ebin").
 -define(RE_PREFIX, "^[^._]").
 
+
 %% ===================================================================
 %% Public API
 %% ===================================================================
@@ -81,111 +89,191 @@
 %%                           'old_inets'}]}.
 %%
 
--spec compile(rebar_app_info:t()) -> 'ok'.
-compile(AppInfo) ->
+%% @equiv compile(AppInfo, []).
+
+-spec compile(rebar_app_info:t()) -> ok.
+compile(AppInfo) when element(1, AppInfo) == app_info_t ->
+    compile(AppInfo, []).
+
+%% @doc compile an individual application.
+
+-spec compile(rebar_app_info:t(), compile_opts()) -> ok.
+compile(AppInfo, CompileOpts) when element(1, AppInfo) == app_info_t ->
     Dir = ec_cnv:to_list(rebar_app_info:out_dir(AppInfo)),
-    compile(rebar_app_info:opts(AppInfo), Dir, filename:join([Dir, "ebin"])).
+    RebarOpts = rebar_app_info:opts(AppInfo),
 
--spec compile(rebar_dict(), file:name(), file:name()) -> 'ok'.
-compile(Opts, Dir, OutDir) ->
-    rebar_base_compiler:run(Opts,
+    rebar_base_compiler:run(RebarOpts,
                             check_files([filename:join(Dir, File)
-                                         || File <- rebar_opts:get(Opts, xrl_first_files, [])]),
+                                         || File <- rebar_opts:get(RebarOpts, xrl_first_files, [])]),
                             filename:join(Dir, "src"), ".xrl", filename:join(Dir, "src"), ".erl",
                             fun compile_xrl/3),
-    rebar_base_compiler:run(Opts,
+    rebar_base_compiler:run(RebarOpts,
                             check_files([filename:join(Dir, File)
-                                         || File <- rebar_opts:get(Opts, yrl_first_files, [])]),
+                                         || File <- rebar_opts:get(RebarOpts, yrl_first_files, [])]),
                             filename:join(Dir, "src"), ".yrl", filename:join(Dir, "src"), ".erl",
                             fun compile_yrl/3),
-    rebar_base_compiler:run(Opts,
+    rebar_base_compiler:run(RebarOpts,
                             check_files([filename:join(Dir, File)
-                                         || File <- rebar_opts:get(Opts, mib_first_files, [])]),
+                                         || File <- rebar_opts:get(RebarOpts, mib_first_files, [])]),
                             filename:join(Dir, "mibs"), ".mib", filename:join([Dir, "priv", "mibs"]), ".bin",
                             fun compile_mib/3),
-    doterl_compile(Opts, Dir, OutDir).
 
--spec compile(rebar_dict(), file:filename(), file:filename(), [file:filename()]) -> 'ok'.
-compile(Opts, Dir, OutDir, More) ->
-    ErlOpts = rebar_opts:erl_opts(Opts),
-    doterl_compile(Opts, Dir, OutDir, More, ErlOpts).
+    SrcDirs = lists:map(fun(SrcDir) -> filename:join(Dir, SrcDir) end,
+                        rebar_dir:src_dirs(RebarOpts, ["src"])),
+    OutDir = filename:join(Dir, outdir(RebarOpts)),
+    compile_dirs(RebarOpts, Dir, SrcDirs, OutDir, CompileOpts),
+
+    ExtraDirs = rebar_dir:extra_src_dirs(RebarOpts),
+    F = fun(D) ->
+        case ec_file:is_dir(filename:join([Dir, D])) of
+            true  -> compile_dirs(RebarOpts, Dir, [D], D, CompileOpts);
+            false -> ok
+        end
+    end,
+    lists:foreach(F, lists:map(fun(SrcDir) -> filename:join(Dir, SrcDir) end, ExtraDirs)).
+
+%% @hidden
+%% these are kept for backwards compatibility but they're bad functions with
+%% bad interfaces you probably shouldn't use
+%% State/RebarOpts have to have src_dirs set and BaseDir must be the parent
+%% directory of those src_dirs
+
+-spec compile(rebar_dict() | rebar_state:t(), file:name(), file:name()) -> ok.
+compile(State, BaseDir, OutDir) when element(1, State) == state_t ->
+    compile(rebar_state:opts(State), BaseDir, OutDir, [{recursive, false}]);
+compile(RebarOpts, BaseDir, OutDir) ->
+    compile(RebarOpts, BaseDir, OutDir, [{recursive, false}]).
+
+%% @hidden
+
+-spec compile(rebar_dict() | rebar_state:t(), file:name(), file:name(), compile_opts()) -> ok.
+compile(State, BaseDir, OutDir, CompileOpts) when element(1, State) == state_t ->
+    compile(rebar_state:opts(State), BaseDir, OutDir, CompileOpts);
+compile(RebarOpts, BaseDir, OutDir, CompileOpts) ->
+    SrcDirs = lists:map(fun(SrcDir) -> filename:join(BaseDir, SrcDir) end,
+                        rebar_dir:src_dirs(RebarOpts, ["src"])),
+    compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, CompileOpts),
+
+    ExtraDirs = rebar_dir:extra_src_dirs(RebarOpts),
+    F = fun(D) ->
+        case ec_file:is_dir(filename:join([BaseDir, D])) of
+            true  -> compile_dirs(RebarOpts, BaseDir, [D], D, CompileOpts);
+            false -> ok
+        end
+    end,
+    lists:foreach(F, lists:map(fun(SrcDir) -> filename:join(BaseDir, SrcDir) end, ExtraDirs)).
+
+%% @equiv compile_dirs(Context, BaseDir, [Dir], Dir, [{recursive, false}]).
+
+-spec compile_dir(rebar_dict() | rebar_state:t(), file:name(), file:name()) -> ok.
+compile_dir(State, BaseDir, Dir) when element(1, State) == state_t ->
+    compile_dir(rebar_state:opts(State), BaseDir, Dir, [{recursive, false}]);
+compile_dir(RebarOpts, BaseDir, Dir) ->
+    compile_dir(RebarOpts, BaseDir, Dir, [{recursive, false}]).
+
+%% @equiv compile_dirs(Context, BaseDir, [Dir], Dir, Opts).
+
+-spec compile_dir(rebar_dict() | rebar_state:t(), file:name(), file:name(), compile_opts()) -> ok.
+compile_dir(State, BaseDir, Dir, Opts) when element(1, State) == state_t ->
+    compile_dirs(rebar_state:opts(State), BaseDir, [Dir], Dir, Opts);
+compile_dir(RebarOpts, BaseDir, Dir, Opts) ->
+    compile_dirs(RebarOpts, BaseDir, [Dir], Dir, Opts).
+
+%% @doc compile a list of directories with the given opts.
+
+-spec compile_dirs(rebar_dict() | rebar_state:t(),
+                   file:filename(),
+                   [file:filename()],
+                   file:filename(),
+                   compile_opts()) -> ok.
+compile_dirs(State, BaseDir, Dirs, OutDir, CompileOpts) when element(1, State) == state_t ->
+    compile_dirs(rebar_state:opts(State), BaseDir, Dirs, OutDir, CompileOpts);
+compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, Opts) ->
+    CompileOpts = parse_opts(Opts),
+  
+    ErlOpts = rebar_opts:erl_opts(RebarOpts),
+    ?DEBUG("erlopts ~p", [ErlOpts]),
+    Recursive = CompileOpts#compile_opts.recursive,
+    AllErlFiles = gather_src(SrcDirs, Recursive),
+    ?DEBUG("files to compile ~p", [AllErlFiles]),
+
+    %% Make sure that outdir is on the path
+    ok = filelib:ensure_dir(filename:join(OutDir, "dummy.beam")),
+    true = code:add_patha(filename:absname(OutDir)),
+
+    G = init_erlcinfo(proplists:get_all_values(i, ErlOpts), AllErlFiles, BaseDir, OutDir),
+
+    NeededErlFiles = needed_files(G, ErlOpts, BaseDir, OutDir, AllErlFiles),
+    {ErlFirstFiles, ErlOptsFirst} = erl_first_files(RebarOpts, ErlOpts, BaseDir, NeededErlFiles),
+    {DepErls, OtherErls} = lists:partition(
+                             fun(Source) -> digraph:in_degree(G, Source) > 0 end,
+                             [File || File <- NeededErlFiles, not lists:member(File, ErlFirstFiles)]),
+    SubGraph = digraph_utils:subgraph(G, DepErls),
+    DepErlsOrdered = digraph_utils:topsort(SubGraph),
+    FirstErls = ErlFirstFiles ++ lists:reverse(DepErlsOrdered),
+    try
+        rebar_base_compiler:run(
+            RebarOpts, FirstErls, OtherErls,
+            fun(S, C) ->
+                    ErlOpts1 = case lists:member(S, ErlFirstFiles) of
+                                   true -> ErlOptsFirst;
+                                   false -> ErlOpts
+                               end,
+                    internal_erl_compile(C, BaseDir, S, OutDir, ErlOpts1)
+            end)
+    after
+        true = digraph:delete(SubGraph),
+        true = digraph:delete(G)
+    end,
+    ok.
+
+%% @doc remove compiled artifacts from an AppDir.
 
--spec clean(file:filename()) -> 'ok'.
-clean(AppDir) ->
-    MibFiles = rebar_utils:find_files(filename:join(AppDir, "mibs"), ?RE_PREFIX".*\\.mib\$"),
+-spec clean(rebar_app_info:t()) -> 'ok'.
+clean(AppInfo) ->
+    AppDir = rebar_app_info:out_dir(AppInfo),
+
+    MibFiles = rebar_utils:find_files(filename:join([AppDir, "mibs"]), ?RE_PREFIX".*\\.mib\$"),
     MIBs = [filename:rootname(filename:basename(MIB)) || MIB <- MibFiles],
     rebar_file_utils:delete_each(
       [filename:join([AppDir, "include",MIB++".hrl"]) || MIB <- MIBs]),
-    lists:foreach(fun(F) -> ok = rebar_file_utils:rm_rf(F) end,
-                  [filename:join(AppDir, "ebin/*.beam"), filename:join(AppDir, "priv/mibs/*.bin")]),
+    ok = rebar_file_utils:rm_rf(filename:join([AppDir, "priv/mibs/*.bin"])),
 
-    YrlFiles = rebar_utils:find_files(filename:join(AppDir, "src"), ?RE_PREFIX".*\\.[x|y]rl\$"),
+    YrlFiles = rebar_utils:find_files(filename:join([AppDir, "src"]), ?RE_PREFIX".*\\.[x|y]rl\$"),
     rebar_file_utils:delete_each(
       [ binary_to_list(iolist_to_binary(re:replace(F, "\\.[x|y]rl$", ".erl")))
         || F <- YrlFiles ]),
 
+    BinDirs = ["ebin"|rebar_dir:extra_src_dirs(rebar_app_info:opts(AppInfo))],
+    ok = clean_dirs(AppDir, BinDirs),
+
     %% Delete the build graph, if any
-    rebar_file_utils:rm_rf(erlcinfo_file(AppDir)),
+    rebar_file_utils:rm_rf(erlcinfo_file(AppDir)).
 
+clean_dirs(_AppDir, []) -> ok;
+clean_dirs(AppDir, [Dir|Rest]) ->
+    ok = rebar_file_utils:rm_rf(filename:join([AppDir, Dir, "*.beam"])),
     %% Erlang compilation is recursive, so it's possible that we have a nested
     %% directory structure in ebin with .beam files within. As such, we want
-    %% to scan whatever is left in the ebin/ directory for sub-dirs which
+    %% to scan whatever is left in the app's out_dir directory for sub-dirs which
     %% satisfy our criteria.
-    BeamFiles = rebar_utils:find_files(filename:join(AppDir, "ebin"), ?RE_PREFIX".*\\.beam\$"),
+    BeamFiles = rebar_utils:find_files(filename:join([AppDir, Dir]), ?RE_PREFIX".*\\.beam\$"),
     rebar_file_utils:delete_each(BeamFiles),
-    lists:foreach(fun(Dir) -> delete_dir(Dir, dirs(Dir)) end, dirs(filename:join(AppDir, "ebin"))),
-    ok.
+    lists:foreach(fun(D) -> delete_dir(D, dirs(D)) end, dirs(filename:join([AppDir, Dir]))),
+    clean_dirs(AppDir, Rest).
+
 
 %% ===================================================================
 %% Internal functions
 %% ===================================================================
 
--spec doterl_compile(rebar_dict(), file:filename(), file:filename()) -> ok.
-doterl_compile(Opts, Dir, ODir) ->
-    ErlOpts = rebar_opts:erl_opts(Opts),
-    doterl_compile(Opts, Dir, ODir, [], ErlOpts).
-
-doterl_compile(Opts, Dir, OutDir, MoreSources, ErlOpts) ->
-    ?DEBUG("erl_opts ~p", [ErlOpts]),
-    %% Support the src_dirs option allowing multiple directories to
-    %% contain erlang source. This might be used, for example, should
-    %% eunit tests be separated from the core application source.
-    SrcDirs = [filename:join(Dir, X) || X <- rebar_dir:all_src_dirs(Opts, ["src"], [])],
-    AllErlFiles = gather_src(SrcDirs, []) ++ MoreSources,
-
-    %% Make sure that ebin/ exists and is on the path
-    ok = filelib:ensure_dir(filename:join(OutDir, "dummy.beam")),
-    true = code:add_patha(filename:absname(OutDir)),
-
-    OutDir1 = proplists:get_value(outdir, ErlOpts, OutDir),
-
-    G = init_erlcinfo(proplists:get_all_values(i, ErlOpts), AllErlFiles, Dir, OutDir),
-
-    NeededErlFiles = needed_files(G, ErlOpts, Dir, OutDir1, AllErlFiles),
-    {ErlFirstFiles, ErlOptsFirst} = erl_first_files(Opts, ErlOpts, Dir, NeededErlFiles),
-    {DepErls, OtherErls} = lists:partition(
-                             fun(Source) -> digraph:in_degree(G, Source) > 0 end,
-                             [File || File <- NeededErlFiles, not lists:member(File, ErlFirstFiles)]),
-    SubGraph = digraph_utils:subgraph(G, DepErls),
-    DepErlsOrdered = digraph_utils:topsort(SubGraph),
-    FirstErls = ErlFirstFiles ++ lists:reverse(DepErlsOrdered),
-    ?DEBUG("Files to compile first: ~p", [FirstErls]),
-    try
-        rebar_base_compiler:run(
-          Opts, FirstErls, OtherErls,
-          fun(S, C) ->
-                  ErlOpts1 = case lists:member(S, ErlFirstFiles) of
-                                 true -> ErlOptsFirst;
-                                 false -> ErlOpts
-                             end,
-                  internal_erl_compile(C, Dir, S, OutDir1, ErlOpts1)
-          end)
-    after
-        true = digraph:delete(SubGraph),
-        true = digraph:delete(G)
-    end,
-    ok.
+gather_src(Dirs, Recursive) ->
+    gather_src(Dirs, [], Recursive).
 
+gather_src([], Srcs, _Recursive) -> Srcs;
+gather_src([Dir|Rest], Srcs, Recursive) ->
+    gather_src(Rest, Srcs ++ rebar_utils:find_files(Dir, ?RE_PREFIX".*\\.erl\$", Recursive), Recursive).
+    
 %% Get files which need to be compiled first, i.e. those specified in erl_first_files
 %% and parse_transform options.  Also produce specific erl_opts for these first
 %% files, so that yet to be compiled parse transformations are excluded from it.
@@ -493,12 +581,7 @@ compile_xrl_yrl(_Opts, Source, Target, AllOpts, Mod) ->
 needs_compile(Source, Target) ->
     filelib:last_modified(Source) > filelib:last_modified(Target).
 
-gather_src([], Srcs) ->
-    ?DEBUG("src_files ~p", [Srcs]),
-    Srcs;
-gather_src([Dir|Rest], Srcs) ->
-    gather_src(
-      Rest, Srcs ++ rebar_utils:find_files(Dir, ?RE_PREFIX".*\\.erl\$")).
+
 
 -spec dirs(file:filename()) -> [file:filename()].
 dirs(Dir) ->
@@ -627,3 +710,13 @@ check_file(File) ->
         false -> ?ABORT("File ~p is missing, aborting\n", [File]);
         true -> File
     end.
+
+outdir(RebarOpts) ->
+    ErlOpts = rebar_opts:erl_opts(RebarOpts),
+    proplists:get_value(outdir, ErlOpts, ?DEFAULT_OUTDIR).
+
+parse_opts(Opts) -> parse_opts(Opts, #compile_opts{}).
+
+parse_opts([], CompileOpts) -> CompileOpts;
+parse_opts([{recursive, Recursive}|Rest], CompileOpts) when Recursive == true; Recursive == false ->
+    parse_opts(Rest, CompileOpts#compile_opts{recursive = Recursive}).
diff --git a/src/rebar_prv_clean.erl b/src/rebar_prv_clean.erl
index a2b537a..6694dc0 100644
--- a/src/rebar_prv_clean.erl
+++ b/src/rebar_prv_clean.erl
@@ -49,6 +49,7 @@ do(State) ->
     Cwd = rebar_dir:get_cwd(),
     rebar_hooks:run_all_hooks(Cwd, pre, ?PROVIDER, Providers, State),
     clean_apps(State, Providers, ProjectApps),
+    clean_extras(State),
     rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State),
 
     {ok, State}.
@@ -66,10 +67,14 @@ clean_apps(State, Providers, Apps) ->
          ?INFO("Cleaning out ~s...", [rebar_app_info:name(AppInfo)]),
          AppDir = rebar_app_info:dir(AppInfo),
          AppInfo1 = rebar_hooks:run_all_hooks(AppDir, pre, ?PROVIDER, Providers, AppInfo, State),
-         rebar_erlc_compiler:clean(rebar_app_info:out_dir(AppInfo1)),
+         rebar_erlc_compiler:clean(AppInfo1),
          rebar_hooks:run_all_hooks(AppDir, post, ?PROVIDER, Providers, AppInfo1, State)
      end || AppInfo <- Apps].
 
+clean_extras(State) ->
+    BaseDir = rebar_dir:base_dir(State),
+    rebar_file_utils:rm_rf(filename:join([BaseDir, "extras"])).
+
 handle_args(State) ->
     {Args, _} = rebar_state:command_parsed_args(State),
     All = proplists:get_value(all, Args, false),
diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl
index 9155d3b..1f4c02d 100644
--- a/src/rebar_prv_common_test.erl
+++ b/src/rebar_prv_common_test.erl
@@ -285,26 +285,7 @@ find_suite_dirs(Suites) ->
 
 copy(State, Dir) ->
     From = reduce_path(Dir),
-    case From == rebar_state:dir(State) of
-        true  -> throw({error, suite_at_project_root});
-        false -> ok
-    end,
-    case retarget_path(State, From) of
-        %% directory lies outside of our project's file structure so
-        %%  don't copy it
-        From   -> From;
-        Target ->
-            %% recursively delete any symlinks in the target directory
-            %%  if it exists so we don't smash files in the linked dirs
-            case ec_file:is_dir(Target) of
-                true  -> remove_links(Target);
-                false -> ok
-            end,
-            case rebar_file_utils:symlink_or_copy(From, Target) of
-               exists -> Target;
-               ok     -> Target
-            end
-    end.
+    retarget_path(State, From).
 
 compile_dir(State, Dir) ->
     NewState = replace_src_dirs(State, [filename:absname(Dir)]),
@@ -350,29 +331,6 @@ reduce_path([_|Acc], [".."|Rest])  -> reduce_path(Acc, Rest);
 reduce_path([], [".."|Rest])       -> reduce_path([], Rest);
 reduce_path(Acc, [Component|Rest]) -> reduce_path([Component|Acc], Rest).
 
-
-remove_links(Path) ->
-    IsDir = ec_file:is_dir(Path),
-    case ec_file:is_symlink(Path) of
-        true when IsDir ->
-            delete_dir_link(Path);
-        false when IsDir ->
-                lists:foreach(fun(ChildPath) ->
-                                      remove_links(ChildPath)
-                              end, dir_entries(Path));
-        _ -> file:delete(Path)
-    end.
-
-delete_dir_link(Path) ->
-    case os:type() of
-        {unix, _} -> file:delete(Path);
-        {win32, _} -> file:del_dir(Path)
-    end.
-
-dir_entries(Path) ->
-    {ok, SubDirs} = file:list_dir(Path),
-    [filename:join(Path, SubDir) || SubDir <- SubDirs].
-
 replace_src_dirs(State, Dirs) ->
     %% replace any `src_dirs` with the test dirs
     ErlOpts = rebar_state:get(State, erl_opts, []),
diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl
index 9811de9..2996aee 100644
--- a/src/rebar_prv_compile.erl
+++ b/src/rebar_prv_compile.erl
@@ -52,8 +52,10 @@ do(State) ->
     ProjectApps2 = build_apps(State, Providers, ProjectApps1),
     State2 = rebar_state:project_apps(State, ProjectApps2),
 
-    ProjAppsPaths = [filename:join(rebar_app_info:out_dir(X), "ebin") || X <- ProjectApps2],
-    State3 = rebar_state:code_paths(State2, all_deps, DepsPaths ++ ProjAppsPaths),
+    %% projects with structures like /apps/foo,/apps/bar,/test
+    build_extra_dirs(State, ProjectApps2),
+
+    State3 = update_code_paths(State2, ProjectApps2, DepsPaths),
 
     rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State2),
     case rebar_state:has_all_artifacts(State3) of
@@ -82,6 +84,30 @@ build_app(State, Providers, AppInfo) ->
     copy_app_dirs(AppInfo, AppDir, OutDir),
     compile(State, Providers, AppInfo).
 
+build_extra_dirs(State, Apps) ->
+    BaseDir = rebar_state:dir(State),
+    F = fun(App) -> rebar_app_info:dir(App) == BaseDir end,
+    %% check that this app hasn't already been dealt with
+    case lists:any(F, Apps) of
+        false ->
+            ProjOpts = rebar_state:opts(State),
+            Extras = rebar_dir:extra_src_dirs(ProjOpts, []),
+            [build_extra_dir(State, Dir) || Dir <- Extras];
+        true  -> ok
+    end.
+
+build_extra_dir(_State, []) -> ok;
+build_extra_dir(State, Dir) ->
+    case ec_file:is_dir(filename:join([rebar_state:dir(State), Dir])) of
+        true ->
+            BaseDir = filename:join([rebar_dir:base_dir(State), "extras"]),
+            OutDir = filename:join([BaseDir, Dir]),
+            filelib:ensure_dir(filename:join([OutDir, "dummy.beam"])),
+            copy(rebar_state:dir(State), BaseDir, Dir),
+            rebar_erlc_compiler:compile_dir(State, BaseDir, OutDir);
+        false -> ok
+    end.
+
 compile(State, AppInfo) ->
     compile(State, rebar_state:providers(State), AppInfo).
 
@@ -104,6 +130,33 @@ compile(State, Providers, AppInfo) ->
 %% Internal functions
 %% ===================================================================
 
+update_code_paths(State, ProjectApps, DepsPaths) ->
+    ProjAppsPaths = paths_for_apps(ProjectApps),
+    ExtrasPaths = paths_for_extras(State, ProjectApps),
+    rebar_state:code_paths(State, all_deps, DepsPaths ++ ProjAppsPaths ++ ExtrasPaths).
+
+paths_for_apps(Apps) -> paths_for_apps(Apps, []).
+
+paths_for_apps([], Acc) -> Acc;
+paths_for_apps([App|Rest], Acc) ->
+    {_SrcDirs, ExtraDirs} = resolve_src_dirs(rebar_app_info:opts(App)),
+    Paths = [filename:join([rebar_app_info:out_dir(App), Dir]) || Dir <- ["ebin"|ExtraDirs]],
+    FilteredPaths = lists:filter(fun ec_file:is_dir/1, Paths),
+    paths_for_apps(Rest, Acc ++ FilteredPaths).
+    
+paths_for_extras(State, Apps) ->
+    F = fun(App) -> rebar_app_info:dir(App) == rebar_state:dir(State) end,
+    %% check that this app hasn't already been dealt with
+    case lists:any(F, Apps) of
+        false -> paths_for_extras(State);
+        true  -> []
+    end.
+
+paths_for_extras(State) ->
+    {_SrcDirs, ExtraDirs} = resolve_src_dirs(rebar_state:opts(State)),
+    Paths = [filename:join([rebar_dir:base_dir(State), "extras", Dir]) || Dir <- ExtraDirs],
+    lists:filter(fun ec_file:is_dir/1, Paths).
+
 has_all_artifacts(AppInfo1) ->
     case rebar_app_info:has_all_artifacts(AppInfo1) of
         {false, File} ->
@@ -120,17 +173,17 @@ copy_app_dirs(AppInfo, OldAppDir, AppDir) ->
             %% copy all files from ebin if it exists
             case filelib:is_dir(EbinDir) of
                 true ->
-                    OutEbin = filename:join(AppDir, "ebin"),
-                    filelib:ensure_dir(filename:join(OutEbin, "dummy.beam")),
-                    rebar_file_utils:cp_r(filelib:wildcard(filename:join(EbinDir, "*")), OutEbin);
+                    OutEbin = filename:join([AppDir, "ebin"]),
+                    filelib:ensure_dir(filename:join([OutEbin, "dummy.beam"])),
+                    rebar_file_utils:cp_r(filelib:wildcard(filename:join([EbinDir, "*"])), OutEbin);
                 false ->
                     ok
             end,
 
-            filelib:ensure_dir(filename:join(AppDir, "dummy")),
+            filelib:ensure_dir(filename:join([AppDir, "dummy"])),
 
             %% link or copy mibs if it exists
-            case filelib:is_dir(filename:join(OldAppDir, "mibs")) of
+            case filelib:is_dir(filename:join([OldAppDir, "mibs"])) of
                 true ->
                     %% If mibs exist it means we must ensure priv exists.
                     %% mibs files are compiled to priv/mibs/
@@ -139,15 +192,57 @@ copy_app_dirs(AppInfo, OldAppDir, AppDir) ->
                 false ->
                     ok
             end,
-
+            {SrcDirs, ExtraDirs} = resolve_src_dirs(rebar_app_info:opts(AppInfo)),
             %% link to src_dirs to be adjacent to ebin is needed for R15 use of cover/xref
-            SrcDirs = rebar_dir:all_src_dirs(rebar_app_info:opts(AppInfo), ["src"], []),
-            [symlink_or_copy(OldAppDir, AppDir, Dir) || Dir <- ["priv", "include"] ++ SrcDirs];
+            [symlink_or_copy(OldAppDir, AppDir, Dir) || Dir <- ["priv", "include"] ++ SrcDirs],
+            %% copy all extra_src_dirs as they build into themselves and linking means they
+            %% are shared across profiles
+            [copy(OldAppDir, AppDir, Dir) || Dir <- ExtraDirs];
         false ->
             ok
     end.
 
 symlink_or_copy(OldAppDir, AppDir, Dir) ->
-    Source = filename:join(OldAppDir, Dir),
-    Target = filename:join(AppDir, Dir),
+    Source = filename:join([OldAppDir, Dir]),
+    Target = filename:join([AppDir, Dir]),
     rebar_file_utils:symlink_or_copy(Source, Target).
+
+copy(OldAppDir, AppDir, Dir) ->
+    Source = filename:join([OldAppDir, Dir]),
+    Target = filename:join([AppDir, Dir]),
+    case ec_file:is_dir(Source) of
+        true  -> copy(Source, Target);
+        false -> ok
+    end.
+
+%% TODO: use ec_file:copy/2 to do this, it preserves timestamps and
+%% may prevent recompilation of files in extra dirs
+copy(Source, Target) ->
+    %% important to do this so no files are copied onto themselves
+    %% which truncates them to zero length on some platforms
+    ok = delete_if_symlink(Target),
+    ok = filelib:ensure_dir(filename:join([Target, "dummy.beam"])),
+    {ok, Files} = rebar_utils:list_dir(Source),
+    case [filename:join([Source, F]) || F <- Files] of
+        []    -> ok;
+        Paths -> rebar_file_utils:cp_r(Paths, Target)
+    end.
+
+delete_if_symlink(Path) ->
+    case ec_file:is_symlink(Path) of
+        true  -> file:delete(Path);
+        false -> ok
+    end.
+
+resolve_src_dirs(Opts) ->
+    SrcDirs = rebar_dir:src_dirs(Opts, ["src"]),
+    ExtraDirs = rebar_dir:extra_src_dirs(Opts, []),
+    normalize_src_dirs(SrcDirs, ExtraDirs).
+
+%% remove duplicates and make sure no directories that exist
+%% in src_dirs also exist in extra_src_dirs
+normalize_src_dirs(SrcDirs, ExtraDirs) ->
+    S = lists:usort(SrcDirs),
+    E = lists:usort(ExtraDirs),
+    {S, lists:subtract(E, S)}.
+
diff --git a/src/rebar_prv_cover.erl b/src/rebar_prv_cover.erl
index fc7457e..0b9b9bb 100644
--- a/src/rebar_prv_cover.erl
+++ b/src/rebar_prv_cover.erl
@@ -103,7 +103,7 @@ analyze(State) ->
 
 get_all_coverdata(CoverDir) ->
     ok = filelib:ensure_dir(filename:join([CoverDir, "dummy.log"])),
-    {ok, Files} = file:list_dir(CoverDir),
+    {ok, Files} = rebar_utils:list_dir(CoverDir),
     rebar_utils:filtermap(fun(FileName) ->
         case filename:extension(FileName) == ".coverdata" of
             true  -> {true, filename:join([CoverDir, FileName])};
@@ -277,8 +277,9 @@ strip_coverdir(File) ->
 
 cover_compile(State, apps) ->
     Apps = filter_checkouts(rebar_state:project_apps(State)),
-    AppDirs = app_ebin_dirs(Apps, []),
-    cover_compile(State, AppDirs);
+    AppDirs = app_dirs(Apps),
+    ExtraDirs = extra_src_dirs(State, Apps),
+    cover_compile(State, AppDirs ++ ExtraDirs);
 cover_compile(State, Dirs) ->
     %% start the cover server if necessary
     {ok, CoverPid} = start_cover(),
@@ -290,9 +291,30 @@ cover_compile(State, Dirs) ->
 
 compile([], Acc) -> lists:reverse(Acc);
 compile([Dir|Rest], Acc) ->
+    ?INFO("covering ~p", [Dir]),
     Result = cover:compile_beam_directory(Dir),
     compile(Rest, [Result|Acc]).
 
+app_dirs(Apps) ->
+    lists:foldl(fun app_ebin_dirs/2, [], Apps).
+
+app_ebin_dirs(App, Acc) ->
+    AppDir = rebar_app_info:ebin_dir(App),
+    ExtraDirs = rebar_dir:extra_src_dirs(rebar_app_info:opts(App), []),
+    OutDir = rebar_app_info:out_dir(App),
+    [filename:join([OutDir, D]) || D <- [AppDir|ExtraDirs]] ++ Acc.
+
+extra_src_dirs(State, Apps) ->
+    BaseDir = rebar_state:dir(State),
+    F = fun(App) -> rebar_app_info:dir(App) == BaseDir end,
+    %% check that this app hasn't already been dealt with
+    Extras = case lists:any(F, Apps) of
+        false -> rebar_dir:extra_src_dirs(rebar_state:opts(State), []);
+        true  -> []
+    end,
+    OutDir = rebar_dir:base_dir(State),
+    [filename:join([OutDir, "extras", D]) || D <- Extras].
+
 filter_checkouts(Apps) -> filter_checkouts(Apps, []).
 
 filter_checkouts([], Acc) -> lists:reverse(Acc);
@@ -302,10 +324,6 @@ filter_checkouts([App|Rest], Acc) ->
         false -> filter_checkouts(Rest, [App|Acc])
     end.
 
-app_ebin_dirs([], Acc) -> Acc;
-app_ebin_dirs([App|Rest], Acc) ->
-    app_ebin_dirs(Rest, [rebar_app_info:ebin_dir(App)|Acc]).
-
 start_cover() ->
     case cover:start() of
         {ok, Pid}                       -> {ok, Pid};
diff --git a/src/rebar_prv_eunit.erl b/src/rebar_prv_eunit.erl
index 959be3f..b748b06 100644
--- a/src/rebar_prv_eunit.erl
+++ b/src/rebar_prv_eunit.erl
@@ -9,7 +9,7 @@
          do/1,
          format_error/1]).
 %% exported solely for tests
--export([compile/1, prepare_tests/1, eunit_opts/1]).
+-export([compile/2, prepare_tests/1, eunit_opts/1]).
 
 -include("rebar.hrl").
 -include_lib("providers/include/providers.hrl").
@@ -39,14 +39,15 @@ init(State) ->
 
 -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
 do(State) ->
-    case compile(State) of
+    Tests = prepare_tests(State),
+    case compile(State, Tests) of
         %% successfully compiled apps
-        {ok, S} -> do_tests(S);
+        {ok, S} -> do(S, Tests);
         %% this should look like a compiler error, not an eunit error
         Error   -> Error
     end.
 
-do_tests(State) ->
+do(State, Tests) ->
     ?INFO("Performing EUnit tests...", []),
 
     rebar_utils:update_code(rebar_state:code_paths(State, all_deps)),
@@ -56,9 +57,9 @@ do_tests(State) ->
     Cwd = rebar_dir:get_cwd(),
     rebar_hooks:run_all_hooks(Cwd, pre, ?PROVIDER, Providers, State),
 
-    case prepare_tests(State) of
-        {ok, Tests} ->
-            case do_tests(State, Tests) of
+    case Tests of
+        {ok, T} ->
+            case run_tests(State, T) of
                 {ok, State1} ->
                     %% Run eunit provider posthooks
                     rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State1),
@@ -73,11 +74,12 @@ do_tests(State) ->
             Error
     end.
 
-do_tests(State, Tests) ->
+run_tests(State, Tests) ->
+    T = translate_paths(State, Tests),
     EUnitOpts = resolve_eunit_opts(State),
-    ?DEBUG("eunit_tests ~p", [Tests]),
+    ?DEBUG("eunit_tests ~p", [T]),
     ?DEBUG("eunit_opts  ~p", [EUnitOpts]),
-    Result = eunit:test(Tests, EUnitOpts),
+    Result = eunit:test(T, EUnitOpts),
     ok = maybe_write_coverdata(State),
     case handle_results(Result) of
         {error, Reason} ->
@@ -114,38 +116,80 @@ safe_define_test_macro(Opts) ->
        false -> [{d, 'TEST'}] ++ Opts
     end.
 
-compile(State) ->
-    %% inject `eunit_first_files` and `eunit_compile_opts` into the applications to be compiled
-    NewState = inject_eunit_state(State),
+compile(State, {ok, Tests}) ->
+    %% inject `eunit_first_files`, `eunit_compile_opts` and any
+    %% directories required by tests into the applications
+    NewState = inject_eunit_state(State, Tests),
 
     case rebar_prv_compile:do(NewState) of
         %% successfully compiled apps
         {ok, S} ->
             ok = maybe_cover_compile(S),
-            ok = maybe_compile_bare_testdir(S),
             {ok, S};
         %% this should look like a compiler error, not an eunit error
         Error   -> Error
-    end.
-
-inject_eunit_state(State) ->
-    Apps = project_apps(State),
-    ModdedApps = lists:map(fun(App) -> inject(State, App) end, Apps),
-    rebar_state:project_apps(State, ModdedApps).
-
-inject(State, App) ->
+    end;
+%% maybe compile even in the face of errors?
+compile(_State, Error) -> Error.
+
+inject_eunit_state(State, Tests) ->
+    Apps = rebar_state:project_apps(State),
+    ModdedApps = lists:map(fun(App) ->
+        NewOpts = inject(rebar_app_info:opts(App), State),
+        rebar_app_info:opts(App, NewOpts)
+    end, Apps),
+    NewOpts = inject(rebar_state:opts(State), State),
+    NewState = rebar_state:opts(State, NewOpts),
+    test_dirs(NewState, ModdedApps, Tests).
+
+inject(Opts, State) ->
     %% append `eunit_compile_opts` to app defined `erl_opts`
-    ErlOpts = rebar_app_info:get(App, erl_opts, []),
+    ErlOpts = rebar_opts:get(Opts, erl_opts, []),
     EUnitOpts = rebar_state:get(State, eunit_compile_opts, []),
-    NewOpts = EUnitOpts ++ ErlOpts,
+    NewErlOpts = EUnitOpts ++ ErlOpts,
     %% append `eunit_first_files` to app defined `erl_first_files`
-    FirstFiles = rebar_app_info:get(App, erl_first_files, []),
+    FirstFiles = rebar_opts:get(Opts, erl_first_files, []),
     EUnitFirstFiles = rebar_state:get(State, eunit_first_files, []),
     NewFirstFiles = EUnitFirstFiles ++ FirstFiles,
-    %% insert the new keys into the app
-    lists:foldl(fun({K, V}, NewApp) -> rebar_app_info:set(NewApp, K, V) end,
-                App,
-                [{erl_opts, NewOpts}, {erl_first_files, NewFirstFiles}]).
+    %% insert the new keys into the opts
+    lists:foldl(fun({K, V}, NewOpts) -> rebar_opts:set(NewOpts, K, V) end,
+                Opts,
+                [{erl_opts, NewErlOpts}, {erl_first_files, NewFirstFiles}]).
+
+test_dirs(State, Apps, []) -> rebar_state:project_apps(State, Apps);
+test_dirs(State, Apps, [{dir, Dir}|Rest]) ->
+    %% insert `Dir` into an app if relative, or the base state if not
+    %% app relative but relative to the root or not at all if outside
+    %% project scope
+    {NewState, NewApps} = maybe_inject_test_dir(State, [], Apps, Dir),
+    test_dirs(NewState, NewApps, Rest);
+test_dirs(State, Apps, [{file, File}|Rest]) ->
+    Dir = filename:dirname(File),
+    {NewState, NewApps} = maybe_inject_test_dir(State, [], Apps, Dir),
+    test_dirs(NewState, NewApps, Rest);
+test_dirs(State, Apps, [_|Rest]) -> test_dirs(State, Apps, Rest).
+
+maybe_inject_test_dir(State, AppAcc, [App|Rest], Dir) ->
+    case rebar_file_utils:path_from_ancestor(Dir, rebar_app_info:dir(App)) of
+        {ok, Path} ->
+            Opts = inject_test_dir(rebar_app_info:opts(App), Path),
+            {State, AppAcc ++ [rebar_app_info:opts(App, Opts)] ++ Rest};
+        {error, badparent} ->
+            maybe_inject_test_dir(State, AppAcc ++ [App], Rest, Dir)
+    end;
+maybe_inject_test_dir(State, AppAcc, [], Dir) ->
+    case rebar_file_utils:path_from_ancestor(Dir, rebar_state:dir(State)) of
+        {ok, Path} ->
+            Opts = inject_test_dir(rebar_state:opts(State), Path),
+            {rebar_state:opts(State, Opts), AppAcc};
+        {error, badparent} ->
+            {State, AppAcc}
+    end.
+
+inject_test_dir(Opts, Dir) ->
+    %% append specified test targets to app defined `extra_src_dirs`
+    ExtraSrcDirs = rebar_opts:get(Opts, extra_src_dirs, []),
+    rebar_opts:set(Opts, extra_src_dirs, ExtraSrcDirs ++ [Dir]).
 
 test_defined([{d, 'TEST'}|_]) -> true;
 test_defined([{d, 'TEST', true}|_]) -> true;
@@ -156,7 +200,7 @@ prepare_tests(State) ->
     %% parse and translate command line tests
     CmdTests = resolve_tests(State),
     CfgTests = rebar_state:get(State, eunit_tests, []),
-    ProjectApps = project_apps(State),
+    ProjectApps = rebar_state:project_apps(State),
     %% prioritize tests to run first trying any command line specified
     %% tests falling back to tests specified in the config file finally
     %% running a default set if no other tests are present
@@ -187,18 +231,6 @@ resolve(Flag, EUnitKey, RawOpts) ->
 normalize(Key, Value) when Key == dir; Key == file -> {Key, Value};
 normalize(Key, Value) -> {Key, list_to_atom(Value)}.
 
-project_apps(State) ->
-    filter_checkouts(rebar_state:project_apps(State)).
-
-filter_checkouts(Apps) -> filter_checkouts(Apps, []).
-
-filter_checkouts([], Acc) -> lists:reverse(Acc);
-filter_checkouts([App|Rest], Acc) ->
-    case rebar_app_info:is_checkout(App) of
-        true  -> filter_checkouts(Rest, Acc);
-        false -> filter_checkouts(Rest, [App|Acc])
-    end.
-
 select_tests(State, ProjectApps, [], []) -> default_tests(State, ProjectApps);
 select_tests(_State, _ProjectApps, [], Tests)  -> Tests;
 select_tests(_State, _ProjectApps, Tests, _) -> Tests.
@@ -212,7 +244,7 @@ default_tests(State, Apps) ->
         %%  included or `test` does not exist
         false -> lists:reverse(Tests);
         %% need to add `test` dir at root to dirs to be included
-        true  -> lists:reverse([{dir, filename:join([rebar_dir:base_dir(State), "test"])}|Tests])
+        true  -> lists:reverse([{dir, BareTest}|Tests])
     end.
 
 set_apps([], Acc) -> Acc;
@@ -268,14 +300,14 @@ validate_app(State, [App|Rest], AppName) ->
         false -> validate_app(State, Rest, AppName)
     end.
 
-validate_dir(_State, Dir) ->
-    case ec_file:is_dir(Dir) of
+validate_dir(State, Dir) ->
+    case ec_file:is_dir(filename:join([rebar_state:dir(State), Dir])) of
         true  -> ok;
         false -> {error, lists:concat(["Directory `", Dir, "' not found."])}
     end.
 
-validate_file(_State, File) ->
-    case ec_file:exists(File) of
+validate_file(State, File) ->
+    case ec_file:exists(filename:join([rebar_state:dir(State), File])) of
         true  -> ok;
         false -> {error, lists:concat(["File `", File, "' not found."])}
     end.
@@ -302,6 +334,31 @@ set_verbose(Opts) ->
         false -> [verbose] ++ Opts
     end.
 
+translate_paths(State, Tests) -> translate_paths(State, Tests, []).
+
+translate_paths(_State, [], Acc) -> lists:reverse(Acc);
+translate_paths(State, [{dir, Dir}|Rest], Acc) ->
+    Apps = rebar_state:project_apps(State),
+    translate_paths(State, Rest, [translate(State, Apps, Dir)|Acc]);
+translate_paths(State, [{file, File}|Rest], Acc) ->
+    Dir = filename:dirname(File),
+    Apps = rebar_state:project_apps(State),
+    translate_paths(State, Rest, [translate(State, Apps, Dir)|Acc]);
+translate_paths(State, [Test|Rest], Acc) ->
+    translate_paths(State, Rest, [Test|Acc]).
+
+translate(State, [App|Rest], Dir) ->
+    case rebar_file_utils:path_from_ancestor(Dir, rebar_app_info:dir(App)) of
+        {ok, Path}         -> {dir, filename:join([rebar_app_info:out_dir(App), Path])};
+        {error, badparent} -> translate(State, Rest, Dir)
+    end;
+translate(State, [], Dir) ->
+    case rebar_file_utils:path_from_ancestor(Dir, rebar_state:dir(State)) of
+        {ok, Path}         -> {dir, filename:join([rebar_dir:base_dir(State), "extras", Path])};
+        %% not relative, leave as is
+        {error, badparent} -> {dir, Dir}
+    end.
+
 maybe_cover_compile(State) ->
     {RawOpts, _} = rebar_state:command_parsed_args(State),
     State1 = case proplists:get_value(cover, RawOpts, false) of
@@ -310,14 +367,6 @@ maybe_cover_compile(State) ->
     end,
     rebar_prv_cover:maybe_cover_compile(State1).
 
-maybe_cover_compile(State, Dir) ->
-    {RawOpts, _} = rebar_state:command_parsed_args(State),
-    State1 = case proplists:get_value(cover, RawOpts, false) of
-        true  -> rebar_state:set(State, cover_enabled, true);
-        false -> State
-    end,
-    rebar_prv_cover:maybe_cover_compile(State1, [Dir]).
-
 maybe_write_coverdata(State) ->
     {RawOpts, _} = rebar_state:command_parsed_args(State),
     State1 = case proplists:get_value(cover, RawOpts, false) of
@@ -326,30 +375,6 @@ maybe_write_coverdata(State) ->
     end,
     rebar_prv_cover:maybe_write_coverdata(State1, ?PROVIDER).
 
-maybe_compile_bare_testdir(State) ->
-    Apps = project_apps(State),
-    BareTest = filename:join([rebar_state:dir(State), "test"]),
-    F = fun(App) -> rebar_app_info:dir(App) == rebar_state:dir(State) end,
-    case ec_file:is_dir(BareTest) andalso not lists:any(F, Apps) of
-        true ->
-            ErlOpts = rebar_state:get(State, erl_opts, []),
-            EUnitOpts = rebar_state:get(State, eunit_compile_opts, []),
-            NewErlOpts = safe_define_test_macro(EUnitOpts ++ ErlOpts),
-            %% append `eunit_first_files` to app defined `erl_first_files`
-            FirstFiles = rebar_state:get(State, erl_first_files, []),
-            EUnitFirstFiles = rebar_state:get(State, eunit_first_files, []),
-            NewFirstFiles = EUnitFirstFiles ++ FirstFiles,
-            Opts = rebar_state:opts(State),
-            NewOpts = lists:foldl(fun({K, V}, Dict) -> rebar_opts:set(Dict, K, V) end,
-                                  Opts,
-                                  [{erl_opts, NewErlOpts}, {erl_first_files, NewFirstFiles}, {src_dirs, ["test"]}]),
-            OutDir = filename:join([rebar_dir:base_dir(State), "test"]),
-            filelib:ensure_dir(filename:join([OutDir, "dummy.beam"])),
-            rebar_erlc_compiler:compile(NewOpts, rebar_state:dir(State), OutDir),
-            maybe_cover_compile(State, [OutDir]);
-        false -> ok
-    end.
-
 handle_results(ok) -> ok;
 handle_results(error) ->
     {error, unknown_error};
diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl
index ccdf960..4fd4bd1 100644
--- a/src/rebar_utils.erl
+++ b/src/rebar_utils.erl
@@ -65,7 +65,8 @@
          escape_double_quotes_weak/1,
          check_min_otp_version/1,
          check_blacklisted_otp_versions/1,
-         info_useless/2]).
+         info_useless/2,
+         list_dir/1]).
 
 %% for internal use only
 -export([otp_release/0]).
@@ -774,3 +775,9 @@ info_useless(Old, New) ->
      || Name <- Old,
         not lists:member(Name, New)],
     ok.
+
+-ifdef(no_list_dir_all).
+list_dir(Dir) -> file:list_dir(Dir).
+-else.
+list_dir(Dir) -> file:list_dir_all(Dir).
+-endif.