From ae0af35e8c8707ea8e60467a422d815bd549b55b Mon Sep 17 00:00:00 2001 From: Maxim Fedorov Date: Thu, 28 Mar 2019 20:08:13 -0700 Subject: Enable parallel build Support for parallel compilation of *.erl file was dropped before 3.0 release. However, our tests for a project containing ~500 source files show substantial gain, lowering compilation time from 58 seconds to 18 on a MacBook Pro 15" (4 cores, 8 threads), and to just 10 seconds on Xeon-D machine. --- src/rebar_compiler.erl | 56 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_compiler.erl b/src/rebar_compiler.erl index 02c74db..b04c2c4 100644 --- a/src/rebar_compiler.erl +++ b/src/rebar_compiler.erl @@ -77,7 +77,7 @@ run(CompilerMod, AppInfo, Label) -> true = digraph:delete(G), compile_each(FirstFiles, FirstFileOpts, BaseOpts, Mappings, CompilerMod), - compile_each(RestFiles, Opts, BaseOpts, Mappings, CompilerMod). + compile_parallel(RestFiles, Opts, BaseOpts, Mappings, CompilerMod). compile_each([], _Opts, _Config, _Outs, _CompilerMod) -> ok; @@ -99,6 +99,60 @@ compile_each([Source | Rest], Opts, Config, Outs, CompilerMod) -> end, compile_each(Rest, Opts, Config, Outs, CompilerMod). +compile_worker(QueuePid, Opts, Config, Outs, CompilerMod) -> + QueuePid ! self(), + receive + {compile, Source} -> + Result = CompilerMod:compile(Source, Outs, Config, Opts), + QueuePid ! {Result, Source}, + compile_worker(QueuePid, Opts, Config, Outs, CompilerMod); + empty -> + ok + end. + +compile_parallel([], _Opts, _BaseOpts, _Mappings, _CompilerMod) -> + ok; +compile_parallel(Targets, Opts, BaseOpts, Mappings, CompilerMod) -> + Self = self(), + F = fun() -> compile_worker(Self, Opts, BaseOpts, Mappings, CompilerMod) end, + Jobs = min(length(Targets), erlang:system_info(schedulers)), + ?DEBUG("Starting ~B compile worker(s)", [Jobs]), + Pids = [spawn_monitor(F) || _I <- lists:seq(1, Jobs)], + compile_queue(Targets, Pids, Opts, BaseOpts, Mappings, CompilerMod). + +compile_queue([], [], _Opts, _Config, _Outs, _CompilerMod) -> + ok; +compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod) -> + receive + Worker when is_pid(Worker), Targets =:= [] -> + Worker ! empty, + compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod); + Worker when is_pid(Worker) -> + Worker ! {compile, hd(Targets)}, + compile_queue(tl(Targets), Pids, Opts, Config, Outs, CompilerMod); + {ok, Source} -> + ?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), filename:basename(Source)]), + compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod); + {{ok, Warnings}, Source} -> + report(Warnings), + ?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), filename:basename(Source)]), + compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod); + {skipped, Source} -> + ?DEBUG("~sSkipped ~s", [rebar_utils:indent(1), filename:basename(Source)]), + compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod); + {Error, Source} -> + NewSource = format_error_source(Source, Config), + ?ERROR("Compiling ~ts failed", [NewSource]), + maybe_report(Error), + ?FAIL; + {'DOWN', Mref, _, Pid, normal} -> + Pids2 = lists:delete({Pid, Mref}, Pids), + compile_queue(Targets, Pids2, Opts, Config, Outs, CompilerMod); + {'DOWN', _Mref, _, _Pid, Info} -> + ?ERROR("Compilation failed: ~p", [Info]), + ?FAIL + end. + %% @doc remove compiled artifacts from an AppDir. -spec clean([module()], rebar_app_info:t()) -> 'ok'. clean(Compilers, AppInfo) -> -- cgit v1.1 From 9f81a5754e0dc4d3ce968d02facdfa5c87dcea3c Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 29 Mar 2019 08:55:59 -0400 Subject: Fix Parallel Compilation This patch does two things: 1. it broadens the interface for the compiler module so that non-first-file modules can possibly be parallelized. This is done by dynamically switching on `[ListOfFiles]`, which remains sequential as before, or `{[SeqPriority], [Parallel]}`, which divides regular files between higher priority ones and those that can be in parallel 2. implements this mechanism in the rebar compiler, based on the erl file digraph. If a file has an in-neighbour, it is depended on by another file. The mechanism therefore makes it so all files that have dependants get compiled in their strict relative sequential order first, and then the undepended-on files get compiled together in parallel. By running: ./rebar3 ct --suite test/rebar_compile_SUITE.erl --case \ recompile_when_parse_transform_inline_changes --repeat 50 the previous iteration of this would rapidly fail, and this one succeeds every time. --- src/rebar_compiler.erl | 13 +++++++++++-- src/rebar_compiler_erl.erl | 12 +++++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/rebar_compiler.erl b/src/rebar_compiler.erl index b04c2c4..55666ba 100644 --- a/src/rebar_compiler.erl +++ b/src/rebar_compiler.erl @@ -21,7 +21,10 @@ out_mappings => out_mappings()}. -callback needed_files(digraph:graph(), [file:filename()], out_mappings(), rebar_app_info:t()) -> - {{[file:filename()], term()}, {[file:filename()], term()}}. + {{[file:filename()], term()}, % ErlFirstFiles (erl_opts global priority) + {[file:filename()] | % [Sequential] + {[file:filename()], [file:filename()]}, % {Sequential, Parallel} + term()}}. -callback dependencies(file:filename(), file:dirname(), [file:dirname()]) -> [file:filename()]. -callback compile(file:filename(), out_mappings(), rebar_dict(), list()) -> ok | {ok, [string()]} | {ok, [string()], [string()]}. @@ -77,7 +80,13 @@ run(CompilerMod, AppInfo, Label) -> true = digraph:delete(G), compile_each(FirstFiles, FirstFileOpts, BaseOpts, Mappings, CompilerMod), - compile_parallel(RestFiles, Opts, BaseOpts, Mappings, CompilerMod). + case RestFiles of + {Sequential, Parallel} -> % new parallelizable form + compile_each(Sequential, Opts, BaseOpts, Mappings, CompilerMod), + compile_parallel(Parallel, Opts, BaseOpts, Mappings, CompilerMod); + _ when is_list(RestFiles) -> % traditional sequential build + compile_each(RestFiles, Opts, BaseOpts, Mappings, CompilerMod) + end. compile_each([], _Opts, _Config, _Outs, _CompilerMod) -> ok; diff --git a/src/rebar_compiler_erl.erl b/src/rebar_compiler_erl.erl index 759305c..1ad16d8 100644 --- a/src/rebar_compiler_erl.erl +++ b/src/rebar_compiler_erl.erl @@ -55,7 +55,14 @@ needed_files(Graph, FoundFiles, _, AppInfo) -> {ErlFirstFiles, ErlOptsFirst} = erl_first_files(RebarOpts, ErlOpts, Dir, NeededErlFiles), SubGraph = digraph_utils:subgraph(Graph, NeededErlFiles), DepErlsOrdered = digraph_utils:topsort(SubGraph), - OtherErls = lists:reverse(DepErlsOrdered), + %% Break out the files required by other modules from those + %% that none other depend of; the former must be sequentially + %% built, the rest is parallelizable. + OtherErls = lists:partition( + fun(Erl) -> digraph:in_degree(Graph, Erl) > 0 end, + lists:reverse([Dep || Dep <- DepErlsOrdered, + not lists:member(Dep, ErlFirstFiles)]) + ), PrivIncludes = [{i, filename:join(OutDir, Src)} || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])], @@ -64,8 +71,7 @@ needed_files(Graph, FoundFiles, _, AppInfo) -> true = digraph:delete(SubGraph), {{ErlFirstFiles, ErlOptsFirst ++ AdditionalOpts}, - {[Erl || Erl <- OtherErls, - not lists:member(Erl, ErlFirstFiles)], ErlOpts ++ AdditionalOpts}}. + {OtherErls, ErlOpts ++ AdditionalOpts}}. dependencies(Source, SourceDir, Dirs) -> {ok, Fd} = file:open(Source, [read]), -- cgit v1.1