From 9f81a5754e0dc4d3ce968d02facdfa5c87dcea3c Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 29 Mar 2019 08:55:59 -0400 Subject: Fix Parallel Compilation This patch does two things: 1. it broadens the interface for the compiler module so that non-first-file modules can possibly be parallelized. This is done by dynamically switching on `[ListOfFiles]`, which remains sequential as before, or `{[SeqPriority], [Parallel]}`, which divides regular files between higher priority ones and those that can be in parallel 2. implements this mechanism in the rebar compiler, based on the erl file digraph. If a file has an in-neighbour, it is depended on by another file. The mechanism therefore makes it so all files that have dependants get compiled in their strict relative sequential order first, and then the undepended-on files get compiled together in parallel. By running: ./rebar3 ct --suite test/rebar_compile_SUITE.erl --case \ recompile_when_parse_transform_inline_changes --repeat 50 the previous iteration of this would rapidly fail, and this one succeeds every time. --- src/rebar_compiler_erl.erl | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'src/rebar_compiler_erl.erl') diff --git a/src/rebar_compiler_erl.erl b/src/rebar_compiler_erl.erl index 759305c..1ad16d8 100644 --- a/src/rebar_compiler_erl.erl +++ b/src/rebar_compiler_erl.erl @@ -55,7 +55,14 @@ needed_files(Graph, FoundFiles, _, AppInfo) -> {ErlFirstFiles, ErlOptsFirst} = erl_first_files(RebarOpts, ErlOpts, Dir, NeededErlFiles), SubGraph = digraph_utils:subgraph(Graph, NeededErlFiles), DepErlsOrdered = digraph_utils:topsort(SubGraph), - OtherErls = lists:reverse(DepErlsOrdered), + %% Break out the files required by other modules from those + %% that none other depend of; the former must be sequentially + %% built, the rest is parallelizable. + OtherErls = lists:partition( + fun(Erl) -> digraph:in_degree(Graph, Erl) > 0 end, + lists:reverse([Dep || Dep <- DepErlsOrdered, + not lists:member(Dep, ErlFirstFiles)]) + ), PrivIncludes = [{i, filename:join(OutDir, Src)} || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])], @@ -64,8 +71,7 @@ needed_files(Graph, FoundFiles, _, AppInfo) -> true = digraph:delete(SubGraph), {{ErlFirstFiles, ErlOptsFirst ++ AdditionalOpts}, - {[Erl || Erl <- OtherErls, - not lists:member(Erl, ErlFirstFiles)], ErlOpts ++ AdditionalOpts}}. + {OtherErls, ErlOpts ++ AdditionalOpts}}. dependencies(Source, SourceDir, Dirs) -> {ok, Fd} = file:open(Source, [read]), -- cgit v1.1