summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/rebar_app_utils.erl1
-rw-r--r--src/rebar_compiler.erl67
-rw-r--r--src/rebar_compiler_erl.erl12
-rw-r--r--src/rebar_packages.erl6
-rw-r--r--src/rebar_prv_deps.erl185
5 files changed, 187 insertions, 84 deletions
diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl
index 5fe5ba6..0ea6ad8 100644
--- a/src/rebar_app_utils.erl
+++ b/src/rebar_app_utils.erl
@@ -34,6 +34,7 @@
validate_application_info/2,
parse_deps/5,
parse_deps/6,
+ parse_dep/6,
expand_deps_sources/2,
dep_to_app/7,
format_error/1]).
diff --git a/src/rebar_compiler.erl b/src/rebar_compiler.erl
index 02c74db..55666ba 100644
--- a/src/rebar_compiler.erl
+++ b/src/rebar_compiler.erl
@@ -21,7 +21,10 @@
out_mappings => out_mappings()}.
-callback needed_files(digraph:graph(), [file:filename()], out_mappings(),
rebar_app_info:t()) ->
- {{[file:filename()], term()}, {[file:filename()], term()}}.
+ {{[file:filename()], term()}, % ErlFirstFiles (erl_opts global priority)
+ {[file:filename()] | % [Sequential]
+ {[file:filename()], [file:filename()]}, % {Sequential, Parallel}
+ term()}}.
-callback dependencies(file:filename(), file:dirname(), [file:dirname()]) -> [file:filename()].
-callback compile(file:filename(), out_mappings(), rebar_dict(), list()) ->
ok | {ok, [string()]} | {ok, [string()], [string()]}.
@@ -77,7 +80,13 @@ run(CompilerMod, AppInfo, Label) ->
true = digraph:delete(G),
compile_each(FirstFiles, FirstFileOpts, BaseOpts, Mappings, CompilerMod),
- compile_each(RestFiles, Opts, BaseOpts, Mappings, CompilerMod).
+ case RestFiles of
+ {Sequential, Parallel} -> % new parallelizable form
+ compile_each(Sequential, Opts, BaseOpts, Mappings, CompilerMod),
+ compile_parallel(Parallel, Opts, BaseOpts, Mappings, CompilerMod);
+ _ when is_list(RestFiles) -> % traditional sequential build
+ compile_each(RestFiles, Opts, BaseOpts, Mappings, CompilerMod)
+ end.
compile_each([], _Opts, _Config, _Outs, _CompilerMod) ->
ok;
@@ -99,6 +108,60 @@ compile_each([Source | Rest], Opts, Config, Outs, CompilerMod) ->
end,
compile_each(Rest, Opts, Config, Outs, CompilerMod).
+compile_worker(QueuePid, Opts, Config, Outs, CompilerMod) ->
+ QueuePid ! self(),
+ receive
+ {compile, Source} ->
+ Result = CompilerMod:compile(Source, Outs, Config, Opts),
+ QueuePid ! {Result, Source},
+ compile_worker(QueuePid, Opts, Config, Outs, CompilerMod);
+ empty ->
+ ok
+ end.
+
+compile_parallel([], _Opts, _BaseOpts, _Mappings, _CompilerMod) ->
+ ok;
+compile_parallel(Targets, Opts, BaseOpts, Mappings, CompilerMod) ->
+ Self = self(),
+ F = fun() -> compile_worker(Self, Opts, BaseOpts, Mappings, CompilerMod) end,
+ Jobs = min(length(Targets), erlang:system_info(schedulers)),
+ ?DEBUG("Starting ~B compile worker(s)", [Jobs]),
+ Pids = [spawn_monitor(F) || _I <- lists:seq(1, Jobs)],
+ compile_queue(Targets, Pids, Opts, BaseOpts, Mappings, CompilerMod).
+
+compile_queue([], [], _Opts, _Config, _Outs, _CompilerMod) ->
+ ok;
+compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod) ->
+ receive
+ Worker when is_pid(Worker), Targets =:= [] ->
+ Worker ! empty,
+ compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod);
+ Worker when is_pid(Worker) ->
+ Worker ! {compile, hd(Targets)},
+ compile_queue(tl(Targets), Pids, Opts, Config, Outs, CompilerMod);
+ {ok, Source} ->
+ ?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), filename:basename(Source)]),
+ compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod);
+ {{ok, Warnings}, Source} ->
+ report(Warnings),
+ ?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), filename:basename(Source)]),
+ compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod);
+ {skipped, Source} ->
+ ?DEBUG("~sSkipped ~s", [rebar_utils:indent(1), filename:basename(Source)]),
+ compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod);
+ {Error, Source} ->
+ NewSource = format_error_source(Source, Config),
+ ?ERROR("Compiling ~ts failed", [NewSource]),
+ maybe_report(Error),
+ ?FAIL;
+ {'DOWN', Mref, _, Pid, normal} ->
+ Pids2 = lists:delete({Pid, Mref}, Pids),
+ compile_queue(Targets, Pids2, Opts, Config, Outs, CompilerMod);
+ {'DOWN', _Mref, _, _Pid, Info} ->
+ ?ERROR("Compilation failed: ~p", [Info]),
+ ?FAIL
+ end.
+
%% @doc remove compiled artifacts from an AppDir.
-spec clean([module()], rebar_app_info:t()) -> 'ok'.
clean(Compilers, AppInfo) ->
diff --git a/src/rebar_compiler_erl.erl b/src/rebar_compiler_erl.erl
index 759305c..1ad16d8 100644
--- a/src/rebar_compiler_erl.erl
+++ b/src/rebar_compiler_erl.erl
@@ -55,7 +55,14 @@ needed_files(Graph, FoundFiles, _, AppInfo) ->
{ErlFirstFiles, ErlOptsFirst} = erl_first_files(RebarOpts, ErlOpts, Dir, NeededErlFiles),
SubGraph = digraph_utils:subgraph(Graph, NeededErlFiles),
DepErlsOrdered = digraph_utils:topsort(SubGraph),
- OtherErls = lists:reverse(DepErlsOrdered),
+ %% Break out the files required by other modules from those
+ %% that none other depend of; the former must be sequentially
+ %% built, the rest is parallelizable.
+ OtherErls = lists:partition(
+ fun(Erl) -> digraph:in_degree(Graph, Erl) > 0 end,
+ lists:reverse([Dep || Dep <- DepErlsOrdered,
+ not lists:member(Dep, ErlFirstFiles)])
+ ),
PrivIncludes = [{i, filename:join(OutDir, Src)}
|| Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])],
@@ -64,8 +71,7 @@ needed_files(Graph, FoundFiles, _, AppInfo) ->
true = digraph:delete(SubGraph),
{{ErlFirstFiles, ErlOptsFirst ++ AdditionalOpts},
- {[Erl || Erl <- OtherErls,
- not lists:member(Erl, ErlFirstFiles)], ErlOpts ++ AdditionalOpts}}.
+ {OtherErls, ErlOpts ++ AdditionalOpts}}.
dependencies(Source, SourceDir, Dirs) ->
{ok, Fd} = file:open(Source, [read]),
diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl
index a260e47..c7bbfac 100644
--- a/src/rebar_packages.erl
+++ b/src/rebar_packages.erl
@@ -238,7 +238,11 @@ update_package(Name, RepoConfig=#{name := Repo}, State) ->
_ = insert_releases(Name, Releases, Repo, ?PACKAGE_TABLE),
{ok, RegistryDir} = rebar_packages:registry_dir(State),
PackageIndex = filename:join(RegistryDir, ?INDEX_FILE),
- ok = ets:tab2file(?PACKAGE_TABLE, PackageIndex);
+ case ets:tab2file(?PACKAGE_TABLE, PackageIndex) of
+ ok -> ok;
+ {error, Error} ->
+ ?WARN("Failed to update package index at ~p: ~p", [PackageIndex, Error])
+ end;
{error, unverified} ->
?WARN(unverified_repo_message(), [Repo]),
fail;
diff --git a/src/rebar_prv_deps.erl b/src/rebar_prv_deps.erl
index 577a859..d00a1c8 100644
--- a/src/rebar_prv_deps.erl
+++ b/src/rebar_prv_deps.erl
@@ -9,7 +9,7 @@
-include("rebar.hrl").
-define(PROVIDER, deps).
--define(DEPS, [app_discovery]).
+-define(DEPS, [install_deps]).
-spec init(rebar_state:t()) -> {ok, rebar_state:t()}.
init(State) ->
@@ -22,98 +22,127 @@ init(State) ->
{deps, ?DEPS},
{example, "rebar3 deps"},
{short_desc, "List dependencies"},
- {desc, "List dependencies. Those not matching lock files "
- "are followed by an asterisk (*)."},
+ {desc, "List dependencies. Those not matching "
+ "the config file are followed by "
+ "an asterisk (*)."},
{opts, []}])),
{ok, State1}.
+
-spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
do(State) ->
Profiles = rebar_state:current_profiles(State),
- List = [{Profile, rebar_state:get(State, {deps, Profile}, [])}
- || Profile <- Profiles],
- [display(State, Profile, Deps) || {Profile, Deps} <- List],
+ [display(State, Profile) || Profile <- Profiles],
{ok, State}.
+
-spec format_error(any()) -> iolist().
format_error(Reason) ->
io_lib:format("~p", [Reason]).
-display(State, default, Deps) ->
- NewDeps = merge(Deps, rebar_state:get(State, deps, [])),
- display_deps(State, NewDeps),
+
+display(State, Profile = default) ->
+ display_profile_deps(State, Profile),
?CONSOLE("", []);
-display(State, Profile, Deps) ->
+display(State, Profile) ->
?CONSOLE("-- ~p --", [Profile]),
- display_deps(State, Deps),
+ display_profile_deps(State, Profile),
?CONSOLE("", []).
-merge(Deps, SourceDeps) ->
- merge1(dedup([normalize(Dep) || Dep <- Deps]),
- [normalize(Dep) || Dep <- SourceDeps]).
-
-normalize(Name) when is_binary(Name) ->
- Name;
-normalize(Name) when is_atom(Name) ->
- atom_to_binary(Name, unicode);
-normalize(Dep) when is_tuple(Dep) ->
- Name = element(1, Dep),
- setelement(1, Dep, normalize(Name)).
-
-merge1(Deps, SourceDeps) ->
- Names = [name(Dep) || Dep <- Deps],
- ToAdd = [Dep || Dep <- SourceDeps,
- not lists:member(name(Dep), Names)],
- Deps ++ ToAdd.
-
-%% Keep the latter one as locks come after regular deps in the list.
-%% This is totally not safe as an assumption, but it's what we got.
-%% We do this by comparing the current element and looking if a
-%% similar named one happens later. If so, drop the current one.
-dedup(Deps) -> dedup(Deps, [name(Dep) || Dep <- Deps]).
-
-dedup([], []) -> [];
-dedup([Dep|Deps], [Name|DepNames]) ->
- case lists:member(Name, DepNames) of
- true -> dedup(Deps, DepNames);
- false -> [Dep | dedup(Deps, DepNames)]
+
+display_profile_deps(State, Profile) ->
+ DepsDir = rebar_prv_install_deps:profile_dep_dir(State, Profile),
+
+ ProfileDeps = rebar_state:get(State, {deps, Profile}, []),
+ % ProfileDeps include those deps from rebar.lock that have been
+ % removed from rebar.config
+ ConfiguredDeps = [parse_dep_without_locks(DepsDir, Dep, State)
+ || Dep <- ProfileDeps],
+ LockedDepsMap = locked_deps_map(State, Profile),
+ [display_dep(State, Dep, LockedDepsMap) || Dep <- ConfiguredDeps].
+
+
+parse_dep_without_locks(DepsDir, Dep, State) ->
+ ParsedDep = rebar_app_utils:parse_dep(Dep, root, DepsDir, State, [], 0),
+ case Dep of
+ {_Name, Src, Level} when is_tuple(Src), is_integer(Level) ->
+ % This Dep is not in rebar.config but in rebar.lock
+ rebar_app_info:source(ParsedDep, undefined);
+ _ ->
+ rebar_app_utils:expand_deps_sources(ParsedDep, State)
+ end.
+
+
+locked_deps_map(State, Profile) ->
+ ParsedDeps = rebar_state:get(State, {parsed_deps, Profile}, []),
+ lists:foldl(fun(Dep, DepsIn) ->
+ case rebar_app_info:is_lock(Dep) of
+ true ->
+ DepName = rebar_app_info:name(Dep),
+ maps:put(rebar_utils:to_binary(DepName), Dep, DepsIn);
+ _ ->
+ DepsIn
+ end
+ end, maps:new(), ParsedDeps).
+
+
+display_dep(State, Dep, LockedDeps) ->
+ Name = rebar_utils:to_binary(rebar_app_info:name(Dep)),
+ NeedsUpdate = rebar_fetch:needs_update(Dep, State),
+ Source = rebar_app_info:source(Dep),
+ LockedSource = case maps:get(Name, LockedDeps, undefined) of
+ undefined -> undefined;
+ LockedDep -> rebar_app_info:source(LockedDep)
+ end,
+
+ display_dep_line(Name, NeedsUpdate, source_text(LockedSource), source_text(Source)).
+
+
+% Dep is a checkout
+display_dep_line(Name, _NeedsUpdate, _LockedSource, Source = checkout) ->
+ ?CONSOLE("~ts* (~ts)", [Name, Source]);
+
+% Dep exists only in lock file
+display_dep_line(Name, _NeedsUpdate, LockedSource, _Source = undefined) ->
+ ?CONSOLE("~ts* (locked ~ts <> none)", [Name, LockedSource]);
+
+% Dep not locked, report whether the disk copy matches the Source
+display_dep_line(Name, true, undefined, Source) ->
+ ?CONSOLE("~ts* (~ts)", [Name, Source]);
+display_dep_line(Name, _, undefined, Source) ->
+ ?CONSOLE("~ts (~ts)", [Name, Source]);
+
+% Dep locked, install_deps provider should have had updated the disk copy with
+% the locked version
+display_dep_line(Name, false, _LockedSource, Source) ->
+ % dep locked and no need to update (LockedSource and Source might not match
+ % because of one using {ref, X} and the other {tag, Y})
+ ?CONSOLE("~ts (locked ~ts)", [Name, Source]);
+display_dep_line(Name, _NeedsUpdate, LockedSource, Source) ->
+ % dep locked with mismatching lock and config files
+ ?CONSOLE("~ts* (locked ~ts <> ~ts)", [Name, LockedSource, Source]).
+
+
+source_text(Source) when is_list(Source); is_atom(Source) ->
+ Source;
+source_text({pkg, _Name, Vsn, _Hash, _RepoConfig}) ->
+ source_text({pkg, _Name, Vsn, _Hash});
+source_text({pkg, _Name, Vsn, _Hash}) ->
+ [<<"package">>, " ", rebar_utils:to_binary(Vsn)];
+source_text(Source) when is_tuple(Source), tuple_size(Source) < 3 ->
+ element(1, Source);
+source_text(Source) when is_tuple(Source) ->
+ Type = element(1, Source),
+ case element(3, Source) of
+ {ref , Ref} ->
+ SmallRef = case rebar_utils:to_binary(Ref) of
+ <<R:7/binary, _/binary>> -> <<R/binary, "...">>;
+ R -> R
+ end,
+ [atom_to_binary(Type, unicode), " source ", SmallRef];
+ {_ , Vsn} ->
+ [atom_to_binary(Type, unicode), " source ", rebar_utils:to_binary(Vsn)];
+ _ ->
+ [atom_to_binary(Type, unicode), " source"]
end.
-name(T) when is_tuple(T) -> element(1, T);
-name(B) when is_binary(B) -> B.
-
-display_deps(State, Deps) ->
- lists:foreach(fun(Dep) -> display_dep(State, Dep) end, Deps).
-
-%% packages
-display_dep(_State, {Name, Vsn}) when is_list(Vsn) ->
- ?CONSOLE("~ts* (package ~ts)", [rebar_utils:to_binary(Name), rebar_utils:to_binary(Vsn)]);
-display_dep(_State, Name) when is_binary(Name) ->
- ?CONSOLE("~ts* (package)", [Name]);
-display_dep(_State, {Name, Source}) when is_tuple(Source) ->
- ?CONSOLE("~ts* (~ts source)", [rebar_utils:to_binary(Name), type(Source)]);
-display_dep(_State, {Name, _Vsn, Source}) when is_tuple(Source) ->
- ?CONSOLE("~ts* (~ts source)", [rebar_utils:to_binary(Name), type(Source)]);
-display_dep(_State, {Name, _Vsn, Source, _Opts}) when is_tuple(Source) ->
- ?CONSOLE("~ts* (~ts source)", [rebar_utils:to_binary(Name), type(Source)]);
-%% Locked
-display_dep(State, {Name, _Source={pkg, _, Vsn}, Level}) when is_integer(Level) ->
- DepsDir = rebar_dir:deps_dir(State),
- AppDir = filename:join([DepsDir, rebar_utils:to_binary(Name)]),
- {ok, AppInfo} = rebar_app_info:discover(AppDir),
- NeedsUpdate = case rebar_fetch:needs_update(AppInfo, State) of
- true -> "*";
- false -> ""
- end,
- ?CONSOLE("~ts~ts (locked package ~ts)", [Name, NeedsUpdate, Vsn]);
-display_dep(State, {Name, Source, Level}) when is_tuple(Source), is_integer(Level) ->
- DepsDir = rebar_dir:deps_dir(State),
- AppDir = filename:join([DepsDir, rebar_utils:to_binary(Name)]),
- {ok, AppInfo} = rebar_app_info:discover(AppDir),
- NeedsUpdate = case rebar_fetch:needs_update(AppInfo, State) of
- true -> "*";
- false -> ""
- end,
- ?CONSOLE("~ts~ts (locked ~ts source)", [Name, NeedsUpdate, type(Source)]).
-
-type(Source) when is_tuple(Source) -> element(1, Source).
generated by cgit v1.1 at 2024-12-26 16:27:18 +0000