diff options
| -rwxr-xr-x | bootstrap | 2 | ||||
| -rw-r--r-- | manpages/rebar3.1 | 18 | ||||
| -rw-r--r-- | rebar.config | 2 | ||||
| -rw-r--r-- | src/rebar_base_compiler.erl | 4 | ||||
| -rw-r--r-- | src/rebar_compiler.erl | 4 | ||||
| -rw-r--r-- | src/rebar_compiler_xrl.erl | 30 | ||||
| -rw-r--r-- | src/rebar_compiler_yrl.erl | 14 | ||||
| -rw-r--r-- | src/rebar_otp_app.erl | 33 | ||||
| -rw-r--r-- | test/rebar_compile_SUITE.erl | 158 |
9 files changed, 227 insertions, 38 deletions
@@ -27,7 +27,7 @@ main(_) -> bootstrap_rebar3(), %% Build rebar.app from rebar.app.src - {ok, App} = rebar_app_info:new(rebar, "3.7.0-rc1", filename:absname("_build/default/lib/rebar/")), + {ok, App} = rebar_app_info:new(rebar, "3.7.4", filename:absname("_build/default/lib/rebar/")), rebar_otp_app:compile(rebar_state:new(), App), %% Because we are compiling files that are loaded already we want to silence diff --git a/manpages/rebar3.1 b/manpages/rebar3.1 index b596925..997d144 100644 --- a/manpages/rebar3.1 +++ b/manpages/rebar3.1 @@ -1,4 +1,4 @@ -.TH "REBAR3" "1" "January 2017" "Erlang" +.TH "REBAR3" "1" "November 2018" "Erlang" .SH NAME @@ -62,8 +62,10 @@ Remove compiled beam files from apps. .IP \fI--profile\fR: Clean under profile. Equivalent to `rebar3 as <profile> clean` .TP -\fBcompile\fR +\fBcompile\fR [\fI-d\fR|\fI--deps_only\fR] Compile apps .app.src and .erl files. +.IP +\fI--deps_only\fR: Only compile dependencies, no project apps will be built. .TP \fBcover\fR [\fI-r\fR|\fI--reset\fR] [\fI-v\fR|\fI--verbose\fR] [\fI-m\fR|\fI--min_coverage\fR] Perform coverage analysis. @@ -131,7 +133,7 @@ Run Common Tests. .IP \fI--include\fR: Directories containing additional include files .IP -\fI--readable\fR: Shows test case names and only displays logs to shell on failures +\fI--readable\fR: Shows test case names and only displays logs to shell on failures (true | compact | false) .IP \fI--verbose\fR: Verbose output .IP @@ -226,7 +228,7 @@ Print paths to build dirs in current profile. \fI--rel\fR: Return the `rel' path of the current profile. .TP \fBpkgs\fR -List available packages. +List information for a package. .TP \fBrelease\fR [\fI-n\fR|\fI--relname\fR] [\fI-v\fR|\fI--relvsn\fR] [\fI-g\fR|\fI--goal\fR] [\fI-u\fR|\fI--upfrom\fR] [\fI-o\fR|\fI--output-dir\fR] [\fI-h\fR|\fI--help\fR] [\fI-l\fR|\fI--lib-dir\fR] [\fI-p\fR|\fI--path\fR] [\fI--default-libs\fR] [\fI-V\fR|\fI--verbose\fR] [\fI-d\fR|\fI--dev-mode\fR] [\fI-i\fR|\fI--include-erts\fR] [\fI-a\fR|\fI--override\fR] [\fI-c\fR|\fI--config\fR] [\fI--overlay_vars\fR] [\fI--vm_args\fR] [\fI--sys_config\fR] [\fI--system_libs\fR] [\fI--version\fR] [\fI-r\fR|\fI--root\fR] Build release of project. @@ -317,7 +319,7 @@ Create relup of releases. \fBreport\fR Provide a crash report to be sent to the rebar3 issues page. .TP -\fBshell\fR [\fI--config\fR] [\fI--name\fR] [\fI--sname\fR] [\fI--setcookie\fR] [\fI--script\fR] [\fI--apps\fR] [\fI--user_drv_args\fR] +\fBshell\fR [\fI--config\fR] [\fI--name\fR] [\fI--sname\fR] [\fI--setcookie\fR] [\fI--script\fR] [\fI--apps\fR] [\fI--start-clean\fR] [\fI--user_drv_args\fR] Run shell with project apps and deps in path. .IP \fI--config\fR: Path to the config file to use. Defaults to {shell, [{config, File}]} and then the relx sys.config file if not specified. @@ -332,6 +334,8 @@ Run shell with project apps and deps in path. .IP \fI--apps\fR: A list of apps to boot before starting the shell. (E.g. --apps app1,app2,app3) Defaults to rebar.config {shell, [{apps, Apps}]} or relx apps if not specified. .IP +\fI--start-clean\fR: Cancel any applications in the 'apps' list or release. +.IP \fI--user_drv_args\fR: Arguments passed to user_drv start function for creating custom shells. .TP \fBtar\fR [\fI-n\fR|\fI--relname\fR] [\fI-v\fR|\fI--relvsn\fR] [\fI-g\fR|\fI--goal\fR] [\fI-u\fR|\fI--upfrom\fR] [\fI-o\fR|\fI--output-dir\fR] [\fI-h\fR|\fI--help\fR] [\fI-l\fR|\fI--lib-dir\fR] [\fI-p\fR|\fI--path\fR] [\fI--default-libs\fR] [\fI-V\fR|\fI--verbose\fR] [\fI-d\fR|\fI--dev-mode\fR] [\fI-i\fR|\fI--include-erts\fR] [\fI-a\fR|\fI--override\fR] [\fI-c\fR|\fI--config\fR] [\fI--overlay_vars\fR] [\fI--vm_args\fR] [\fI--sys_config\fR] [\fI--system_libs\fR] [\fI--version\fR] [\fI-r\fR|\fI--root\fR] @@ -429,5 +433,9 @@ How much color to show in the terminal. Defaults to \fIhigh\fR. \fBREBAR_CONFIG\fR Name of rebar configuration files. Defaults to \fIrebar.config\fR +.TP +\fBREBAR_GIT_CLONE_OPTIONS\fR +Arguments to add after each \fIgit clone\fR operation. For example, the value \fI--reference ~/.cache/repos.reference\fR allows to create a cache of all fetched repositories across builds + .SH Configuration File Options See \fIhttp://www.rebar3.org/v3.0/docs/configuration\fR diff --git a/rebar.config b/rebar.config index 202bb0d..61efd8d 100644 --- a/rebar.config +++ b/rebar.config @@ -33,7 +33,7 @@ {erl_opts, [{platform_define, "^[0-9]+", namespaced_types}, {platform_define, "^(19|2)", rand_only}, {platform_define, "^2", unicode_str}, - {platform_define, "^2", filelib_find_source}, + {platform_define, "^(2[1-9])|(20\\\\.3)", filelib_find_source}, {platform_define, "^(R|1|20)", fun_stacktrace}, warnings_as_errors ]}. diff --git a/src/rebar_base_compiler.erl b/src/rebar_base_compiler.erl index 2fb3a12..ad81c86 100644 --- a/src/rebar_base_compiler.erl +++ b/src/rebar_base_compiler.erl @@ -96,14 +96,14 @@ run(Config, FirstFiles, SourceDir, SourceExt, TargetDir, TargetExt, TargetDir :: file:filename(), SourceExt :: string(), TargetExt :: string(). -run(Config, FirstFiles, SourceDirs, SourceExt, TargetDir, TargetExt, +run(Config, FirstFiles, SourceDir, SourceExt, TargetDir, TargetExt, Compile3Fn, Opts) -> %% Convert simple extension to proper regex SourceExtRe = "^(?!\\._).*\\" ++ SourceExt ++ [$$], Recursive = proplists:get_value(recursive, Opts, true), %% Find all possible source files - FoundFiles = rebar_utils:find_files_in_dirs(SourceDirs, SourceExtRe, Recursive), + FoundFiles = rebar_utils:find_files(SourceDir, SourceExtRe, Recursive), %% Remove first files from found files RestFiles = [Source || Source <- FoundFiles, not lists:member(Source, FirstFiles)], diff --git a/src/rebar_compiler.erl b/src/rebar_compiler.erl index 092f898..7da265c 100644 --- a/src/rebar_compiler.erl +++ b/src/rebar_compiler.erl @@ -20,10 +20,12 @@ src_ext => extension(), out_mappings => out_mappings()}. -callback needed_files(digraph:graph(), [file:filename()], out_mappings(), - rebar_app_info:t()) -> [file:filename()]. + rebar_app_info:t()) -> + {{[file:filename()], term()}, {[file:filename()], term()}}. -callback dependencies(file:filename(), file:dirname(), [file:dirname()]) -> [file:filename()]. -callback compile(file:filename(), out_mappings(), rebar_dict(), list()) -> ok | {ok, [string()]} | {ok, [string()], [string()]}. +-callback clean([file:filename()], rebar_app_info:t()) -> _. -define(DAG_VSN, 2). -define(DAG_FILE, "source.dag"). diff --git a/src/rebar_compiler_xrl.erl b/src/rebar_compiler_xrl.erl index 23ed1eb..35447ed 100644 --- a/src/rebar_compiler_xrl.erl +++ b/src/rebar_compiler_xrl.erl @@ -8,6 +8,8 @@ compile/4, clean/2]). +-export([update_opts/2]). + context(AppInfo) -> Dir = rebar_app_info:dir(AppInfo), Mappings = [{".erl", filename:join([Dir, "src"])}], @@ -25,28 +27,38 @@ needed_files(_, FoundFiles, Mappings, AppInfo) -> rebar_compiler:needs_compile(Source, ".erl", Mappings)], Opts = rebar_opts:get(rebar_app_info:opts(AppInfo), xrl_opts, []), + Opts1 = update_opts(Opts, AppInfo), - {{FirstFiles, Opts}, {RestFiles, Opts}}. + {{FirstFiles, Opts1}, {RestFiles, Opts1}}. dependencies(_, _, _) -> []. -compile(Source, [{_, OutDir}], _, Opts) -> - BaseName = filename:basename(Source), - Target = filename:join([OutDir, BaseName]), - AllOpts = [{parserfile, Target} | Opts], - AllOpts1 = [{includefile, filename:join(OutDir, I)} || {includefile, I} <- AllOpts, - filename:pathtype(I) =:= relative], - case leex:file(Source, AllOpts1 ++ [{return, true}]) of +compile(Source, [{_, _}], _, Opts) -> + case leex:file(Source, [{return, true} | Opts]) of {ok, _} -> ok; {ok, _Mod, Ws} -> rebar_compiler:ok_tuple(Source, Ws); {error, Es, Ws} -> - rebar_compiler:error_tuple(Source, Es, Ws, AllOpts1) + rebar_compiler:error_tuple(Source, Es, Ws, Opts) end. clean(XrlFiles, _AppInfo) -> rebar_file_utils:delete_each( [rebar_utils:to_list(re:replace(F, "\\.xrl$", ".erl", [unicode])) || F <- XrlFiles]). + +%% make includefile options absolute paths +update_opts(Opts, AppInfo) -> + OutDir = rebar_app_info:out_dir(AppInfo), + lists:map(fun({includefile, I}) -> + case filename:pathtype(I) =:= relative of + true -> + {includefile, filename:join(OutDir, I)}; + false -> + {includefile, I} + end; + (O) -> + O + end, Opts). diff --git a/src/rebar_compiler_yrl.erl b/src/rebar_compiler_yrl.erl index 9c1767e..8e52d0e 100644 --- a/src/rebar_compiler_yrl.erl +++ b/src/rebar_compiler_yrl.erl @@ -25,24 +25,24 @@ needed_files(_, FoundFiles, Mappings, AppInfo) -> rebar_compiler:needs_compile(Source, ".erl", Mappings)], Opts = rebar_opts:get(rebar_app_info:opts(AppInfo), yrl_opts, []), - {{FirstFiles, Opts}, {RestFiles, Opts}}. + Opts1 = rebar_compiler_xrl:update_opts(Opts, AppInfo), + + {{FirstFiles, Opts1}, {RestFiles, Opts1}}. dependencies(_, _, _) -> []. compile(Source, [{_, OutDir}], _, Opts) -> - BaseName = filename:basename(Source), + BaseName = filename:basename(Source, ".yrl"), Target = filename:join([OutDir, BaseName]), - AllOpts = [{parserfile, Target} | Opts], - AllOpts1 = [{includefile, filename:join(OutDir, I)} || {includefile, I} <- AllOpts, - filename:pathtype(I) =:= relative], - case yecc:file(Source, AllOpts1 ++ [{return, true}]) of + AllOpts = [{parserfile, Target}, {return, true} | Opts], + case yecc:file(Source, AllOpts) of {ok, _} -> ok; {ok, _Mod, Ws} -> rebar_compiler:ok_tuple(Source, Ws); {error, Es, Ws} -> - rebar_compiler:error_tuple(Source, Es, Ws, AllOpts1) + rebar_compiler:error_tuple(Source, Es, Ws, AllOpts) end. clean(YrlFiles, _AppInfo) -> diff --git a/src/rebar_otp_app.erl b/src/rebar_otp_app.erl index e14975f..952271b 100644 --- a/src/rebar_otp_app.erl +++ b/src/rebar_otp_app.erl @@ -164,17 +164,32 @@ validate_name(AppName, File) -> ebin_modules(AppInfo, Dir) -> Beams = lists:sort(rebar_utils:beams(filename:join(Dir, "ebin"))), - SrcDirs = rebar_dir:src_dirs(rebar_app_info:opts(AppInfo), ["src"]), - FindSourceRules = [{".beam", ".erl", - [{"ebin", SrcDir} || SrcDir <- SrcDirs]}], - Filtered = lists:filter(fun(Beam) -> - rebar_utils:find_source(filename:basename(Beam), - filename:dirname(Beam), - FindSourceRules) - =/= {error, not_found} - end, Beams), + ExtraDirs = extra_dirs(AppInfo), + F = fun(Beam) -> not in_extra_dir(AppInfo, Beam, ExtraDirs) end, + Filtered = lists:filter(F, Beams), [rebar_utils:beam_to_mod(N) || N <- Filtered]. +extra_dirs(State) -> + Extras = rebar_dir:extra_src_dirs(rebar_app_info:opts(State)), + SrcDirs = rebar_dir:src_dirs(rebar_app_info:opts(State), ["src"]), + %% remove any dirs that are defined in `src_dirs` from `extra_src_dirs` + Extras -- SrcDirs. + +in_extra_dir(AppInfo, Beam, Dirs) -> + lists:any(fun(Dir) -> lists:prefix(filename:join([rebar_app_info:out_dir(AppInfo), Dir]), + beam_src(Beam)) end, + Dirs). + +beam_src(Beam) -> + case beam_lib:chunks(Beam, [compile_info]) of + {ok, {_mod, Chunks}} -> + CompileInfo = proplists:get_value(compile_info, Chunks, []), + proplists:get_value(source, CompileInfo, []); + {error, beam_lib, Reason} -> + ?WARN("Couldn't read debug info from ~p for reason: ~p", [Beam, Reason]), + [] + end. + ensure_registered(AppData) -> case lists:keyfind(registered, 1, AppData) of false -> diff --git a/test/rebar_compile_SUITE.erl b/test/rebar_compile_SUITE.erl index 4d51f2a..a300690 100644 --- a/test/rebar_compile_SUITE.erl +++ b/test/rebar_compile_SUITE.erl @@ -852,7 +852,13 @@ dont_recompile_yrl_or_xrl(Config) -> XrlBeam = filename:join([EbinDir, filename:basename(Xrl, ".xrl") ++ ".beam"]), YrlBeam = filename:join([EbinDir, filename:basename(Yrl, ".yrl") ++ ".beam"]), - rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name}]}), + Hrl = filename:join([AppDir, "include", "some_header.hrl"]), + ok = filelib:ensure_dir(Hrl), + HrlBody = yeccpre_hrl(), + ok = ec_file:write(Hrl, HrlBody), + RebarConfig = [{yrl_opts, [{includefile, "include/some_header.hrl"}]}], + + rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], {ok, [{app, Name}]}), XrlModTime = filelib:last_modified(XrlErl), YrlModTime = filelib:last_modified(YrlErl), @@ -862,7 +868,7 @@ dont_recompile_yrl_or_xrl(Config) -> timer:sleep(1000), - rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name}]}), + rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], {ok, [{app, Name}]}), NewXrlModTime = filelib:last_modified(XrlErl), NewYrlModTime = filelib:last_modified(YrlErl), @@ -2151,7 +2157,13 @@ recursive(Config) -> EbinDir = filename:join([AppDir, "_build", "default", "lib", Name, "ebin"]), {ok, Files} = rebar_utils:list_dir(EbinDir), - ?assert(lists:member("rec.beam",Files)). + ?assert(lists:member("rec.beam",Files)), + + %% check that rec is in modules list of .app file + AppFile = filename:join(EbinDir, Name++".app"), + {ok, [{application, _, List}]} = file:consult(AppFile), + {modules, Modules} = lists:keyfind(modules, 1, List), + ?assert(lists:member(rec, Modules)). no_recursive(Config) -> AppDir = ?config(apps, Config), @@ -2208,3 +2220,143 @@ regex_filter_regression(Config) -> {ok, [{file, Expected}]}), ok. +%% + +%% a copy of lib/parsetools/include/yeccpre.hrl so we can test yrl includefile +yeccpre_hrl() -> + <<"-type yecc_ret() :: {'error', _} | {'ok', _}. + +-spec parse(Tokens :: list()) -> yecc_ret(). +parse(Tokens) -> + yeccpars0(Tokens, {no_func, no_line}, 0, [], []). + +-spec parse_and_scan({function() | {atom(), atom()}, [_]} + | {atom(), atom(), [_]}) -> yecc_ret(). +parse_and_scan({F, A}) -> + yeccpars0([], {{F, A}, no_line}, 0, [], []); +parse_and_scan({M, F, A}) -> + Arity = length(A), + yeccpars0([], {{fun M:F/Arity, A}, no_line}, 0, [], []). + +-spec format_error(any()) -> [char() | list()]. +format_error(Message) -> + case io_lib:deep_char_list(Message) of + true -> + Message; + _ -> + io_lib:write(Message) + end. + +%% To be used in grammar files to throw an error message to the parser +%% toplevel. Doesn't have to be exported! +-compile({nowarn_unused_function, return_error/2}). +-spec return_error(integer(), any()) -> no_return(). +return_error(Line, Message) -> + throw({error, {Line, ?MODULE, Message}}). + +-define(CODE_VERSION, \"1.4\"). + +yeccpars0(Tokens, Tzr, State, States, Vstack) -> + try yeccpars1(Tokens, Tzr, State, States, Vstack) + catch + error:Error -> + try yecc_error_type(Error, []) of + Desc -> + erlang:raise(error, {yecc_bug, ?CODE_VERSION, Desc}, + []) + catch _:_ -> erlang:raise(error, Error, []) + end; + %% Probably thrown from return_error/2: + throw: {error, {_Line, ?MODULE, _M}} = Error -> + Error + end. + +yecc_error_type(function_clause, _) -> + not_implemented. + +yeccpars1([Token | Tokens], Tzr, State, States, Vstack) -> + yeccpars2(State, element(1, Token), States, Vstack, Token, Tokens, Tzr); +yeccpars1([], {{F, A},_Line}, State, States, Vstack) -> + case apply(F, A) of + {ok, Tokens, Endline} -> + yeccpars1(Tokens, {{F, A}, Endline}, State, States, Vstack); + {eof, Endline} -> + yeccpars1([], {no_func, Endline}, State, States, Vstack); + {error, Descriptor, _Endline} -> + {error, Descriptor} + end; +yeccpars1([], {no_func, no_line}, State, States, Vstack) -> + Line = 999999, + yeccpars2(State, '$end', States, Vstack, yecc_end(Line), [], + {no_func, Line}); +yeccpars1([], {no_func, Endline}, State, States, Vstack) -> + yeccpars2(State, '$end', States, Vstack, yecc_end(Endline), [], + {no_func, Endline}). + +%% yeccpars1/7 is called from generated code. +%% +%% When using the {includefile, Includefile} option, make sure that +%% yeccpars1/7 can be found by parsing the file without following +%% include directives. yecc will otherwise assume that an old +%% yeccpre.hrl is included (one which defines yeccpars1/5). +yeccpars1(State1, State, States, Vstack, Token0, [Token | Tokens], Tzr) -> + yeccpars2(State, element(1, Token), [State1 | States], + [Token0 | Vstack], Token, Tokens, Tzr); +yeccpars1(State1, State, States, Vstack, Token0, [], {{_F,_A}, _Line}=Tzr) -> + yeccpars1([], Tzr, State, [State1 | States], [Token0 | Vstack]); +yeccpars1(State1, State, States, Vstack, Token0, [], {no_func, no_line}) -> + Line = yecctoken_end_location(Token0), + yeccpars2(State, '$end', [State1 | States], [Token0 | Vstack], + yecc_end(Line), [], {no_func, Line}); +yeccpars1(State1, State, States, Vstack, Token0, [], {no_func, Line}) -> + yeccpars2(State, '$end', [State1 | States], [Token0 | Vstack], + yecc_end(Line), [], {no_func, Line}). + +%% For internal use only. +yecc_end({Line,_Column}) -> + {'$end', Line}; +yecc_end(Line) -> + {'$end', Line}. + +yecctoken_end_location(Token) -> + try erl_anno:end_location(element(2, Token)) of + undefined -> yecctoken_location(Token); + Loc -> Loc + catch _:_ -> yecctoken_location(Token) + end. + +-compile({nowarn_unused_function, yeccerror/1}). +yeccerror(Token) -> + Text = yecctoken_to_string(Token), + Location = yecctoken_location(Token), + {error, {Location, ?MODULE, [\"syntax error before: \", Text]}}. + +-compile({nowarn_unused_function, yecctoken_to_string/1}). +yecctoken_to_string(Token) -> + try erl_scan:text(Token) of + undefined -> yecctoken2string(Token); + Txt -> Txt + catch _:_ -> yecctoken2string(Token) + end. + +yecctoken_location(Token) -> + try erl_scan:location(Token) + catch _:_ -> element(2, Token) + end. + +-compile({nowarn_unused_function, yecctoken2string/1}). +yecctoken2string({atom, _, A}) -> io_lib:write_atom(A); +yecctoken2string({integer,_,N}) -> io_lib:write(N); +yecctoken2string({float,_,F}) -> io_lib:write(F); +yecctoken2string({char,_,C}) -> io_lib:write_char(C); +yecctoken2string({var,_,V}) -> io_lib:format(\"~s\", [V]); +yecctoken2string({string,_,S}) -> io_lib:write_string(S); +yecctoken2string({reserved_symbol, _, A}) -> io_lib:write(A); +yecctoken2string({_Cat, _, Val}) -> io_lib:format(\"~p\", [Val]); +yecctoken2string({dot, _}) -> \"'.'\"; +yecctoken2string({'$end', _}) -> []; +yecctoken2string({Other, _}) when is_atom(Other) -> + io_lib:write_atom(Other); +yecctoken2string(Other) -> + io_lib:format(\"~p\", [Other]). +">>. |