diff options
189 files changed, 13977 insertions, 3251 deletions
diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..7e2f83d --- /dev/null +++ b/.editorconfig @@ -0,0 +1,13 @@ +# EditorConfig file: http://EditorConfig.org + +# Top-most EditorConfig file. +root = true + +# Unix-style, newlines, indent style of 4 spaces, with a newline ending every file. +[*] +end_of_line = lf +insert_final_newline = true +charset = utf-8 +trim_trailing_whitespace = true +indent_style = space +indent_size = 4 diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..6d5a0c2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,31 @@ +### Pre-Check ### + +- If you are filing for a bug, please do a quick search in current issues first +- For bugs, mention if you are willing or interested in helping fix the issue +- For questions or support, it helps to include context around your project or problem +- Think of a descriptive title (more descriptive than 'feature X is broken' unless it is fully broken) + +### Environment ### + +- Add the result of `rebar3 report` to your message: + +``` +$ rebar3 report "my failing command" +... +``` + +- Verify whether the version of rebar3 you're running is the latest release (see https://github.com/erlang/rebar3/releases) +- If possible, include information about your project and its structure. Open source projects or examples are always easier to debug. + If you can provide an example code base to reproduce the issue on, we will generally be able to provide more help, and faster. + +### Current behaviour ### + +Describe the current behaviour. In case of a failure, crash, or exception, please include the result of running the command with debug information: + +``` +DEBUG=1 rebar3 <my failing command> +``` + +### Expected behaviour ### + +Describe what you expected to happen. @@ -1,6 +1,7 @@ _checkouts .rebar3 rebar3 +rebar3.cmd _build .depsolver_plt *.beam diff --git a/.travis.yml b/.travis.yml index dbb4f26..32d51ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,25 +1,48 @@ -sudo: false language: erlang -install: 'true' -otp_release: -- 19.0 -- 18.0 -- 17.5 -- R16B03-1 -- R15B03 -before_script: "./bootstrap" -script: "./rebar3 ct" +matrix: + include: + - os: linux + otp_release: 17.5 + - os: linux + otp_release: 18.3 + - os: linux + otp_release: 19.3 + - os: linux + otp_release: 20.0 + - os: linux + otp_release: 21.0 + - os: osx + language: generic +before_script: + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; fi + ## should eventually use a tap that has previous erlang versions here + ## as this only uses the latest erlang available via brew + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install erlang; fi +script: "./bootstrap && ./rebar3 ct" branches: only: - master + - hex_core cache: directories: - "$HOME/.cache/rebar3/hex/default" deploy: - provider: releases - api_key: - secure: MjloYuaQF3cd3Oab57zqwPDLPqt5MDgBIrRLpXOQwNovr2tnkKd4aJK3QJ3pTxvZievjgl+qIYI1IZyjuRV37nkjAfMw14iig959wi0k8XTJoMdylVxE5X7hk4SiWhX/ycnJx3C28PPw1OitGTF76HAJDMgEelNdoNt+hvjvDEo= - file: rebar3 - on: - repo: erlang/rebar3 - tags: true + - provider: releases + api_key: + secure: MjloYuaQF3cd3Oab57zqwPDLPqt5MDgBIrRLpXOQwNovr2tnkKd4aJK3QJ3pTxvZievjgl+qIYI1IZyjuRV37nkjAfMw14iig959wi0k8XTJoMdylVxE5X7hk4SiWhX/ycnJx3C28PPw1OitGTF76HAJDMgEelNdoNt+hvjvDEo= + file: rebar3 + on: + repo: erlang/rebar3 + tags: true + - provider: s3 + access_key_id: AKIAJAPYAQEFYCYSNL7Q + secret_access_key: + secure: "BUv2KQABv0Q4e8DAVNBRTc/lXHWt27yCN46Fdgo1IrcSSIiP+hq2yXzQcXLbPwkEu6pxUZQtL3mvKbt6l7uw3wFrcRfFAi1PGTITAW8MTmxtwcZIBcHSk3XOzDbkK+fYYcaddszmt7hDzzEFPtmYXiNgnaMIVeynhQLgcCcIRRQ=" + skip_cleanup: true + local-dir: _build/prod/bin + bucket: "rebar3-nightly" + acl: public_read + on: + repo: erlang/rebar3 + branch: master + condition: $TRAVIS_OTP_RELEASE = "17.5" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8077813..16de1e5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -162,7 +162,7 @@ $ ./rebar3 ct ``` Most tests are named according to their module name followed by the `_SUITE` -suffi. Providers are made shorter, such that `rebar_prv_new` is tested in +suffix. Providers are made shorter, such that `rebar_prv_new` is tested in `rebar_new_SUITE`. Most tests in the test suite will rely on calling Rebar3 in its API form, @@ -285,7 +285,7 @@ The reviewer may ask you to later squash the commits together to provide a clean commit history before merging in the feature. It's important to write a proper commit title and description. The commit title -should fir around 50 characters; it is the first line of the commit text. The +should be no more than 50 characters; it is the first line of the commit text. The second line of the commit text must be left blank. The third line and beyond is the commit message. You should write a commit message. If you do, wrap all lines at 72 characters. You should explain what the commit does, what @@ -299,7 +299,7 @@ maintainers. When opening a pull request, explain what the patch is doing and if it makes sense, why the proposed implementation was chosen. Try to use well-defined commits (one feature per commit) so that reading -them and testing them is easier for reviewers and while bissecting the code +them and testing them is easier for reviewers and while bisecting the code base for issues. During the review process, you may be asked to correct or edit a few things @@ -56,7 +56,7 @@ best experience you can get. A [getting started guide is maintained on the official documentation website](http://www.rebar3.org/docs/getting-started), but installing rebar3 can be done by any of the ways described below -Nightly compiled version: +Latest stable compiled version: ```bash $ wget https://s3.amazonaws.com/rebar3/rebar3 && chmod +x rebar3 ``` @@ -69,7 +69,7 @@ $ cd rebar3 $ ./bootstrap ``` -Stable versions can be obtained from the [releases page](https://github.com/erlang/rebar3/releases). +Stable versions can also be obtained from the [releases page](https://github.com/erlang/rebar3/releases). The rebar3 escript can also extract itself with a run script under the user's home directory: @@ -77,11 +77,12 @@ The rebar3 escript can also extract itself with a run script under the user's ho $ ./rebar3 local install ===> Extracting rebar3 libs to ~/.cache/rebar3/lib... ===> Writing rebar3 run script ~/.cache/rebar3/bin/rebar3... -===> Add to $PATH for use: export PATH=$PATH:~/.cache/rebar3/bin +===> Add to $PATH for use: export PATH=~/.cache/rebar3/bin:$PATH ``` To keep it up to date after you've installed rebar3 this way you can use `rebar3 local upgrade` which -fetches the latest nightly and extracts to the same place as above. +fetches the latest stable release and extracts to the same place as above. A [nightly version can +also be obtained](https://s3.amazonaws.com/rebar3-nightly/rebar3) if desired. Rebar3 may also be available on various OS-specific package managers such as FreeBSD Ports. Those are maintained by the community and Rebar3 maintainers @@ -89,7 +90,7 @@ themselves are generally not involved in that process. If you do not have a full Erlang install, we using [erln8](http://erln8.github.io/erln8/) or [kerl](https://github.com/yrashk/kerl). For binary packages use those provided -by [Erlang Solutions](https://www.erlang-solutions.com/downloads/download-erlang-otp), +by [Erlang Solutions](https://www.erlang-solutions.com/resources/download.html), but be sure to choose the "Standard" download option or you'll have issues building projects. @@ -127,7 +128,7 @@ others via the plugin ecosystem: | Tarballs | Releases can be packaged into tarballs ready to be deployed. | | Templates | Configurable templates ship out of the box (try `rebar3 new` for a list or `rebar3 new help <template>` for a specific one). [Custom templates](http://www.rebar3.org/docs/using-templates) are also supported, and plugins can also add their own. | | Unstable namespace | We use a namespace to provide commands that are still in flux, allowing to test more experimental features we are working on. See `rebar3 unstable`. | -| Xref | Run cross reference analysis on the project with [xref](http://www.erlang.org/doc/apps/tools/xref_chapter.html) by calling `rebar3 xref`. | +| Xref | Run cross-reference analysis on the project with [xref](http://www.erlang.org/doc/apps/tools/xref_chapter.html) by calling `rebar3 xref`. | ## Migrating From rebar2 @@ -146,7 +147,7 @@ relx. ## Additional Resources -In case of problems that cannot be solved through documentation or examples, you +In the case of problems that cannot be solved through documentation or examples, you may want to try to contact members of the community for help. The community is also where you want to go for questions about how to extend rebar, fill in bug reports, and so on. @@ -156,7 +157,7 @@ list](http://lists.basho.com/pipermail/rebar_lists.basho.com/). If you need quick feedback, you can try the #rebar channel on [irc.freenode.net](http://freenode.net). Be sure to check the [documentation](http://www.rebar3.org/v3.0/docs) first, just to be sure you're not -asking about things with well known answers. +asking about things with well-known answers. For bug reports, roadmaps, and issues, visit the [github issues page](https://github.com/erlang/rebar3/issues). @@ -137,3 +137,8 @@ Stefan Grundmann Carlos Eduardo de Paula Derek Brown Heinz N. Gies +Roberto Aloi +Andrew McRobb +Drew Varner +Niklas Johansson +Bryan Paxton diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 0000000..fa44ffa --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,20 @@ +version: "{build}" +branches: + only: + - master +environment: + matrix: + - erlang_vsn: 19.2 + - erlang_vsn: 18.3 + - erlang_vsn: 17.5 +install: +- ps: choco install erlang --version $env:erlang_vsn +build_script: +- ps: cmd.exe /c 'bootstrap.bat' +test_script: +- ps: cmd.exe /c 'rebar3.cmd ct' +notifications: +- provider: GitHubPullRequest + on_build_success: true + on_build_failure: true + on_build_status_changed: false @@ -16,7 +16,10 @@ main(_) -> ,{getopt, []} ,{cf, []} ,{erlware_commons, ["ec_dictionary.erl", "ec_vsn.erl"]} - ,{certifi, []}], + ,{parse_trans, ["parse_trans.erl", "parse_trans_pp.erl", + "parse_trans_codegen.erl"]} + ,{certifi, []} + ,{hex_core, []}], Deps = get_deps(), [fetch_and_compile(Dep, Deps) || Dep <- BaseDeps], @@ -24,7 +27,7 @@ main(_) -> bootstrap_rebar3(), %% Build rebar.app from rebar.app.src - {ok, App} = rebar_app_info:new(rebar, "3.2.0", filename:absname("_build/default/lib/rebar/")), + {ok, App} = rebar_app_info:new(rebar, "3.7.5", filename:absname("_build/default/lib/rebar/")), rebar_otp_app:compile(rebar_state:new(), App), %% Because we are compiling files that are loaded already we want to silence @@ -33,14 +36,6 @@ main(_) -> setup_env(), os:putenv("REBAR_PROFILE", "bootstrap"), - RegistryFile = default_registry_file(), - case filelib:is_file(RegistryFile) of - true -> - ok; - false -> - rebar3:run(["update"]) - end, - {ok, State} = rebar3:run(["compile"]), reset_env(), os:putenv("REBAR_PROFILE", ""), @@ -49,29 +44,10 @@ main(_) -> code:add_pathsa(DepsPaths), rebar3:run(["clean", "-a"]), - rebar3:run(["escriptize"]), + rebar3:run(["as", "prod", "escriptize"]), %% Done with compile, can turn back on error logger - error_logger:tty(true), - - %% Finally, update executable perms for our script on *nix, - %% or write out script files on win32. - ec_file:copy("_build/default/bin/rebar3", "./rebar3"), - case os:type() of - {unix,_} -> - [] = os:cmd("chmod u+x rebar3"), - ok; - {win32,_} -> - write_windows_scripts(), - ok; - _ -> - ok - end. - -default_registry_file() -> - {ok, [[Home]]} = init:get_argument(home), - CacheDir = filename:join([Home, ".cache", "rebar3"]), - filename:join([CacheDir, "hex", "default", "registry"]). + error_logger:tty(true). fetch_and_compile({Name, ErlFirstFiles}, Deps) -> case lists:keyfind(Name, 1, Deps) of @@ -94,13 +70,14 @@ fetch({pkg, Name, Vsn}, App) -> false -> CDN = "https://repo.hex.pm/tarballs", Package = binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>), - Url = string:join([CDN, Package], "/"), + Url = join([CDN, Package], "/"), case request(Url) of {ok, Binary} -> {ok, Contents} = extract(Binary), ok = erl_tar:extract({binary, Contents}, [{cwd, Dir}, compressed]); - _ -> - io:format("Error: Unable to fetch package ~p ~p~n", [Name, Vsn]) + {error, {Reason, _}} -> + ReasonText = re:replace(atom_to_list(Reason), "_", " ", [global,{return,list}]), + io:format("Error: Unable to fetch package ~s ~s: ~s~n", [Name, Vsn, ReasonText]) end; true -> io:format("Dependency ~s already exists~n", [Name]) @@ -112,8 +89,10 @@ extract(Binary) -> {ok, Contents}. request(Url) -> + HttpOptions = [{relaxed, true} | get_proxy_auth()], + case httpc:request(get, {Url, []}, - [{relaxed, true}], + HttpOptions, [{body_format, binary}], rebar) of {ok, {{_Version, 200, _Reason}, _Headers, Body}} -> @@ -147,8 +126,9 @@ set_httpc_options(_, []) -> ok; set_httpc_options(Scheme, Proxy) -> - {ok, {_, _, Host, Port, _, _}} = http_uri:parse(Proxy), - httpc:set_options([{Scheme, {{Host, Port}, []}}], rebar). + {ok, {_, UserInfo, Host, Port, _, _}} = http_uri:parse(Proxy), + httpc:set_options([{Scheme, {{Host, Port}, []}}], rebar), + set_proxy_auth(UserInfo). compile(App, FirstFiles) -> Dir = filename:join(filename:absname("_build/default/lib/"), App), @@ -178,9 +158,10 @@ compile_file(File, Opts) -> bootstrap_rebar3() -> filelib:ensure_dir("_build/default/lib/rebar/ebin/dummy.beam"), code:add_path("_build/default/lib/rebar/ebin/"), - ok = symlink_or_copy(filename:absname("src"), - filename:absname("_build/default/lib/rebar/src")), - Sources = ["src/rebar_resource.erl" | filelib:wildcard("src/*.erl")], + Res = symlink_or_copy(filename:absname("src"), + filename:absname("_build/default/lib/rebar/src")), + true = Res == ok orelse Res == exists, + Sources = ["src/rebar_resource_v2.erl", "src/rebar_resource.erl" | filelib:wildcard("src/*.erl")], [compile_file(X, [{outdir, "_build/default/lib/rebar/ebin/"} ,return | additional_defines()]) || X <- Sources], code:add_patha(filename:absname("_build/default/lib/rebar/ebin")). @@ -189,6 +170,8 @@ bootstrap_rebar3() -> -define(FMT(Str, Args), lists:flatten(io_lib:format(Str, Args))). %%/rebar.hrl %%rebar_file_utils +-include_lib("kernel/include/file.hrl"). + symlink_or_copy(Source, Target) -> Link = case os:type() of {win32, _} -> @@ -200,55 +183,63 @@ symlink_or_copy(Source, Target) -> ok -> ok; {error, eexist} -> - ok; + exists; {error, _} -> - cp_r([Source], Target) + case os:type() of + {win32, _} -> + S = unicode:characters_to_list(Source), + T = unicode:characters_to_list(Target), + case filelib:is_dir(S) of + true -> + win32_symlink_or_copy(S, T); + false -> + cp_r([S], T) + end; + _ -> + case filelib:is_dir(Target) of + true -> + ok; + false -> + cp_r([Source], Target) + end + end end. -make_relative_path(Source, Target) -> - do_make_relative_path(filename:split(Source), filename:split(Target)). - -do_make_relative_path([H|T1], [H|T2]) -> - do_make_relative_path(T1, T2); -do_make_relative_path(Source, Target) -> - Base = lists:duplicate(max(length(Target) - 1, 0), ".."), - filename:join(Base ++ Source). - +-spec cp_r(list(string()), file:filename()) -> 'ok'. cp_r([], _Dest) -> ok; cp_r(Sources, Dest) -> case os:type() of {unix, _} -> - EscSources = [escape_path(Src) || Src <- Sources], - SourceStr = string:join(EscSources, " "), - os:cmd(?FMT("cp -R ~s \"~s\"", [SourceStr, Dest])), + EscSources = [escape_chars(Src) || Src <- Sources], + SourceStr = join(EscSources, " "), + {ok, []} = sh(?FMT("cp -Rp ~ts \"~ts\"", + [SourceStr, escape_double_quotes(Dest)]), + [{use_stdout, false}, abort_on_error]), ok; {win32, _} -> lists:foreach(fun(Src) -> ok = cp_r_win32(Src,Dest) end, Sources), ok end. -xcopy_win32(Source,Dest)-> - R = os:cmd(?FMT("xcopy \"~s\" \"~s\" /q /y /e 2> nul", - [filename:nativename(Source), filename:nativename(Dest)])), - case length(R) > 0 of - %% when xcopy fails, stdout is empty and and error message is printed - %% to stderr (which is redirected to nul) +%% @private Compatibility function for windows +win32_symlink_or_copy(Source, Target) -> + Res = sh(?FMT("cmd /c mklink /j \"~ts\" \"~ts\"", + [escape_double_quotes(filename:nativename(Target)), + escape_double_quotes(filename:nativename(Source))]), + [{use_stdout, false}, return_on_error]), + case win32_mklink_ok(Res, Target) of true -> ok; - false -> - {error, lists:flatten( - io_lib:format("Failed to xcopy from ~s to ~s~n", - [Source, Dest]))} + false -> cp_r_win32(Source, drop_last_dir_from_path(Target)) end. cp_r_win32({true, SourceDir}, {true, DestDir}) -> %% from directory to directory - SourceBase = filename:basename(SourceDir), - ok = case file:make_dir(filename:join(DestDir, SourceBase)) of + ok = case file:make_dir(DestDir) of {error, eexist} -> ok; Other -> Other end, - ok = xcopy_win32(SourceDir, filename:join(DestDir, SourceBase)); + ok = xcopy_win32(SourceDir, DestDir); cp_r_win32({false, Source} = S,{true, DestDir}) -> %% from file to directory cp_r_win32(S, {false, filename:join(DestDir, filename:basename(Source))}); @@ -282,10 +273,231 @@ cp_r_win32(Source,Dest) -> end, filelib:wildcard(Source)), ok. -escape_path(Str) -> - re:replace(Str, "([ ()?])", "\\\\&", [global, {return, list}]). +%% drops the last 'node' of the filename, presumably the last dir such as 'src' +%% this is because cp_r_win32/2 automatically adds the dir name, to appease +%% robocopy and be more uniform with POSIX +drop_last_dir_from_path([]) -> + []; +drop_last_dir_from_path(Path) -> + case lists:droplast(filename:split(Path)) of + [] -> []; + Dirs -> filename:join(Dirs) + end. + +%% @private specifically pattern match against the output +%% of the windows 'mklink' shell call; different values from +%% what win32_ok/1 handles +win32_mklink_ok({ok, _}, _) -> + true; +win32_mklink_ok({error,{1,"Local NTFS volumes are required to complete the operation.\n"}}, _) -> + false; +win32_mklink_ok({error,{1,"Cannot create a file when that file already exists.\n"}}, Target) -> + % File or dir is already in place; find if it is already a symlink (true) or + % if it is a directory (copy-required; false) + is_symlink(Target); +win32_mklink_ok(_, _) -> + false. + +xcopy_win32(Source,Dest)-> + %% "xcopy \"~ts\" \"~ts\" /q /y /e 2> nul", Changed to robocopy to + %% handle long names. May have issues with older windows. + Cmd = case filelib:is_dir(Source) of + true -> + %% For robocopy, copying /a/b/c/ to /d/e/f/ recursively does not + %% create /d/e/f/c/*, but rather copies all files to /d/e/f/*. + %% The usage we make here expects the former, not the later, so we + %% must manually add the last fragment of a directory to the `Dest` + %% in order to properly replicate POSIX platforms + NewDest = filename:join([Dest, filename:basename(Source)]), + ?FMT("robocopy \"~ts\" \"~ts\" /e 1> nul", + [escape_double_quotes(filename:nativename(Source)), + escape_double_quotes(filename:nativename(NewDest))]); + false -> + ?FMT("robocopy \"~ts\" \"~ts\" \"~ts\" /e 1> nul", + [escape_double_quotes(filename:nativename(filename:dirname(Source))), + escape_double_quotes(filename:nativename(Dest)), + escape_double_quotes(filename:basename(Source))]) + end, + Res = sh(Cmd, [{use_stdout, false}, return_on_error]), + case win32_ok(Res) of + true -> ok; + false -> + {error, lists:flatten( + io_lib:format("Failed to copy ~ts to ~ts~n", + [Source, Dest]))} + end. + +is_symlink(Filename) -> + {ok, Info} = file:read_link_info(Filename), + Info#file_info.type == symlink. + +win32_ok({ok, _}) -> true; +win32_ok({error, {Rc, _}}) when Rc<9; Rc=:=16 -> true; +win32_ok(_) -> false. + %%/rebar_file_utils +%%rebar_utils + +%% escape\ as\ a\ shell\? +escape_chars(Str) when is_atom(Str) -> + escape_chars(atom_to_list(Str)); +escape_chars(Str) -> + re:replace(Str, "([ ()?`!$&;\"\'])", "\\\\&", + [global, {return, list}, unicode]). + +%% "escape inside these" +escape_double_quotes(Str) -> + re:replace(Str, "([\"\\\\`!$&*;])", "\\\\&", + [global, {return, list}, unicode]). + +sh(Command0, Options0) -> + DefaultOptions = [{use_stdout, false}], + Options = [expand_sh_flag(V) + || V <- proplists:compact(Options0 ++ DefaultOptions)], + + ErrorHandler = proplists:get_value(error_handler, Options), + OutputHandler = proplists:get_value(output_handler, Options), + + Command = lists:flatten(patch_on_windows(Command0, proplists:get_value(env, Options, []))), + PortSettings = proplists:get_all_values(port_settings, Options) ++ + [exit_status, {line, 16384}, use_stdio, stderr_to_stdout, hide, eof], + Port = open_port({spawn, Command}, PortSettings), + + try + case sh_loop(Port, OutputHandler, []) of + {ok, _Output} = Ok -> + Ok; + {error, {_Rc, _Output}=Err} -> + ErrorHandler(Command, Err) + end + after + port_close(Port) + end. + +sh_loop(Port, Fun, Acc) -> + receive + {Port, {data, {eol, Line}}} -> + sh_loop(Port, Fun, Fun(Line ++ "\n", Acc)); + {Port, {data, {noeol, Line}}} -> + sh_loop(Port, Fun, Fun(Line, Acc)); + {Port, eof} -> + Data = lists:flatten(lists:reverse(Acc)), + receive + {Port, {exit_status, 0}} -> + {ok, Data}; + {Port, {exit_status, Rc}} -> + {error, {Rc, Data}} + end + end. + +expand_sh_flag(return_on_error) -> + {error_handler, + fun(_Command, Err) -> + {error, Err} + end}; +expand_sh_flag(abort_on_error) -> + {error_handler, + fun log_and_abort/2}; +expand_sh_flag({use_stdout, false}) -> + {output_handler, + fun(Line, Acc) -> + [Line | Acc] + end}; +expand_sh_flag({cd, _CdArg} = Cd) -> + {port_settings, Cd}; +expand_sh_flag({env, _EnvArg} = Env) -> + {port_settings, Env}. + +%% We do the shell variable substitution ourselves on Windows and hope that the +%% command doesn't use any other shell magic. +patch_on_windows(Cmd, Env) -> + case os:type() of + {win32,nt} -> + Cmd1 = "cmd /q /c " + ++ lists:foldl(fun({Key, Value}, Acc) -> + expand_env_variable(Acc, Key, Value) + end, Cmd, Env), + %% Remove left-over vars + re:replace(Cmd1, "\\\$\\w+|\\\${\\w+}", "", + [global, {return, list}, unicode]); + _ -> + Cmd + end. + +%% @doc Given env. variable `FOO' we want to expand all references to +%% it in `InStr'. References can have two forms: `$FOO' and `${FOO}' +%% The end of form `$FOO' is delimited with whitespace or EOL +-spec expand_env_variable(string(), string(), term()) -> string(). +expand_env_variable(InStr, VarName, RawVarValue) -> + case chr(InStr, $$) of + 0 -> + %% No variables to expand + InStr; + _ -> + ReOpts = [global, unicode, {return, list}], + VarValue = re:replace(RawVarValue, "\\\\", "\\\\\\\\", ReOpts), + %% Use a regex to match/replace: + %% Given variable "FOO": match $FOO\s | $FOOeol | ${FOO} + RegEx = io_lib:format("\\\$(~ts(\\W|$)|{~ts})", [VarName, VarName]), + re:replace(InStr, RegEx, [VarValue, "\\2"], ReOpts) + end. + +-spec log_and_abort(string(), {integer(), string()}) -> no_return(). +log_and_abort(Command, {Rc, Output}) -> + io:format("sh(~ts)~n" + "failed with return code ~w and the following output:~n" + "~ts", [Command, Rc, Output]), + throw(bootstrap_abort). + +%%/rebar_utils + +%%rebar_dir +make_relative_path(Source, Target) -> + AbsSource = make_normalized_path(Source), + AbsTarget = make_normalized_path(Target), + do_make_relative_path(filename:split(AbsSource), filename:split(AbsTarget)). + +%% @private based on fragments of paths, replace the number of common +%% segments by `../' bits, and add the rest of the source alone after it +-spec do_make_relative_path([string()], [string()]) -> file:filename(). +do_make_relative_path([H|T1], [H|T2]) -> + do_make_relative_path(T1, T2); +do_make_relative_path(Source, Target) -> + Base = lists:duplicate(max(length(Target) - 1, 0), ".."), + filename:join(Base ++ Source). + +make_normalized_path(Path) -> + AbsPath = make_absolute_path(Path), + Components = filename:split(AbsPath), + make_normalized_path(Components, []). + +make_absolute_path(Path) -> + case filename:pathtype(Path) of + absolute -> + Path; + relative -> + {ok, Dir} = file:get_cwd(), + filename:join([Dir, Path]); + volumerelative -> + Volume = hd(filename:split(Path)), + {ok, Dir} = file:get_cwd(Volume), + filename:join([Dir, Path]) + end. + +-spec make_normalized_path([string()], [string()]) -> file:filename(). +make_normalized_path([], NormalizedPath) -> + filename:join(lists:reverse(NormalizedPath)); +make_normalized_path([H|T], NormalizedPath) -> + case H of + "." when NormalizedPath == [], T == [] -> make_normalized_path(T, ["."]); + "." -> make_normalized_path(T, NormalizedPath); + ".." when NormalizedPath == [] -> make_normalized_path(T, [".."]); + ".." when hd(NormalizedPath) =/= ".." -> make_normalized_path(T, tl(NormalizedPath)); + _ -> make_normalized_path(T, [H|NormalizedPath]) + end. +%%/rebar_dir + setup_env() -> %% We don't need or want relx providers loaded yet application:load(rebar), @@ -301,14 +513,6 @@ reset_env() -> application:unload(rebar), application:load(rebar). -write_windows_scripts() -> - CmdScript= - "@echo off\r\n" - "setlocal\r\n" - "set rebarscript=%~f0\r\n" - "escript.exe \"%rebarscript:.cmd=%\" %*\r\n", - ok = file:write_file("rebar3.cmd", CmdScript). - get_deps() -> case file:consult("rebar.lock") of {ok, [[]]} -> @@ -354,7 +558,12 @@ format_error(AbsSource, Extra, {Mod, Desc}) -> io_lib:format("~s: ~s~s~n", [AbsSource, Extra, ErrorDesc]). additional_defines() -> - [{d, D} || {Re, D} <- [{"^[0-9]+", namespaced_types}, {"^R1[4|5]", deprecated_crypto}, {"^((1[8|9])|2)", rand_module}], is_otp_release(Re)]. + [{d, D} || {Re, D} <- [{"^[0-9]+", namespaced_types}, + {"^R1[4|5]", deprecated_crypto}, + {"^2", unicode_str}, + {"^(R|1|20)", fun_stacktrace}, + {"^((1[8|9])|2)", rand_module}], + is_otp_release(Re)]. is_otp_release(ArchRegex) -> case re:run(otp_release(), ArchRegex, [{capture, none}]) of @@ -402,3 +611,33 @@ otp_release1(Rel) -> binary:bin_to_list(Vsn, {0, Size - 1}) end end. + +set_proxy_auth([]) -> + ok; +set_proxy_auth(UserInfo) -> + [Username, Password] = re:split(UserInfo, ":", + [{return, list}, {parts,2}, unicode]), + %% password may contain url encoded characters, need to decode them first + put(proxy_auth, [{proxy_auth, {Username, http_uri:decode(Password)}}]). + +get_proxy_auth() -> + case get(proxy_auth) of + undefined -> []; + ProxyAuth -> ProxyAuth + end. + +%% string:join/2 copy; string:join/2 is getting obsoleted +%% and replaced by lists:join/2, but lists:join/2 is too new +%% for version support (only appeared in 19.0) so it cannot be +%% used. Instead we just adopt join/2 locally and hope it works +%% for most unicode use cases anyway. +join([], Sep) when is_list(Sep) -> + []; +join([H|T], Sep) -> + H ++ lists:append([Sep ++ X || X <- T]). + +%% Same for chr; no non-deprecated equivalent in OTP20+ +chr(S, C) when is_integer(C) -> chr(S, C, 1). +chr([C|_Cs], C, I) -> I; +chr([_|Cs], C, I) -> chr(Cs, C, I+1); +chr([], _C, _I) -> 0. diff --git a/manpages/commands b/manpages/commands new file mode 100644 index 0000000..b4db50a --- /dev/null +++ b/manpages/commands @@ -0,0 +1,25 @@ +f(), +P = application:get_env(rebar, providers, []), +S = lists:foldl(fun(P, S) -> {ok, S2} = P:init(S), S2 end, rebar_state:new(), P), +PS = rebar_state:providers(S), +DP = lists:keysort(2,providers:get_providers_by_namespace(default, PS)), +f(Str), +Str = [begin + Name = element(2,Pn), + Desc = element(8,Pn), + Opts = element(10,Pn), + OptShort = [case {Short,Long} of + {undefined,undefined} -> ""; + {undefined,_} -> ["[\\fI--",Long,"\\fR] "]; + {_,undefined} -> ["[\\fI-",Short,"\\fR] "]; + {_,_} -> ["[\\fI-",Short,"\\fR|\\fI--",Long,"\\fR] "] + end || {_,Short,Long,_,_Desc} <- Opts], + OptLong = [case {Short,Long} of + {undefined,undefined} -> ""; + {_,undefined} -> [".IP\n\\fI-",Short,"\\fR: ", Desc, "\n"]; + {_,_} -> [".IP\n\\fI--",Long,"\\fR: ", Desc, "\n"] + end || {_,Short,Long,_,Desc} <- Opts], + [".TP\n", + "\\fB", atom_to_list(element(2,Pn)), "\\fR ", OptShort, "\n", + Desc, "\n", OptLong] end || Pn <- DP, element(5,Pn) == true], +file:write_file("commands.out", Str). diff --git a/manpages/rebar3.1 b/manpages/rebar3.1 new file mode 100644 index 0000000..997d144 --- /dev/null +++ b/manpages/rebar3.1 @@ -0,0 +1,441 @@ +.TH "REBAR3" "1" "November 2018" "Erlang" + +.SH NAME + +\fBrebar3\fR \- tool for working with Erlang projects + +.SH "SYNOPSIS" + +\fBrebar3\fR \fB\-\-version\fR + +.br +\fBrebar3\fR \fBhelp\fR + +.br +\fBrebar3\fR \fIcommand\fR [\fIoptions\fR] \.\.\. + +.SH "DESCRIPTION" + +Rebar3 is an Erlang tool that makes it easy to create, develop, and release Erlang libraries, applications, and systems in a repeatable manner\. + +Full documentation at \fIhttp://www.rebar3.org/\fR + +.SH "ESSENTIAL COMMANDS" + +For the full command list, see the \fICOMMANDS\fR section\. + +.P +With \fBrebar3 help <task>\fR, commands and plugins will display their own help information\. + +.TP +\fBcompile\fR +Compile the current project + +.TP +\fBnew\fR (\fBhelp [\fItemplate\fR])|\fItemplate\fR +Show information about templates or use one + +.TP +\fBupdate\fR +Fetch the newest version of the Hex index + +.TP +\fBdo\fR \fIcommand\fR[,\fIcommand\fR,...] +Run one or more commands in a sequence + +.TP +\fBshell\fR +Start the current project in a REPL\. You can then use \fBr3:do(\fItask\fR)\fR to call it on the project without dropping the current state. + +.SH "COMMANDS" + +. this section generated by running the contents of 'commands' in rebar3 shell + +.TP +\fBas\fR +Higher order provider for running multiple tasks in a sequence as a certain profiles. +.TP +\fBclean\fR [\fI-a\fR|\fI--all\fR] [\fI-p\fR|\fI--profile\fR] +Remove compiled beam files from apps. +.IP +\fI--all\fR: Clean all apps include deps +.IP +\fI--profile\fR: Clean under profile. Equivalent to `rebar3 as <profile> clean` +.TP +\fBcompile\fR [\fI-d\fR|\fI--deps_only\fR] +Compile apps .app.src and .erl files. +.IP +\fI--deps_only\fR: Only compile dependencies, no project apps will be built. +.TP +\fBcover\fR [\fI-r\fR|\fI--reset\fR] [\fI-v\fR|\fI--verbose\fR] [\fI-m\fR|\fI--min_coverage\fR] +Perform coverage analysis. +.IP +\fI--reset\fR: Reset all coverdata. +.IP +\fI--verbose\fR: Print coverage analysis. +.IP +\fI--min_coverage\fR: Mandate a coverage percentage required to succeed (0..100) +.TP +\fBct\fR [\fI--dir\fR] [\fI--suite\fR] [\fI--group\fR] [\fI--case\fR] [\fI--label\fR] [\fI--config\fR] [\fI--spec\fR] [\fI--join_specs\fR] [\fI--allow_user_terms\fR] [\fI--logdir\fR] [\fI--logopts\fR] [\fI--verbosity\fR] [\fI-c\fR|\fI--cover\fR] [\fI--cover_export_name\fR] [\fI--repeat\fR] [\fI--duration\fR] [\fI--until\fR] [\fI--force_stop\fR] [\fI--basic_html\fR] [\fI--stylesheet\fR] [\fI--decrypt_key\fR] [\fI--decrypt_file\fR] [\fI--abort_if_missing_suites\fR] [\fI--multiply_timetraps\fR] [\fI--scale_timetraps\fR] [\fI--create_priv_dir\fR] [\fI--include\fR] [\fI--readable\fR] [\fI-v\fR|\fI--verbose\fR] [\fI--name\fR] [\fI--sname\fR] [\fI--setcookie\fR] [\fI--sys_config\fR] [\fI--compile_only\fR] [\fI--retry\fR] +Run Common Tests. +.IP +\fI--dir\fR: List of additional directories containing test suites +.IP +\fI--suite\fR: List of test suites to run +.IP +\fI--group\fR: List of test groups to run +.IP +\fI--case\fR: List of test cases to run +.IP +\fI--label\fR: Test label +.IP +\fI--config\fR: List of config files +.IP +\fI--spec\fR: List of test specifications +.IP +\fI--join_specs\fR: Merge all test specifications and perform a single test run +.IP +\fI--allow_user_terms\fR: Allow user defined config values in config files +.IP +\fI--logdir\fR: Log folder +.IP +\fI--logopts\fR: Options for common test logging +.IP +\fI--verbosity\fR: Verbosity +.IP +\fI--cover\fR: Generate cover data +.IP +\fI--cover_export_name\fR: Base name of the coverdata file to write +.IP +\fI--repeat\fR: How often to repeat tests +.IP +\fI--duration\fR: Max runtime (format: HHMMSS) +.IP +\fI--until\fR: Run until (format: HHMMSS) +.IP +\fI--force_stop\fR: Force stop on test timeout (true | false | skip_rest) +.IP +\fI--basic_html\fR: Show basic HTML +.IP +\fI--stylesheet\fR: CSS stylesheet to apply to html output +.IP +\fI--decrypt_key\fR: Path to key for decrypting config +.IP +\fI--decrypt_file\fR: Path to file containing key for decrypting config +.IP +\fI--abort_if_missing_suites\fR: Abort if suites are missing +.IP +\fI--multiply_timetraps\fR: +.IP +\fI--scale_timetraps\fR: Scale timetraps +.IP +\fI--create_priv_dir\fR: Create priv dir (auto_per_run | auto_per_tc | manual_per_tc) +.IP +\fI--include\fR: Directories containing additional include files +.IP +\fI--readable\fR: Shows test case names and only displays logs to shell on failures (true | compact | false) +.IP +\fI--verbose\fR: Verbose output +.IP +\fI--name\fR: Gives a long name to the node +.IP +\fI--sname\fR: Gives a short name to the node +.IP +\fI--setcookie\fR: Sets the cookie if the node is distributed +.IP +\fI--sys_config\fR: List of application config files +.IP +\fI--compile_only\fR: Compile modules in the project with the test configuration but do not run the tests +.IP +\fI--retry\fR: Experimental feature. If any specification for previously failing test is found, runs them. +.TP +\fBdeps\fR +List dependencies +.TP +\fBdialyzer\fR [\fI-u\fR|\fI--update-plt\fR] [\fI-s\fR|\fI--succ-typings\fR] +Run the Dialyzer analyzer on the project. +.IP +\fI--update-plt\fR: Enable updating the PLT. Default: true +.IP +\fI--succ-typings\fR: Enable success typing analysis. Default: true +.TP +\fBdo\fR +Higher order provider for running multiple tasks in a sequence. +.TP +\fBedoc\fR +Generate documentation using edoc. +.TP +\fBescriptize\fR +Generate escript archive. +.TP +\fBeunit\fR [\fI--app\fR] [\fI--application\fR] [\fI-c\fR|\fI--cover\fR] [\fI--cover_export_name\fR] [\fI-d\fR|\fI--dir\fR] [\fI-f\fR|\fI--file\fR] [\fI-m\fR|\fI--module\fR] [\fI-s\fR|\fI--suite\fR] [\fI-v\fR|\fI--verbose\fR] [\fI--name\fR] [\fI--sname\fR] [\fI--setcookie\fR] +Run EUnit Tests. +.IP +\fI--app\fR: Comma separated list of application test suites to run. Equivalent to `[{application, App}]`. +.IP +\fI--application\fR: Comma separated list of application test suites to run. Equivalent to `[{application, App}]`. +.IP +\fI--cover\fR: Generate cover data. Defaults to false. +.IP +\fI--cover_export_name\fR: Base name of the coverdata file to write +.IP +\fI--dir\fR: Comma separated list of dirs to load tests from. Equivalent to `[{dir, Dir}]`. +.IP +\fI--file\fR: Comma separated list of files to load tests from. Equivalent to `[{file, File}]`. +.IP +\fI--module\fR: Comma separated list of modules to load tests from. Equivalent to `[{module, Module}]`. +.IP +\fI--suite\fR: Comma separated list of modules to load tests from. Equivalent to `[{module, Module}]`. +.IP +\fI--verbose\fR: Verbose output. Defaults to false. +.IP +\fI--name\fR: Gives a long name to the node +.IP +\fI--sname\fR: Gives a short name to the node +.IP +\fI--setcookie\fR: Sets the cookie if the node is distributed +.TP +\fBget-deps\fR +Fetch dependencies. +.TP +\fBhelp\fR +Display a list of tasks or help for a given task or subtask. +.TP +\fBnew\fR [\fI-f\fR|\fI--force\fR] +Create new project from templates. +.IP +\fI--force\fR: overwrite existing files +.TP +\fBpath\fR [\fI--app\fR] [\fI--base\fR] [\fI--bin\fR] [\fI--ebin\fR] [\fI--lib\fR] [\fI--priv\fR] [\fI-s\fR|\fI--separator\fR] [\fI--src\fR] [\fI--rel\fR] +Print paths to build dirs in current profile. +.IP +\fI--app\fR: Comma separated list of applications to return paths for. +.IP +\fI--base\fR: Return the `base' path of the current profile. +.IP +\fI--bin\fR: Return the `bin' path of the current profile. +.IP +\fI--ebin\fR: Return all `ebin' paths of the current profile's applications. +.IP +\fI--lib\fR: Return the `lib' path of the current profile. +.IP +\fI--priv\fR: Return the `priv' path of the current profile's applications. +.IP +\fI--separator\fR: In case of multiple return paths, the separator character to use to join them. +.IP +\fI--src\fR: Return the `src' path of the current profile's applications. +.IP +\fI--rel\fR: Return the `rel' path of the current profile. +.TP +\fBpkgs\fR +List information for a package. +.TP +\fBrelease\fR [\fI-n\fR|\fI--relname\fR] [\fI-v\fR|\fI--relvsn\fR] [\fI-g\fR|\fI--goal\fR] [\fI-u\fR|\fI--upfrom\fR] [\fI-o\fR|\fI--output-dir\fR] [\fI-h\fR|\fI--help\fR] [\fI-l\fR|\fI--lib-dir\fR] [\fI-p\fR|\fI--path\fR] [\fI--default-libs\fR] [\fI-V\fR|\fI--verbose\fR] [\fI-d\fR|\fI--dev-mode\fR] [\fI-i\fR|\fI--include-erts\fR] [\fI-a\fR|\fI--override\fR] [\fI-c\fR|\fI--config\fR] [\fI--overlay_vars\fR] [\fI--vm_args\fR] [\fI--sys_config\fR] [\fI--system_libs\fR] [\fI--version\fR] [\fI-r\fR|\fI--root\fR] +Build release of project. +.IP +\fI--relname\fR: Specify the name for the release that will be generated +.IP +\fI--relvsn\fR: Specify the version for the release +.IP +\fI--goal\fR: Specify a target constraint on the system. These are usually the OTP +.IP +\fI--upfrom\fR: Only valid with relup target, specify the release to upgrade from +.IP +\fI--output-dir\fR: The output directory for the release. This is `./` by default. +.IP +\fI--help\fR: Print usage +.IP +\fI--lib-dir\fR: Additional dir that should be searched for OTP Apps +.IP +\fI--path\fR: Additional dir to add to the code path +.IP +\fI--default-libs\fR: Whether to use the default system added lib dirs (means you must add them all manually). Default is true +.IP +\fI--verbose\fR: Verbosity level, maybe between 0 and 3 +.IP +\fI--dev-mode\fR: Symlink the applications and configuration into the release instead of copying +.IP +\fI--include-erts\fR: If true include a copy of erts used to build with, if a path include erts at that path. If false, do not include erts +.IP +\fI--override\fR: Provide an app name and a directory to override in the form <appname>:<app directory> +.IP +\fI--config\fR: The path to a config file +.IP +\fI--overlay_vars\fR: Path to a file of overlay variables +.IP +\fI--vm_args\fR: Path to a file to use for vm.args +.IP +\fI--sys_config\fR: Path to a file to use for sys.config +.IP +\fI--system_libs\fR: Path to dir of Erlang system libs +.IP +\fI--version\fR: Print relx version +.IP +\fI--root\fR: The project root directory +.TP +\fBrelup\fR [\fI-n\fR|\fI--relname\fR] [\fI-v\fR|\fI--relvsn\fR] [\fI-g\fR|\fI--goal\fR] [\fI-u\fR|\fI--upfrom\fR] [\fI-o\fR|\fI--output-dir\fR] [\fI-h\fR|\fI--help\fR] [\fI-l\fR|\fI--lib-dir\fR] [\fI-p\fR|\fI--path\fR] [\fI--default-libs\fR] [\fI-V\fR|\fI--verbose\fR] [\fI-d\fR|\fI--dev-mode\fR] [\fI-i\fR|\fI--include-erts\fR] [\fI-a\fR|\fI--override\fR] [\fI-c\fR|\fI--config\fR] [\fI--overlay_vars\fR] [\fI--vm_args\fR] [\fI--sys_config\fR] [\fI--system_libs\fR] [\fI--version\fR] [\fI-r\fR|\fI--root\fR] +Create relup of releases. +.IP +\fI--relname\fR: Specify the name for the release that will be generated +.IP +\fI--relvsn\fR: Specify the version for the release +.IP +\fI--goal\fR: Specify a target constraint on the system. These are usually the OTP +.IP +\fI--upfrom\fR: Only valid with relup target, specify the release to upgrade from +.IP +\fI--output-dir\fR: The output directory for the release. This is `./` by default. +.IP +\fI--help\fR: Print usage +.IP +\fI--lib-dir\fR: Additional dir that should be searched for OTP Apps +.IP +\fI--path\fR: Additional dir to add to the code path +.IP +\fI--default-libs\fR: Whether to use the default system added lib dirs (means you must add them all manually). Default is true +.IP +\fI--verbose\fR: Verbosity level, maybe between 0 and 3 +.IP +\fI--dev-mode\fR: Symlink the applications and configuration into the release instead of copying +.IP +\fI--include-erts\fR: If true include a copy of erts used to build with, if a path include erts at that path. If false, do not include erts +.IP +\fI--override\fR: Provide an app name and a directory to override in the form <appname>:<app directory> +.IP +\fI--config\fR: The path to a config file +.IP +\fI--overlay_vars\fR: Path to a file of overlay variables +.IP +\fI--vm_args\fR: Path to a file to use for vm.args +.IP +\fI--sys_config\fR: Path to a file to use for sys.config +.IP +\fI--system_libs\fR: Path to dir of Erlang system libs +.IP +\fI--version\fR: Print relx version +.IP +\fI--root\fR: The project root directory +.TP +\fBreport\fR +Provide a crash report to be sent to the rebar3 issues page. +.TP +\fBshell\fR [\fI--config\fR] [\fI--name\fR] [\fI--sname\fR] [\fI--setcookie\fR] [\fI--script\fR] [\fI--apps\fR] [\fI--start-clean\fR] [\fI--user_drv_args\fR] +Run shell with project apps and deps in path. +.IP +\fI--config\fR: Path to the config file to use. Defaults to {shell, [{config, File}]} and then the relx sys.config file if not specified. +.IP +\fI--name\fR: Gives a long name to the node. +.IP +\fI--sname\fR: Gives a short name to the node. +.IP +\fI--setcookie\fR: Sets the cookie if the node is distributed. +.IP +\fI--script\fR: Path to an escript file to run before starting the project apps. Defaults to rebar.config {shell, [{script_file, File}]} if not specified. +.IP +\fI--apps\fR: A list of apps to boot before starting the shell. (E.g. --apps app1,app2,app3) Defaults to rebar.config {shell, [{apps, Apps}]} or relx apps if not specified. +.IP +\fI--start-clean\fR: Cancel any applications in the 'apps' list or release. +.IP +\fI--user_drv_args\fR: Arguments passed to user_drv start function for creating custom shells. +.TP +\fBtar\fR [\fI-n\fR|\fI--relname\fR] [\fI-v\fR|\fI--relvsn\fR] [\fI-g\fR|\fI--goal\fR] [\fI-u\fR|\fI--upfrom\fR] [\fI-o\fR|\fI--output-dir\fR] [\fI-h\fR|\fI--help\fR] [\fI-l\fR|\fI--lib-dir\fR] [\fI-p\fR|\fI--path\fR] [\fI--default-libs\fR] [\fI-V\fR|\fI--verbose\fR] [\fI-d\fR|\fI--dev-mode\fR] [\fI-i\fR|\fI--include-erts\fR] [\fI-a\fR|\fI--override\fR] [\fI-c\fR|\fI--config\fR] [\fI--overlay_vars\fR] [\fI--vm_args\fR] [\fI--sys_config\fR] [\fI--system_libs\fR] [\fI--version\fR] [\fI-r\fR|\fI--root\fR] +Tar archive of release built of project. +.IP +\fI--relname\fR: Specify the name for the release that will be generated +.IP +\fI--relvsn\fR: Specify the version for the release +.IP +\fI--goal\fR: Specify a target constraint on the system. These are usually the OTP +.IP +\fI--upfrom\fR: Only valid with relup target, specify the release to upgrade from +.IP +\fI--output-dir\fR: The output directory for the release. This is `./` by default. +.IP +\fI--help\fR: Print usage +.IP +\fI--lib-dir\fR: Additional dir that should be searched for OTP Apps +.IP +\fI--path\fR: Additional dir to add to the code path +.IP +\fI--default-libs\fR: Whether to use the default system added lib dirs (means you must add them all manually). Default is true +.IP +\fI--verbose\fR: Verbosity level, maybe between 0 and 3 +.IP +\fI--dev-mode\fR: Symlink the applications and configuration into the release instead of copying +.IP +\fI--include-erts\fR: If true include a copy of erts used to build with, if a path include erts at that path. If false, do not include erts +.IP +\fI--override\fR: Provide an app name and a directory to override in the form <appname>:<app directory> +.IP +\fI--config\fR: The path to a config file +.IP +\fI--overlay_vars\fR: Path to a file of overlay variables +.IP +\fI--vm_args\fR: Path to a file to use for vm.args +.IP +\fI--sys_config\fR: Path to a file to use for sys.config +.IP +\fI--system_libs\fR: Path to dir of Erlang system libs +.IP +\fI--version\fR: Print relx version +.IP +\fI--root\fR: The project root directory +.TP +\fBtree\fR [\fI-v\fR|\fI--verbose\fR] +Print dependency tree. +.IP +\fI--verbose\fR: Print repo and branch/tag/ref for git and hg deps +.TP +\fBunlock\fR +Unlock dependencies. +.TP +\fBupdate\fR +Update package index. +.TP +\fBupgrade\fR +Upgrade dependencies. +.TP +\fBversion\fR +Print version for rebar and current Erlang. +.TP +\fBxref\fR +Run cross reference analysis. + +.SH ENVIRONMENT + +Environment variables allow overall rebar3 control across command boundaries. + +.TP +\fBREBAR_PROFILE\fR +Choose a default profile. Defaults to \fBdefault\fR + +.TP +\fBHEX_CDN\fR +Pick an alternative hex mirror. + +.TP +\fBREBAR_CACHE_DIR\fR +Location of the directory for local cache. Defaults to \fIhex.pm\fB. + +.TP +\fBQUIET\fR +Only display errors. + +.TP +\fBDEBUG\fR +Display debug information. + +.TP +\fBREBAR_COLOR\fR=\fIhigh\fR|\fIlow\fR +How much color to show in the terminal. Defaults to \fIhigh\fR. + +.TP +\fBREBAR_CONFIG\fR +Name of rebar configuration files. Defaults to \fIrebar.config\fR + +.TP +\fBREBAR_GIT_CLONE_OPTIONS\fR +Arguments to add after each \fIgit clone\fR operation. For example, the value \fI--reference ~/.cache/repos.reference\fR allows to create a cache of all fetched repositories across builds + +.SH Configuration File Options +See \fIhttp://www.rebar3.org/v3.0/docs/configuration\fR diff --git a/pr2relnotes.sh b/pr2relnotes.sh index 28712bb..9049e82 100755 --- a/pr2relnotes.sh +++ b/pr2relnotes.sh @@ -13,7 +13,7 @@ awk -v url=$url ' /^commit / {mode="new"} # merge commit default message - / +Merge pull request/ { + mode=="new" && / +Merge pull request/ { page_id=substr($4, 2, length($4)-1); mode="started"; next; diff --git a/priv/shell-completion/fish/rebar3.fish b/priv/shell-completion/fish/rebar3.fish index fd28c97..9cd2c82 100644 --- a/priv/shell-completion/fish/rebar3.fish +++ b/priv/shell-completion/fish/rebar3.fish @@ -147,7 +147,7 @@ complete -f -c 'rebar3' -n '__fish_rebar3_using_command new' -s f -l force -d "O complete -f -c 'rebar3' -n '__fish_rebar3_using_command new' -a help -d "Display all variables and arguments for each template" complete -f -c 'rebar3' -n '__fish_rebar3_needs_command' -a paths -d "Print paths to build dirs in current profile." -complete -f -c 'rebar3' -n '__fish_rebar3_needs_command paths' -l app -d "Comma seperated list of applications to return paths for." +complete -f -c 'rebar3' -n '__fish_rebar3_needs_command paths' -l app -d "Comma separated list of applications to return paths for." complete -f -c 'rebar3' -n '__fish_rebar3_needs_command paths' -l base -d "Return the `base` path of the current profile." complete -f -c 'rebar3' -n '__fish_rebar3_needs_command paths' -l bin -d "Return the `bin` path of the current profile." complete -f -c 'rebar3' -n '__fish_rebar3_needs_command paths' -l ebin -d "Return all `ebin` paths of the current profile`s applications." diff --git a/priv/shell-completion/zsh/_rebar3 b/priv/shell-completion/zsh/_rebar3 index 0abc340..490a824 100644 --- a/priv/shell-completion/zsh/_rebar3 +++ b/priv/shell-completion/zsh/_rebar3 @@ -110,7 +110,7 @@ _rebar3 () { ;; (path) _arguments \ - '(--app)--app[Comma seperated list of applications to return paths for.]:apps' \ + '(--app)--app[Comma separated list of applications to return paths for.]:apps' \ '(--base)--base[Return the `base` path of the current profile.]' \ '(--bin)--bin[Return the `bin` path of the current profile.]' \ '(--ebin)--ebin[Return all `ebin` paths of the current profile`s applications.]' \ diff --git a/priv/templates/LICENSE b/priv/templates/LICENSE index 41588ab..59e1345 100644 --- a/priv/templates/LICENSE +++ b/priv/templates/LICENSE @@ -1,29 +1,191 @@ -Copyright (c) {{copyright_year}}, {{author_name}} <{{author_email}}>. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -* Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -* The names of its contributors may not be used to endorse or promote - products derived from this software without specific prior written - permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright {{copyright_year}}, {{author_name}} <{{author_email}}>. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/priv/templates/Makefile b/priv/templates/Makefile index d3c3767..5d490fd 100644 --- a/priv/templates/Makefile +++ b/priv/templates/Makefile @@ -6,9 +6,9 @@ BASEDIR := $(abspath $(CURDIR)/..) PROJECT ?= $(notdir $(BASEDIR)) PROJECT := $(strip $(PROJECT)) -ERTS_INCLUDE_DIR ?= $(shell erl -noshell -s init stop -eval "io:format(\"~s/erts-~s/include/\", [code:root_dir(), erlang:system_info(version)]).") -ERL_INTERFACE_INCLUDE_DIR ?= $(shell erl -noshell -s init stop -eval "io:format(\"~s\", [code:lib_dir(erl_interface, include)]).") -ERL_INTERFACE_LIB_DIR ?= $(shell erl -noshell -s init stop -eval "io:format(\"~s\", [code:lib_dir(erl_interface, lib)]).") +ERTS_INCLUDE_DIR ?= $(shell erl -noshell -s init stop -eval "io:format(\"~ts/erts-~ts/include/\", [code:root_dir(), erlang:system_info(version)]).") +ERL_INTERFACE_INCLUDE_DIR ?= $(shell erl -noshell -s init stop -eval "io:format(\"~ts\", [code:lib_dir(erl_interface, include)]).") +ERL_INTERFACE_LIB_DIR ?= $(shell erl -noshell -s init stop -eval "io:format(\"~ts\", [code:lib_dir(erl_interface, lib)]).") C_SRC_DIR = $(CURDIR) C_SRC_OUTPUT ?= $(CURDIR)/../priv/$(PROJECT).so diff --git a/priv/templates/app.template b/priv/templates/app.template index d0bebd7..7d68b2c 100644 --- a/priv/templates/app.template +++ b/priv/templates/app.template @@ -6,7 +6,7 @@ {template, "app.erl", "{{name}}/src/{{name}}_app.erl"}. {template, "sup.erl", "{{name}}/src/{{name}}_sup.erl"}. {template, "otp_app.app.src", "{{name}}/src/{{name}}.app.src"}. -{template, "rebar.config", "{{name}}/rebar.config"}. +{template, "app_rebar.config", "{{name}}/rebar.config"}. {template, "gitignore", "{{name}}/.gitignore"}. {template, "LICENSE", "{{name}}/LICENSE"}. {template, "README.md", "{{name}}/README.md"}. diff --git a/priv/templates/app_rebar.config b/priv/templates/app_rebar.config new file mode 100644 index 0000000..a3f5b8c --- /dev/null +++ b/priv/templates/app_rebar.config @@ -0,0 +1,7 @@ +{erl_opts, [debug_info]}. +{deps, []}. + +{shell, [ + % {config, "config/sys.config"}, + {apps, [{{name}}]} +]}. diff --git a/priv/templates/escript_rebar.config b/priv/templates/escript_rebar.config index ef498a8..3ed7150 100644 --- a/priv/templates/escript_rebar.config +++ b/priv/templates/escript_rebar.config @@ -5,7 +5,7 @@ [{{name}}]}. {escript_main_app, {{name}}}. {escript_name, {{name}}}. -{escript_emu_args, "%%! +sbtu +A0\n"}. +{escript_emu_args, "%%! +sbtu +A1\n"}. %% Profiles {profiles, [{test, diff --git a/priv/templates/gitignore b/priv/templates/gitignore index 617aab8..9d7e9c5 100644 --- a/priv/templates/gitignore +++ b/priv/templates/gitignore @@ -13,4 +13,11 @@ erl_crash.dump .rebar logs _build +<<<<<<< HEAD rebar.lock +======= +.idea +*.iml +rebar3.crashdump +*~ +>>>>>>> upstream/master diff --git a/priv/templates/otp_app.app.src b/priv/templates/otp_app.app.src index c18f82c..6040089 100644 --- a/priv/templates/otp_app.app.src +++ b/priv/templates/otp_app.app.src @@ -1,8 +1,9 @@ -{application, {{name}}, - [{description, "{{desc}}"}, +{{=@@ @@=}} +{application, @@name@@, + [{description, "@@desc@@"}, {vsn, "0.1.0"}, {registered, []}, - {mod, { {{name}}_app, []}}, + {mod, {@@name@@_app, []}}, {applications, [kernel, stdlib @@ -11,6 +12,6 @@ {modules, []}, {maintainers, []}, - {licenses, []}, + {licenses, ["Apache 2.0"]}, {links, []} ]}. diff --git a/priv/templates/otp_lib.app.src b/priv/templates/otp_lib.app.src index 5b98a51..aa31966 100644 --- a/priv/templates/otp_lib.app.src +++ b/priv/templates/otp_lib.app.src @@ -10,6 +10,6 @@ {modules, []}, {maintainers, []}, - {licenses, []}, + {licenses, ["Apache 2.0"]}, {links, []} ]}. diff --git a/priv/templates/plugin_README.md b/priv/templates/plugin_README.md index 7f9ba84..1176d95 100644 --- a/priv/templates/plugin_README.md +++ b/priv/templates/plugin_README.md @@ -1,7 +1,8 @@ -{{name}} +{{=@@ @@=}} +@@name@@ ===== -{{desc}} +@@desc@@ Build ----- @@ -14,13 +15,13 @@ Use Add the plugin to your rebar config: {plugins, [ - { {{name}}, ".*", {git, "git@host:user/{{name}}.git", {tag, "0.1.0"}}} + {@@name@@, {git, "https://host/user/@@name@@.git", {tag, "0.1.0"}}} ]}. Then just call your plugin directly in an existing application: - $ rebar3 {{name}} - ===> Fetching {{name}} - ===> Compiling {{name}} + $ rebar3 @@name@@ + ===> Fetching @@name@@ + ===> Compiling @@name@@ <Plugin Output> diff --git a/priv/templates/relx_rebar.config b/priv/templates/relx_rebar.config index 81b7dbe..30a629f 100644 --- a/priv/templates/relx_rebar.config +++ b/priv/templates/relx_rebar.config @@ -1,8 +1,9 @@ +{{=@@ @@=}} {erl_opts, [debug_info]}. {deps, []}. -{relx, [{release, { {{name}}, "0.1.0" }, - [{{name}}, +{relx, [{release, {@@name@@, "0.1.0"}, + [@@name@@, sasl]}, {sys_config, "./config/sys.config"}, diff --git a/priv/templates/sup.erl b/priv/templates/sup.erl index a2e7209..020021c 100644 --- a/priv/templates/sup.erl +++ b/priv/templates/sup.erl @@ -1,9 +1,10 @@ +{{=@@ @@=}} %%%------------------------------------------------------------------- -%% @doc {{name}} top level supervisor. +%% @doc @@name@@ top level supervisor. %% @end %%%------------------------------------------------------------------- --module({{name}}_sup). +-module(@@name@@_sup). -behaviour(supervisor). @@ -26,9 +27,12 @@ start_link() -> %% Supervisor callbacks %%==================================================================== +%% Child :: #{id => Id, start => {M, F, A}} +%% Optional keys are restart, shutdown, type, modules. +%% Before OTP 18 tuples must be used to specify a child. e.g. %% Child :: {Id,StartFunc,Restart,Shutdown,Type,Modules} init([]) -> - {ok, { {one_for_all, 0, 1}, []} }. + {ok, {{one_for_all, 0, 1}, []}}. %%==================================================================== %% Internal functions diff --git a/priv/templates/sys.config b/priv/templates/sys.config index d892fd6..983801f 100644 --- a/priv/templates/sys.config +++ b/priv/templates/sys.config @@ -1,3 +1,4 @@ +{{=@@ @@=}} [ - { {{name}}, []} + {@@name@@, []} ]. diff --git a/priv/templates/umbrella.template b/priv/templates/umbrella.template new file mode 100644 index 0000000..a2855d3 --- /dev/null +++ b/priv/templates/umbrella.template @@ -0,0 +1,15 @@ +{description, "OTP structure for executable programs (alias of 'release' template)"}. +{variables, [ + {name, "myapp", "Name of the OTP release. An app with this name will also be created."}, + {desc, "An OTP application", "Short description of the release's main app's purpose"} +]}. +{template, "app.erl", "{{name}}/{{apps_dir}}/{{name}}/src/{{name}}_app.erl"}. +{template, "sup.erl", "{{name}}/{{apps_dir}}/{{name}}/src/{{name}}_sup.erl"}. +{template, "otp_app.app.src", "{{name}}/{{apps_dir}}/{{name}}/src/{{name}}.app.src"}. +{template, "relx_rebar.config", "{{name}}/rebar.config"}. +{template, "sys.config", "{{name}}/config/sys.config"}. +{template, "vm.args", "{{name}}/config/vm.args"}. +{template, "gitignore", "{{name}}/.gitignore"}. +{template, "LICENSE", "{{name}}/LICENSE"}. +{template, "README.md", "{{name}}/README.md"}. + diff --git a/rebar.config b/rebar.config index 0489572..61efd8d 100644 --- a/rebar.config +++ b/rebar.config @@ -1,19 +1,30 @@ %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- %% ex: ts=4 sw=4 ft=erlang et -{deps, [{erlware_commons, "0.21.0"}, - {ssl_verify_hostname, "1.0.5"}, - {certifi, "0.4.0"}, - {providers, "1.6.0"}, - {getopt, "0.8.2"}, - {bbmustache, "1.0.4"}, - {relx, "3.19.0"}, - {cf, "0.2.1"}, - {cth_readable, "1.2.3"}, - {eunit_formatters, "0.3.1"}]}. +{deps, [{erlware_commons, "1.3.0"}, + {ssl_verify_fun, "1.1.3"}, + {certifi, "2.3.1"}, + {parse_trans, "3.3.0"}, % force otp-21 compat + {providers, "1.7.0"}, + {getopt, "1.0.1"}, + {bbmustache, "1.6.0"}, + {relx, "3.27.0"}, + {cf, "0.2.2"}, + {cth_readable, "1.4.2"}, + {hex_core, "0.2.0"}, + {eunit_formatters, "0.5.0"}]}. + +{post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", + escriptize, + "cp \"$REBAR_BUILD_DIR/bin/rebar3\" ./rebar3"}, + {"win32", + escriptize, + "robocopy \"%REBAR_BUILD_DIR%/bin/\" ./ rebar3* " + "/njs /njh /nfl /ndl & exit /b 0"} % silence things + ]}. {escript_name, rebar3}. -{escript_emu_args, "%%! +sbtu +A0\n"}. +{escript_emu_args, "%%! +sbtu +A1\n"}. %% escript_incl_extra is for internal rebar-private use only. %% Do not use outside rebar. Config interface is not stable. {escript_incl_extra, [{"relx/priv/templates/*", "_build/default/lib/"}, @@ -21,49 +32,57 @@ {erl_opts, [{platform_define, "^[0-9]+", namespaced_types}, {platform_define, "^(19|2)", rand_only}, - no_debug_info, - warnings_as_errors]}. + {platform_define, "^2", unicode_str}, + {platform_define, "^(2[1-9])|(20\\\\.3)", filelib_find_source}, + {platform_define, "^(R|1|20)", fun_stacktrace}, + warnings_as_errors + ]}. %% Use OTP 18+ when dialyzing rebar3 -{dialyzer, [{warnings, [unknown]}]}. +{dialyzer, [ + {warnings, [unknown]}, + {exclude_mods, [rebar_prv_alias]} +]}. %% Profiles {profiles, [{test, [ - {deps, [{meck, "0.8.2"}]}, - {erl_opts, [debug_info]} + {deps, [{meck, "0.8.12"}]}, + {erl_opts, [debug_info, nowarn_export_all]} ] }, + {systest, [ + {erl_opts, [debug_info, nowarn_export_all]}, + {ct_opts, [{dir, "systest"}]} + ]}, {bootstrap, []}, - {dialyze, [{overrides, [{add, erlware_commons, [{erl_opts, [debug_info]}]}, - {add, ssl_verify_hostname, [{erl_opts, [debug_info]}]}, - {add, certifi, [{erl_opts, [debug_info]}]}, - {add, providers, [{erl_opts, [debug_info]}]}, - {add, getopt, [{erl_opts, [debug_info]}]}, - {add, bbmustache, [{erl_opts, [debug_info]}]}, - {add, relx, [{erl_opts, [debug_info]}]}, - {add, cf, [{erl_opts, [debug_info]}]}, - {add, cth_readable, [{erl_opts, [debug_info]}]}, - {add, eunit_formatters, [{erl_opts, [debug_info]}]}]}, - {erl_opts, [debug_info]}]} + {prod, [ + {escript_incl_extra, [ + {"relx/priv/templates/*", "_build/prod/lib/"}, + {"rebar/priv/templates/*", "_build/prod/lib/"} + ]}, + {erl_opts, [no_debug_info]}, + {overrides, [ + {override, erlware_commons, [ + {erl_opts, [{platform_define, "^[0-9]+", namespaced_types}, + {platform_define, "^R1[4|5]", deprecated_crypto}, + {platform_define, "^((1[8|9])|2)", rand_module}, + {platform_define, "^2", unicode_str}, + {platform_define, "^(R|1|20)", fun_stacktrace}, + no_debug_info, + warnings_as_errors]}, + {deps, []}, {plugins, []}]}, + {add, ssl_verify_hostname, [{erl_opts, [no_debug_info]}]}, + {add, certifi, [{erl_opts, [no_debug_info]}]}, + {add, cf, [{erl_opts, [no_debug_info]}]}, + {add, cth_readable, [{erl_opts, [no_debug_info]}]}, + {add, eunit_formatters, [{erl_opts, [no_debug_info]}]}, + {override, bbmustache, [ + {erl_opts, [no_debug_info, {platform_define, "^[0-9]+", namespaced_types}]}, + {deps, []}, {plugins, []}]}, + {add, getopt, [{erl_opts, [no_debug_info]}]}, + {add, providers, [{erl_opts, [no_debug_info]}]}, + {add, relx, [{erl_opts, [no_debug_info]}]}]} + ]} ]}. - -%% Overrides -{overrides, [{override, erlware_commons, [{erl_opts, [{platform_define, "^[0-9]+", namespaced_types}, - {platform_define, "^R1[4|5]", deprecated_crypto}, - {platform_define, "^((1[8|9])|2)", rand_module}, - no_debug_info, - warnings_as_errors]}, - {deps, []}, {plugins, []}]}, - {add, ssl_verify_hostname, [{erl_opts, [no_debug_info]}]}, - {add, certifi, [{erl_opts, [no_debug_info]}]}, - {add, cf, [{erl_opts, [no_debug_info]}]}, - {add, cth_readable, [{erl_opts, [no_debug_info]}]}, - {add, eunit_formatters, [{erl_opts, [no_debug_info]}]}, - {override, bbmustache, [{erl_opts, [no_debug_info, - {platform_define, "^[0-9]+", namespaced_types}]}, - {deps, []}, {plugins, []}]}, - {add, getopt, [{erl_opts, [no_debug_info]}]}, - {add, providers, [{erl_opts, [no_debug_info]}]}, - {add, relx, [{erl_opts, [no_debug_info]}]}]}. diff --git a/rebar.config.sample b/rebar.config.sample index f57f8dc..c255e49 100644 --- a/rebar.config.sample +++ b/rebar.config.sample @@ -57,6 +57,9 @@ %% is `false' {cover_enabled, false}. +%% Modules to exclude from cover +{cover_excl_mods, []}. + %% Options to pass to cover provider {cover_opts, [verbose]}. @@ -128,7 +131,8 @@ %% Paths to miscellaneous Erlang files to compile for an app %% without including them in its modules list {extra_src_dirs, []}. - +%% Path where custom rebar3 templates could be found +{template_dir, []}. %% == EDoc == @@ -140,7 +144,7 @@ %% == Escript == %% name of the main OTP application to boot -{escript_main_app, application} +{escript_main_app, application}. %% Name of the resulting escript executable {escript_name, "application"}. %% apps (other than main and deps) to be included @@ -154,7 +158,7 @@ %% == EUnit == %% eunit:test(Tests) -{eunit_tests, [{application, rebar3}]} +{eunit_tests, [{application, rebar3}]}. %% Options for eunit:test(Tests, Opts) {eunit_opts, [verbose]}. %% Additional compile options for eunit. erl_opts is also used @@ -194,7 +198,7 @@ %% Only clean, ct, compile, eunit, release, and tar can be hooked around %% runs 'clean' before 'compile' -{provider_hooks, [{pre, [{compile, clean}]}]} +{provider_hooks, [{pre, [{compile, clean}]}]}. %% == Releases == @@ -1,24 +1,28 @@ {"1.1.0", -[{<<"bbmustache">>,{pkg,<<"bbmustache">>,<<"1.0.4">>},0}, - {<<"certifi">>,{pkg,<<"certifi">>,<<"0.4.0">>},0}, - {<<"cf">>,{pkg,<<"cf">>,<<"0.2.1">>},0}, - {<<"cth_readable">>,{pkg,<<"cth_readable">>,<<"1.2.3">>},0}, - {<<"erlware_commons">>,{pkg,<<"erlware_commons">>,<<"0.21.0">>},0}, - {<<"eunit_formatters">>,{pkg,<<"eunit_formatters">>,<<"0.3.1">>},0}, - {<<"getopt">>,{pkg,<<"getopt">>,<<"0.8.2">>},0}, - {<<"providers">>,{pkg,<<"providers">>,<<"1.6.0">>},0}, - {<<"relx">>,{pkg,<<"relx">>,<<"3.19.0">>},0}, - {<<"ssl_verify_hostname">>,{pkg,<<"ssl_verify_hostname">>,<<"1.0.5">>},0}]}. +[{<<"bbmustache">>,{pkg,<<"bbmustache">>,<<"1.6.0">>},0}, + {<<"certifi">>,{pkg,<<"certifi">>,<<"2.3.1">>},0}, + {<<"cf">>,{pkg,<<"cf">>,<<"0.2.2">>},0}, + {<<"cth_readable">>,{pkg,<<"cth_readable">>,<<"1.4.2">>},0}, + {<<"erlware_commons">>,{pkg,<<"erlware_commons">>,<<"1.3.0">>},0}, + {<<"eunit_formatters">>,{pkg,<<"eunit_formatters">>,<<"0.5.0">>},0}, + {<<"getopt">>,{pkg,<<"getopt">>,<<"1.0.1">>},0}, + {<<"hex_core">>,{pkg,<<"hex_core">>,<<"0.2.0">>},0}, + {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.3.0">>},0}, + {<<"providers">>,{pkg,<<"providers">>,<<"1.7.0">>},0}, + {<<"relx">>,{pkg,<<"relx">>,<<"3.27.0">>},0}, + {<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.3">>},0}]}. [ {pkg_hash,[ - {<<"bbmustache">>, <<"7BA94F971C5AFD7B6617918A4BB74705E36CAB36EB84B19B6A1B7EE06427AA38">>}, - {<<"certifi">>, <<"A7966EFB868B179023618D29A407548F70C52466BF1849B9E8EBD0E34B7EA11F">>}, - {<<"cf">>, <<"69D0B1349FD4D7D4DC55B7F407D29D7A840BF9A1EF5AF529F1EBE0CE153FC2AB">>}, - {<<"cth_readable">>, <<"293120673DFF82F0768612C5282E35C40CACC1B6F94FE99077438FD3749D0E27">>}, - {<<"erlware_commons">>, <<"A04433071AD7D112EDEFC75AC77719DD3E6753E697AC09428FC83D7564B80B15">>}, - {<<"eunit_formatters">>, <<"7A6FC351EB5B873E2356B8852EB751E20C13A72FBCA03393CF682B8483509573">>}, - {<<"getopt">>, <<"B17556DB683000BA50370B16C0619DF1337E7AF7ECBF7D64FBF8D1D6BCE3109B">>}, - {<<"providers">>, <<"DB0E2F9043AE60C0155205FCD238D68516331D0E5146155E33D1E79DC452964A">>}, - {<<"relx">>, <<"286DD5244B4786F56AAC75D5C8E2D1FB4CFD306810D4EC8548F3AE1B3AADB8F7">>}, - {<<"ssl_verify_hostname">>, <<"2E73E068CD6393526F9FA6D399353D7C9477D6886BA005F323B592D389FB47BE">>}]} + {<<"bbmustache">>, <<"7AC372AEC621A69C369DF237FBD9986CAABCDD6341089FE5F42E5A7A4AC706B8">>}, + {<<"certifi">>, <<"D0F424232390BF47D82DA8478022301C561CF6445B5B5FB6A84D49A9E76D2639">>}, + {<<"cf">>, <<"7F2913FFF90ABCABD0F489896CFEB0B0674F6C8DF6C10B17A83175448029896C">>}, + {<<"cth_readable">>, <<"0F57B4EB7DA7F5438F422312245F9143A1B3118C11B6BAE5C3D1391C9EE88322">>}, + {<<"erlware_commons">>, <<"1705CF2AB4212EF235C21971A55E22E2A39055C05B9C65C8848126865F42A07A">>}, + {<<"eunit_formatters">>, <<"6A9133943D36A465D804C1C5B6E6839030434B8879C5600D7DDB5B3BAD4CCB59">>}, + {<<"getopt">>, <<"C73A9FA687B217F2FF79F68A3B637711BB1936E712B521D8CE466B29CBF7808A">>}, + {<<"hex_core">>, <<"3A7EACCFB8ADD3FF05D950C10ED5BDB5D0C48C988EBBC5D7AE2A55498F0EFF1B">>}, + {<<"parse_trans">>, <<"09765507A3C7590A784615CFD421D101AEC25098D50B89D7AA1D66646BC571C1">>}, + {<<"providers">>, <<"BBF730563914328EC2511D205E6477A94831DB7297DE313B3872A2B26C562EAB">>}, + {<<"relx">>, <<"96CC7663EDCC02A8117AB0C64FE6D15BE79760C08726ABEAD1DAACE11BFBF75D">>}, + {<<"ssl_verify_fun">>, <<"6C49665D4326E26CD4A5B7BD54AA442B33DADFB7C5D59A0D0CD0BF5534BBFBD7">>}]} ]. diff --git a/src/cth_retry.erl b/src/cth_retry.erl new file mode 100644 index 0000000..7056c71 --- /dev/null +++ b/src/cth_retry.erl @@ -0,0 +1,159 @@ +-module(cth_retry). + +%% Callbacks +-export([id/1]). +-export([init/2]). + +-export([pre_init_per_suite/3]). +-export([post_init_per_suite/4]). +-export([pre_end_per_suite/3]). +-export([post_end_per_suite/4]). + +-export([pre_init_per_group/3]). +-export([post_init_per_group/4]). +-export([pre_end_per_group/3]). +-export([post_end_per_group/4]). + +-export([pre_init_per_testcase/3]). +-export([post_end_per_testcase/4]). + +-export([on_tc_fail/3]). +-export([on_tc_skip/3, on_tc_skip/4]). + +-export([terminate/1]). + +-record(state, {id, suite, groups, acc=[]}). + +%% @doc Return a unique id for this CTH. +id(_Opts) -> + {?MODULE, make_ref()}. + +%% @doc Always called before any other callback function. Use this to initiate +%% any common state. +init(Id, _Opts) -> + {ok, #state{id=Id}}. + +%% @doc Called before init_per_suite is called. +pre_init_per_suite(Suite,Config,State) -> + {Config, State#state{suite=Suite, groups=[]}}. + +%% @doc Called after init_per_suite. +post_init_per_suite(_Suite,_Config,Return,State) -> + {Return, State}. + +%% @doc Called before end_per_suite. +pre_end_per_suite(_Suite,Config,State) -> + {Config, State}. + +%% @doc Called after end_per_suite. +post_end_per_suite(_Suite,_Config,Return,State) -> + {Return, State#state{suite=undefined, groups=[]}}. + +%% @doc Called before each init_per_group. +pre_init_per_group(_Group,Config,State) -> + {Config, State}. + +%% @doc Called after each init_per_group. +post_init_per_group(Group,_Config,Return, State=#state{groups=Groups}) -> + {Return, State#state{groups=[Group|Groups]}}. + +%% @doc Called after each end_per_group. +pre_end_per_group(_Group,Config,State) -> + {Config, State}. + +%% @doc Called after each end_per_group. +post_end_per_group(_Group,_Config,Return, State=#state{groups=Groups}) -> + {Return, State#state{groups=tl(Groups)}}. + +%% @doc Called before each test case. +pre_init_per_testcase(_TC,Config,State) -> + {Config, State}. + +%% @doc Called after each test case. +post_end_per_testcase(_TC,_Config,ok,State) -> + {ok, State}; +post_end_per_testcase(TC,_Config,Error,State=#state{suite=Suite, groups=Groups, acc=Acc}) -> + Test = case TC of + {_Group, Case} -> Case; + TC -> TC + end, + {Error, State#state{acc=[{Suite, Groups, Test}|Acc]}}. + +%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group, +%% post_end_per_group and post_end_per_testcase if the suite, group or test case failed. +on_tc_fail(_TC, _Reason, State) -> + State. + +%% @doc Called when a test case is skipped by either user action +%% or due to an init function failing. (>= 19.3) +on_tc_skip(Suite, TC, {tc_auto_skip, _}, State=#state{suite=Suite, groups=Groups, acc=Acc}) -> + NewAcc = case TC of + init_per_testcase -> Acc; + end_per_testcase -> Acc; + {init_per_group,_} -> Acc; + {end_per_group, _} -> Acc; + init_per_suite -> Acc; + end_per_suite -> Acc; + {_Group, Case} -> [{Suite, Groups, Case}|Acc]; + TC -> [{Suite, Groups, TC}|Acc] + end, + State#state{suite=Suite, acc=NewAcc}; +on_tc_skip(Suite, _TC, _Reason, State) -> + State#state{suite=Suite}. + +%% @doc Called when a test case is skipped by either user action +%% or due to an init function failing. (Pre-19.3) +on_tc_skip(TC, {tc_auto_skip, _}, State=#state{suite=Suite, groups=Groups, acc=Acc}) -> + NewAcc = case TC of + init_per_testcase -> Acc; + end_per_testcase -> Acc; + {init_per_group,_} -> Acc; + {end_per_group, _} -> Acc; + init_per_suite -> Acc; + end_per_suite -> Acc; + {_Group, Case} -> [{Suite, Groups, Case}|Acc]; + TC -> [{Suite, Groups, TC}|Acc] + end, + State#state{acc=NewAcc}; +on_tc_skip(_TC, _Reason, State) -> + State. + +%% @doc Called when the scope of the CTH is done +terminate(#state{acc=[]}) -> + ok; +terminate(#state{acc=Acc}) -> + Spec = to_spec(Acc), + {ok, Cwd} = file:get_cwd(), + Path = filename:join(lists:droplast(filename:split(Cwd))++["retry.spec"]), + io:format(user, + "EXPERIMENTAL: Writing retry specification at ~s~n" + " call rebar3 ct with '--retry' to re-run failing cases.~n", + [Path]), + file:write_file(Path, Spec), + ok. + +%%% Helpers +to_spec(List) -> + [to_spec_entry(X) || X <- merge(List)]. + +merge([]) -> []; +merge([{Suite, Groups, Case}|T]) when is_atom(Case) -> + merge([{Suite, Groups, [Case]}|T]); +merge([{Suite, Groups, Cases}, {Suite, Groups, Case} | T]) -> + merge([{Suite, Groups, [Case|Cases]}|T]); +merge([{Suite, Groups, Cases} | T]) -> + [{Suite, Groups, Cases} | merge(T)]. + +to_spec_entry({Suite, [], Cases}) -> + Dir = filename:dirname(proplists:get_value(source, Suite:module_info(compile))), + io_lib:format("~p.~n", [{cases, Dir, Suite, Cases}]); +to_spec_entry({Suite, Groups, Cases}) -> + Dir = filename:dirname(proplists:get_value(source, Suite:module_info(compile))), + ExpandedGroups = expand_groups(lists:reverse(Groups)), + io_lib:format("~p.~n", [{groups, Dir, Suite, ExpandedGroups, {cases,Cases}}]). + +expand_groups([Group]) -> + {Group, []}; +expand_groups([H|T]) -> + {H,[],[expand_groups(T)]}. + @@ -1,7 +1,58 @@ -%%% external alias for rebar_agent +%%% @doc external alias for `rebar_agent' for more convenient +%%% calls from a shell. -module(r3). --export([do/1, do/2]). +-export([do/1, do/2, async_do/1, async_do/2, break/0, resume/0]). +-export(['$handle_undefined_function'/2]). +-include("rebar.hrl"). +%% @doc alias for `rebar_agent:do/1' +-spec do(atom()) -> ok | {error, term()}. do(Command) -> rebar_agent:do(Command). +%% @doc alias for `rebar_agent:do/2' +-spec do(atom(), atom()) -> ok | {error, term()}. do(Namespace, Command) -> rebar_agent:do(Namespace, Command). + +%% @async_doc alias for `rebar_agent:async_do/1' +-spec async_do(atom()) -> ok | {error, term()}. +async_do(Command) -> rebar_agent:async_do(Command). + +%% @async_doc alias for `rebar_agent:async_do/2' +-spec async_do(atom(), atom()) -> ok | {error, term()}. +async_do(Namespace, Command) -> rebar_agent:async_do(Namespace, Command). + +break() -> + case whereis(rebar_agent) of % is the shell running + undefined -> + ok; + Pid -> + {dictionary, Dict} = process_info(Pid, dictionary), + case lists:keyfind(cmd_type, 1, Dict) of + {cmd_type, async} -> + Self = self(), + Ref = make_ref(), + spawn_link(fun() -> + register(r3_breakpoint_handler, self()), + receive + resume -> + Self ! Ref + end + end), + io:format(user, "~n=== BREAK ===~n", []), + receive + Ref -> ok + end; + _ -> + ?DEBUG("ignoring breakpoint since command is not run " + "in async mode", []), + ok + end + end. + +resume() -> + r3_breakpoint_handler ! resume, + ok. + +%% @private defer to rebar_agent +'$handle_undefined_function'(Cmd, Args) -> + rebar_agent:'$handle_undefined_function'(Cmd, Args). diff --git a/src/rebar.app.src b/src/rebar.app.src index bd0f871..6058efc 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -8,6 +8,7 @@ {registered, []}, {applications, [kernel, stdlib, + hipe, sasl, compiler, crypto, @@ -23,12 +24,13 @@ erlware_commons, providers, bbmustache, - ssl_verify_hostname, + ssl_verify_fun, certifi, cth_readable, relx, cf, inets, + hex_core, eunit_formatters]}, {env, [ %% Default log level @@ -38,6 +40,9 @@ {pkg, rebar_pkg_resource}, {hg, rebar_hg_resource}]}, + {compilers, [rebar_compiler_xrl, rebar_compiler_yrl, + rebar_compiler_mib, rebar_compiler_erl]}, + {providers, [rebar_prv_app_discovery, rebar_prv_as, rebar_prv_bare_compile, @@ -52,6 +57,7 @@ rebar_prv_edoc, rebar_prv_escriptize, rebar_prv_eunit, + rebar_prv_get_deps, rebar_prv_help, rebar_prv_install_deps, rebar_prv_local_install, @@ -65,6 +71,7 @@ rebar_prv_release, rebar_prv_relup, rebar_prv_report, + rebar_prv_repos, rebar_prv_shell, rebar_prv_state, rebar_prv_tar, @@ -72,6 +79,7 @@ rebar_prv_update, rebar_prv_upgrade, rebar_prv_version, - rebar_prv_xref]} + rebar_prv_xref, + rebar_prv_alias]} % must run last to prevent overloads ]} ]}. diff --git a/src/rebar.hrl b/src/rebar.hrl index f96ed5e..f11302d 100644 --- a/src/rebar.hrl +++ b/src/rebar.hrl @@ -22,17 +22,38 @@ -define(DEFAULT_PLUGINS_DIR, "plugins"). -define(DEFAULT_TEST_DEPS_DIR, "test/lib"). -define(DEFAULT_RELEASE_DIR, "rel"). --define(DEFAULT_CONFIG_FILE, "rebar.config"). -define(CONFIG_VERSION, "1.1.0"). -define(DEFAULT_CDN, "https://repo.hex.pm/"). -define(REMOTE_PACKAGE_DIR, "tarballs"). --define(REMOTE_REGISTRY_FILE, "registry.ets.gz"). -define(LOCK_FILE, "rebar.lock"). - --define(PACKAGE_INDEX_VERSION, 3). +-define(DEFAULT_COMPILER_SOURCE_FORMAT, relative). +-define(PACKAGE_INDEX_VERSION, 5). -define(PACKAGE_TABLE, package_index). -define(INDEX_FILE, "packages.idx"). --define(REGISTRY_FILE, "registry"). +-define(HEX_AUTH_FILE, "hex.config"). +-define(PUBLIC_HEX_REPO, <<"hexpm">>). + +%% ignore this function in all modules +%% not every module that exports it and relies on it being called implements provider +-ignore_xref([{format_error, 1}]). + +%% the package record is used in a select match spec which upsets dialyzer +%% this is the suggested workaround from Tobias +%% http://erlang.org/pipermail/erlang-questions/2009-February/041445.html +-type ms_field() :: '$1' | '_'. + +%% TODO: change package and requirement keys to be required (:=) after dropping support for OTP-18 +-record(package, {key :: {unicode:unicode_binary() | ms_field(), unicode:unicode_binary() | ms_field(), + unicode:unicode_binary() | ms_field()}, + checksum :: binary() | ms_field(), + retired :: boolean() | ms_field(), + dependencies :: [#{package => unicode:unicode_binary(), + requirement => unicode:unicode_binary()}] | ms_field()}). + +-record(resource, {type :: atom(), + module :: module(), + state :: term(), + implementation :: rebar_resource | rebar_resource_v2}). -ifdef(namespaced_types). -type rebar_dict() :: dict:dict(). @@ -52,6 +73,12 @@ -type rebar_set() :: set(). -endif. +-ifdef(fun_stacktrace). +-define(WITH_STACKTRACE(T, R, S), T:R -> S = erlang:get_stacktrace(),). +-else. +-define(WITH_STACKTRACE(T, R, S), T:R:S ->). +-endif. + -define(GRAPH_VSN, 2). -type v() :: {digraph:vertex(), term()} | 'false'. -type e() :: {digraph:vertex(), digraph:vertex()}. diff --git a/src/rebar3.erl b/src/rebar3.erl index d3ea15f..a490a15 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -24,6 +24,16 @@ %% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN %% THE SOFTWARE. %% ------------------------------------------------------------------- +%% +%% @doc Main module for rebar3. Supports two interfaces; one for escripts, +%% and one for usage as a library (although rebar3 makes a lot of +%% assumptions about its environment, making it a bit tricky to use as +%% a lib). +%% +%% This module's job is mostly to set up the root environment for rebar3 +%% and handle global options (mostly all from the ENV) and make them +%% accessible to the rest of the run. +%% @end -module(rebar3). -export([main/0, @@ -43,27 +53,28 @@ %% Public API %% ==================================================================== -%% For running with: -%% erl +sbtu +A0 -noinput -mode minimal -boot start_clean -s rebar3 main -extra "$@" +%% @doc For running with: +%% erl +sbtu +A1 -noinput -mode minimal -boot start_clean -s rebar3 main -extra "$@" -spec main() -> no_return(). main() -> List = init:get_plain_arguments(), main(List). -%% escript Entry point +%% @doc escript Entry point -spec main(list()) -> no_return(). main(Args) -> try run(Args) of {ok, _State} -> erlang:halt(0); Error -> - handle_error(Error) + handle_error(Error, []) catch - _:Error -> - handle_error(Error) + ?WITH_STACKTRACE(_,Error,Stacktrace) + handle_error(Error, Stacktrace) end. -%% Erlang-API entry point +%% @doc Erlang-API entry point +-spec run(rebar_state:t(), [string()]) -> {ok, rebar_state:t()} | {error, term()}. run(BaseState, Commands) -> start_and_load_apps(api), BaseState1 = rebar_state:set(BaseState, task, Commands), @@ -78,6 +89,11 @@ run(BaseState, Commands) -> %% Internal functions %% ==================================================================== +%% @private sets up the rebar3 environment based on the command line +%% arguments passed, if they have any relevance; used to translate +%% from the escript call-site into a common one with the library +%% usage. +-spec run([any(), ...]) -> {ok, rebar_state:t()} | {error, term()}. run(RawArgs) -> start_and_load_apps(command_line), @@ -87,7 +103,7 @@ run(RawArgs) -> case erlang:system_info(version) of "6.1" -> ?WARN("Due to a filelib bug in Erlang 17.1 it is recommended" - "you update to a newer release.", []); + "you update to a newer release.", []); _ -> ok end, @@ -95,7 +111,14 @@ run(RawArgs) -> {BaseState2, _Args1} = set_options(BaseState1, {[], []}), run_aux(BaseState2, RawArgs). +%% @private Junction point between the CLI and library entry points. +%% From here on the module's role is a shared path here to finish +%% up setting the environment for the run. +-spec run_aux(rebar_state:t(), [string()]) -> + {ok, rebar_state:t()} | {error, term()}. run_aux(State, RawArgs) -> + io:setopts([{encoding, unicode}]), + %% Profile override; can only support one profile State1 = case os:getenv("REBAR_PROFILE") of false -> State; @@ -108,6 +131,7 @@ run_aux(State, RawArgs) -> rebar_utils:check_min_otp_version(rebar_state:get(State1, minimum_otp_vsn, undefined)), rebar_utils:check_blacklisted_otp_versions(rebar_state:get(State1, blacklisted_otp_vsns, undefined)), + %% Change the default hex CDN State2 = case os:getenv("HEX_CDN") of false -> State1; @@ -115,8 +139,17 @@ run_aux(State, RawArgs) -> rebar_state:set(State1, rebar_packages_cdn, CDN) end, + Compilers = application:get_env(rebar, compilers, []), + State0 = rebar_state:compilers(State2, Compilers), + + %% TODO: this means use of REBAR_PROFILE=profile will replace the repos with + %% the repos defined in the profile. But it will not work with `as profile`. + %% Maybe it shouldn't work with either to be consistent? + Resources = application:get_env(rebar, resources, []), + State2_ = rebar_state:create_resources(Resources, State0), + %% bootstrap test profile - State3 = rebar_state:add_to_profile(State2, test, test_state(State1)), + State3 = rebar_state:add_to_profile(State2_, test, test_state(State1)), %% Process each command, resetting any state between each one BaseDir = rebar_state:get(State, base_dir, ?DEFAULT_BASE_DIR), @@ -142,30 +175,43 @@ run_aux(State, RawArgs) -> State10 = rebar_state:code_paths(State9, default, code:get_path()), - rebar_core:init_command(rebar_state:command_args(State10, Args), Task). + case rebar_core:init_command(rebar_state:command_args(State10, Args), Task) of + {ok, State11} -> + case rebar_state:get(State11, caller, command_line) of + api -> + rebar_paths:unset_paths([deps, plugins], State11), + {ok, State11}; + _ -> + {ok, State11} + end; + Other -> + Other + end. + + +%% @doc set up base configuration having to do with verbosity, where +%% to find config files, and so on, and return an internal rebar3 state term. +-spec init_config() -> rebar_state:t(). init_config() -> + rebar_utils:set_httpc_options(), + %% Initialize logging system Verbosity = log_level(), ok = rebar_log:init(command_line, Verbosity), - Config = case os:getenv("REBAR_CONFIG") of - false -> - rebar_config:consult_file(?DEFAULT_CONFIG_FILE); - ConfigFile -> - rebar_config:consult_file(ConfigFile) - end, + Config = rebar_config:consult_root(), Config1 = rebar_config:merge_locks(Config, rebar_config:consult_lock_file(?LOCK_FILE)), %% If $HOME/.config/rebar3/rebar.config exists load and use as global config GlobalConfigFile = rebar_dir:global_config(), State = case filelib:is_regular(GlobalConfigFile) of true -> - ?DEBUG("Load global config file ~s", [GlobalConfigFile]), + ?DEBUG("Load global config file ~ts", [GlobalConfigFile]), try state_from_global_config(Config1, GlobalConfigFile) catch _:_ -> - ?WARN("Global config ~s exists but can not be read. Ignoring global config values.", [GlobalConfigFile]), + ?WARN("Global config ~ts exists but can not be read. Ignoring global config values.", [GlobalConfigFile]), rebar_state:new(Config1) end; false -> @@ -193,6 +239,17 @@ init_config() -> %% Initialize vsn cache rebar_state:set(State1, vsn_cache, dict:new()). +%% @doc Parse basic rebar3 arguments to find the top-level task +%% to be run; this parsing is only partial from the point of view that +%% runs done with arguments like `as $PROFILE do $TASK' will just +%% return `as', which is then in charge of doing a more dynamic +%% dispatch. +%% If no arguments are given, the `help' task is returned. +%% If special arguments like `-h' or `-v' are translated to `help' +%% and `version' tasks. +%% The unparsed parts of arguments are returned in: +%% `{Task, Rest}'. +-spec parse_args([string()]) -> {atom(), [string()]}. parse_args([]) -> parse_args(["help"]); parse_args([H | Rest]) when H =:= "-h" @@ -204,6 +261,8 @@ parse_args([H | Rest]) when H =:= "-v" parse_args([Task | RawRest]) -> {list_to_atom(Task), RawRest}. +%% @private actually not too sure what this does anymore. +-spec set_options(rebar_state:t(),{[any()],[any()]}) -> {rebar_state:t(),[any()]}. set_options(State, {Options, NonOptArgs}) -> GlobalDefines = proplists:get_all_values(defines, Options), @@ -216,9 +275,8 @@ set_options(State, {Options, NonOptArgs}) -> {rebar_state:set(State2, task, Task), NonOptArgs}. -%% -%% get log level based on getopt option -%% +%% @doc get log level based on getopt options and ENV +-spec log_level() -> integer(). log_level() -> case os:getenv("QUIET") of Q when Q == false; Q == "" -> @@ -233,18 +291,16 @@ log_level() -> rebar_log:error_level() end. -%% -%% show version information and halt -%% +%% @doc show version information +-spec version() -> ok. version() -> {ok, Vsn} = application:get_key(rebar, vsn), - ?CONSOLE("rebar ~s on Erlang/OTP ~s Erts ~s", + ?CONSOLE("rebar ~ts on Erlang/OTP ~ts Erts ~ts", [Vsn, erlang:system_info(otp_release), erlang:system_info(version)]). +%% @private set global flag based on getopt option boolean value %% TODO: Actually make it 'global' -%% -%% set global flag based on getopt option boolean value -%% +-spec set_global_flag(rebar_state:t(), list(), term()) -> rebar_state:t(). set_global_flag(State, Options, Flag) -> Value = case proplists:get_bool(Flag, Options) of true -> @@ -254,9 +310,9 @@ set_global_flag(State, Options, Flag) -> end, rebar_state:set(State, Flag, Value). -%% -%% options accepted via getopt -%% + +%% @doc options accepted via getopt +-spec global_option_spec_list() -> [{atom(), char(), string(), atom(), string()}, ...]. global_option_spec_list() -> [ %% {Name, ShortOpt, LongOpt, ArgSpec, HelpMsg} @@ -265,38 +321,46 @@ global_option_spec_list() -> {task, undefined, undefined, string, "Task to run."} ]. -handle_error(rebar_abort) -> +%% @private translate unhandled errors and internal return codes into proper +%% erroneous program exits. +-spec handle_error(term(), term()) -> no_return(). +handle_error(rebar_abort, _) -> erlang:halt(1); -handle_error({error, rebar_abort}) -> +handle_error({error, rebar_abort}, _) -> erlang:halt(1); -handle_error({error, {Module, Reason}}) -> +handle_error({error, {Module, Reason}}, Stacktrace) -> case code:which(Module) of non_existing -> - ?CRASHDUMP("~p: ~p~n~p~n~n", [Module, Reason, erlang:get_stacktrace()]), + ?CRASHDUMP("~p: ~p~n~p~n~n", [Module, Reason, Stacktrace]), ?ERROR("Uncaught error in rebar_core. Run with DEBUG=1 to stacktrace or consult rebar3.crashdump", []), ?DEBUG("Uncaught error: ~p ~p", [Module, Reason]), ?INFO("When submitting a bug report, please include the output of `rebar3 report \"your command\"`", []); _ -> - ?ERROR("~s", [Module:format_error(Reason)]) + ?ERROR("~ts", [Module:format_error(Reason)]) end, erlang:halt(1); -handle_error({error, Error}) when is_list(Error) -> - ?ERROR("~s", [Error]), +handle_error({error, Error}, _) when is_list(Error) -> + ?ERROR("~ts", [Error]), erlang:halt(1); -handle_error(Error) -> +handle_error(Error, StackTrace) -> %% Nothing should percolate up from rebar_core; %% Dump this error to console - ?CRASHDUMP("Error: ~p~n~p~n~n", [Error, erlang:get_stacktrace()]), + ?CRASHDUMP("Error: ~p~n~p~n~n", [Error, StackTrace]), ?ERROR("Uncaught error in rebar_core. Run with DEBUG=1 to see stacktrace or consult rebar3.crashdump", []), ?DEBUG("Uncaught error: ~p", [Error]), - case erlang:get_stacktrace() of + case StackTrace of [] -> ok; Trace -> - ?DEBUG("Stack trace to the error location: ~p", [Trace]) + ?DEBUG("Stack trace to the error location:~n~p", [Trace]) end, ?INFO("When submitting a bug report, please include the output of `rebar3 report \"your command\"`", []), erlang:halt(1). +%% @private Boot Erlang dependencies; problem is that escripts don't auto-boot +%% stuff the way releases do and we have to do it by hand. +%% This also lets us detect and show nicer errors when a critical lib is +%% not supported +-spec start_and_load_apps(command_line|api) -> term(). start_and_load_apps(Caller) -> _ = application:load(rebar), %% Make sure crypto is running @@ -304,9 +368,12 @@ start_and_load_apps(Caller) -> ensure_running(asn1, Caller), ensure_running(public_key, Caller), ensure_running(ssl, Caller), - inets:start(), + ensure_running(inets, Caller), inets:start(httpc, [{profile, rebar}]). +%% @doc Make sure a required app is running, or display an error message +%% and abort if there's a problem. +-spec ensure_running(atom(), command_line|api) -> ok | no_return(). ensure_running(App, Caller) -> case application:start(App) of ok -> ok; @@ -323,40 +390,55 @@ ensure_running(App, Caller) -> throw(rebar_abort) end. +-spec state_from_global_config([term()], file:filename()) -> rebar_state:t(). state_from_global_config(Config, GlobalConfigFile) -> - rebar_utils:set_httpc_options(), GlobalConfigTerms = rebar_config:consult_file(GlobalConfigFile), GlobalConfig = rebar_state:new(GlobalConfigTerms), %% We don't want to worry about global plugin install state effecting later %% usage. So we throw away the global profile state used for plugin install. - GlobalConfigThrowAway = rebar_state:current_profiles(GlobalConfig, [global]), - GlobalState = case rebar_state:get(GlobalConfigThrowAway, plugins, []) of + GlobalConfigThrowAway0 = rebar_state:current_profiles(GlobalConfig, [global]), + + Resources = application:get_env(rebar, resources, []), + GlobalConfigThrowAway = rebar_state:create_resources(Resources, GlobalConfigThrowAway0), + + Compilers = application:get_env(rebar, compilers, []), + GlobalConfigThrowAway1 = rebar_state:compilers(GlobalConfigThrowAway, Compilers), + + GlobalState = case rebar_state:get(GlobalConfigThrowAway1, plugins, []) of [] -> - GlobalConfigThrowAway; + GlobalConfigThrowAway1; GlobalPluginsToInstall -> rebar_plugins:handle_plugins(global, GlobalPluginsToInstall, - GlobalConfigThrowAway) + GlobalConfigThrowAway1) end, GlobalPlugins = rebar_state:providers(GlobalState), GlobalConfig2 = rebar_state:set(GlobalConfig, plugins, []), - GlobalConfig3 = rebar_state:set(GlobalConfig2, {plugins, global}, rebar_state:get(GlobalConfigThrowAway, plugins, [])), + GlobalConfig3 = rebar_state:set(GlobalConfig2, {plugins, global}, + rebar_state:get(GlobalConfigThrowAway1, plugins, [])), rebar_state:providers(rebar_state:new(GlobalConfig3, Config), GlobalPlugins). +-spec test_state(rebar_state:t()) -> [{'extra_src_dirs',[string()]} | {'erl_opts',[any()]}]. test_state(State) -> - ErlOpts = rebar_state:get(State, erl_opts, []), + %% Fetch the test profile's erl_opts only + Opts = rebar_state:opts(State), + Profiles = rebar_opts:get(Opts, profiles, []), + ProfileOpts = proplists:get_value(test, Profiles, []), + ErlOpts = proplists:get_value(erl_opts, ProfileOpts, []), TestOpts = safe_define_test_macro(ErlOpts), [{extra_src_dirs, ["test"]}, {erl_opts, TestOpts}]. +-spec safe_define_test_macro([any()]) -> [any()] | [{'d',atom()} | any()]. safe_define_test_macro(Opts) -> %% defining a compile macro twice results in an exception so %% make sure 'TEST' is only defined once case test_defined(Opts) of - true -> []; - false -> [{d, 'TEST'}] + true -> Opts; + false -> [{d, 'TEST'}|Opts] end. +-spec test_defined([{d, atom()} | {d, atom(), term()} | term()]) -> boolean(). test_defined([{d, 'TEST'}|_]) -> true; test_defined([{d, 'TEST', true}|_]) -> true; test_defined([_|Rest]) -> test_defined(Rest); diff --git a/src/rebar_agent.erl b/src/rebar_agent.erl index 95818d8..b4734f1 100644 --- a/src/rebar_agent.erl +++ b/src/rebar_agent.erl @@ -1,5 +1,8 @@ +%%% @doc Runs a process that holds a rebar3 state and can be used +%%% to statefully maintain loaded project state into a running VM. -module(rebar_agent). --export([start_link/1, do/1, do/2]). +-export([start_link/1, do/1, do/2, async_do/1, async_do/2]). +-export(['$handle_undefined_function'/2]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, code_change/3, terminate/2]). @@ -10,47 +13,125 @@ cwd, show_warning=true}). +%% @doc boots an agent server; requires a full rebar3 state already. +%% By default (within rebar3), this isn't called; `rebar_prv_shell' +%% enters and transforms into this module +-spec start_link(rebar_state:t()) -> {ok, pid()}. start_link(State) -> gen_server:start_link({local, ?MODULE}, ?MODULE, State, []). +%% @doc runs a given command in the agent's context. +-spec do(atom()) -> ok | {error, term()}. do(Command) when is_atom(Command) -> - gen_server:call(?MODULE, {cmd, Command}, infinity). + gen_server:call(?MODULE, {cmd, Command}, infinity); +do(Args) when is_list(Args) -> + gen_server:call(?MODULE, {cmd, default, do, Args}, infinity). +%% @doc runs a given command in the agent's context, under a given +%% namespace. +-spec do(atom(), atom()) -> ok | {error, term()}. do(Namespace, Command) when is_atom(Namespace), is_atom(Command) -> - gen_server:call(?MODULE, {cmd, Namespace, Command}, infinity). + gen_server:call(?MODULE, {cmd, Namespace, Command}, infinity); +do(Namespace, Args) when is_atom(Namespace), is_list(Args) -> + gen_server:call(?MODULE, {cmd, Namespace, do, Args}, infinity). +-spec async_do(atom()) -> ok | {error, term()}. +async_do(Command) when is_atom(Command) -> + gen_server:cast(?MODULE, {cmd, Command}); +async_do(Args) when is_list(Args) -> + gen_server:cast(?MODULE, {cmd, default, do, Args}). + +-spec async_do(atom(), atom()) -> ok. +async_do(Namespace, Command) when is_atom(Namespace), is_atom(Command) -> + gen_server:cast(?MODULE, {cmd, Namespace, Command}); +async_do(Namespace, Args) when is_atom(Namespace), is_list(Args) -> + gen_server:cast(?MODULE, {cmd, Namespace, do, Args}). + +'$handle_undefined_function'(Cmd, [Namespace, Args]) -> + gen_server:call(?MODULE, {cmd, Namespace, Cmd, Args}, infinity); +'$handle_undefined_function'(Cmd, [Args]) -> + gen_server:call(?MODULE, {cmd, default, Cmd, Args}, infinity); +'$handle_undefined_function'(Cmd, []) -> + gen_server:call(?MODULE, {cmd, default, Cmd}, infinity). + +%%%%%%%%%%%%%%%%% +%%% CALLBACKS %%% +%%%%%%%%%%%%%%%%% + +%% @private init(State) -> Cwd = rebar_dir:get_cwd(), {ok, #state{state=State, cwd=Cwd}}. +%% @private handle_call({cmd, Command}, _From, State=#state{state=RState, cwd=Cwd}) -> MidState = maybe_show_warning(State), - {Res, NewRState} = run(default, Command, RState, Cwd), + put(cmd_type, sync), + {Res, NewRState} = run(default, Command, "", RState, Cwd), + put(cmd_type, undefined), {reply, Res, MidState#state{state=NewRState}, hibernate}; handle_call({cmd, Namespace, Command}, _From, State = #state{state=RState, cwd=Cwd}) -> MidState = maybe_show_warning(State), - {Res, NewRState} = run(Namespace, Command, RState, Cwd), + put(cmd_type, sync), + {Res, NewRState} = run(Namespace, Command, "", RState, Cwd), + put(cmd_type, undefined), + {reply, Res, MidState#state{state=NewRState}, hibernate}; +handle_call({cmd, Namespace, Command, Args}, _From, State = #state{state=RState, cwd=Cwd}) -> + MidState = maybe_show_warning(State), + put(cmd_type, sync), + {Res, NewRState} = run(Namespace, Command, Args, RState, Cwd), + put(cmd_type, undefined), {reply, Res, MidState#state{state=NewRState}, hibernate}; handle_call(_Call, _From, State) -> {noreply, State}. +%% @private +handle_cast({cmd, Command}, State=#state{state=RState, cwd=Cwd}) -> + MidState = maybe_show_warning(State), + put(cmd_type, async), + {_, NewRState} = run(default, Command, "", RState, Cwd), + put(cmd_type, undefined), + {noreply, MidState#state{state=NewRState}, hibernate}; +handle_cast({cmd, Namespace, Command}, State = #state{state=RState, cwd=Cwd}) -> + MidState = maybe_show_warning(State), + put(cmd_type, async), + {_, NewRState} = run(Namespace, Command, "", RState, Cwd), + put(cmd_type, undefined), + {noreply, MidState#state{state=NewRState}, hibernate}; +handle_cast({cmd, Namespace, Command, Args}, State = #state{state=RState, cwd=Cwd}) -> + MidState = maybe_show_warning(State), + put(cmd_type, async), + {_, NewRState} = run(Namespace, Command, Args, RState, Cwd), + put(cmd_type, undefined), + {noreply, MidState#state{state=NewRState}, hibernate}; handle_cast(_Cast, State) -> {noreply, State}. +%% @private handle_info(_Info, State) -> {noreply, State}. +%% @private code_change(_OldVsn, State, _Extra) -> {ok, State}. +%% @private terminate(_Reason, _State) -> ok. -run(Namespace, Command, RState, Cwd) -> +%%%%%%%%%%%%%%% +%%% PRIVATE %%% +%%%%%%%%%%%%%%% + +%% @private runs the actual command and maintains the state changes +-spec run(atom(), atom(), string(), rebar_state:t(), file:filename()) -> + {ok, rebar_state:t()} | {{error, term()}, rebar_state:t()}. +run(Namespace, Command, StrArgs, RState, Cwd) -> try case rebar_dir:get_cwd() of Cwd -> - Args = [atom_to_list(Namespace), atom_to_list(Command)], + PArgs = getopt:tokenize(StrArgs), + Args = [atom_to_list(Namespace), atom_to_list(Command)] ++ PArgs, CmdState0 = refresh_state(RState, Cwd), CmdState1 = rebar_state:set(CmdState0, task, atom_to_list(Command)), CmdState = rebar_state:set(CmdState1, caller, api), @@ -69,57 +150,168 @@ run(Namespace, Command, RState, Cwd) -> {{error, cwd_changed}, RState} end catch - Type:Reason -> - ?DEBUG("Agent Stacktrace: ~p", [erlang:get_stacktrace()]), + ?WITH_STACKTRACE(Type, Reason, Stacktrace) + ?DEBUG("Agent Stacktrace: ~p", [Stacktrace]), {{error, {Type, Reason}}, RState} end. +%% @private function to display a warning for the feature only once +-spec maybe_show_warning(#state{}) -> #state{}. maybe_show_warning(S=#state{show_warning=true}) -> ?WARN("This feature is experimental and may be modified or removed at any time.", []), S#state{show_warning=false}; maybe_show_warning(State) -> State. +%% @private based on a rebar3 state term, reload paths in a way +%% that makes sense. +-spec refresh_paths(rebar_state:t()) -> ok. refresh_paths(RState) -> - ToRefresh = (rebar_state:code_paths(RState, all_deps) - ++ [filename:join([rebar_app_info:out_dir(App), "test"]) - || App <- rebar_state:project_apps(RState)] - %% make sure to never reload self; halt()s the VM - ) -- [filename:dirname(code:which(?MODULE))], + RefreshPaths = application:get_env(rebar, refresh_paths, [all_deps, test]), + ToRefresh = parse_refresh_paths(RefreshPaths, RState, []), %% Modules from apps we can't reload without breaking functionality - Blacklist = [ec_cmd_log, providers, cf, cth_readable], + Blacklist = lists:usort( + application:get_env(rebar, refresh_paths_blacklist, []) + ++ [rebar, erlware_commons, providers, cf, cth_readable]), %% Similar to rebar_utils:update_code/1, but also forces a reload %% of used modules. Also forces to reload all of ebin/ instead %% of just the modules in the .app file, because 'extra_src_dirs' %% allows to load and compile files that are not to be kept %% in the app file. - lists:foreach(fun(Path) -> - Name = filename:basename(Path, "/ebin"), - Files = filelib:wildcard(filename:join([Path, "*.beam"])), - Modules = [list_to_atom(filename:basename(F, ".beam")) - || F <- Files], - App = list_to_atom(Name), + [refresh_path(Path, Blacklist) || Path <- ToRefresh], + ok. + +refresh_path(Path, Blacklist) -> + Name = filename:basename(Path, "/ebin"), + App = list_to_atom(Name), + case App of + test -> % skip + code:add_patha(Path), + ok; + _ -> application:load(App), case application:get_key(App, modules) of undefined -> - code:add_patha(Path), - ok; - {ok, Mods} -> - case {length(Mods), length(Mods -- Blacklist)} of - {X,X} -> - ?DEBUG("reloading ~p from ~s", [Modules, Path]), - code:replace_path(App, Path), - [begin code:purge(M), code:delete(M), code:load_file(M) end - || M <- Modules]; - {_,_} -> + code:add_patha(Path); + {ok, _Mods} -> + case lists:member(App, Blacklist) of + false -> + refresh_path_do(Path, App); + true -> ?DEBUG("skipping app ~p, stable copy required", [App]) end end - end, ToRefresh). + end. +refresh_path_do(Path, App) -> + Files = filelib:wildcard(filename:join([Path, "*.beam"])), + Modules = [list_to_atom(filename:basename(F, ".beam")) + || F <- Files], + ?DEBUG("reloading ~p from ~ts", [Modules, Path]), + code:replace_path(App, Path), + reload_modules(Modules). + +%% @private parse refresh_paths option +%% no_deps means only project_apps's ebin path +%% no_test means no test path +%% OtherPath. +parse_refresh_paths([all_deps | RefreshPaths], RState, Acc) -> + Paths = rebar_state:code_paths(RState, all_deps), + parse_refresh_paths(RefreshPaths, RState, Paths ++ Acc); +parse_refresh_paths([project_apps | RefreshPaths], RState, Acc) -> + Paths = [filename:join([rebar_app_info:out_dir(App), "ebin"]) + || App <- rebar_state:project_apps(RState)], + parse_refresh_paths(RefreshPaths, RState, Paths ++ Acc); +parse_refresh_paths([test | RefreshPaths], RState, Acc) -> + Paths = [filename:join([rebar_app_info:out_dir(App), "test"]) + || App <- rebar_state:project_apps(RState)], + parse_refresh_paths(RefreshPaths, RState, Paths ++ Acc); +parse_refresh_paths([RefreshPath0 | RefreshPaths], RState, Acc) when is_list(RefreshPath0) -> + case filelib:is_dir(RefreshPath0) of + true -> + RefreshPath0 = + case filename:basename(RefreshPath0) of + "ebin" -> RefreshPath0; + _ -> filename:join([RefreshPath0, "ebin"]) + end, + parse_refresh_paths(RefreshPaths, RState, [RefreshPath0 | Acc]); + false -> + parse_refresh_paths(RefreshPaths, RState, Acc) + end; +parse_refresh_paths([_ | RefreshPaths], RState, Acc) -> + parse_refresh_paths(RefreshPaths, RState, Acc); +parse_refresh_paths([], _RState, Acc) -> + lists:usort(Acc). +%% @private from a disk config, reload and reapply with the current +%% profiles; used to find changes in the config from a prior run. +-spec refresh_state(rebar_state:t(), file:filename()) -> rebar_state:t(). refresh_state(RState, _Dir) -> lists:foldl( fun(F, State) -> F(State) end, rebar3:init_config(), [fun(S) -> rebar_state:apply_profiles(S, rebar_state:current_profiles(RState)) end] ). + +%% @private takes a list of modules and reloads them +-spec reload_modules([module()]) -> term(). +reload_modules([]) -> noop; +reload_modules(Modules0) -> + Modules = [M || M <- Modules0, is_changed(M)], + reload_modules(Modules, erlang:function_exported(code, prepare_loading, 1)). + +%% @spec is_changed(atom()) -> boolean() +%% @doc true if the loaded module is a beam with a vsn attribute +%% and does not match the on-disk beam file, returns false otherwise. +is_changed(M) -> + try + module_vsn(M:module_info(attributes)) =/= module_vsn(code:get_object_code(M)) + catch _:_ -> + false + end. + +module_vsn({M, Beam, _Fn}) -> + % Because the vsn can set by -vsn(X) in module. + % So didn't use beam_lib:version/1 to get the vsn. + % So if set -vsn(X) in module, it will always reload the module. + {ok, {M, <<Vsn:128>>}} = beam_lib:md5(Beam), + Vsn; +module_vsn(Attrs) when is_list(Attrs) -> + {_, Vsn} = lists:keyfind(vsn, 1, Attrs), + Vsn. + +%% @private reloading modules, when there are modules to actually reload +reload_modules(Modules, true) -> + %% OTP 19 and later -- use atomic loading and ignore unloadable mods + case code:prepare_loading(Modules) of + {ok, Prepared} -> + [code:purge(M) || M <- Modules], + code:finish_loading(Prepared); + {error, ModRsns} -> + Blacklist = + lists:foldr(fun({ModError, Error}, Acc) -> + case Error of + % perhaps cover other cases of failure? + on_load_not_allowed -> + reload_modules([ModError], false), + [ModError|Acc]; + _ -> + ?DEBUG("Module ~p failed to atomic load because ~p", [ModError, Error]), + [ModError|Acc] + end + end, + [], ModRsns + ), + reload_modules(Modules -- Blacklist, true) + end; +reload_modules(Modules, false) -> + %% Older versions, use a more ad-hoc mechanism. + lists:foreach(fun(M) -> + code:delete(M), + code:purge(M), + case code:load_file(M) of + {module, M} -> ok; + {error, Error} -> + ?DEBUG("Module ~p failed to load because ~p", [M, Error]) + end + end, Modules + ). diff --git a/src/rebar_api.erl b/src/rebar_api.erl index 6ebc500..00eb054 100644 --- a/src/rebar_api.erl +++ b/src/rebar_api.erl @@ -1,4 +1,4 @@ -%%% Packages rebar.hrl features and macros into a more generic API +%%% @doc Packages rebar.hrl features and macros into a more generic API %%% that can be used by plugin builders. -module(rebar_api). -include("rebar.hrl"). @@ -9,6 +9,8 @@ expand_env_variable/3, get_arch/0, wordsize/0, + set_paths/2, + unset_paths/2, add_deps_to_path/1, restore_code_path/1, processing_base_dir/1, @@ -30,42 +32,77 @@ abort() -> ?FAIL. abort(Str, Args) -> ?ABORT(Str, Args). %% @doc Prints to the console, including a newline +-spec console(string(), list()) -> ok. console(Str, Args) -> ?CONSOLE(Str, Args). %% @doc logs with severity `debug' +-spec debug(string(), list()) -> ok. debug(Str, Args) -> ?DEBUG(Str, Args). + %% @doc logs with severity `info' +-spec info(string(), list()) -> ok. info(Str, Args) -> ?INFO(Str, Args). + %% @doc logs with severity `warn' +-spec warn(string(), list()) -> ok. warn(Str, Args) -> ?WARN(Str, Args). + %% @doc logs with severity `error' +-spec error(string(), list()) -> ok. error(Str, Args) -> ?ERROR(Str, Args). -%% -%% Given env. variable FOO we want to expand all references to -%% it in InStr. References can have two forms: $FOO and ${FOO} -%% The end of form $FOO is delimited with whitespace or eol -%% +%% @doc Given env. variable `FOO' we want to expand all references to +%% it in `InStr'. References can have two forms: `$FOO' and `${FOO}' +%% The end of form `$FOO' is delimited with whitespace or EOL +-spec expand_env_variable(string(), string(), term()) -> string(). expand_env_variable(InStr, VarName, RawVarValue) -> rebar_utils:expand_env_variable(InStr, VarName, RawVarValue). +%% @doc returns the sytem architecture, in strings like +%% `"19.0.4-x86_64-unknown-linux-gnu-64"'. +-spec get_arch() -> string(). get_arch() -> rebar_utils:get_arch(). +%% @doc returns the size of a word on the system, as a string +-spec wordsize() -> string(). wordsize() -> rebar_utils:wordsize(). +%% @doc Set code paths. Takes arguments of the form +%% `[plugins, deps]' or `[deps, plugins]' and ensures the +%% project's app and dependencies are set in the right order +%% for the next bit of execution +-spec set_paths(rebar_paths:targets(), rebar_state:t()) -> ok. +set_paths(List, State) -> + rebar_paths:set_paths(List, State). + +%% @doc Unsets code paths. Takes arguments of the form +%% `[plugins, deps]' or `[deps, plugins]' and ensures the +%% paths are no longer active. +-spec unset_paths(rebar_paths:targets(), rebar_state:t()) -> ok. +unset_paths(List, State) -> + rebar_paths:unset_paths(List, State). -%% Add deps to the code path +%% @doc Add deps to the code path +-spec add_deps_to_path(rebar_state:t()) -> ok. add_deps_to_path(State) -> code:add_pathsa(rebar_state:code_paths(State, all_deps)). -%% Revert to only having the beams necessary for running rebar3 and plugins in the path +%% @doc Revert to only having the beams necessary for running rebar3 and +%% plugins in the path +-spec restore_code_path(rebar_state:t()) -> true | {error, term()}. restore_code_path(State) -> rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)). +%% @doc checks if the current working directory is the base directory +%% for the project. +-spec processing_base_dir(rebar_state:t()) -> boolean(). processing_base_dir(State) -> rebar_dir:processing_base_dir(State). +%% @doc returns the SSL options adequate for the project based on +%% its configuration, including for validation of certs. +-spec ssl_opts(string()) -> [term()]. ssl_opts(Url) -> - rebar_pkg_resource:ssl_opts(Url). + rebar_utils:ssl_opts(Url). diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index 67acf54..21dea29 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -1,3 +1,5 @@ +%%% @doc utility functions to do the basic discovery of apps +%%% and layout for the project. -module(rebar_app_discover). -export([do/2, @@ -5,16 +7,23 @@ find_unbuilt_apps/1, find_apps/1, find_apps/2, + find_apps/4, find_app/2, find_app/3]). -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). +%% @doc from the base directory, find all the applications +%% at the top level and their dependencies based on the configuration +%% and profile information. +-spec do(rebar_state:t(), [file:filename()]) -> rebar_state:t(). do(State, LibDirs) -> BaseDir = rebar_state:dir(State), Dirs = [filename:join(BaseDir, LibDir) || LibDir <- LibDirs], - Apps = find_apps(Dirs, all), + RebarOpts = rebar_state:opts(State), + SrcDirs = rebar_dir:src_dirs(RebarOpts, ["src"]), + Apps = find_apps(Dirs, SrcDirs, all, State), ProjectDeps = rebar_state:deps_names(State), DepsDir = rebar_dir:deps_dir(State), CurrentProfiles = rebar_state:current_profiles(State), @@ -43,18 +52,22 @@ do(State, LibDirs) -> Name = rebar_app_info:name(AppInfo), case enable(State, AppInfo) of true -> - {AppInfo1, StateAcc1} = merge_deps(AppInfo, StateAcc), + {AppInfo1, StateAcc1} = merge_opts(AppInfo, StateAcc), OutDir = filename:join(DepsDir, Name), AppInfo2 = rebar_app_info:out_dir(AppInfo1, OutDir), ProjectDeps1 = lists:delete(Name, ProjectDeps), rebar_state:project_apps(StateAcc1 ,rebar_app_info:deps(AppInfo2, ProjectDeps1)); false -> - ?INFO("Ignoring ~s", [Name]), + ?INFO("Ignoring ~ts", [Name]), StateAcc end end, State1, SortedApps). +%% @doc checks whether there is an app at the top level (and returns its +%% name) or the 'root' atom in case we're in an umbrella project. +-spec define_root_app([rebar_app_info:t()], rebar_state:t()) -> + root | binary(). define_root_app(Apps, State) -> RootDir = rebar_dir:root_dir(State), case ec_lists:find(fun(X) -> @@ -67,36 +80,46 @@ define_root_app(Apps, State) -> root end. +%% @doc formatting errors from the module. +-spec format_error(term()) -> iodata(). format_error({module_list, File}) -> io_lib:format("Error reading module list from ~p~n", [File]); format_error({missing_module, Module}) -> io_lib:format("Module defined in app file missing: ~p~n", [Module]). -merge_deps(AppInfo, State) -> +%% @doc merges configuration of a project app and the top level state +%% some configuration like erl_opts must be merged into a subapp's opts +%% while plugins and hooks need to be kept defined to only either the +%% top level state or an individual application. +-spec merge_opts(rebar_app_info:t(), rebar_state:t()) -> + {rebar_app_info:t(), rebar_state:t()}. +merge_opts(AppInfo, State) -> %% These steps make sure that hooks and artifacts are run in the context of %% the application they are defined at. If an umbrella structure is used and - %% they are deifned at the top level they will instead run in the context of + %% they are defined at the top level they will instead run in the context of %% the State and at the top level, not as part of an application. - Default = reset_hooks(rebar_state:default(State)), - {C, State1} = project_app_config(AppInfo, State), - AppInfo0 = rebar_app_info:update_opts(AppInfo, Default, C), + CurrentProfiles = rebar_state:current_profiles(State), + {AppInfo1, State1} = maybe_reset_hooks_plugins(AppInfo, State), - CurrentProfiles = rebar_state:current_profiles(State1), - Name = rebar_app_info:name(AppInfo0), + Name = rebar_app_info:name(AppInfo1), %% We reset the opts here to default so no profiles are applied multiple times - AppInfo1 = rebar_app_info:apply_overrides(rebar_state:get(State1, overrides, []), AppInfo0), - AppInfo2 = rebar_app_info:apply_profiles(AppInfo1, CurrentProfiles), + AppInfo2 = rebar_app_info:apply_overrides(rebar_state:get(State1, overrides, []), AppInfo1), + AppInfo3 = rebar_app_info:apply_profiles(AppInfo2, CurrentProfiles), %% Will throw an exception if checks fail - rebar_app_info:verify_otp_vsn(AppInfo2), + rebar_app_info:verify_otp_vsn(AppInfo3), State2 = lists:foldl(fun(Profile, StateAcc) -> - handle_profile(Profile, Name, AppInfo2, StateAcc) + handle_profile(Profile, Name, AppInfo3, StateAcc) end, State1, lists:reverse(CurrentProfiles)), - {AppInfo2, State2}. + {AppInfo3, State2}. +%% @doc Applies a given profile for an app, ensuring the deps +%% match the context it will require. +-spec handle_profile(atom(), binary(), rebar_app_info:t(), rebar_state:t()) -> + rebar_state:t(). handle_profile(Profile, Name, AppInfo, State) -> TopParsedDeps = rebar_state:get(State, {parsed_deps, Profile}, {[], []}), TopLevelProfileDeps = rebar_state:get(State, {deps, Profile}, []), @@ -113,6 +136,12 @@ handle_profile(Profile, Name, AppInfo, State) -> State2 = rebar_state:set(State1, {deps, Profile}, ProfileDeps2), rebar_state:set(State2, {parsed_deps, Profile}, TopParsedDeps++ParsedDeps). +%% @doc parses all the known dependencies for a given profile +-spec parse_profile_deps(Profile, Name, Deps, Opts, rebar_state:t()) -> [rebar_app_info:t()] when + Profile :: atom(), + Name :: binary(), + Deps :: [term()], % TODO: refine types + Opts :: term(). % TODO: refine types parse_profile_deps(Profile, Name, Deps, Opts, State) -> DepsDir = rebar_prv_install_deps:profile_dep_dir(State, Profile), Locks = rebar_state:get(State, {locks, Profile}, []), @@ -123,77 +152,195 @@ parse_profile_deps(Profile, Name, Deps, Opts, State) -> ,Locks ,1). -project_app_config(AppInfo, State) -> - C = rebar_config:consult(rebar_app_info:dir(AppInfo)), +%% reset the State hooks if there is a top level application +-spec maybe_reset_hooks_plugins(AppInfo, State) -> {AppInfo, State} when + AppInfo :: rebar_app_info:t(), + State :: rebar_state:t(). +maybe_reset_hooks_plugins(AppInfo, State) -> Dir = rebar_app_info:dir(AppInfo), - Opts = maybe_reset_hooks(Dir, rebar_state:opts(State), State), - {C, rebar_state:opts(State, Opts)}. - -%% Here we check if the app is at the root of the project. -%% If it is, then drop the hooks from the config so they aren't run twice -maybe_reset_hooks(Dir, Opts, State) -> + CurrentProfiles = rebar_state:current_profiles(State), case ec_file:real_dir_path(rebar_dir:root_dir(State)) of Dir -> - reset_hooks(Opts); + Opts = reset_hooks(rebar_state:opts(State), CurrentProfiles), + State1 = rebar_state:opts(State, Opts), + + %% set plugins to empty since this is an app at the top level + %% and top level plugins are installed in run_aux + AppInfo1 = rebar_app_info:set(rebar_app_info:set(AppInfo, {plugins,default}, []), plugins, []), + + {AppInfo1, State1}; _ -> - Opts + %% if not in the top root directory then we need to merge in the + %% default state opts to this subapp's opts + Default = reset_hooks(rebar_state:default(State), CurrentProfiles), + AppInfo1 = rebar_app_info:update_opts(AppInfo, Default), + {AppInfo1, State} end. -reset_hooks(Opts) -> - lists:foldl(fun(Key, OptsAcc) -> - rebar_opts:set(OptsAcc, Key, []) - end, Opts, [post_hooks, pre_hooks, provider_hooks, artifacts]). --spec all_app_dirs(list(file:name())) -> list(file:name()). +%% @doc make the hooks empty for a given set of options +-spec reset_hooks(Opts, Profiles) -> + Opts when + Opts :: rebar_dict(), + Profiles :: [atom()]. +reset_hooks(Opts, CurrentProfiles) -> + AllHooks = [post_hooks, pre_hooks, provider_hooks, artifacts], + Opts1 = lists:foldl(fun(Key, OptsAcc) -> + rebar_opts:set(OptsAcc, Key, []) + end, Opts, AllHooks), + Profiles = rebar_opts:get(Opts1, profiles, []), + Profiles1 = lists:map(fun({P, ProfileOpts}) -> + case lists:member(P, CurrentProfiles) of + true -> + {P, [X || X={Key, _} <- ProfileOpts, + not lists:member(Key, AllHooks)]}; + false -> + {P, ProfileOpts} + end + end, Profiles), + rebar_opts:set(Opts1, profiles, Profiles1). + +%% @private find the directories for all apps, while detecting their source dirs +%% Returns the app dir with the respective src_dirs for them, in that order, +%% for every app found. +-spec all_app_dirs([file:name()]) -> [{file:name(), [file:name()]}]. all_app_dirs(LibDirs) -> lists:flatmap(fun(LibDir) -> - app_dirs(LibDir) + {_, SrcDirs} = find_config_src(LibDir, ["src"]), + app_dirs(LibDir, SrcDirs) end, LibDirs). -app_dirs(LibDir) -> - Path1 = filename:join([LibDir, - "src", - "*.app.src"]), - - Path2 = filename:join([LibDir, - "src", - "*.app.src.script"]), - - Path3 = filename:join([LibDir, - "ebin", - "*.app"]), +%% @private find the directories for all apps based on their source dirs +%% Returns the app dir with the respective src_dirs for them, in that order, +%% for every app found. +-spec all_app_dirs([file:name()], [file:name()]) -> [{file:name(), [file:name()]}]. +all_app_dirs(LibDirs, SrcDirs) -> + lists:flatmap(fun(LibDir) -> app_dirs(LibDir, SrcDirs) end, LibDirs). + +%% @private find the directories based on the library directories. +%% Returns the app dir with the respective src_dirs for them, in that order, +%% for every app found. +%% +%% The function returns the src directories since they might have been +%% detected in a top-level loop and we want to skip further detection +%% starting now. +-spec app_dirs([file:name()], [file:name()]) -> [{file:name(), [file:name()]}]. +app_dirs(LibDir, SrcDirs) -> + Paths = lists:append([ + [filename:join([LibDir, SrcDir, "*.app.src"]), + filename:join([LibDir, SrcDir, "*.app.src.script"])] + || SrcDir <- SrcDirs + ]), + EbinPath = filename:join([LibDir, "ebin", "*.app"]), lists:usort(lists:foldl(fun(Path, Acc) -> - Files = filelib:wildcard(ec_cnv:to_list(Path)), - [app_dir(File) || File <- Files] ++ Acc - end, [], [Path1, Path2, Path3])). + Files = filelib:wildcard(rebar_utils:to_list(Path)), + [{app_dir(File), SrcDirs} + || File <- Files] ++ Acc + end, [], [EbinPath | Paths])). +%% @doc find all apps that haven't been built in a list of directories +-spec find_unbuilt_apps([file:filename_all()]) -> [rebar_app_info:t()]. find_unbuilt_apps(LibDirs) -> find_apps(LibDirs, invalid). +%% @doc for each directory passed, find all apps that are valid. +%% Returns all the related app info records. -spec find_apps([file:filename_all()]) -> [rebar_app_info:t()]. find_apps(LibDirs) -> find_apps(LibDirs, valid). +%% @doc for each directory passed, find all apps according +%% to the validity rule passed in. Returns all the related +%% app info records. -spec find_apps([file:filename_all()], valid | invalid | all) -> [rebar_app_info:t()]. find_apps(LibDirs, Validate) -> - rebar_utils:filtermap(fun(AppDir) -> - find_app(AppDir, Validate) - end, all_app_dirs(LibDirs)). - + rebar_utils:filtermap( + fun({AppDir, AppSrcDirs}) -> + find_app(rebar_app_info:new(), AppDir, AppSrcDirs, Validate) + end, + all_app_dirs(LibDirs) + ). + +%% @doc for each directory passed, with the configured source directories, +%% find all apps according to the validity rule passed in. +%% Returns all the related app info records. +-spec find_apps([file:filename_all()], [file:filename_all()], valid | invalid | all, rebar_state:t()) -> [rebar_app_info:t()]. +find_apps(LibDirs, SrcDirs, Validate, State) -> + rebar_utils:filtermap( + fun({AppDir, AppSrcDirs}) -> + find_app(rebar_app_info:new(), AppDir, AppSrcDirs, Validate, State) + end, + all_app_dirs(LibDirs, SrcDirs) + ). + +%% @doc check that a given app in a directory is there, and whether it's +%% valid or not based on the second argument. Returns the related +%% app info record. -spec find_app(file:filename_all(), valid | invalid | all) -> {true, rebar_app_info:t()} | false. find_app(AppDir, Validate) -> - find_app(rebar_app_info:new(), AppDir, Validate). - + {Config, SrcDirs} = find_config_src(AppDir, ["src"]), + AppInfo = rebar_app_info:update_opts(rebar_app_info:dir(rebar_app_info:new(), AppDir), + dict:new(), Config), + find_app_(AppInfo, AppDir, SrcDirs, Validate). + +%% @doc check that a given app in a directory is there, and whether it's +%% valid or not based on the second argument. Returns the related +%% app info record. +-spec find_app(rebar_app_info:t(), file:filename_all(), valid | invalid | all) -> + {true, rebar_app_info:t()} | false. find_app(AppInfo, AppDir, Validate) -> + %% if no src dir is passed, figure it out from the app info, with a default + %% of src/ + AppOpts = rebar_app_info:opts(AppInfo), + SrcDirs = rebar_dir:src_dirs(AppOpts, ["src"]), + find_app_(AppInfo, AppDir, SrcDirs, Validate). + +%% @doc check that a given app in a directory is there, and whether it's +%% valid or not based on the second argument. The third argument includes +%% the directories where source files can be located. Returns the related +%% app info record. +-spec find_app(rebar_app_info:t(), file:filename_all(), + [file:filename_all()], valid | invalid | all, rebar_state:t()) -> + {true, rebar_app_info:t()} | false. +find_app(AppInfo, AppDir, SrcDirs, Validate, State) -> + AppInfo1 = case ec_file:real_dir_path(rebar_dir:root_dir(State)) of + AppDir -> + Opts = rebar_state:opts(State), + rebar_app_info:default(rebar_app_info:opts(AppInfo, Opts), Opts); + _ -> + Config = rebar_config:consult(AppDir), + rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), Config) + end, + find_app_(AppInfo1, AppDir, SrcDirs, Validate). + +find_app(AppInfo, AppDir, SrcDirs, Validate) -> + Config = rebar_config:consult(AppDir), + AppInfo1 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), Config), + find_app_(AppInfo1, AppDir, SrcDirs, Validate). + +-spec find_app_(rebar_app_info:t(), file:filename_all(), + [file:filename_all()], valid | invalid | all) -> + {true, rebar_app_info:t()} | false. +find_app_(AppInfo, AppDir, SrcDirs, Validate) -> AppFile = filelib:wildcard(filename:join([AppDir, "ebin", "*.app"])), - AppSrcFile = filelib:wildcard(filename:join([AppDir, "src", "*.app.src"])), - AppSrcScriptFile = filelib:wildcard(filename:join([AppDir, "src", "*.app.src.script"])), + AppSrcFile = lists:append( + [filelib:wildcard(filename:join([AppDir, SrcDir, "*.app.src"])) + || SrcDir <- SrcDirs] + ), + AppSrcScriptFile = lists:append( + [filelib:wildcard(filename:join([AppDir, SrcDir, "*.app.src.script"])) + || SrcDir <- SrcDirs] + ), try_handle_app_file(AppInfo, AppFile, AppDir, AppSrcFile, AppSrcScriptFile, Validate). +%% @doc find the directory that an appfile has +-spec app_dir(file:filename()) -> file:filename(). app_dir(AppFile) -> filename:join(rebar_utils:droplast(filename:split(filename:dirname(AppFile)))). +%% @doc populates an app info record based on an app directory and its +%% app file. -spec create_app_info(rebar_app_info:t(), file:name(), file:name()) -> rebar_app_info:t(). create_app_info(AppInfo, AppDir, AppFile) -> [{application, AppName, AppDetails}] = rebar_config:consult_app_file(AppFile), @@ -215,8 +362,15 @@ create_app_info(AppInfo, AppDir, AppFile) -> end, rebar_app_info:dir(rebar_app_info:valid(AppInfo2, Valid), AppDir). -%% Read in and parse the .app file if it is availabe. Do the same for +%% @doc Read in and parse the .app file if it is availabe. Do the same for %% the .app.src file if it exists. +-spec try_handle_app_file(AppInfo, AppFile, AppDir, AppSrcFile, AppSrcScriptFile, valid | invalid | all) -> + {true, AppInfo} | false when + AppInfo :: rebar_app_info:t(), + AppFile :: file:filename(), + AppDir :: file:filename(), + AppSrcFile :: file:filename(), + AppSrcScriptFile :: file:filename(). try_handle_app_file(AppInfo, [], AppDir, [], AppSrcScriptFile, Validate) -> try_handle_app_src_file(AppInfo, [], AppDir, AppSrcScriptFile, Validate); try_handle_app_file(AppInfo, [], AppDir, AppSrcFile, _, Validate) -> @@ -254,32 +408,64 @@ try_handle_app_file(AppInfo0, [File], AppDir, AppSrcFile, _, Validate) -> end catch throw:{error, {Module, Reason}} -> - ?DEBUG("Falling back to app.src file because .app failed: ~s", [Module:format_error(Reason)]), + ?DEBUG("Falling back to app.src file because .app failed: ~ts", [Module:format_error(Reason)]), try_handle_app_src_file(AppInfo0, File, AppDir, AppSrcFile, Validate) end; try_handle_app_file(_AppInfo, Other, _AppDir, _AppSrcFile, _, _Validate) -> throw({error, {multiple_app_files, Other}}). -%% Read in the .app.src file if we aren't looking for a valid (already built) app -try_handle_app_src_file(_AppInfo, _, _AppDir, [], _Validate) -> - false; +%% @doc Read in the .app.src file if we aren't looking for a valid (already +%% built) app. +-spec try_handle_app_src_file(AppInfo, AppFile, AppDir, AppSrcFile, valid | invalid | all) -> + {true, AppInfo} | false when + AppInfo :: rebar_app_info:t(), + AppFile :: file:filename(), + AppDir :: file:filename(), + AppSrcFile :: file:filename(). +try_handle_app_src_file(AppInfo, _, _AppDir, [], _Validate) -> + %% if .app and .app.src are not found check for a mix config file + %% it is assumed a plugin will build the application, including + %% a .app after this step + case filelib:is_file(filename:join(rebar_app_info:dir(AppInfo), "mix.exs")) of + true -> + {true, rebar_app_info:project_type(AppInfo, mix)}; + false -> + false + end; try_handle_app_src_file(_AppInfo, _, _AppDir, _AppSrcFile, valid) -> false; try_handle_app_src_file(AppInfo, _, AppDir, [File], Validate) when Validate =:= invalid - ; Validate =:= all -> - AppInfo1 = create_app_info(AppInfo, AppDir, File), + ; Validate =:= all -> + AppInfo1 = rebar_app_info:app_file(AppInfo, undefined), + AppInfo2 = create_app_info(AppInfo1, AppDir, File), case filename:extension(File) of ".script" -> - {true, rebar_app_info:app_file_src_script(AppInfo1, File)}; + {true, rebar_app_info:app_file_src_script(AppInfo2, File)}; _ -> - {true, rebar_app_info:app_file_src(AppInfo1, File)} + {true, rebar_app_info:app_file_src(AppInfo2, File)} end; try_handle_app_src_file(_AppInfo, _, _AppDir, Other, _Validate) -> throw({error, {multiple_app_files, Other}}). +%% @doc checks whether the given app is not blacklisted in the config. +-spec enable(rebar_state:t(), rebar_app_info:t()) -> boolean(). enable(State, AppInfo) -> not lists:member(to_atom(rebar_app_info:name(AppInfo)), rebar_state:get(State, excluded_apps, [])). +%% @private convert a binary to an atom. +-spec to_atom(binary()) -> atom(). to_atom(Bin) -> list_to_atom(binary_to_list(Bin)). + +%% @private when looking for unknown apps, it's possible they have a +%% rebar.config file specifying non-standard src_dirs. Check for a +%% possible config file and extract src_dirs from it. +find_config_src(AppDir, Default) -> + case rebar_config:consult(AppDir) of + [] -> + {[], Default}; + Terms -> + %% TODO: handle profiles I guess, but we don't have that info + {Terms, proplists:get_value(src_dirs, Terms, Default)} + end. diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index cf3b82e..d051a15 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -7,6 +7,8 @@ new/4, new/5, update_opts/3, + update_opts/2, + update_opts_deps/2, discover/1, name/1, name/2, @@ -22,7 +24,6 @@ parent/2, original_vsn/1, original_vsn/2, - ebin_dir/1, priv_dir/1, applications/1, applications/2, @@ -36,6 +37,8 @@ dir/2, out_dir/1, out_dir/2, + ebin_dir/1, + ebin_dir/2, default/1, default/2, opts/1, @@ -43,16 +46,18 @@ get/2, get/3, set/3, - resource_type/1, - resource_type/2, source/1, source/2, + project_type/1, + project_type/2, is_lock/1, is_lock/2, is_checkout/1, is_checkout/2, valid/1, valid/2, + is_available/1, + is_available/2, verify_otp_vsn/1, has_all_artifacts/1, @@ -66,13 +71,16 @@ -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). --export_type([t/0]). +-export_type([t/0, + project_type/0]). + +-type project_type() :: rebar3 | mix | undefined. --record(app_info_t, {name :: binary(), +-record(app_info_t, {name :: binary() | undefined, app_file_src :: file:filename_all() | undefined, app_file_src_script:: file:filename_all() | undefined, app_file :: file:filename_all() | undefined, - original_vsn :: binary() | string() | undefined, + original_vsn :: binary() | undefined, parent=root :: binary() | root, app_details=[] :: list(), applications=[] :: list(), @@ -83,11 +91,13 @@ dep_level=0 :: integer(), dir :: file:name(), out_dir :: file:name(), - resource_type :: pkg | src, + ebin_dir :: file:name(), source :: string() | tuple() | checkout | undefined, is_lock=false :: boolean(), is_checkout=false :: boolean(), - valid :: boolean()}). + valid :: boolean() | undefined, + project_type :: project_type(), + is_available=false :: boolean()}). %%============================================================================ %% types @@ -103,54 +113,64 @@ new() -> #app_info_t{}. +%% @doc Build a new app info value with only the app name set. -spec new(atom() | binary() | string()) -> {ok, t()}. new(AppName) -> - {ok, #app_info_t{name=ec_cnv:to_binary(AppName)}}. + {ok, #app_info_t{name=rebar_utils:to_binary(AppName)}}. +%% @doc Build a new app info value with only the name and version set. -spec new(atom() | binary() | string(), binary() | string()) -> {ok, t()}. new(AppName, Vsn) -> - {ok, #app_info_t{name=ec_cnv:to_binary(AppName), + {ok, #app_info_t{name=rebar_utils:to_binary(AppName), original_vsn=Vsn}}. %% @doc build a complete version of the app info with all fields set. -spec new(atom() | binary() | string(), binary() | string(), file:name()) -> {ok, t()}. new(AppName, Vsn, Dir) -> - {ok, #app_info_t{name=ec_cnv:to_binary(AppName), + {ok, #app_info_t{name=rebar_utils:to_binary(AppName), original_vsn=Vsn, - dir=ec_cnv:to_list(Dir), - out_dir=ec_cnv:to_list(Dir)}}. + dir=rebar_utils:to_list(Dir), + out_dir=rebar_utils:to_list(Dir), + ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin")}}. %% @doc build a complete version of the app info with all fields set. -spec new(atom() | binary() | string(), binary() | string(), file:name(), list()) -> {ok, t()}. new(AppName, Vsn, Dir, Deps) -> - {ok, #app_info_t{name=ec_cnv:to_binary(AppName), + {ok, #app_info_t{name=rebar_utils:to_binary(AppName), original_vsn=Vsn, - dir=ec_cnv:to_list(Dir), - out_dir=ec_cnv:to_list(Dir), + dir=rebar_utils:to_list(Dir), + out_dir=rebar_utils:to_list(Dir), + ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin"), deps=Deps}}. %% @doc build a complete version of the app info with all fields set. -spec new(atom() | binary(), atom() | binary() | string(), binary() | string(), file:name(), list()) -> {ok, t()}. new(Parent, AppName, Vsn, Dir, Deps) -> - {ok, #app_info_t{name=ec_cnv:to_binary(AppName), + {ok, #app_info_t{name=rebar_utils:to_binary(AppName), parent=Parent, original_vsn=Vsn, - dir=ec_cnv:to_list(Dir), - out_dir=ec_cnv:to_list(Dir), + dir=rebar_utils:to_list(Dir), + out_dir=rebar_utils:to_list(Dir), + ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin"), deps=Deps}}. +%% @doc update the opts based on the contents of a config +%% file for the app +-spec update_opts(t(), rebar_dict(), [any()]) -> t(). update_opts(AppInfo, Opts, Config) -> - LockDeps = case resource_type(AppInfo) of - pkg -> - Deps = deps(AppInfo), - [{{locks, default}, Deps}, {{deps, default}, Deps}]; + LockDeps = case source(AppInfo) of + Tuple when is_tuple(Tuple) andalso element(1, Tuple) =:= pkg -> + %% Deps are set separate for packages + %% instead of making it seem we have no deps + %% don't set anything here. + []; _ -> - deps_from_config(dir(AppInfo), Config) + deps_from_config(dir(AppInfo), proplists:get_value(deps, Config, [])) end, Plugins = proplists:get_value(plugins, Config, []), @@ -160,13 +180,32 @@ update_opts(AppInfo, Opts, Config) -> NewOpts = rebar_opts:merge_opts(LocalOpts, Opts), - AppInfo#app_info_t{opts=NewOpts - ,default=NewOpts}. + AppInfo#app_info_t{opts=NewOpts, + default=NewOpts}. -deps_from_config(Dir, Config) -> +%% @doc update current app info opts by merging in a new dict of opts +-spec update_opts(t(), rebar_dict()) -> t(). +update_opts(AppInfo=#app_info_t{opts=LocalOpts}, Opts) -> + NewOpts = rebar_opts:merge_opts(LocalOpts, Opts), + AppInfo#app_info_t{opts=NewOpts, + default=NewOpts}. + +%% @doc update the opts based on new deps, usually from an app's hex registry metadata +-spec update_opts_deps(t(), [any()]) -> t(). +update_opts_deps(AppInfo=#app_info_t{opts=Opts}, Deps) -> + LocalOpts = dict:from_list([{{locks, default}, Deps}, {{deps, default}, Deps}]), + NewOpts = rebar_opts:merge_opts(LocalOpts, Opts), + AppInfo#app_info_t{opts=NewOpts, + default=NewOpts, + deps=Deps}. + + +%% @private extract the deps for an app in `Dir' based on its config file data +-spec deps_from_config(file:filename(), [any()]) -> [{tuple(), any()}, ...]. +deps_from_config(Dir, ConfigDeps) -> case rebar_config:consult_lock_file(filename:join(Dir, ?LOCK_FILE)) of [] -> - [{{deps, default}, proplists:get_value(deps, Config, [])}]; + [{{deps, default}, ConfigDeps}]; D -> %% We want the top level deps only from the lock file. %% This ensures deterministic overrides for configs. @@ -184,30 +223,48 @@ discover(Dir) -> not_found end. +%% @doc get the name of the app. -spec name(t()) -> binary(). name(#app_info_t{name=Name}) -> Name. +%% @doc set the name of the app. -spec name(t(), atom() | binary() | string()) -> t(). name(AppInfo=#app_info_t{}, AppName) -> - AppInfo#app_info_t{name=ec_cnv:to_binary(AppName)}. + AppInfo#app_info_t{name=rebar_utils:to_binary(AppName)}. +%% @doc get the dictionary of options for the app. +-spec opts(t()) -> rebar_dict(). opts(#app_info_t{opts=Opts}) -> Opts. +%% @doc set the dictionary of options for the app. +-spec opts(t(), rebar_dict()) -> t(). opts(AppInfo, Opts) -> AppInfo#app_info_t{opts=Opts}. +%% @doc get the dictionary of options under the default profile. +%% Represents a root set prior to applying other profiles. +-spec default(t()) -> rebar_dict(). default(#app_info_t{default=Default}) -> Default. +%% @doc set the dictionary of options under the default profile. +%% Useful when re-applying profile. +-spec default(t(), rebar_dict()) -> t(). default(AppInfo, Default) -> AppInfo#app_info_t{default=Default}. +%% @doc look up a value in the dictionary of options; fails if +%% the key for it does not exist. +-spec get(t(), term()) -> term(). get(AppInfo, Key) -> {ok, Value} = dict:find(Key, AppInfo#app_info_t.opts), Value. +%% @doc look up a value in the dictionary of options; returns +%% a `Default' value otherwise. +-spec get(t(), term(), term()) -> term(). get(AppInfo, Key, Default) -> case dict:find(Key, AppInfo#app_info_t.opts) of {ok, Value} -> @@ -216,31 +273,35 @@ get(AppInfo, Key, Default) -> Default end. +%% @doc sets a given value in the dictionary of options for the app. -spec set(t(), any(), any()) -> t(). set(AppInfo=#app_info_t{opts=Opts}, Key, Value) -> AppInfo#app_info_t{opts = dict:store(Key, Value, Opts)}. +%% @doc finds the .app.src file for an app, if any. -spec app_file_src(t()) -> file:filename_all() | undefined. -app_file_src(#app_info_t{app_file_src=undefined, dir=Dir, name=Name}) -> - AppFileSrc = filename:join([ec_cnv:to_list(Dir), "src", ec_cnv:to_list(Name)++".app.src"]), - case filelib:is_file(AppFileSrc) of - true -> - AppFileSrc; - false -> - undefined +app_file_src(#app_info_t{app_file_src=undefined, dir=Dir, name=Name, opts=Opts}) -> + CandidatePaths = [filename:join([rebar_utils:to_list(Dir), Src, rebar_utils:to_list(Name)++".app.src"]) + || Src <- rebar_opts:get(Opts, src_dirs, ["src"])], + case lists:dropwhile(fun(Path) -> not filelib:is_file(Path) end, CandidatePaths) of + [] -> undefined; + [AppFileSrc|_] -> AppFileSrc end; app_file_src(#app_info_t{app_file_src=AppFileSrc}) -> - ec_cnv:to_list(AppFileSrc). + rebar_utils:to_list(AppFileSrc). +%% @doc sets the .app.src file for an app. An app without such a file +%% can explicitly be set with `undefined'. -spec app_file_src(t(), file:filename_all() | undefined) -> t(). app_file_src(AppInfo=#app_info_t{}, undefined) -> AppInfo#app_info_t{app_file_src=undefined}; app_file_src(AppInfo=#app_info_t{}, AppFileSrc) -> - AppInfo#app_info_t{app_file_src=ec_cnv:to_list(AppFileSrc)}. + AppInfo#app_info_t{app_file_src=rebar_utils:to_list(AppFileSrc)}. +%% @doc finds the .app.src.script file for an app, if any. -spec app_file_src_script(t()) -> file:filename_all() | undefined. app_file_src_script(#app_info_t{app_file_src_script=undefined, dir=Dir, name=Name}) -> - AppFileSrcScript = filename:join([ec_cnv:to_list(Dir), "src", ec_cnv:to_list(Name)++".app.src.script"]), + AppFileSrcScript = filename:join([rebar_utils:to_list(Dir), "src", rebar_utils:to_list(Name)++".app.src.script"]), case filelib:is_file(AppFileSrcScript) of true -> AppFileSrcScript; @@ -248,17 +309,20 @@ app_file_src_script(#app_info_t{app_file_src_script=undefined, dir=Dir, name=Nam undefined end; app_file_src_script(#app_info_t{app_file_src_script=AppFileSrcScript}) -> - ec_cnv:to_list(AppFileSrcScript). + rebar_utils:to_list(AppFileSrcScript). +%% @doc sets the .app.src.script file for an app. An app without such a file +%% can explicitly be set with `undefined'. -spec app_file_src_script(t(), file:filename_all()) -> t(). app_file_src_script(AppInfo=#app_info_t{}, undefined) -> AppInfo#app_info_t{app_file_src_script=undefined}; app_file_src_script(AppInfo=#app_info_t{}, AppFileSrcScript) -> - AppInfo#app_info_t{app_file_src_script=ec_cnv:to_list(AppFileSrcScript)}. + AppInfo#app_info_t{app_file_src_script=rebar_utils:to_list(AppFileSrcScript)}. +%% @doc finds the .app file for an app, if any. -spec app_file(t()) -> file:filename_all() | undefined. app_file(#app_info_t{app_file=undefined, out_dir=Dir, name=Name}) -> - AppFile = filename:join([ec_cnv:to_list(Dir), "ebin", ec_cnv:to_list(Name)++".app"]), + AppFile = filename:join([rebar_utils:to_list(Dir), "ebin", rebar_utils:to_list(Name)++".app"]), case filelib:is_file(AppFile) of true -> AppFile; @@ -268,136 +332,209 @@ app_file(#app_info_t{app_file=undefined, out_dir=Dir, name=Name}) -> app_file(#app_info_t{app_file=AppFile}) -> AppFile. --spec app_file(t(), file:filename_all()) -> t(). +%% @doc sets the .app file for an app. +-spec app_file(t(), file:filename_all() | undefined) -> t(). app_file(AppInfo=#app_info_t{}, AppFile) -> AppInfo#app_info_t{app_file=AppFile}. +%% @doc returns the information stored in the app's app file, +%% or if none, from the .app.src file. -spec app_details(t()) -> list(). app_details(AppInfo=#app_info_t{app_details=[]}) -> case app_file(AppInfo) of undefined -> - rebar_file_utils:try_consult(app_file_src(AppInfo)); + try rebar_config:consult_app_file(app_file_src(AppInfo)) of + [] -> []; + [{application, _Name, AppDetails}] -> AppDetails + catch + _:_ -> + [] + end; AppFile -> - try - rebar_file_utils:try_consult(AppFile) + try rebar_file_utils:try_consult(AppFile) of + [] -> []; + [{application, _Name, AppDetails}] -> AppDetails catch throw:{error, {Module, Reason}} -> - ?DEBUG("Warning, falling back to .app.src because of: ~s", + ?DEBUG("Warning, falling back to .app.src because of: ~ts", [Module:format_error(Reason)]), - rebar_file_utils:try_consult(app_file_src(AppInfo)) + case rebar_config:consult_app_file(app_file_src(AppInfo)) of + [] -> []; + [{application, _Name, AppDetails}] -> AppDetails + end end end; app_details(#app_info_t{app_details=AppDetails}) -> AppDetails. +%% @doc stores the information that would be returned from the +%% app file, when reading from `app_details/1'. -spec app_details(t(), list()) -> t(). app_details(AppInfo=#app_info_t{}, AppDetails) -> AppInfo#app_info_t{app_details=AppDetails}. +%% @doc returns the app's parent in the dep tree. +-spec parent(t()) -> root | binary(). parent(#app_info_t{parent=Parent}) -> Parent. +%% @doc sets the app's parent. -spec parent(t(), binary() | root) -> t(). parent(AppInfo=#app_info_t{}, Parent) -> AppInfo#app_info_t{parent=Parent}. --spec original_vsn(t()) -> string(). +%% @doc returns the original version of the app (unevaluated if +%% asking for a semver) +-spec original_vsn(t()) -> binary(). original_vsn(#app_info_t{original_vsn=Vsn}) -> Vsn. --spec original_vsn(t(), string()) -> t(). +%% @doc stores the original version of the app (unevaluated if +%% asking for a semver) +-spec original_vsn(t(), binary() | string()) -> t(). original_vsn(AppInfo=#app_info_t{}, Vsn) -> AppInfo#app_info_t{original_vsn=Vsn}. +%% @doc returns the list of applications the app depends on. -spec applications(t()) -> list(). applications(#app_info_t{applications=Applications}) -> Applications. +%% @doc sets the list of applications the app depends on. +%% Should be obtained from the app file. -spec applications(t(), list()) -> t(). applications(AppInfo=#app_info_t{}, Applications) -> AppInfo#app_info_t{applications=Applications}. +%% @doc returns the list of active profiles -spec profiles(t()) -> list(). profiles(#app_info_t{profiles=Profiles}) -> Profiles. +%% @doc sets the list of active profiles -spec profiles(t(), list()) -> t(). profiles(AppInfo=#app_info_t{}, Profiles) -> AppInfo#app_info_t{profiles=Profiles}. +%% @doc returns the list of dependencies -spec deps(t()) -> list(). deps(#app_info_t{deps=Deps}) -> Deps. +%% @doc sets the list of dependencies. -spec deps(t(), list()) -> t(). deps(AppInfo=#app_info_t{}, Deps) -> AppInfo#app_info_t{deps=Deps}. -dep_level(AppInfo=#app_info_t{}, Level) -> - AppInfo#app_info_t{dep_level=Level}. - +%% @doc returns the level the app has in the lock files or in the +%% dep tree. +-spec dep_level(t()) -> non_neg_integer(). dep_level(#app_info_t{dep_level=Level}) -> Level. +%% @doc sets the level the app has in the lock files or in the +%% dep tree. +-spec dep_level(t(), non_neg_integer()) -> t(). +dep_level(AppInfo=#app_info_t{}, Level) -> + AppInfo#app_info_t{dep_level=Level}. + +%% @doc returns the directory that contains the app. -spec dir(t()) -> file:name(). dir(#app_info_t{dir=Dir}) -> Dir. +%% @doc sets the directory that contains the app. -spec dir(t(), file:name()) -> t(). dir(AppInfo=#app_info_t{out_dir=undefined}, Dir) -> - AppInfo#app_info_t{dir=ec_cnv:to_list(Dir), - out_dir=ec_cnv:to_list(Dir)}; + AppInfo#app_info_t{dir=rebar_utils:to_list(Dir), + out_dir=rebar_utils:to_list(Dir)}; dir(AppInfo=#app_info_t{}, Dir) -> - AppInfo#app_info_t{dir=ec_cnv:to_list(Dir)}. + AppInfo#app_info_t{dir=rebar_utils:to_list(Dir)}. +%% @doc returns the directory where build artifacts for the app +%% should go -spec out_dir(t()) -> file:name(). out_dir(#app_info_t{out_dir=OutDir}) -> OutDir. +%% @doc sets the directory where build artifacts for the app +%% should go -spec out_dir(t(), file:name()) -> t(). out_dir(AppInfo=#app_info_t{}, OutDir) -> - AppInfo#app_info_t{out_dir=ec_cnv:to_list(OutDir)}. + AppInfo#app_info_t{out_dir=rebar_utils:to_list(OutDir), + ebin_dir=filename:join(rebar_utils:to_list(OutDir), "ebin")}. +%% @doc gets the directory where ebin files for the app should go -spec ebin_dir(t()) -> file:name(). -ebin_dir(#app_info_t{out_dir=OutDir}) -> - ec_cnv:to_list(filename:join(OutDir, "ebin")). - +ebin_dir(#app_info_t{ebin_dir=undefined, + out_dir=OutDir}) -> + filename:join(rebar_utils:to_list(OutDir), "ebin"); +ebin_dir(#app_info_t{ebin_dir=EbinDir}) -> + EbinDir. + +%% @doc sets the directory where beam files should go +-spec ebin_dir(t(), file:name()) -> t(). +ebin_dir(AppInfo, EbinDir) -> + AppInfo#app_info_t{ebin_dir=EbinDir}. + +%% @doc gets the directory where private files for the app should go -spec priv_dir(t()) -> file:name(). priv_dir(#app_info_t{out_dir=OutDir}) -> - ec_cnv:to_list(filename:join(OutDir, "priv")). + rebar_utils:to_list(filename:join(OutDir, "priv")). --spec resource_type(t(), pkg | src) -> t(). -resource_type(AppInfo=#app_info_t{}, Type) -> - AppInfo#app_info_t{resource_type=Type}. - --spec resource_type(t()) -> pkg | src. -resource_type(#app_info_t{resource_type=ResourceType}) -> - ResourceType. +%% @doc finds the source specification for the app +-spec source(t()) -> string() | tuple(). +source(#app_info_t{source=Source}) -> + Source. +%% @doc sets the source specification for the app -spec source(t(), string() | tuple() | checkout) -> t(). source(AppInfo=#app_info_t{}, Source) -> AppInfo#app_info_t{source=Source}. --spec source(t()) -> string() | tuple(). -source(#app_info_t{source=Source}) -> - Source. +%% @doc returns the lock status for the app +-spec is_lock(t()) -> boolean(). +is_lock(#app_info_t{is_lock=IsLock}) -> + IsLock. +%% @doc sets the lock status for the app -spec is_lock(t(), boolean()) -> t(). is_lock(AppInfo=#app_info_t{}, IsLock) -> AppInfo#app_info_t{is_lock=IsLock}. --spec is_lock(t()) -> boolean(). -is_lock(#app_info_t{is_lock=IsLock}) -> - IsLock. +%% @doc returns whether the app is a checkout app or not +-spec is_checkout(t()) -> boolean(). +is_checkout(#app_info_t{is_checkout=IsCheckout}) -> + IsCheckout. +%% @doc sets whether the app is a checkout app or not -spec is_checkout(t(), boolean()) -> t(). is_checkout(AppInfo=#app_info_t{}, IsCheckout) -> AppInfo#app_info_t{is_checkout=IsCheckout}. --spec is_checkout(t()) -> boolean(). -is_checkout(#app_info_t{is_checkout=IsCheckout}) -> - IsCheckout. +%% @doc returns whether the app source exists in the deps dir +-spec is_available(t()) -> boolean(). +is_available(#app_info_t{is_available=IsAvailable}) -> + IsAvailable. + +%% @doc sets whether the app's source is available +%% only set if the app's source is found in the expected dep directory +-spec is_available(t(), boolean()) -> t(). +is_available(AppInfo=#app_info_t{}, IsAvailable) -> + AppInfo#app_info_t{is_available=IsAvailable}. + +%% @doc +-spec project_type(t()) -> atom(). +project_type(#app_info_t{project_type=ProjectType}) -> + ProjectType. + +%% @doc +-spec project_type(t(), atom()) -> t(). +project_type(AppInfo=#app_info_t{}, ProjectType) -> + AppInfo#app_info_t{project_type=ProjectType}. + +%% @doc returns whether the app is valid (built) or not -spec valid(t()) -> boolean(). valid(AppInfo=#app_info_t{valid=undefined}) -> case rebar_app_utils:validate_application_info(AppInfo) =:= true @@ -410,14 +547,22 @@ valid(AppInfo=#app_info_t{valid=undefined}) -> valid(#app_info_t{valid=Valid}) -> Valid. +%% @doc sets whether the app is valid (built) or not. If left unset, +%% rebar3 will do the detection of the status itself. -spec valid(t(), boolean()) -> t(). valid(AppInfo=#app_info_t{}, Valid) -> AppInfo#app_info_t{valid=Valid}. +%% @doc checks whether the app can be built with the current +%% Erlang/OTP version. If the check fails, the function raises +%% an exception and displays an error. +-spec verify_otp_vsn(t()) -> ok | no_return(). verify_otp_vsn(AppInfo) -> rebar_utils:check_min_otp_version(rebar_app_info:get(AppInfo, minimum_otp_vsn, undefined)), rebar_utils:check_blacklisted_otp_versions(rebar_app_info:get(AppInfo, blacklisted_otp_vsns, [])). +%% @doc checks whether all the build artifacts for an app to be considered +%% valid are present. -spec has_all_artifacts(#app_info_t{}) -> true | {false, file:filename()}. has_all_artifacts(AppInfo) -> Artifacts = rebar_app_info:get(AppInfo, artifacts, []), @@ -427,13 +572,17 @@ has_all_artifacts(AppInfo) -> ,{out_dir, OutDir}], all(OutDir, Context, Artifacts). +%% @private checks that all files/artifacts in the directory are found. +%% Template evaluation must happen and a bbmustache context needs to +%% be provided. +-spec all(file:filename(), term(), [string()]) -> true | {false, string()}. all(_, _, []) -> true; all(Dir, Context, [File|Artifacts]) -> FilePath = filename:join(Dir, rebar_templater:render(File, Context)), case filelib:is_regular(FilePath) of false -> - ?DEBUG("Missing artifact ~s", [FilePath]), + ?DEBUG("Missing artifact ~ts", [FilePath]), {false, File}; true -> all(Dir, Context, Artifacts) @@ -441,15 +590,23 @@ all(Dir, Context, [File|Artifacts]) -> %%%%% +%% @doc given a set of override rules, modify the app info accordingly +-spec apply_overrides(list(), t()) -> t(). apply_overrides(Overrides, AppInfo) -> Name = binary_to_atom(rebar_app_info:name(AppInfo), utf8), Opts = rebar_opts:apply_overrides(opts(AppInfo), Name, Overrides), AppInfo#app_info_t{default=Opts, opts=Opts}. +%% @doc adds a new profile with its own config to the app data +-spec add_to_profile(t(), atom(), [{_,_}]) -> t(). add_to_profile(AppInfo, Profile, KVs) when is_atom(Profile), is_list(KVs) -> Opts = rebar_opts:add_to_profile(opts(AppInfo), Profile, KVs), AppInfo#app_info_t{opts=Opts}. +%% @doc applies and merges the profile configuration in the specified order +%% of profiles (or for a single profile) and returns an app info record +%% with the resulting configuration +-spec apply_profiles(t(), atom() | [atom(),...]) -> t(). apply_profiles(AppInfo, Profile) when not is_list(Profile) -> apply_profiles(AppInfo, [Profile]); apply_profiles(AppInfo, [default]) -> @@ -481,9 +638,13 @@ apply_profiles(AppInfo=#app_info_t{default = Defaults, profiles=CurrentProfiles} end, Defaults, AppliedProfiles), AppInfo#app_info_t{profiles = AppliedProfiles, opts=NewOpts}. +%% @private drops duplicated profile definitions +-spec deduplicate(list()) -> list(). deduplicate(Profiles) -> do_deduplicate(lists:reverse(Profiles), []). +%% @private drops duplicated profile definitions +-spec do_deduplicate(list(), list()) -> list(). do_deduplicate([], Acc) -> Acc; do_deduplicate([Head | Rest], Acc) -> diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index d256cac..5fe5ba6 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -34,6 +34,7 @@ validate_application_info/2, parse_deps/5, parse_deps/6, + expand_deps_sources/2, dep_to_app/7, format_error/1]). @@ -44,10 +45,14 @@ %% Public API %% =================================================================== +%% @doc finds the proper app info record for a given app name in a list of +%% such records. -spec find(binary(), [rebar_app_info:t()]) -> {ok, rebar_app_info:t()} | error. find(Name, Apps) -> ec_lists:find(fun(App) -> rebar_app_info:name(App) =:= Name end, Apps). +%% @doc finds the proper app info record for a given app name at a given version +%% in a list of such records. -spec find(binary(), binary(), [rebar_app_info:t()]) -> {ok, rebar_app_info:t()} | error. find(Name, Vsn, Apps) -> ec_lists:find(fun(App) -> @@ -55,11 +60,18 @@ find(Name, Vsn, Apps) -> andalso rebar_app_info:original_vsn(App) =:= Vsn end, Apps). +%% @doc checks if a given file is .app.src file is_app_src(Filename) -> %% If removing the extension .app.src yields a shorter name, %% this is an .app.src file. Filename =/= filename:rootname(Filename, ".app.src"). +%% @doc translates the name of the .app.src[.script] file to where +%% its .app counterpart should be stored. +-spec app_src_to_app(OutDir, SrcFilename) -> OutFilename when + OutDir :: file:filename(), + SrcFilename :: file:filename(), + OutFilename :: file:filename(). app_src_to_app(OutDir, Filename) -> AppFile = case lists:suffix(".app.src", Filename) of @@ -72,10 +84,16 @@ app_src_to_app(OutDir, Filename) -> filelib:ensure_dir(AppFile), AppFile. +%% @doc checks whether the .app file has all the required data to be valid, +%% and cross-references it with compiled modules on disk -spec validate_application_info(rebar_app_info:t()) -> boolean(). validate_application_info(AppInfo) -> validate_application_info(AppInfo, rebar_app_info:app_details(AppInfo)). +%% @doc checks whether the .app file has all the required data to be valid +%% and cross-references it with compiled modules on disk. +%% The app info is passed explicitly as a second argument. +-spec validate_application_info(rebar_app_info:t(), list()) -> boolean(). validate_application_info(AppInfo, AppDetail) -> EbinDir = rebar_app_info:ebin_dir(AppInfo), case rebar_app_info:app_file(AppInfo) of @@ -90,13 +108,37 @@ validate_application_info(AppInfo, AppDetail) -> end end. --spec parse_deps(binary(), list(), rebar_state:t(), list(), integer()) -> [rebar_app_info:t()]. +%% @doc parses all dependencies from the root of the project +-spec parse_deps(Dir, Deps, State, Locks, Level) -> [rebar_app_info:t()] when + Dir :: file:filename(), + Deps :: [tuple() | atom() | binary()], % TODO: meta to source() | lock() + State :: rebar_state:t(), + Locks :: [tuple()], % TODO: meta to [lock()] + Level :: non_neg_integer(). parse_deps(DepsDir, Deps, State, Locks, Level) -> parse_deps(root, DepsDir, Deps, State, Locks, Level). +%% @doc runs `parse_dep/6' for a set of dependencies. +-spec parse_deps(Parent, Dir, Deps, State, Locks, Level) -> [rebar_app_info:t()] when + Parent :: root | binary(), + Dir :: file:filename(), + Deps :: [tuple() | atom() | binary()], % TODO: meta to source() | lock() + State :: rebar_state:t(), + Locks :: [tuple()], % TODO: meta to [lock()] + Level :: non_neg_integer(). parse_deps(Parent, DepsDir, Deps, State, Locks, Level) -> [parse_dep(Dep, Parent, DepsDir, State, Locks, Level) || Dep <- Deps]. +%% @doc for a given dep, return its app info record. The function +%% also has to choose whether to define the dep from its immediate spec +%% (if it is a newer thing) or from the locks specified in the lockfile. +-spec parse_dep(Dep, Parent, Dir, State, Locks, Level) -> rebar_app_info:t() when + Dep :: tuple() | atom() | binary(), % TODO: meta to source() | lock() + Parent :: root | binary(), + Dir :: file:filename(), + State :: rebar_state:t(), + Locks :: [tuple()], % TODO: meta to [lock()] + Level :: non_neg_integer(). parse_dep(Dep, Parent, DepsDir, State, Locks, Level) -> Name = case Dep of Dep when is_tuple(Dep) -> @@ -104,7 +146,7 @@ parse_dep(Dep, Parent, DepsDir, State, Locks, Level) -> Dep -> Dep end, - case lists:keyfind(ec_cnv:to_binary(Name), 1, Locks) of + case lists:keyfind(rebar_utils:to_binary(Name), 1, Locks) of false -> parse_dep(Parent, Dep, DepsDir, false, State); LockedDep -> @@ -117,19 +159,29 @@ parse_dep(Dep, Parent, DepsDir, State, Locks, Level) -> end end. +%% @doc converts a dependency definition and a location for it on disk +%% into an app info tuple representing it. +-spec parse_dep(Parent, Dep, Dir, IsLock, State) -> rebar_app_info:t() when + Parent :: root | binary(), + Dep :: tuple() | atom() | binary(), % TODO: meta to source() | lock() + Dir :: file:filename(), + IsLock :: boolean(), + State :: rebar_state:t(). parse_dep(Parent, {Name, Vsn, {pkg, PkgName}}, DepsDir, IsLock, State) -> - {PkgName1, PkgVsn} = {ec_cnv:to_binary(PkgName), ec_cnv:to_binary(Vsn)}, + {PkgName1, PkgVsn} = {rebar_utils:to_binary(PkgName), + rebar_utils:to_binary(Vsn)}, dep_to_app(Parent, DepsDir, Name, PkgVsn, {pkg, PkgName1, PkgVsn, undefined}, IsLock, State); parse_dep(Parent, {Name, {pkg, PkgName}}, DepsDir, IsLock, State) -> %% Package dependency with different package name from app name - dep_to_app(Parent, DepsDir, Name, undefined, {pkg, ec_cnv:to_binary(PkgName), undefined, undefined}, IsLock, State); + dep_to_app(Parent, DepsDir, Name, undefined, {pkg, rebar_utils:to_binary(PkgName), undefined, undefined}, IsLock, State); parse_dep(Parent, {Name, Vsn}, DepsDir, IsLock, State) when is_list(Vsn); is_binary(Vsn) -> %% Versioned Package dependency - {PkgName, PkgVsn} = {ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}, + {PkgName, PkgVsn} = {rebar_utils:to_binary(Name), + rebar_utils:to_binary(Vsn)}, dep_to_app(Parent, DepsDir, PkgName, PkgVsn, {pkg, PkgName, PkgVsn, undefined}, IsLock, State); parse_dep(Parent, Name, DepsDir, IsLock, State) when is_atom(Name); is_binary(Name) -> %% Unversioned package dependency - dep_to_app(Parent, DepsDir, ec_cnv:to_binary(Name), undefined, {pkg, ec_cnv:to_binary(Name), undefined, undefined}, IsLock, State); + dep_to_app(Parent, DepsDir, rebar_utils:to_binary(Name), undefined, {pkg, rebar_utils:to_binary(Name), undefined, undefined}, IsLock, State); parse_dep(Parent, {Name, Source}, DepsDir, IsLock, State) when is_tuple(Source) -> dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); parse_dep(Parent, {Name, _Vsn, Source}, DepsDir, IsLock, State) when is_tuple(Source) -> @@ -152,62 +204,81 @@ parse_dep(Parent, {Name, Source, Level}, DepsDir, IsLock, State) when is_tuple(S parse_dep(_, Dep, _, _, _) -> throw(?PRV_ERROR({parse_dep, Dep})). +%% @doc convert a dependency that has just been fetched into +%% an app info record related to it +-spec dep_to_app(Parent, Dir, Name, Vsn, Source, IsLock, State) -> rebar_app_info:t() when + Parent :: root | binary(), + Dir :: file:filename(), + Name :: binary(), + Vsn :: iodata() | undefined, + Source :: tuple(), + IsLock :: boolean(), + State :: rebar_state:t(). dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> - CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), + CheckoutsDir = rebar_utils:to_list(rebar_dir:checkouts_dir(State, Name)), AppInfo = case rebar_app_info:discover(CheckoutsDir) of - {ok, App} -> - rebar_app_info:source(rebar_app_info:is_checkout(App, true), checkout); - not_found -> - Dir = ec_cnv:to_list(filename:join(DepsDir, Name)), - {ok, AppInfo0} = - case rebar_app_info:discover(Dir) of - {ok, App} -> - {ok, rebar_app_info:parent(App, Parent)}; - not_found -> - rebar_app_info:new(Parent, Name, Vsn, Dir, []) - end, - update_source(AppInfo0, Source, State) - end, - C = rebar_config:consult(rebar_app_info:dir(AppInfo)), - AppInfo1 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), C), - Overrides = rebar_state:get(State, overrides, []), - AppInfo2 = rebar_app_info:set(AppInfo1, overrides, rebar_app_info:get(AppInfo, overrides, [])++Overrides), - AppInfo3 = rebar_app_info:apply_overrides(rebar_app_info:get(AppInfo2, overrides, []), AppInfo2), - AppInfo4 = rebar_app_info:apply_profiles(AppInfo3, [default, prod]), - AppInfo5 = rebar_app_info:profiles(AppInfo4, [default]), + {ok, App} -> + rebar_app_info:source(rebar_app_info:is_checkout(App, true), checkout); + not_found -> + Dir = rebar_utils:to_list(filename:join(DepsDir, Name)), + {ok, AppInfo0} = + case rebar_app_info:discover(Dir) of + {ok, App} -> + App1 = rebar_app_info:name(App, Name), + {ok, rebar_app_info:is_available(rebar_app_info:parent(App1, Parent), + true)}; + not_found -> + rebar_app_info:new(Parent, Name, Vsn, Dir, []) + end, + rebar_app_info:source(AppInfo0, Source) + end, + Overrides = rebar_app_info:get(AppInfo, overrides, []) ++ rebar_state:get(State, overrides, []), + AppInfo2 = rebar_app_info:set(AppInfo, overrides, Overrides), + AppInfo5 = rebar_app_info:profiles(AppInfo2, [default]), rebar_app_info:is_lock(AppInfo5, IsLock). +%% @doc Takes a given application app_info record along with the project. +%% If the app is a package, resolve and expand the package definition. +-spec expand_deps_sources(rebar_app_info:t(), rebar_state:t()) -> + rebar_app_info:t(). +expand_deps_sources(Dep, State) -> + update_source(Dep, rebar_app_info:source(Dep), State). + +%% @doc sets the source for a given dependency or app along with metadata +%% around version if required. +-spec update_source(rebar_app_info:t(), Source, rebar_state:t()) -> + rebar_app_info:t() when + Source :: rebar_resource_v2:source(). update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) -> - {PkgName1, PkgVsn1} = case PkgVsn of - undefined -> - get_package(PkgName, "0", State); - <<"~>", Vsn/binary>> -> - [Vsn1] = binary:split(Vsn, [<<" ">>], [trim_all, global]), - get_package(PkgName, Vsn1, State); - _ -> - {PkgName, PkgVsn} - end, - %% store the expected hash for the dependency - Hash1 = case Hash of - undefined -> % unknown, define the hash since we know the dep - rebar_packages:registry_checksum({pkg, PkgName1, PkgVsn1, Hash}, State); - _ -> % keep as is - Hash - end, - AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName1, PkgVsn1, Hash1}), - Deps = rebar_packages:deps(PkgName1 - ,PkgVsn1 - ,State), - AppInfo2 = rebar_app_info:resource_type(rebar_app_info:deps(AppInfo1, Deps), pkg), - rebar_app_info:original_vsn(AppInfo2, PkgVsn1); + case rebar_packages:resolve_version(PkgName, PkgVsn, Hash, + ?PACKAGE_TABLE, State) of + {ok, Package, RepoConfig} -> + #package{key={_, PkgVsn1, _}, + checksum=Hash1, + dependencies=Deps, + retired=Retired} = Package, + maybe_warn_retired(PkgName, PkgVsn1, Hash, Retired), + PkgVsn2 = list_to_binary(lists:flatten(ec_semver:format(PkgVsn1))), + AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName, PkgVsn2, Hash1, RepoConfig}), + rebar_app_info:update_opts_deps(AppInfo1, Deps); + not_found -> + throw(?PRV_ERROR({missing_package, PkgName, PkgVsn})); + {error, {invalid_vsn, InvalidVsn}} -> + throw(?PRV_ERROR({invalid_vsn, PkgName, InvalidVsn})) + end; update_source(AppInfo, Source, _State) -> rebar_app_info:source(AppInfo, Source). - -format_error({missing_package, Package}) -> - io_lib:format("Package not found in registry: ~s", [Package]); +%% @doc convert a given exception's payload into an io description. +-spec format_error(any()) -> iolist(). +format_error({missing_package, Name, undefined}) -> + io_lib:format("Package not found in any repo: ~ts", [rebar_utils:to_binary(Name)]); +format_error({missing_package, Name, Constraint}) -> + io_lib:format("Package not found in any repo: ~ts ~ts", [Name, Constraint]); format_error({parse_dep, Dep}) -> io_lib:format("Failed parsing dep ~p", [Dep]); +format_error({invalid_vsn, Dep, InvalidVsn}) -> + io_lib:format("Dep ~ts has invalid version ~ts", [Dep, InvalidVsn]); format_error(Error) -> io_lib:format("~p", [Error]). @@ -215,18 +286,38 @@ format_error(Error) -> %% Internal functions %% =================================================================== -get_package(Dep, Vsn, State) -> - case rebar_packages:find_highest_matching(Dep, Vsn, ?PACKAGE_TABLE, State) of - {ok, HighestDepVsn} -> - {Dep, HighestDepVsn}; - none -> - throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Dep)})) - end. +maybe_warn_retired(_, _, _, false) -> + ok; +maybe_warn_retired(_, _, Hash, _) when is_binary(Hash) -> + %% don't warn if this is a lock + ok; +maybe_warn_retired(Name, Vsn, _, R=#{reason := Reason}) -> + Message = maps:get(message, R, ""), + ?WARN("Warning: package ~s-~s is retired: (~s) ~s", + [Name, ec_semver:format(Vsn), retire_reason(Reason), Message]); +maybe_warn_retired(_, _, _, _) -> + ok. + +%% TODO: move to hex_core +retire_reason('RETIRED_OTHER') -> + "other"; +retire_reason('RETIRED_INVALID') -> + "invalid"; +retire_reason('RETIRED_SECURITY') -> + "security"; +retire_reason('RETIRED_DEPRECATED') -> + "deprecated"; +retire_reason('RETIRED_RENAMED') -> + "renamed"; +retire_reason(_Other) -> + "other". +%% @private checks that all the beam files have been properly +%% created. -spec has_all_beams(file:filename_all(), [module()]) -> true | ?PRV_ERROR({missing_module, module()}). has_all_beams(EbinDir, [Module | ModuleList]) -> - BeamFile = filename:join([EbinDir, ec_cnv:to_list(Module) ++ ".beam"]), + BeamFile = filename:join([EbinDir, rebar_utils:to_list(Module) ++ ".beam"]), case filelib:is_file(BeamFile) of true -> has_all_beams(EbinDir, ModuleList); diff --git a/src/rebar_base_compiler.erl b/src/rebar_base_compiler.erl index 6b8c7ca..ad81c86 100644 --- a/src/rebar_base_compiler.erl +++ b/src/rebar_base_compiler.erl @@ -33,28 +33,73 @@ run/8, ok_tuple/2, error_tuple/4, + report/1, + maybe_report/1, format_error_source/2]). --define(DEFAULT_COMPILER_SOURCE_FORMAT, relative). +-type desc() :: term(). +-type loc() :: {line(), col()} | line(). +-type line() :: integer(). +-type col() :: integer(). +-type err_or_warn() :: {module(), desc()} | {loc(), module(), desc()}. + +-type compile_fn_ret() :: ok | {ok, [string()]} | skipped | term(). +-type compile_fn() :: fun((file:filename(), [{_,_}] | rebar_dict()) -> compile_fn_ret()). +-type compile_fn3() :: fun((file:filename(), file:filename(), [{_,_}] | rebar_dict()) + -> compile_fn_ret()). +-type error_tuple() :: {error, [string()], [string()]}. +-export_type([compile_fn/0, compile_fn_ret/0, error_tuple/0]). %% =================================================================== %% Public API %% =================================================================== +%% @doc Runs a compile job, applying `compile_fn()' to all files, +%% starting with `First' files, and then `RestFiles'. +-spec run(rebar_dict() | [{_,_}] , [First], [Next], compile_fn()) -> + compile_fn_ret() when + First :: file:filename(), + Next :: file:filename(). run(Config, FirstFiles, RestFiles, CompileFn) -> %% Compile the first files in sequence compile_each(FirstFiles++RestFiles, Config, CompileFn). +%% @doc Runs a compile job, applying `compile_fn3()' to all files, +%% starting with `First' files, and then the other content of `SourceDir'. +%% Files looked for are those ending in `SourceExt'. Results of the +%% compilation are put in `TargetDir' with the base file names +%% postfixed with `SourceExt'. +-spec run(rebar_dict() | [{_,_}] , [First], SourceDir, SourceExt, + TargetDir, TargetExt, compile_fn3()) -> compile_fn_ret() when + First :: file:filename(), + SourceDir :: file:filename(), + TargetDir :: file:filename(), + SourceExt :: string(), + TargetExt :: string(). run(Config, FirstFiles, SourceDir, SourceExt, TargetDir, TargetExt, Compile3Fn) -> run(Config, FirstFiles, SourceDir, SourceExt, TargetDir, TargetExt, Compile3Fn, [check_last_mod]). +%% @doc Runs a compile job, applying `compile_fn3()' to all files, +%% starting with `First' files, and then the other content of `SourceDir'. +%% Files looked for are those ending in `SourceExt'. Results of the +%% compilation are put in `TargetDir' with the base file names +%% postfixed with `SourceExt'. +%% Additional compile options can be passed in the last argument as +%% a proplist. +-spec run(rebar_dict() | [{_,_}] , [First], SourceDir, SourceExt, + TargetDir, TargetExt, compile_fn3(), [term()]) -> compile_fn_ret() when + First :: file:filename(), + SourceDir :: file:filename(), + TargetDir :: file:filename(), + SourceExt :: string(), + TargetExt :: string(). run(Config, FirstFiles, SourceDir, SourceExt, TargetDir, TargetExt, Compile3Fn, Opts) -> %% Convert simple extension to proper regex - SourceExtRe = "^[^._].*\\" ++ SourceExt ++ [$$], + SourceExtRe = "^(?!\\._).*\\" ++ SourceExt ++ [$$], Recursive = proplists:get_value(recursive, Opts, true), %% Find all possible source files @@ -68,44 +113,43 @@ run(Config, FirstFiles, SourceDir, SourceExt, TargetDir, TargetExt, run(Config, FirstFiles, RestFiles, fun(S, C) -> - Target = target_file(S, SourceDir, SourceExt, + Target = target_file(S, SourceExt, TargetDir, TargetExt), simple_compile_wrapper(S, Target, Compile3Fn, C, CheckLastMod) end). +%% @doc Format good compiler results with warnings to work with +%% module internals. Assumes that warnings are not treated as errors. +-spec ok_tuple(file:filename(), [string()]) -> {ok, [string()]}. ok_tuple(Source, Ws) -> {ok, format_warnings(Source, Ws)}. +%% @doc format error and warning strings for a given source file +%% according to user preferences. +-spec error_tuple(file:filename(), [Err], [Warn], rebar_dict() | [{_,_}]) -> + error_tuple() when + Err :: string(), + Warn :: string(). error_tuple(Source, Es, Ws, Opts) -> {error, format_errors(Source, Es), format_warnings(Source, Ws, Opts)}. +%% @doc from a given path, and based on the user-provided options, +%% format the file path according to the preferences. +-spec format_error_source(file:filename(), rebar_dict() | [{_,_}]) -> + file:filename(). format_error_source(Path, Opts) -> - Type = case rebar_opts:get(Opts, compiler_source_format, - ?DEFAULT_COMPILER_SOURCE_FORMAT) of - V when V == absolute; V == relative; V == build -> - V; - Other -> - ?WARN("Invalid argument ~p for compiler_source_format - " - "assuming ~s~n", [Other, ?DEFAULT_COMPILER_SOURCE_FORMAT]), - ?DEFAULT_COMPILER_SOURCE_FORMAT - end, - case Type of - absolute -> resolve_linked_source(Path); - build -> Path; - relative -> - Cwd = rebar_dir:get_cwd(), - rebar_dir:make_relative_path(resolve_linked_source(Path), Cwd) - end. - -resolve_linked_source(Src) -> - {Dir, Base} = rebar_file_utils:split_dirname(Src), - filename:join(rebar_file_utils:resolve_link(Dir), Base). + rebar_dir:format_source_file_name(Path, Opts). %% =================================================================== %% Internal functions %% =================================================================== +%% @private if a check for last modifications is required, do the verification +%% and possibly skip the compile job. +-spec simple_compile_wrapper(Source, Target, compile_fn3(), [{_,_}] | rebar_dict(), boolean()) -> compile_fn_ret() when + Source :: file:filename(), + Target :: file:filename(). simple_compile_wrapper(Source, Target, Compile3Fn, Config, false) -> Compile3Fn(Source, Target, Config); simple_compile_wrapper(Source, Target, Compile3Fn, Config, true) -> @@ -116,51 +160,76 @@ simple_compile_wrapper(Source, Target, Compile3Fn, Config, true) -> skipped end. -target_file(SourceFile, SourceDir, SourceExt, TargetDir, TargetExt) -> - BaseFile = remove_common_path(SourceFile, SourceDir), - filename:join([TargetDir, filename:basename(BaseFile, SourceExt) ++ TargetExt]). - -remove_common_path(Fname, Path) -> - remove_common_path1(filename:split(Fname), filename:split(Path)). - -remove_common_path1([Part | RestFilename], [Part | RestPath]) -> - remove_common_path1(RestFilename, RestPath); -remove_common_path1(FilenameParts, _) -> - filename:join(FilenameParts). +%% @private take a basic source set of file fragments and a target location, +%% create a file path and name for a compile artifact. +-spec target_file(SourceFile, SourceExt, TargetDir, TargetExt) -> File when + SourceFile :: file:filename(), + TargetDir :: file:filename(), + SourceExt :: string(), + TargetExt :: string(), + File :: file:filename(). +target_file(SourceFile, SourceExt, TargetDir, TargetExt) -> + %% BaseFile = remove_common_path(SourceFile, SourceDir), + filename:join([TargetDir, filename:basename(SourceFile, SourceExt) ++ TargetExt]). +%% @private runs the compile function `CompileFn' on every file +%% passed internally, along with the related project configuration. +%% If any errors are encountered, they're reported to stdout. +-spec compile_each([file:filename()], Config, CompileFn) -> Ret | no_return() when + Config :: [{_,_}] | rebar_dict(), + CompileFn :: compile_fn(), + Ret :: compile_fn_ret(). compile_each([], _Config, _CompileFn) -> ok; compile_each([Source | Rest], Config, CompileFn) -> case CompileFn(Source, Config) of ok -> - ?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), filename:basename(Source)]); + ?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]); {ok, Warnings} -> report(Warnings), - ?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), filename:basename(Source)]); + ?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]); skipped -> - ?DEBUG("~sSkipped ~s", [rebar_utils:indent(1), filename:basename(Source)]); + ?DEBUG("~tsSkipped ~ts", [rebar_utils:indent(1), filename:basename(Source)]); Error -> NewSource = format_error_source(Source, Config), - ?ERROR("Compiling ~s failed", [NewSource]), + ?ERROR("Compiling ~ts failed", [NewSource]), maybe_report(Error), ?DEBUG("Compilation failed: ~p", [Error]), ?FAIL end, compile_each(Rest, Config, CompileFn). +%% @private Formats and returns errors ready to be output. +-spec format_errors(string(), [err_or_warn()]) -> [string()]. format_errors(Source, Errors) -> format_errors(Source, "", Errors). +%% @private Formats and returns warning strings ready to be output. +-spec format_warnings(string(), [err_or_warn()]) -> [string()]. format_warnings(Source, Warnings) -> format_warnings(Source, Warnings, []). +%% @private Formats and returns warnings; chooses the distinct format they +%% may have based on whether `warnings_as_errors' option is on. +-spec format_warnings(string(), [err_or_warn()], rebar_dict() | [{_,_}]) -> [string()]. format_warnings(Source, Warnings, Opts) -> - Prefix = case lists:member(warnings_as_errors, Opts) of + %% `Opts' can be passed in both as a list or a dictionary depending + %% on whether the first call to rebar_erlc_compiler was done with + %% the type `rebar_dict()' or `rebar_state:t()'. + LookupFn = if is_list(Opts) -> fun lists:member/2 + ; true -> fun dict:is_key/2 + end, + Prefix = case LookupFn(warnings_as_errors, Opts) of true -> ""; false -> "Warning: " end, format_errors(Source, Prefix, Warnings). +%% @private output compiler errors if they're judged to be reportable. +-spec maybe_report(Reportable | term()) -> ok when + Reportable :: {{error, error_tuple()}, Source} | error_tuple() | ErrProps, + ErrProps :: [{error, string()} | Source, ...], + Source :: {source, string()}. maybe_report({{error, {error, _Es, _Ws}=ErrorsAndWarnings}, {source, _}}) -> maybe_report(ErrorsAndWarnings); maybe_report([{error, E}, {source, S}]) -> @@ -171,21 +240,39 @@ maybe_report({error, Es, Ws}) -> maybe_report(_) -> ok. +%% @private Outputs a bunch of strings, including a newline +-spec report([string()]) -> ok. report(Messages) -> - lists:foreach(fun(Msg) -> io:format("~s~n", [Msg]) end, Messages). + lists:foreach(fun(Msg) -> io:format("~ts~n", [Msg]) end, Messages). +%% private format compiler errors into proper outputtable strings +-spec format_errors(_, Extra, [err_or_warn()]) -> [string()] when + Extra :: string(). format_errors(_MainSource, Extra, Errors) -> - [begin - [format_error(Source, Extra, Desc) || Desc <- Descs] - end + [[format_error(Source, Extra, Desc) || Desc <- Descs] || {Source, Descs} <- Errors]. +%% @private format compiler errors into proper outputtable strings +-spec format_error(file:filename(), Extra, err_or_warn()) -> string() when + Extra :: string(). +format_error(Source, Extra, {Line, Mod=epp, Desc={include,lib,File}}) -> + %% Special case for include file errors, overtaking the default one + BaseDesc = Mod:format_error(Desc), + Friendly = case filename:split(File) of + [Lib, "include", _] -> + io_lib:format("; Make sure ~s is in your app " + "file's 'applications' list", [Lib]); + _ -> + "" + end, + FriendlyDesc = BaseDesc ++ Friendly, + ?FMT("~ts:~w: ~ts~ts~n", [Source, Line, Extra, FriendlyDesc]); format_error(Source, Extra, {{Line, Column}, Mod, Desc}) -> ErrorDesc = Mod:format_error(Desc), - ?FMT("~s:~w:~w: ~s~s~n", [Source, Line, Column, Extra, ErrorDesc]); + ?FMT("~ts:~w:~w: ~ts~ts~n", [Source, Line, Column, Extra, ErrorDesc]); format_error(Source, Extra, {Line, Mod, Desc}) -> ErrorDesc = Mod:format_error(Desc), - ?FMT("~s:~w: ~s~s~n", [Source, Line, Extra, ErrorDesc]); + ?FMT("~ts:~w: ~ts~ts~n", [Source, Line, Extra, ErrorDesc]); format_error(Source, Extra, {Mod, Desc}) -> ErrorDesc = Mod:format_error(Desc), - ?FMT("~s: ~s~s~n", [Source, Extra, ErrorDesc]). + ?FMT("~ts: ~ts~ts~n", [Source, Extra, ErrorDesc]). diff --git a/src/rebar_compiler.erl b/src/rebar_compiler.erl new file mode 100644 index 0000000..7da265c --- /dev/null +++ b/src/rebar_compiler.erl @@ -0,0 +1,315 @@ +-module(rebar_compiler). + +-export([compile_all/2, + clean/2, + + needs_compile/3, + ok_tuple/2, + error_tuple/4, + maybe_report/1, + format_error_source/2, + report/1]). + +-include("rebar.hrl"). + +-type extension() :: string(). +-type out_mappings() :: [{extension(), file:filename()}]. + +-callback context(rebar_app_info:t()) -> #{src_dirs => [file:dirname()], + include_dirs => [file:dirname()], + src_ext => extension(), + out_mappings => out_mappings()}. +-callback needed_files(digraph:graph(), [file:filename()], out_mappings(), + rebar_app_info:t()) -> + {{[file:filename()], term()}, {[file:filename()], term()}}. +-callback dependencies(file:filename(), file:dirname(), [file:dirname()]) -> [file:filename()]. +-callback compile(file:filename(), out_mappings(), rebar_dict(), list()) -> + ok | {ok, [string()]} | {ok, [string()], [string()]}. +-callback clean([file:filename()], rebar_app_info:t()) -> _. + +-define(DAG_VSN, 2). +-define(DAG_FILE, "source.dag"). +-type dag_v() :: {digraph:vertex(), term()} | 'false'. +-type dag_e() :: {digraph:vertex(), digraph:vertex()}. +-type dag() :: {list(dag_v()), list(dag_e()), list(string())}. +-record(dag, {vsn = ?DAG_VSN :: pos_integer(), + info = {[], [], []} :: dag()}). + +-define(RE_PREFIX, "^(?!\\._)"). + +compile_all(Compilers, AppInfo) -> + EbinDir = rebar_utils:to_list(rebar_app_info:ebin_dir(AppInfo)), + %% Make sure that outdir is on the path + ok = rebar_file_utils:ensure_dir(EbinDir), + true = code:add_patha(filename:absname(EbinDir)), + + %% necessary for erlang:function_exported/3 to work as expected + %% called here for clarity as it's required by both opts_changed/2 + %% and erl_compiler_opts_set/0 in needed_files + _ = code:ensure_loaded(compile), + + lists:foreach(fun(CompilerMod) -> + run(CompilerMod, AppInfo), + run_on_extra_src_dirs(CompilerMod, AppInfo, fun run/2) + end, Compilers), + ok. + +run(CompilerMod, AppInfo) -> + #{src_dirs := SrcDirs, + include_dirs := InclDirs, + src_ext := SrcExt, + out_mappings := Mappings} = CompilerMod:context(AppInfo), + + BaseDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)), + EbinDir = rebar_utils:to_list(rebar_app_info:ebin_dir(AppInfo)), + + BaseOpts = rebar_app_info:opts(AppInfo), + AbsInclDirs = [filename:join(BaseDir, InclDir) || InclDir <- InclDirs], + FoundFiles = find_source_files(BaseDir, SrcExt, SrcDirs, BaseOpts), + + OutDir = rebar_app_info:out_dir(AppInfo), + AbsSrcDirs = [filename:join(BaseDir, SrcDir) || SrcDir <- SrcDirs], + G = init_dag(CompilerMod, AbsInclDirs, AbsSrcDirs, FoundFiles, OutDir, EbinDir), + {{FirstFiles, FirstFileOpts}, {RestFiles, Opts}} = CompilerMod:needed_files(G, FoundFiles, + Mappings, AppInfo), + true = digraph:delete(G), + + compile_each(FirstFiles, FirstFileOpts, BaseOpts, Mappings, CompilerMod), + compile_each(RestFiles, Opts, BaseOpts, Mappings, CompilerMod). + +compile_each([], _Opts, _Config, _Outs, _CompilerMod) -> + ok; +compile_each([Source | Rest], Opts, Config, Outs, CompilerMod) -> + case CompilerMod:compile(Source, Outs, Config, Opts) of + ok -> + ?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]); + {ok, Warnings} -> + report(Warnings), + ?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]); + skipped -> + ?DEBUG("~tsSkipped ~ts", [rebar_utils:indent(1), filename:basename(Source)]); + Error -> + NewSource = format_error_source(Source, Config), + ?ERROR("Compiling ~ts failed", [NewSource]), + maybe_report(Error), + ?DEBUG("Compilation failed: ~p", [Error]), + ?FAIL + end, + compile_each(Rest, Opts, Config, Outs, CompilerMod). + +%% @doc remove compiled artifacts from an AppDir. +-spec clean([module()], rebar_app_info:t()) -> 'ok'. +clean(Compilers, AppInfo) -> + lists:foreach(fun(CompilerMod) -> + clean_(CompilerMod, AppInfo), + run_on_extra_src_dirs(CompilerMod, AppInfo, fun clean_/2) + end, Compilers). + +clean_(CompilerMod, AppInfo) -> + #{src_dirs := SrcDirs, + src_ext := SrcExt} = CompilerMod:context(AppInfo), + BaseDir = rebar_app_info:dir(AppInfo), + Opts = rebar_app_info:opts(AppInfo), + EbinDir = rebar_app_info:ebin_dir(AppInfo), + + FoundFiles = find_source_files(BaseDir, SrcExt, SrcDirs, Opts), + CompilerMod:clean(FoundFiles, AppInfo), + rebar_file_utils:rm_rf(dag_file(CompilerMod, EbinDir)). + +-spec needs_compile(filename:all(), extension(), [{extension(), file:dirname()}]) -> boolean(). +needs_compile(Source, OutExt, Mappings) -> + Ext = filename:extension(Source), + BaseName = filename:basename(Source, Ext), + {_, OutDir} = lists:keyfind(OutExt, 1, Mappings), + Target = filename:join(OutDir, BaseName++OutExt), + filelib:last_modified(Source) > filelib:last_modified(Target). + +run_on_extra_src_dirs(CompilerMod, AppInfo, Fun) -> + ExtraDirs = rebar_dir:extra_src_dirs(rebar_app_info:opts(AppInfo), []), + run_on_extra_src_dirs(ExtraDirs, CompilerMod, AppInfo, Fun). + +run_on_extra_src_dirs([], _CompilerMod, _AppInfo, _Fun) -> + ok; +run_on_extra_src_dirs([Dir | Rest], CompilerMod, AppInfo, Fun) -> + case filelib:is_dir(filename:join(rebar_app_info:dir(AppInfo), Dir)) of + true -> + EbinDir = filename:join(rebar_app_info:out_dir(AppInfo), Dir), + AppInfo1 = rebar_app_info:ebin_dir(AppInfo, EbinDir), + AppInfo2 = rebar_app_info:set(AppInfo1, src_dirs, [Dir]), + AppInfo3 = rebar_app_info:set(AppInfo2, extra_src_dirs, ["src"]), + Fun(CompilerMod, AppInfo3); + _ -> + ok + end, + run_on_extra_src_dirs(Rest, CompilerMod, AppInfo, Fun). + +%% These functions are here for the ultimate goal of getting rid of +%% rebar_base_compiler. This can't be done because of existing plugins. + +ok_tuple(Source, Ws) -> + rebar_base_compiler:ok_tuple(Source, Ws). + +error_tuple(Source, Es, Ws, Opts) -> + rebar_base_compiler:error_tuple(Source, Es, Ws, Opts). + +maybe_report(Reportable) -> + rebar_base_compiler:maybe_report(Reportable). + +format_error_source(Path, Opts) -> + rebar_base_compiler:format_error_source(Path, Opts). + +report(Messages) -> + rebar_base_compiler:report(Messages). + +%% private functions + +find_source_files(BaseDir, SrcExt, SrcDirs, Opts) -> + SourceExtRe = "^(?!\\._).*\\" ++ SrcExt ++ [$$], + lists:flatmap(fun(SrcDir) -> + Recursive = rebar_dir:recursive(Opts, SrcDir), + rebar_utils:find_files_in_dirs([filename:join(BaseDir, SrcDir)], SourceExtRe, Recursive) + end, SrcDirs). + +dag_file(CompilerMod, Dir) -> + filename:join([rebar_dir:local_cache_dir(Dir), CompilerMod, ?DAG_FILE]). + +%% private graph functions + +%% Get dependency graph of given Erls files and their dependencies (header files, +%% parse transforms, behaviours etc.) located in their directories or given +%% InclDirs. Note that last modification times stored in vertices already respect +%% dependencies induced by given graph G. +init_dag(Compiler, InclDirs, SrcDirs, Erls, Dir, EbinDir) -> + G = digraph:new([acyclic]), + try restore_dag(Compiler, G, InclDirs, Dir) + catch + _:_ -> + ?WARN("Failed to restore ~ts file. Discarding it.~n", [dag_file(Compiler, Dir)]), + file:delete(dag_file(Compiler, Dir)) + end, + Dirs = lists:usort(InclDirs ++ SrcDirs), + %% A source file may have been renamed or deleted. Remove it from the graph + %% and remove any beam file for that source if it exists. + Modified = maybe_rm_beams_and_edges(G, EbinDir, Erls), + Modified1 = lists:foldl(update_dag_fun(G, Compiler, Dirs), Modified, Erls), + if Modified1 -> store_dag(Compiler, G, InclDirs, Dir); not Modified1 -> ok end, + G. + +maybe_rm_beams_and_edges(G, Dir, Files) -> + Vertices = digraph:vertices(G), + case lists:filter(fun(File) -> + case filename:extension(File) =:= ".erl" of + true -> + maybe_rm_beam_and_edge(G, Dir, File); + false -> + false + end + end, lists:sort(Vertices) -- lists:sort(Files)) of + [] -> + false; + _ -> + true + end. + +maybe_rm_beam_and_edge(G, OutDir, Source) -> + %% This is NOT a double check it is the only check that the source file is actually gone + case filelib:is_regular(Source) of + true -> + %% Actually exists, don't delete + false; + false -> + Target = target_base(OutDir, Source) ++ ".beam", + ?DEBUG("Source ~ts is gone, deleting previous beam file if it exists ~ts", [Source, Target]), + file:delete(Target), + digraph:del_vertex(G, Source), + true + end. + + +target_base(OutDir, Source) -> + filename:join(OutDir, filename:basename(Source, ".erl")). + +restore_dag(Compiler, G, InclDirs, Dir) -> + case file:read_file(dag_file(Compiler, Dir)) of + {ok, Data} -> + % Since externally passed InclDirs can influence dependency graph (see + % modify_dag), we have to check here that they didn't change. + #dag{vsn=?DAG_VSN, info={Vs, Es, InclDirs}} = + binary_to_term(Data), + lists:foreach( + fun({V, LastUpdated}) -> + digraph:add_vertex(G, V, LastUpdated) + end, Vs), + lists:foreach( + fun({_, V1, V2, _}) -> + digraph:add_edge(G, V1, V2) + end, Es); + {error, _} -> + ok + end. + +store_dag(Compiler, G, InclDirs, Dir) -> + Vs = lists:map(fun(V) -> digraph:vertex(G, V) end, digraph:vertices(G)), + Es = lists:map(fun(E) -> digraph:edge(G, E) end, digraph:edges(G)), + File = dag_file(Compiler, Dir), + ok = filelib:ensure_dir(File), + Data = term_to_binary(#dag{info={Vs, Es, InclDirs}}, [{compressed, 2}]), + file:write_file(File, Data). + +update_dag(G, Compiler, Dirs, Source) -> + case digraph:vertex(G, Source) of + {_, LastUpdated} -> + case filelib:last_modified(Source) of + 0 -> + %% The file doesn't exist anymore, + %% erase it from the graph. + %% All the edges will be erased automatically. + digraph:del_vertex(G, Source), + modified; + LastModified when LastUpdated < LastModified -> + modify_dag(G, Compiler, Source, LastModified, filename:dirname(Source), Dirs); + _ -> + Modified = lists:foldl( + update_dag_fun(G, Compiler, Dirs), + false, digraph:out_neighbours(G, Source)), + MaxModified = update_max_modified_deps(G, Source), + case Modified orelse MaxModified > LastUpdated of + true -> modified; + false -> unmodified + end + end; + false -> + modify_dag(G, Compiler, Source, filelib:last_modified(Source), filename:dirname(Source), Dirs) + end. + +modify_dag(G, Compiler, Source, LastModified, SourceDir, Dirs) -> + AbsIncls = Compiler:dependencies(Source, SourceDir, Dirs), + digraph:add_vertex(G, Source, LastModified), + digraph:del_edges(G, digraph:out_edges(G, Source)), + lists:foreach( + fun(Incl) -> + update_dag(G, Compiler, Dirs, Incl), + digraph:add_edge(G, Source, Incl) + end, AbsIncls), + modified. + +update_dag_fun(G, Compiler, Dirs) -> + fun(Erl, Modified) -> + case update_dag(G, Compiler, Dirs, Erl) of + modified -> true; + unmodified -> Modified + end + end. + +update_max_modified_deps(G, Source) -> + MaxModified = + lists:foldl(fun(File, Acc) -> + case digraph:vertex(G, File) of + {_, MaxModified} when MaxModified > Acc -> + MaxModified; + _ -> + Acc + end + end, 0, [Source | digraph:out_neighbours(G, Source)]), + digraph:add_vertex(G, Source, MaxModified), + MaxModified. diff --git a/src/rebar_compiler_erl.erl b/src/rebar_compiler_erl.erl new file mode 100644 index 0000000..0a560cd --- /dev/null +++ b/src/rebar_compiler_erl.erl @@ -0,0 +1,359 @@ +-module(rebar_compiler_erl). + +-behaviour(rebar_compiler). + +-export([context/1, + needed_files/4, + dependencies/3, + compile/4, + clean/2]). + +-include("rebar.hrl"). + +context(AppInfo) -> + EbinDir = rebar_app_info:ebin_dir(AppInfo), + Mappings = [{".beam", EbinDir}], + + OutDir = rebar_app_info:dir(AppInfo), + SrcDirs = rebar_dir:src_dirs(rebar_app_info:opts(AppInfo), ["src"]), + ExistingSrcDirs = lists:filter(fun(D) -> + ec_file:is_dir(filename:join(OutDir, D)) + end, SrcDirs), + + RebarOpts = rebar_app_info:opts(AppInfo), + ErlOpts = rebar_opts:erl_opts(RebarOpts), + ErlOptIncludes = proplists:get_all_values(i, ErlOpts), + InclDirs = lists:map(fun(Incl) -> filename:absname(Incl) end, ErlOptIncludes), + + #{src_dirs => ExistingSrcDirs, + include_dirs => [filename:join([OutDir, "include"]) | InclDirs], + src_ext => ".erl", + out_mappings => Mappings}. + + +needed_files(Graph, FoundFiles, _, AppInfo) -> + OutDir = rebar_app_info:out_dir(AppInfo), + Dir = rebar_app_info:dir(AppInfo), + EbinDir = rebar_app_info:ebin_dir(AppInfo), + RebarOpts = rebar_app_info:opts(AppInfo), + ErlOpts = rebar_opts:erl_opts(RebarOpts), + ?DEBUG("erlopts ~p", [ErlOpts]), + ?DEBUG("files to compile ~p", [FoundFiles]), + + %% Make sure that the ebin dir is on the path + ok = rebar_file_utils:ensure_dir(EbinDir), + true = code:add_patha(filename:absname(EbinDir)), + + {ParseTransforms, Rest} = split_source_files(FoundFiles, ErlOpts), + NeededErlFiles = case needed_files(Graph, ErlOpts, RebarOpts, OutDir, EbinDir, ParseTransforms) of + [] -> + needed_files(Graph, ErlOpts, RebarOpts, OutDir, EbinDir, Rest); + _ -> + %% at least one parse transform in the opts needs updating, so recompile all + FoundFiles + end, + {ErlFirstFiles, ErlOptsFirst} = erl_first_files(RebarOpts, ErlOpts, Dir, NeededErlFiles), + SubGraph = digraph_utils:subgraph(Graph, NeededErlFiles), + DepErlsOrdered = digraph_utils:topsort(SubGraph), + OtherErls = lists:reverse(DepErlsOrdered), + + PrivIncludes = [{i, filename:join(OutDir, Src)} + || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])], + AdditionalOpts = PrivIncludes ++ [{i, filename:join(OutDir, "include")}, {i, OutDir}, return], + + true = digraph:delete(SubGraph), + + {{ErlFirstFiles, ErlOptsFirst ++ AdditionalOpts}, + {[Erl || Erl <- OtherErls, + not lists:member(Erl, ErlFirstFiles)], ErlOpts ++ AdditionalOpts}}. + +dependencies(Source, SourceDir, Dirs) -> + {ok, Fd} = file:open(Source, [read]), + Incls = parse_attrs(Fd, [], SourceDir), + AbsIncls = expand_file_names(Incls, Dirs), + ok = file:close(Fd), + AbsIncls. + +compile(Source, [{_, OutDir}], Config, ErlOpts) -> + case compile:file(Source, [{outdir, OutDir} | ErlOpts]) of + {ok, _Mod} -> + ok; + {ok, _Mod, []} -> + ok; + {ok, _Mod, Ws} -> + FormattedWs = format_error_sources(Ws, Config), + rebar_compiler:ok_tuple(Source, FormattedWs); + {error, Es, Ws} -> + error_tuple(Source, Es, Ws, Config, ErlOpts); + error -> + error + end. + +clean(Files, AppInfo) -> + EbinDir = rebar_app_info:ebin_dir(AppInfo), + [begin + Source = filename:basename(File, ".erl"), + Target = target_base(EbinDir, Source) ++ ".beam", + file:delete(Target) + end || File <- Files]. + +%% + +error_tuple(Module, Es, Ws, AllOpts, Opts) -> + FormattedEs = format_error_sources(Es, AllOpts), + FormattedWs = format_error_sources(Ws, AllOpts), + rebar_compiler:error_tuple(Module, FormattedEs, FormattedWs, Opts). + +format_error_sources(Es, Opts) -> + [{rebar_compiler:format_error_source(Src, Opts), Desc} + || {Src, Desc} <- Es]. + +%% Get files which need to be compiled first, i.e. those specified in erl_first_files +%% and parse_transform options. Also produce specific erl_opts for these first +%% files, so that yet to be compiled parse transformations are excluded from it. +erl_first_files(Opts, ErlOpts, Dir, NeededErlFiles) -> + ErlFirstFilesConf = rebar_opts:get(Opts, erl_first_files, []), + valid_erl_first_conf(ErlFirstFilesConf), + NeededSrcDirs = lists:usort(lists:map(fun filename:dirname/1, NeededErlFiles)), + %% NOTE: order of files here is important! + ErlFirstFiles = + [filename:join(Dir, File) || File <- ErlFirstFilesConf, + lists:member(filename:join(Dir, File), NeededErlFiles)], + {ParseTransforms, ParseTransformsErls} = + lists:unzip(lists:flatmap( + fun(PT) -> + PTerls = [filename:join(D, module_to_erl(PT)) || D <- NeededSrcDirs], + [{PT, PTerl} || PTerl <- PTerls, lists:member(PTerl, NeededErlFiles)] + end, proplists:get_all_values(parse_transform, ErlOpts))), + ErlOptsFirst = lists:filter(fun({parse_transform, PT}) -> + not lists:member(PT, ParseTransforms); + (_) -> + true + end, ErlOpts), + {ErlFirstFiles ++ ParseTransformsErls, ErlOptsFirst}. + +split_source_files(SourceFiles, ErlOpts) -> + ParseTransforms = proplists:get_all_values(parse_transform, ErlOpts), + lists:partition(fun(Source) -> + lists:member(filename_to_atom(Source), ParseTransforms) + end, SourceFiles). + +filename_to_atom(F) -> list_to_atom(filename:rootname(filename:basename(F))). + +%% Get subset of SourceFiles which need to be recompiled, respecting +%% dependencies induced by given graph G. +needed_files(Graph, ErlOpts, RebarOpts, Dir, OutDir, SourceFiles) -> + lists:filter(fun(Source) -> + TargetBase = target_base(OutDir, Source), + Target = TargetBase ++ ".beam", + PrivIncludes = [{i, filename:join(Dir, Src)} + || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])], + AllOpts = [{outdir, filename:dirname(Target)} + ,{i, filename:join(Dir, "include")} + ,{i, Dir}] ++ PrivIncludes ++ ErlOpts, + digraph:vertex(Graph, Source) > {Source, filelib:last_modified(Target)} + orelse opts_changed(AllOpts, TargetBase) + orelse erl_compiler_opts_set() + end, SourceFiles). + +target_base(OutDir, Source) -> + filename:join(OutDir, filename:basename(Source, ".erl")). + +opts_changed(NewOpts, Target) -> + TotalOpts = case erlang:function_exported(compile, env_compiler_options, 0) of + true -> NewOpts ++ compile:env_compiler_options(); + false -> NewOpts + end, + case compile_info(Target) of + {ok, Opts} -> lists:any(fun effects_code_generation/1, lists:usort(TotalOpts) -- lists:usort(Opts)); + _ -> true + end. + +effects_code_generation(Option) -> + case Option of + beam -> false; + report_warnings -> false; + report_errors -> false; + return_errors-> false; + return_warnings-> false; + report -> false; + warnings_as_errors -> false; + binary -> false; + verbose -> false; + {cwd,_} -> false; + {outdir, _} -> false; + _ -> true + end. + +compile_info(Target) -> + case beam_lib:chunks(Target, [compile_info]) of + {ok, {_mod, Chunks}} -> + CompileInfo = proplists:get_value(compile_info, Chunks, []), + {ok, proplists:get_value(options, CompileInfo, [])}; + {error, beam_lib, Reason} -> + ?WARN("Couldn't read debug info from ~p for reason: ~p", [Target, Reason]), + {error, Reason} + end. + +erl_compiler_opts_set() -> + EnvSet = case os:getenv("ERL_COMPILER_OPTIONS") of + false -> false; + _ -> true + end, + %% return false if changed env opts would have been caught in opts_changed/2 + EnvSet andalso not erlang:function_exported(compile, env_compiler_options, 0). + +valid_erl_first_conf(FileList) -> + Strs = filter_file_list(FileList), + case rebar_utils:is_list_of_strings(Strs) of + true -> true; + false -> ?ABORT("An invalid file list (~p) was provided as part of your erl_first_files directive", + [FileList]) + end. + +filter_file_list(FileList) -> + Atoms = lists:filter( fun(X) -> is_atom(X) end, FileList), + case Atoms of + [] -> + FileList; + _ -> + atoms_in_erl_first_files_warning(Atoms), + lists:filter( fun(X) -> not(is_atom(X)) end, FileList) + end. + +atoms_in_erl_first_files_warning(Atoms) -> + W = "You have provided atoms as file entries in erl_first_files; " + "erl_first_files only expects lists of filenames as strings. " + "The following modules (~p) may not work as expected and it is advised " + "that you change these entires to string format " + "(e.g., \"src/module.erl\") ", + ?WARN(W, [Atoms]). + +module_to_erl(Mod) -> + atom_to_list(Mod) ++ ".erl". + +parse_attrs(Fd, Includes, Dir) -> + case io:parse_erl_form(Fd, "") of + {ok, Form, _Line} -> + case erl_syntax:type(Form) of + attribute -> + NewIncludes = process_attr(Form, Includes, Dir), + parse_attrs(Fd, NewIncludes, Dir); + _ -> + parse_attrs(Fd, Includes, Dir) + end; + {eof, _} -> + Includes; + _Err -> + parse_attrs(Fd, Includes, Dir) + end. + +process_attr(Form, Includes, Dir) -> + AttrName = erl_syntax:atom_value(erl_syntax:attribute_name(Form)), + process_attr(AttrName, Form, Includes, Dir). + +process_attr(import, Form, Includes, _Dir) -> + case erl_syntax_lib:analyze_import_attribute(Form) of + {Mod, _Funs} -> + [module_to_erl(Mod)|Includes]; + Mod -> + [module_to_erl(Mod)|Includes] + end; +process_attr(file, Form, Includes, _Dir) -> + {File, _} = erl_syntax_lib:analyze_file_attribute(Form), + [File|Includes]; +process_attr(include, Form, Includes, _Dir) -> + [FileNode] = erl_syntax:attribute_arguments(Form), + File = erl_syntax:string_value(FileNode), + [File|Includes]; +process_attr(include_lib, Form, Includes, Dir) -> + [FileNode] = erl_syntax:attribute_arguments(Form), + RawFile = erl_syntax:string_value(FileNode), + maybe_expand_include_lib_path(RawFile, Dir) ++ Includes; +process_attr(behavior, Form, Includes, _Dir) -> + process_attr(behaviour, Form, Includes, _Dir); +process_attr(behaviour, Form, Includes, _Dir) -> + [FileNode] = erl_syntax:attribute_arguments(Form), + File = module_to_erl(erl_syntax:atom_value(FileNode)), + [File|Includes]; +process_attr(compile, Form, Includes, _Dir) -> + [Arg] = erl_syntax:attribute_arguments(Form), + case erl_syntax:concrete(Arg) of + {parse_transform, Mod} -> + [module_to_erl(Mod)|Includes]; + {core_transform, Mod} -> + [module_to_erl(Mod)|Includes]; + L when is_list(L) -> + lists:foldl( + fun({parse_transform, Mod}, Acc) -> + [module_to_erl(Mod)|Acc]; + ({core_transform, Mod}, Acc) -> + [module_to_erl(Mod)|Acc]; + (_, Acc) -> + Acc + end, Includes, L); + _ -> + Includes + end; +process_attr(_, _Form, Includes, _Dir) -> + Includes. + +%% NOTE: If, for example, one of the entries in Files, refers to +%% gen_server.erl, that entry will be dropped. It is dropped because +%% such an entry usually refers to the beam file, and we don't pass a +%% list of OTP src dirs for finding gen_server.erl's full path. Also, +%% if gen_server.erl was modified, it's not rebar's task to compile a +%% new version of the beam file. Therefore, it's reasonable to drop +%% such entries. Also see process_attr(behaviour, Form, Includes). +-spec expand_file_names([file:filename()], + [file:filename()]) -> [file:filename()]. +expand_file_names(Files, Dirs) -> + %% We check if Files exist by itself or within the directories + %% listed in Dirs. + %% Return the list of files matched. + lists:flatmap( + fun(Incl) -> + case filelib:is_regular(Incl) of + true -> + [Incl]; + false -> + rebar_utils:find_files_in_dirs(Dirs, Incl, true) + end + end, Files). + +%% Given a path like "stdlib/include/erl_compile.hrl", return +%% "OTP_INSTALL_DIR/lib/erlang/lib/stdlib-x.y.z/include/erl_compile.hrl". +%% Usually a simple [Lib, SubDir, File1] = filename:split(File) should +%% work, but to not crash when an unusual include_lib path is used, +%% utilize more elaborate logic. +maybe_expand_include_lib_path(File, Dir) -> + File1 = filename:basename(File), + case filename:split(filename:dirname(File)) of + [_] -> + warn_and_find_path(File, Dir); + [Lib | SubDir] -> + case code:lib_dir(list_to_atom(Lib), list_to_atom(filename:join(SubDir))) of + {error, bad_name} -> + warn_and_find_path(File, Dir); + AppDir -> + [filename:join(AppDir, File1)] + end + end. + +%% The use of -include_lib was probably incorrect by the user but lets try to make it work. +%% We search in the outdir and outdir/../include to see if the header exists. +warn_and_find_path(File, Dir) -> + SrcHeader = filename:join(Dir, File), + case filelib:is_regular(SrcHeader) of + true -> + [SrcHeader]; + false -> + IncludeDir = filename:join(rebar_utils:droplast(filename:split(Dir))++["include"]), + IncludeHeader = filename:join(IncludeDir, File), + case filelib:is_regular(IncludeHeader) of + true -> + [filename:join(IncludeDir, File)]; + false -> + [] + end + end. diff --git a/src/rebar_compiler_mib.erl b/src/rebar_compiler_mib.erl new file mode 100644 index 0000000..499976d --- /dev/null +++ b/src/rebar_compiler_mib.erl @@ -0,0 +1,101 @@ +-module(rebar_compiler_mib). + +-behaviour(rebar_compiler). + +-export([context/1, + needed_files/4, + dependencies/3, + compile/4, + clean/2]). + +-include("rebar.hrl"). +-include_lib("stdlib/include/erl_compile.hrl"). + +context(AppInfo) -> + Dir = rebar_app_info:dir(AppInfo), + Mappings = [{".bin", filename:join([Dir, "priv", "mibs"])}, + {".hrl", filename:join(Dir, "include")}], + + #{src_dirs => ["mibs"], + include_dirs => [], + src_ext => ".mib", + out_mappings => Mappings}. + +needed_files(_, FoundFiles, _, AppInfo) -> + RebarOpts = rebar_app_info:opts(AppInfo), + MibFirstConf = rebar_opts:get(RebarOpts, mib_first_files, []), + valid_mib_first_conf(MibFirstConf), + Dir = rebar_app_info:dir(AppInfo), + MibFirstFiles = [filename:join(Dir, File) || File <- MibFirstConf], + + %% Remove first files from found files + RestFiles = [Source || Source <- FoundFiles, not lists:member(Source, MibFirstFiles)], + + Opts = rebar_opts:get(rebar_app_info:opts(AppInfo), mib_opts, []), + {{MibFirstFiles, Opts}, {RestFiles, Opts}}. + +valid_mib_first_conf(FileList) -> + Strs = filter_file_list(FileList), + case rebar_utils:is_list_of_strings(Strs) of + true -> true; + false -> ?ABORT("An invalid file list (~p) was provided as part of your mib_first_files directive", + [FileList]) + end. + +filter_file_list(FileList) -> + Atoms = lists:filter( fun(X) -> is_atom(X) end, FileList), + case Atoms of + [] -> + FileList; + _ -> + atoms_in_mib_first_files_warning(Atoms), + lists:filter( fun(X) -> not(is_atom(X)) end, FileList) + end. + +atoms_in_mib_first_files_warning(Atoms) -> + W = "You have provided atoms as file entries in mib_first_files; " + "mib_first_files only expects lists of filenames as strings. " + "The following MIBs (~p) may not work as expected and it is advised " + "that you change these entires to string format " + "(e.g., \"mibs/SOME-MIB.mib\") ", + ?WARN(W, [Atoms]). + + +dependencies(_, _, _) -> + []. + +compile(Source, OutDirs, _, Opts) -> + {_, BinOut} = lists:keyfind(".bin", 1, OutDirs), + {_, HrlOut} = lists:keyfind(".hrl", 1, OutDirs), + + ok = rebar_file_utils:ensure_dir(BinOut), + ok = rebar_file_utils:ensure_dir(HrlOut), + Mib = filename:join(BinOut, filename:basename(Source, ".mib")), + HrlFilename = Mib ++ ".hrl", + + AllOpts = [{outdir, BinOut}, {i, [BinOut]}] ++ Opts, + + case snmpc:compile(Source, AllOpts) of + {ok, _} -> + MibToHrlOpts = + case proplists:get_value(verbosity, AllOpts, undefined) of + undefined -> + #options{specific = [], + cwd = rebar_dir:get_cwd()}; + Verbosity -> + #options{specific = [{verbosity, Verbosity}], + cwd = rebar_dir:get_cwd()} + end, + ok = snmpc:mib_to_hrl(Mib, Mib, MibToHrlOpts), + rebar_file_utils:mv(HrlFilename, HrlOut), + ok; + {error, compilation_failed} -> + ?FAIL + end. + +clean(MibFiles, AppInfo) -> + AppDir = rebar_app_info:dir(AppInfo), + MIBs = [filename:rootname(filename:basename(MIB)) || MIB <- MibFiles], + rebar_file_utils:delete_each( + [filename:join([AppDir, "include", MIB++".hrl"]) || MIB <- MIBs]), + ok = rebar_file_utils:rm_rf(filename:join([AppDir, "priv/mibs/*.bin"])). diff --git a/src/rebar_compiler_xrl.erl b/src/rebar_compiler_xrl.erl new file mode 100644 index 0000000..35447ed --- /dev/null +++ b/src/rebar_compiler_xrl.erl @@ -0,0 +1,64 @@ +-module(rebar_compiler_xrl). + +-behaviour(rebar_compiler). + +-export([context/1, + needed_files/4, + dependencies/3, + compile/4, + clean/2]). + +-export([update_opts/2]). + +context(AppInfo) -> + Dir = rebar_app_info:dir(AppInfo), + Mappings = [{".erl", filename:join([Dir, "src"])}], + #{src_dirs => ["src"], + include_dirs => [], + src_ext => ".xrl", + out_mappings => Mappings}. + +needed_files(_, FoundFiles, Mappings, AppInfo) -> + FirstFiles = [], + + %% Remove first files from found files + RestFiles = [Source || Source <- FoundFiles, + not lists:member(Source, FirstFiles), + rebar_compiler:needs_compile(Source, ".erl", Mappings)], + + Opts = rebar_opts:get(rebar_app_info:opts(AppInfo), xrl_opts, []), + Opts1 = update_opts(Opts, AppInfo), + + {{FirstFiles, Opts1}, {RestFiles, Opts1}}. + +dependencies(_, _, _) -> + []. + +compile(Source, [{_, _}], _, Opts) -> + case leex:file(Source, [{return, true} | Opts]) of + {ok, _} -> + ok; + {ok, _Mod, Ws} -> + rebar_compiler:ok_tuple(Source, Ws); + {error, Es, Ws} -> + rebar_compiler:error_tuple(Source, Es, Ws, Opts) + end. + +clean(XrlFiles, _AppInfo) -> + rebar_file_utils:delete_each( + [rebar_utils:to_list(re:replace(F, "\\.xrl$", ".erl", [unicode])) + || F <- XrlFiles]). + +%% make includefile options absolute paths +update_opts(Opts, AppInfo) -> + OutDir = rebar_app_info:out_dir(AppInfo), + lists:map(fun({includefile, I}) -> + case filename:pathtype(I) =:= relative of + true -> + {includefile, filename:join(OutDir, I)}; + false -> + {includefile, I} + end; + (O) -> + O + end, Opts). diff --git a/src/rebar_compiler_yrl.erl b/src/rebar_compiler_yrl.erl new file mode 100644 index 0000000..8e52d0e --- /dev/null +++ b/src/rebar_compiler_yrl.erl @@ -0,0 +1,51 @@ +-module(rebar_compiler_yrl). + +-behaviour(rebar_compiler). + +-export([context/1, + needed_files/4, + dependencies/3, + compile/4, + clean/2]). + +context(AppInfo) -> + Dir = rebar_app_info:dir(AppInfo), + Mappings = [{".erl", filename:join([Dir, "src"])}], + #{src_dirs => ["src"], + include_dirs => [], + src_ext => ".yrl", + out_mappings => Mappings}. + +needed_files(_, FoundFiles, Mappings, AppInfo) -> + FirstFiles = [], + + %% Remove first files from found files + RestFiles = [Source || Source <- FoundFiles, + not lists:member(Source, FirstFiles), + rebar_compiler:needs_compile(Source, ".erl", Mappings)], + + Opts = rebar_opts:get(rebar_app_info:opts(AppInfo), yrl_opts, []), + Opts1 = rebar_compiler_xrl:update_opts(Opts, AppInfo), + + {{FirstFiles, Opts1}, {RestFiles, Opts1}}. + +dependencies(_, _, _) -> + []. + +compile(Source, [{_, OutDir}], _, Opts) -> + BaseName = filename:basename(Source, ".yrl"), + Target = filename:join([OutDir, BaseName]), + AllOpts = [{parserfile, Target}, {return, true} | Opts], + case yecc:file(Source, AllOpts) of + {ok, _} -> + ok; + {ok, _Mod, Ws} -> + rebar_compiler:ok_tuple(Source, Ws); + {error, Es, Ws} -> + rebar_compiler:error_tuple(Source, Es, Ws, AllOpts) + end. + +clean(YrlFiles, _AppInfo) -> + rebar_file_utils:delete_each( + [rebar_utils:to_list(re:replace(F, "\\.yrl$", ".erl", [unicode])) + || F <- YrlFiles]). diff --git a/src/rebar_config.erl b/src/rebar_config.erl index b50c030..2651ca1 100644 --- a/src/rebar_config.erl +++ b/src/rebar_config.erl @@ -26,7 +26,8 @@ %% ------------------------------------------------------------------- -module(rebar_config). --export([consult/1 +-export([consult_root/0 + ,consult/1 ,consult_app_file/1 ,consult_file/1 ,consult_lock_file/1 @@ -39,17 +40,31 @@ -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). +-define(DEFAULT_CONFIG_FILE, "rebar.config"). + %% =================================================================== %% Public API %% =================================================================== +%% @doc reads the default config file at the top of a full project +-spec consult_root() -> [any()]. +consult_root() -> + consult_file(config_file()). + +%% @doc reads the default config file in a given directory. -spec consult(file:name()) -> [any()]. consult(Dir) -> consult_file(filename:join(Dir, ?DEFAULT_CONFIG_FILE)). +%% @doc reads a given app file, including the `.script' variations, +%% if any can be found. +-spec consult_app_file(file:filename()) -> [any()]. consult_app_file(File) -> consult_file_(File). +%% @doc reads the lock file for the project, and re-formats its +%% content to match the internals for rebar3. +-spec consult_lock_file(file:filename()) -> [any()]. % TODO: refine lock() consult_lock_file(File) -> Terms = consult_file_(File), case Terms of @@ -59,7 +74,7 @@ consult_lock_file(File) -> read_attrs(beta, Locks, []); [{Vsn, Locks}|Attrs] when is_list(Locks) -> % versioned lock file %% Because this is the first version of rebar3 to introduce a lock - %% file, all versionned lock files with a different versions have + %% file, all versioned lock files with a different version have %% to be newer. case Vsn of ?CONFIG_VERSION -> @@ -73,6 +88,11 @@ consult_lock_file(File) -> read_attrs(Vsn, Locks, Attrs) end. +%% @private outputs a warning for a newer lockfile format than supported +%% at most once. +%% The warning can also be cancelled by configuring the `warn_config_vsn' +%% OTP env variable. +-spec warn_vsn_once() -> ok. warn_vsn_once() -> Warn = application:get_env(rebar, warn_config_vsn) =/= {ok, false}, application:set_env(rebar, warn_config_vsn, false), @@ -86,6 +106,11 @@ warn_vsn_once() -> end. +%% @doc Converts the internal format for locks into the multi-version +%% compatible one used within rebar3 lock files. +%% @end +%% TODO: refine type for lock() +-spec write_lock_file(file:filename(), [any()]) -> ok | {error, term()}. write_lock_file(LockFile, Locks) -> {NewLocks, Attrs} = write_attrs(Locks), %% Write locks in the beta format, at least until it's been long @@ -95,34 +120,46 @@ write_lock_file(LockFile, Locks) -> file:write_file(LockFile, io_lib:format("~p.~n", [NewLocks])); _ -> file:write_file(LockFile, - io_lib:format("{~p,~n~p}.~n[~n~s~n].~n", + io_lib:format("{~p,~n~p}.~n[~n~ts~n].~n", [?CONFIG_VERSION, NewLocks, format_attrs(Attrs)])) end. -%% Attributes have a special formatting to ensure there's only one per -%% line in terms of pkg_hash, so we disturb source diffing as little -%% as possible. +%% @private Because attributes for packages are fairly large, there is the need +%% for a special formatting to ensure there's only one entry per lock file +%% line and that diffs are generally stable. +-spec format_attrs([term()]) -> iodata(). format_attrs([]) -> []; format_attrs([{pkg_hash, Vals}|T]) -> [io_lib:format("{pkg_hash,[~n",[]), format_hashes(Vals), "]}", - maybe_comma(T) | format_attrs(T)]; -format_attrs([H|T]) -> - [io_lib:format("~p~s", [H, maybe_comma(T)]) | format_attrs(T)]. + maybe_comma(T) | format_attrs(T)]. +%% @private format hashing in order to disturb source diffing as little +%% as possible +-spec format_hashes([term()]) -> iodata(). format_hashes([]) -> []; format_hashes([{Pkg,Hash}|T]) -> [" {", io_lib:format("~p",[Pkg]), ", ", io_lib:format("~p", [Hash]), "}", maybe_comma(T) | format_hashes(T)]. +%% @private add a comma if we're not done with the full list of terms +%% to convert. +-spec maybe_comma([term()]) -> iodata(). maybe_comma([]) -> ""; maybe_comma([_|_]) -> io_lib:format(",~n", []). +%% @private extract attributes from the lock file and integrate them +%% into the full-blow internal lock format +%% @end +%% TODO: refine typings for lock() +-spec read_attrs(_, [any()], [any()]) -> [any()]. read_attrs(_Vsn, Locks, Attrs) -> %% Beta copy does not know how to expand attributes, but %% is ready to support it. expand_locks(Locks, extract_pkg_hashes(Attrs)). +%% @private extract the package hashes from lockfile attributes, if any. +-spec extract_pkg_hashes(list()) -> [binary()]. extract_pkg_hashes(Attrs) -> Props = case Attrs of [First|_] -> First; @@ -130,6 +167,11 @@ extract_pkg_hashes(Attrs) -> end, proplists:get_value(pkg_hash, Props, []). +%% @private extract attributes from the lock file and integrate them +%% into the full-blow internal lock format +%% @end +%% TODO: refine typings for lock() +-spec expand_locks(list(), list()) -> list(). expand_locks([], _Hashes) -> []; expand_locks([{Name, {pkg,PkgName,Vsn}, Lvl} | Locks], Hashes) -> @@ -138,6 +180,9 @@ expand_locks([{Name, {pkg,PkgName,Vsn}, Lvl} | Locks], Hashes) -> expand_locks([Lock|Locks], Hashes) -> [Lock | expand_locks(Locks, Hashes)]. +%% @private split up extra attributes for locks out of the internal lock +%% structure for backwards compatibility reasons +-spec write_attrs(list()) -> {list(), list()}. write_attrs(Locks) -> %% No attribute known that needs to be taken out of the structure, %% just return terms as is. @@ -147,6 +192,9 @@ write_attrs(Locks) -> _ -> {NewLocks, [{pkg_hash, lists:sort(Hashes)}]} end. +%% @private split up extra attributes for locks out of the internal lock +%% structure for backwards compatibility reasons +-spec split_locks(list(), list(), [{_,binary()}]) -> {list(), list()}. split_locks([], Locks, Hashes) -> {lists:reverse(Locks), Hashes}; split_locks([{Name, {pkg,PkgName,Vsn,undefined}, Lvl} | Locks], LAcc, HAcc) -> @@ -156,11 +204,17 @@ split_locks([{Name, {pkg,PkgName,Vsn,Hash}, Lvl} | Locks], LAcc, HAcc) -> split_locks([Lock|Locks], LAcc, HAcc) -> split_locks(Locks, [Lock|LAcc], HAcc). +%% @doc reads a given config file, including the `.script' variations, +%% if any can be found, and asserts that the config format is in +%% a key-value format. +-spec consult_file(file:filename()) -> [{_,_}]. consult_file(File) -> Terms = consult_file_(File), true = verify_config_format(Terms), Terms. +%% @private reads a given file; if the file has a `.script'-postfixed +%% counterpart, it is evaluated along with the original file. -spec consult_file_(file:name()) -> [any()]. consult_file_(File) when is_binary(File) -> consult_file_(binary_to_list(File)); @@ -180,6 +234,9 @@ consult_file_(File) -> end end. +%% @private checks that a list is in a key-value format. +%% Raises an exception in any other case. +-spec verify_config_format([{_,_}]) -> true. verify_config_format([]) -> true; verify_config_format([{_Key, _Value} | T]) -> @@ -187,11 +244,14 @@ verify_config_format([{_Key, _Value} | T]) -> verify_config_format([Term | _]) -> throw(?PRV_ERROR({bad_config_format, Term})). -%% no lockfile +%% @doc takes an existing configuration and the content of a lockfile +%% and merges the locks into the config. +-spec merge_locks([{_,_}], list()) -> [{_,_}]. merge_locks(Config, []) -> +%% no lockfile Config; -%% lockfile with entries merge_locks(Config, Locks) -> + %% lockfile with entries ConfigDeps = proplists:get_value(deps, Config, []), %% We want the top level deps only from the lock file. %% This ensures deterministic overrides for configs. @@ -201,6 +261,8 @@ merge_locks(Config, Locks) -> NewDeps = find_newly_added(ConfigDeps, Locks), [{{locks, default}, Locks}, {{deps, default}, NewDeps++Deps} | Config]. +%% @doc convert a given exception's payload into an io description. +-spec format_error(any()) -> iolist(). format_error({bad_config_format, Term}) -> io_lib:format("Unable to parse config. Term is not in {Key, Value} format:~n~p", [Term]); format_error({bad_dep_name, Dep}) -> @@ -210,6 +272,8 @@ format_error({bad_dep_name, Dep}) -> %% Internal functions %% =================================================================== +%% @private consults a consult file, then executes its related script file +%% with the data returned from the consult. -spec consult_and_eval(File::file:name_all(), Script::file:name_all()) -> {ok, Terms::[term()]} | {error, Reason::term()}. @@ -226,21 +290,31 @@ consult_and_eval(File, Script) -> {ok, Term} -> {ok, [Term]}; Error -> + ?ERROR("Error evaluating configuration script at ~p:~n~p~n", + [Script, Error]), Error end. +%% @private drops the .script extension from a filename. +-spec remove_script_ext(file:filename()) -> file:filename(). remove_script_ext(F) -> filename:rootname(F, ".script"). +%% @private sets up bindings for evaluations from a KV list. +-spec bs([{_,_}]) -> erl_eval:binding_struct(). bs(Vars) -> lists:foldl(fun({K,V}, Bs) -> erl_eval:add_binding(K, V, Bs) end, erl_eval:new_bindings(), Vars). -%% Find deps that have been added to the config after the lock was created +%% @private Find deps that have been added to the config after the lock was created +-spec find_newly_added(list(), list()) -> list(). find_newly_added(ConfigDeps, LockedDeps) -> [D || {true, D} <- [check_newly_added(Dep, LockedDeps) || Dep <- ConfigDeps]]. +%% @private checks if a given dependency is not within the lock file. +%% TODO: refine types for dependencies +-spec check_newly_added(term(), list()) -> false | {true, term()}. check_newly_added({_, _}=Dep, LockedDeps) -> check_newly_added_(Dep, LockedDeps); check_newly_added({_, _, {pkg, _}}=Dep, LockedDeps) -> @@ -250,6 +324,10 @@ check_newly_added({Name, _, Source}, LockedDeps) -> check_newly_added(Dep, LockedDeps) -> check_newly_added_(Dep, LockedDeps). +%% @private checks if a given dependency is not within the lock file. +%% TODO: refine types for dependencies +%% @end +-spec check_newly_added_(term(), list()) -> false | {true, term()}. %% get [raw] deps out of the way check_newly_added_({Name, Source, Opts}, LockedDeps) when is_tuple(Source), is_list(Opts) -> @@ -283,7 +361,7 @@ check_newly_added_({Name, Source}, LockedDeps) -> false end; check_newly_added_(Dep, LockedDeps) when is_atom(Dep) -> - Name = ec_cnv:to_binary(Dep), + Name = rebar_utils:to_binary(Dep), case lists:keyfind(Name, 1, LockedDeps) of false -> {true, Name}; @@ -292,11 +370,22 @@ check_newly_added_(Dep, LockedDeps) when is_atom(Dep) -> 0 -> {true, Name}; _ -> - ?WARN("Newly added dep ~s is locked at a lower level. " - "If you really want to unlock it, use 'rebar3 upgrade ~s'", + ?WARN("Newly added dep ~ts is locked at a lower level. " + "If you really want to unlock it, use 'rebar3 upgrade ~ts'", [Name, Name]), false end end; check_newly_added_(Dep, _) -> throw(?PRV_ERROR({bad_dep_name, Dep})). + +%% @private returns the name/path of the default config file, or its +%% override from the OS ENV var `REBAR_CONFIG'. +-spec config_file() -> file:filename(). +config_file() -> + case os:getenv("REBAR_CONFIG") of + false -> + ?DEFAULT_CONFIG_FILE; + ConfigFile -> + ConfigFile + end. diff --git a/src/rebar_core.erl b/src/rebar_core.erl index da8c3e6..6a1cdbf 100644 --- a/src/rebar_core.erl +++ b/src/rebar_core.erl @@ -24,6 +24,8 @@ %% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN %% THE SOFTWARE. %% ------------------------------------------------------------------- +%% @doc Module providing core functionality about command dispatch, namespacing, +%% and chaining for rebar3. -module(rebar_core). -export([init_command/2, process_namespace/2, process_command/2, do/2, format_error/1]). @@ -31,6 +33,12 @@ -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). +%% @doc initial command set up; based on the first fragment of the +%% command, dispatch to special environments. The keywords for +%% `do' and `as' are implicitly reserved here, barring them from +%% being used as other commands or namespaces. +-spec init_command(rebar_state:t(), atom()) -> + {ok, rebar_state:t()} | {error, term()}. init_command(State, do) -> process_command(rebar_state:namespace(State, default), do); init_command(State, as) -> @@ -43,6 +51,14 @@ init_command(State, Command) -> {error, Reason} end. +%% @doc parse the commands starting at the namespace level; +%% a namespace is found if the first keyword to match is not +%% belonging to an existing provider, and iff the keyword also +%% matches a registered namespace. +%% The command to run is returned last; for namespaces, some +%% magic is done implicitly calling `do' as an indirect dispatcher. +-spec process_namespace(rebar_state:t(), atom()) -> + {error, term()} | {ok, rebar_state:t(), atom()}. process_namespace(_State, as) -> {error, "Namespace 'as' is forbidden"}; process_namespace(State, Command) -> @@ -61,7 +77,15 @@ process_namespace(State, Command) -> {ok, rebar_state:namespace(State, default), Command} end. --spec process_command(rebar_state:t(), atom()) -> {ok, rebar_state:t()} | {error, string()} | {error, {module(), any()}}. +%% @doc Dispatches a given command based on the current state. +%% This requires mapping a command name to a specific provider. +%% `as' and `do' are still treated as special providers here. +%% Basic profile application may also be run. +%% +%% The function also takes care of expanding a provider to its +%% dependencies in the proper order. +-spec process_command(rebar_state:t(), atom()) -> + {ok, rebar_state:t()} | {error, string()} | {error, {module(), any()}}. process_command(State, Command) -> %% ? rebar_prv_install_deps:setup_env(State), Providers = rebar_state:providers(State), @@ -95,19 +119,24 @@ process_command(State, Command) -> State2 = rebar_state:command_parsed_args(State1, Args), do(TargetProviders, State2); {error, {invalid_option, Option}} -> - {error, io_lib:format("Invalid option ~s on task ~p", [Option, Command])}; + {error, io_lib:format("Invalid option ~ts on task ~p", [Option, Command])}; {error, {invalid_option_arg, {Option, Arg}}} -> - {error, io_lib:format("Invalid argument ~s to option ~s", [Arg, Option])}; + {error, io_lib:format("Invalid argument ~ts to option ~ts", [Arg, Option])}; {error, {missing_option_arg, Option}} -> - {error, io_lib:format("Missing argument to option ~s", [Option])} + {error, io_lib:format("Missing argument to option ~ts", [Option])} end end end. --spec do([{atom(), atom()}], rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()} | {error, {module(), any()}}. +%% @doc execute the selected providers. If a chain of providers +%% has been returned, run them one after the other, while piping +%% the state from the first into the next one. +-spec do([{atom(), atom()}], rebar_state:t()) -> + {ok, rebar_state:t()} | {error, string()} | {error, {module(), any()}}. do([], State) -> {ok, State}; do([ProviderName | Rest], State) -> + ?DEBUG("Provider: ~p", [ProviderName]), %% Special providers like 'as', 'do' or some hooks may be passed %% as a tuple {Namespace, Name}, otherwise not. Handle them %% on a per-need basis. @@ -128,8 +157,7 @@ do([ProviderName | Rest], State) -> {error, Error} -> {error, Error} catch - error:undef -> - Stack = erlang:get_stacktrace(), + ?WITH_STACKTRACE(error,undef,Stack) case Stack of [{ProviderName, do, [_], _}|_] -> %% This should really only happen if a plugin provider doesn't export do/1 @@ -142,7 +170,9 @@ do([ProviderName | Rest], State) -> {error, ProviderName} end. +%% @doc convert a given exception's payload into an io description. +-spec format_error(any()) -> iolist(). format_error({bad_provider_namespace, {Namespace, Name}}) -> - io_lib:format("Undefined command ~s in namespace ~s", [Name, Namespace]); + io_lib:format("Undefined command ~ts in namespace ~ts", [Name, Namespace]); format_error({bad_provider_namespace, Name}) -> - io_lib:format("Undefined command ~s", [Name]). + io_lib:format("Undefined command ~ts", [Name]). diff --git a/src/rebar_dialyzer_format.erl b/src/rebar_dialyzer_format.erl index b30c4dc..cb0e958 100644 --- a/src/rebar_dialyzer_format.erl +++ b/src/rebar_dialyzer_format.erl @@ -16,21 +16,22 @@ -include("rebar.hrl"). --export([format_warnings/1]). +-export([format_warnings/2]). %% Formats a list of warnings in a nice per file way. Note that we reverse %% the list at the end to 'undo' the reversal by foldl -format_warnings(Warnings) -> - {_, Res} = lists:foldl(fun format_warning_/2, {undefined, []}, Warnings), +format_warnings(Opts, Warnings) -> + Fold = fun(Warning, Acc) -> format_warning_(Opts, Warning, Acc) end, + {_, Res} = lists:foldl(Fold, {undefined, []}, Warnings), lists:reverse(Res). %% If the last seen file is and the file of this warning are the same %% we skip the file header -format_warning_(Warning = {_Tag, {File, Line}, Msg}, {File, Acc}) -> +format_warning_(_Opts, Warning = {_Tag, {File, Line}, Msg}, {File, Acc}) -> try String = message_to_string(Msg), - {File, [lists:flatten(fmt("~!c~4w~!!: ~s", [Line, String])) | Acc]} + {File, [lists:flatten(fmt("~!c~4w~!!: ~ts", [Line, String])) | Acc]} catch Error:Reason -> ?DEBUG("Failed to pretty format warning: ~p:~p", @@ -39,22 +40,23 @@ format_warning_(Warning = {_Tag, {File, Line}, Msg}, {File, Acc}) -> end; %% With a new file detencted we also write a file header. -format_warning_(Warning = {_Tag, {File, Line}, Msg}, {_LastFile, Acc}) -> +format_warning_(Opts, Warning = {_Tag, {SrcFile, Line}, Msg}, {_LastFile, Acc}) -> try + File = rebar_dir:format_source_file_name(SrcFile, Opts), Base = filename:basename(File), Dir = filename:dirname(File), Root = filename:rootname(Base), Ext = filename:extension(Base), - Path = re:replace(Dir, "^.*/_build/", "_build/", [{return, list}]), - Base1 = fmt("~!_c~s~!!~!__~s", [Root, Ext]), - F = fmt("~!__~s", [filename:join(Path, Base1)]), + Path = re:replace(Dir, "^.*/_build/", "_build/", [{return, list}, unicode]), + Base1 = fmt("~!_c~ts~!!~!__~ts", [Root, Ext]), + F = fmt("~!__~ts", [filename:join(Path, Base1)]), String = message_to_string(Msg), - {File, [lists:flatten(fmt("~n~s~n~!c~4w~!!: ~s", [F, Line, String])) | Acc]} + {SrcFile, [lists:flatten(fmt("~n~ts~n~!c~4w~!!: ~ts", [F, Line, String])) | Acc]} catch - Error:Reason -> + ?WITH_STACKTRACE(Error, Reason, Stacktrace) ?DEBUG("Failed to pretty format warning: ~p:~p~n~p", - [Error, Reason, erlang:get_stacktrace()]), - {File, [dialyzer:format_warning(Warning, fullpath) | Acc]} + [Error, Reason, Stacktrace]), + {SrcFile, [dialyzer:format_warning(Warning, fullpath) | Acc]} end. fmt(Fmt) -> @@ -70,53 +72,53 @@ fmt(Fmt, Args) -> %%----- Warnings for general discrepancies ---------------- message_to_string({apply, [Args, ArgNs, FailReason, SigArgs, SigRet, Contract]}) -> - fmt("~!^Fun application with arguments ~!!~s ", + fmt("~!^Fun application with arguments ~!!~ts ", [bad_arg(ArgNs, Args)]) ++ call_or_apply_to_string(ArgNs, FailReason, SigArgs, SigRet, Contract); message_to_string({app_call, [M, F, Args, Culprit, ExpectedType, FoundType]}) -> - fmt("~!^The call~!! ~s:~s~s ~!^requires that" - "~!! ~s ~!^is of type ~!g~s~!^ not ~!r~s", + fmt("~!^The call~!! ~ts:~ts~ts ~!^requires that" + "~!! ~ts ~!^is of type ~!g~ts~!^ not ~!r~ts", [M, F, Args, Culprit, ExpectedType, FoundType]); message_to_string({bin_construction, [Culprit, Size, Seg, Type]}) -> - fmt("~!^Binary construction will fail since the ~!b~s~!^ field~!!" - " ~s~!^ in segment~!! ~s~!^ has type~!! ~s", + fmt("~!^Binary construction will fail since the ~!b~ts~!^ field~!!" + " ~ts~!^ in segment~!! ~ts~!^ has type~!! ~ts", [Culprit, Size, Seg, Type]); message_to_string({call, [M, F, Args, ArgNs, FailReason, SigArgs, SigRet, Contract]}) -> - fmt("~!^The call~!! ~w:~w~s ", [M, F, bad_arg(ArgNs, Args)]) ++ + fmt("~!^The call~!! ~w:~w~ts ", [M, F, bad_arg(ArgNs, Args)]) ++ call_or_apply_to_string(ArgNs, FailReason, SigArgs, SigRet, Contract); message_to_string({call_to_missing, [M, F, A]}) -> fmt("~!^Call to missing or unexported function ~!!~w:~w/~w", [M, F, A]); message_to_string({exact_eq, [Type1, Op, Type2]}) -> - fmt("~!^The test ~!!~s ~s ~s~!^ can never evaluate to 'true'", + fmt("~!^The test ~!!~ts ~ts ~ts~!^ can never evaluate to 'true'", [Type1, Op, Type2]); message_to_string({fun_app_args, [Args, Type]}) -> - fmt("~!^Fun application with arguments ~!!~s~!^ will fail" - " since the function has type ~!!~s", [Args, Type]); + fmt("~!^Fun application with arguments ~!!~ts~!^ will fail" + " since the function has type ~!!~ts", [Args, Type]); message_to_string({fun_app_no_fun, [Op, Type, Arity]}) -> - fmt("~!^Fun application will fail since ~!!~s ~!^::~!! ~s" + fmt("~!^Fun application will fail since ~!!~ts ~!^::~!! ~ts" " is not a function of arity ~!!~w", [Op, Type, Arity]); message_to_string({guard_fail, []}) -> "~!^Clause guard cannot succeed.~!!"; message_to_string({guard_fail, [Arg1, Infix, Arg2]}) -> - fmt("~!^Guard test ~!!~s ~s ~s~!^ can never succeed", + fmt("~!^Guard test ~!!~ts ~ts ~ts~!^ can never succeed", [Arg1, Infix, Arg2]); message_to_string({neg_guard_fail, [Arg1, Infix, Arg2]}) -> - fmt("~!^Guard test not(~!!~s ~s ~s~!^) can never succeed", + fmt("~!^Guard test not(~!!~ts ~ts ~ts~!^) can never succeed", [Arg1, Infix, Arg2]); message_to_string({guard_fail, [Guard, Args]}) -> - fmt("~!^Guard test ~!!~w~s~!^ can never succeed", + fmt("~!^Guard test ~!!~w~ts~!^ can never succeed", [Guard, Args]); message_to_string({neg_guard_fail, [Guard, Args]}) -> - fmt("~!^Guard test not(~!!~w~s~!^) can never succeed", + fmt("~!^Guard test not(~!!~w~ts~!^) can never succeed", [Guard, Args]); message_to_string({guard_fail_pat, [Pat, Type]}) -> - fmt("~!^Clause guard cannot succeed. The ~!!~s~!^ was matched" - " against the type ~!!~s", [Pat, Type]); + fmt("~!^Clause guard cannot succeed. The ~!!~ts~!^ was matched" + " against the type ~!!~ts", [Pat, Type]); message_to_string({improper_list_constr, [TlType]}) -> fmt("~!^Cons will produce an improper list" - " since its ~!b2~!!nd~!^ argument is~!! ~s", [TlType]); + " since its ~!b2~!!nd~!^ argument is~!! ~ts", [TlType]); message_to_string({no_return, [Type|Name]}) -> NameString = case Name of @@ -124,59 +126,59 @@ message_to_string({no_return, [Type|Name]}) -> [F, A] -> fmt("~!^Function ~!r~w/~w ", [F, A]) end, case Type of - no_match -> fmt("~s~!^has no clauses that will ever match",[NameString]); - only_explicit -> fmt("~s~!^only terminates with explicit exception", [NameString]); - only_normal -> fmt("~s~!^has no local return", [NameString]); - both -> fmt("~s~!^has no local return", [NameString]) + no_match -> fmt("~ts~!^has no clauses that will ever match",[NameString]); + only_explicit -> fmt("~ts~!^only terminates with explicit exception", [NameString]); + only_normal -> fmt("~ts~!^has no local return", [NameString]); + both -> fmt("~ts~!^has no local return", [NameString]) end; message_to_string({record_constr, [RecConstr, FieldDiffs]}) -> - fmt("~!^Record construction ~!!~s~!^ violates the" - " declared type of field ~!!~s", [RecConstr, FieldDiffs]); + fmt("~!^Record construction ~!!~ts~!^ violates the" + " declared type of field ~!!~ts", [RecConstr, FieldDiffs]); message_to_string({record_constr, [Name, Field, Type]}) -> fmt("~!^Record construction violates the declared type for ~!!#~w{}~!^" - " since ~!!~s~!^ cannot be of type ~!!~s", + " since ~!!~ts~!^ cannot be of type ~!!~ts", [Name, Field, Type]); message_to_string({record_matching, [String, Name]}) -> - fmt("~!^The ~!!~s~!^ violates the" + fmt("~!^The ~!!~ts~!^ violates the" " declared type for ~!!#~w{}", [String, Name]); message_to_string({record_match, [Pat, Type]}) -> - fmt("~!^Matching of ~!!~s~!^ tagged with a record name violates the" - " declared type of ~!!~s", [Pat, Type]); + fmt("~!^Matching of ~!!~ts~!^ tagged with a record name violates the" + " declared type of ~!!~ts", [Pat, Type]); message_to_string({pattern_match, [Pat, Type]}) -> - fmt("~!^The ~s~!^ can never match the type ~!g~s", + fmt("~!^The ~ts~!^ can never match the type ~!g~ts", [bad_pat(Pat), Type]); message_to_string({pattern_match_cov, [Pat, Type]}) -> - fmt("~!^The ~s~!^ can never match since previous" - " clauses completely covered the type ~!g~s", + fmt("~!^The ~ts~!^ can never match since previous" + " clauses completely covered the type ~!g~ts", [bad_pat(Pat), Type]); message_to_string({unmatched_return, [Type]}) -> - fmt("~!^Expression produces a value of type ~!!~s~!^," + fmt("~!^Expression produces a value of type ~!!~ts~!^," " but this value is unmatched", [Type]); message_to_string({unused_fun, [F, A]}) -> fmt("~!^Function ~!r~w/~w~!!~!^ will never be called", [F, A]); %%----- Warnings for specs and contracts ------------------- message_to_string({contract_diff, [M, F, _A, Contract, Sig]}) -> - fmt("~!^Type specification ~!!~w:~w~s~!^" - " is not equal to the success typing: ~!!~w:~w~s", + fmt("~!^Type specification ~!!~w:~w~ts~!^" + " is not equal to the success typing: ~!!~w:~w~ts", [M, F, Contract, M, F, Sig]); message_to_string({contract_subtype, [M, F, _A, Contract, Sig]}) -> - fmt("~!^Type specification ~!!~w:~w~s~!^" - " is a subtype of the success typing: ~!!~w:~w~s", + fmt("~!^Type specification ~!!~w:~w~ts~!^" + " is a subtype of the success typing: ~!!~w:~w~ts", [M, F, Contract, M, F, Sig]); message_to_string({contract_supertype, [M, F, _A, Contract, Sig]}) -> - fmt("~!^Type specification ~!!~w:~w~s~!^" - " is a supertype of the success typing: ~!!~w:~w~s", + fmt("~!^Type specification ~!!~w:~w~ts~!^" + " is a supertype of the success typing: ~!!~w:~w~ts", [M, F, Contract, M, F, Sig]); message_to_string({contract_range, [Contract, M, F, ArgStrings, Line, CRet]}) -> - fmt("~!^The contract ~!!~w:~w~s~!^ cannot be right because the" - " inferred return for ~!!~w~s~!^ on line ~!!~w~!^ is ~!!~s", + fmt("~!^The contract ~!!~w:~w~ts~!^ cannot be right because the" + " inferred return for ~!!~w~ts~!^ on line ~!!~w~!^ is ~!!~ts", [M, F, Contract, F, ArgStrings, Line, CRet]); message_to_string({invalid_contract, [M, F, A, Sig]}) -> fmt("~!^Invalid type specification for function~!! ~w:~w/~w." - "~!^ The success typing is~!! ~s", [M, F, A, Sig]); + "~!^ The success typing is~!! ~ts", [M, F, A, Sig]); message_to_string({extra_range, [M, F, A, ExtraRanges, SigRange]}) -> fmt("~!^The specification for ~!!~w:~w/~w~!^ states that the function" - " might also return ~!!~s~!^ but the inferred return is ~!!~s", + " might also return ~!!~ts~!^ but the inferred return is ~!!~ts", [M, F, A, ExtraRanges, SigRange]); message_to_string({overlapping_contract, [M, F, A]}) -> fmt("~!^Overloaded contract for ~!!~w:~w/~w~!^ has overlapping" @@ -187,62 +189,62 @@ message_to_string({spec_missing_fun, [M, F, A]}) -> [M, F, A]); %%----- Warnings for opaque type violations ------------------- message_to_string({call_with_opaque, [M, F, Args, ArgNs, ExpArgs]}) -> - fmt("~!^The call ~!!~w:~w~s~!^ contains ~!!~s~!^ when ~!!~s", + fmt("~!^The call ~!!~w:~w~ts~!^ contains ~!!~ts~!^ when ~!!~ts", [M, F, bad_arg(ArgNs, Args), form_positions(ArgNs), form_expected(ExpArgs)]); message_to_string({call_without_opaque, [M, F, Args, [{N,_,_}|_] = ExpectedTriples]}) -> - fmt("~!^The call ~!!~w:~w~s ~!^does not have~!! ~s", + fmt("~!^The call ~!!~w:~w~ts ~!^does not have~!! ~ts", [M, F, bad_arg(N, Args), form_expected_without_opaque(ExpectedTriples)]); message_to_string({opaque_eq, [Type, _Op, OpaqueType]}) -> - fmt("~!^Attempt to test for equality between a term of type ~!!~s~!^" - " and a term of opaque type ~!!~s", [Type, OpaqueType]); + fmt("~!^Attempt to test for equality between a term of type ~!!~ts~!^" + " and a term of opaque type ~!!~ts", [Type, OpaqueType]); message_to_string({opaque_guard, [Arg1, Infix, Arg2, ArgNs]}) -> - fmt("~!^Guard test ~!!~s ~s ~s~!^ contains ~!!~s", + fmt("~!^Guard test ~!!~ts ~ts ~ts~!^ contains ~!!~ts", [Arg1, Infix, Arg2, form_positions(ArgNs)]); message_to_string({opaque_guard, [Guard, Args]}) -> - fmt("~!^Guard test ~!!~w~s~!^ breaks the opaqueness of its" + fmt("~!^Guard test ~!!~w~ts~!^ breaks the opaqueness of its" " argument", [Guard, Args]); message_to_string({opaque_match, [Pat, OpaqueType, OpaqueTerm]}) -> Term = if OpaqueType =:= OpaqueTerm -> "the term"; true -> OpaqueTerm end, - fmt("~!^The attempt to match a term of type ~!!~s~!^ against the" - "~!! ~s~!^ breaks the opaqueness of ~!!~s", + fmt("~!^The attempt to match a term of type ~!!~ts~!^ against the" + "~!! ~ts~!^ breaks the opaqueness of ~!!~ts", [OpaqueType, Pat, Term]); message_to_string({opaque_neq, [Type, _Op, OpaqueType]}) -> - fmt("~!^Attempt to test for inequality between a term of type ~!!~s" - "~!^ and a term of opaque type ~!!~s", [Type, OpaqueType]); + fmt("~!^Attempt to test for inequality between a term of type ~!!~ts" + "~!^ and a term of opaque type ~!!~ts", [Type, OpaqueType]); message_to_string({opaque_type_test, [Fun, Args, Arg, ArgType]}) -> - fmt("~!^The type test ~!!~s~s~!^ breaks the opaqueness of the term " - "~!!~s~s", [Fun, Args, Arg, ArgType]); + fmt("~!^The type test ~!!~ts~ts~!^ breaks the opaqueness of the term " + "~!!~ts~ts", [Fun, Args, Arg, ArgType]); message_to_string({opaque_size, [SizeType, Size]}) -> - fmt("~!^The size ~!!~s~!^ breaks the opaqueness of ~!!~s", + fmt("~!^The size ~!!~ts~!^ breaks the opaqueness of ~!!~ts", [SizeType, Size]); message_to_string({opaque_call, [M, F, Args, Culprit, OpaqueType]}) -> - fmt("~!^The call ~!!~s:~s~s~!^ breaks the opaqueness of the term~!!" - " ~s :: ~s", [M, F, Args, Culprit, OpaqueType]); + fmt("~!^The call ~!!~ts:~ts~ts~!^ breaks the opaqueness of the term~!!" + " ~ts :: ~ts", [M, F, Args, Culprit, OpaqueType]); %%----- Warnings for concurrency errors -------------------- message_to_string({race_condition, [M, F, Args, Reason]}) -> - fmt("~!^The call ~!!~w:~w~s ~s", [M, F, Args, Reason]); + fmt("~!^The call ~!!~w:~w~ts ~ts", [M, F, Args, Reason]); %%----- Warnings for behaviour errors -------------------- message_to_string({callback_type_mismatch, [B, F, A, ST, CT]}) -> - fmt("~!^The inferred return type of~!! ~w/~w (~s) ~!^" - "has nothing in common with~!! ~s, ~!^which is the expected" + fmt("~!^The inferred return type of~!! ~w/~w (~ts) ~!^" + "has nothing in common with~!! ~ts, ~!^which is the expected" " return type for the callback of~!! ~w ~!^behaviour", [F, A, ST, CT, B]); message_to_string({callback_arg_type_mismatch, [B, F, A, N, ST, CT]}) -> - fmt("~!^The inferred type for the~!! ~s ~!^argument of~!!" - " ~w/~w (~s) ~!^is not a supertype of~!! ~s~!^, which is" + fmt("~!^The inferred type for the~!! ~ts ~!^argument of~!!" + " ~w/~w (~ts) ~!^is not a supertype of~!! ~ts~!^, which is" "expected type for this argument in the callback of the~!! ~w " "~!^behaviour", [ordinal(N), F, A, ST, CT, B]); message_to_string({callback_spec_type_mismatch, [B, F, A, ST, CT]}) -> - fmt("~!^The return type ~!!~s~!^ in the specification of ~!!" - "~w/~w~!^ is not a subtype of ~!!~s~!^, which is the expected" + fmt("~!^The return type ~!!~ts~!^ in the specification of ~!!" + "~w/~w~!^ is not a subtype of ~!!~ts~!^, which is the expected" " return type for the callback of ~!!~w~!^ behaviour", [ST, F, A, CT, B]); message_to_string({callback_spec_arg_type_mismatch, [B, F, A, N, ST, CT]}) -> - fmt("~!^The specified type for the ~!!~s~!^ argument of ~!!" - "~w/~w (~s)~!^ is not a supertype of ~!!~s~!^, which is" + fmt("~!^The specified type for the ~!!~ts~!^ argument of ~!!" + "~w/~w (~ts)~!^ is not a supertype of ~!!~ts~!^, which is" " expected type for this argument in the callback of the ~!!~w" "~!^ behaviour", [ordinal(N), F, A, ST, CT, B]); message_to_string({callback_missing, [B, F, A]}) -> @@ -272,26 +274,26 @@ call_or_apply_to_string(ArgNs, FailReason, SigArgs, SigRet, true -> %% We do not know which argument(s) caused the failure fmt("~!^will never return since the success typing arguments" - " are ~!!~s", [SigArgs]); + " are ~!!~ts", [SigArgs]); false -> fmt("~!^will never return since it differs in the~!!" - " ~s ~!^argument from the success typing" - " arguments:~!! ~s", + " ~ts ~!^argument from the success typing" + " arguments:~!! ~ts", [PositionString, good_arg(ArgNs, SigArgs)]) end; only_contract -> case (ArgNs =:= []) orelse IsOverloaded of true -> %% We do not know which arguments caused the failure - fmt("~!^breaks the contract~!! ~s", [good_arg(ArgNs, Contract)]); + fmt("~!^breaks the contract~!! ~ts", [good_arg(ArgNs, Contract)]); false -> - fmt("~!^breaks the contract~!! ~s ~!^in the~!!" - " ~s ~!^argument", + fmt("~!^breaks the contract~!! ~ts ~!^in the~!!" + " ~ts ~!^argument", [good_arg(ArgNs, Contract), PositionString]) end; both -> fmt("~!^will never return since the success typing is " - "~!!~s ~!^->~!! ~s ~!^and the contract is ~!!~s", + "~!!~ts ~!^->~!! ~ts ~!^and the contract is ~!!~ts", [good_arg(ArgNs, SigArgs), SigRet, good_arg(ArgNs, Contract)]) end. @@ -299,8 +301,8 @@ call_or_apply_to_string(ArgNs, FailReason, SigArgs, SigRet, form_positions(ArgNs) -> ArgS = form_position_string(ArgNs), case ArgNs of - [_] -> fmt("~!^an opaque term as ~!!~s~!^ argument", [ArgS]); - [_,_|_] -> fmt("~!^opaque terms as ~!!~s~!^ arguments", [ArgS]) + [_] -> fmt("~!^an opaque term as ~!!~ts~!^ argument", [ArgS]); + [_,_|_] -> fmt("~!^opaque terms as ~!!~ts~!^ arguments", [ArgS]) end. %% We know which positions N are to blame; @@ -308,9 +310,9 @@ form_positions(ArgNs) -> form_expected_without_opaque([{N, T, TStr}]) -> FStr = case erl_types:t_is_opaque(T) of true -> - "~!^an opaque term of type~!g ~s ~!^as "; + "~!^an opaque term of type~!g ~ts ~!^as "; false -> - "~!^a term of type ~!g~s ~!^(with opaque subterms) as " + "~!^a term of type ~!g~ts ~!^(with opaque subterms) as " end ++ form_position_string([N]) ++ "~!^ argument", fmt(FStr, [TStr]); @@ -323,9 +325,9 @@ form_expected(ExpectedArgs) -> [T] -> TS = erl_types:t_to_string(T), case erl_types:t_is_opaque(T) of - true -> fmt("~!^an opaque term of type ~!!~s~!^ is" + true -> fmt("~!^an opaque term of type ~!!~ts~!^ is" " expected", [TS]); - false -> fmt("~!^a structured term of type ~!!~s~!^ is" + false -> fmt("~!^a structured term of type ~!!~ts~!^ is" " expected", [TS]) end; [_,_|_] -> fmt("~!^terms of different types are expected in these" @@ -338,7 +340,7 @@ form_position_string(ArgNs) -> [N1] -> ordinal(N1); [_,_|_] -> [Last|Prevs] = lists:reverse(ArgNs), - ", " ++ Head = lists:flatten([fmt(", ~s",[ordinal(N)]) || + ", " ++ Head = lists:flatten([fmt(", ~ts",[ordinal(N)]) || N <- lists:reverse(Prevs)]), Head ++ " and " ++ ordinal(Last) end. @@ -350,11 +352,11 @@ ordinal(N) when is_integer(N) -> fmt("~!B~w~!!th", [N]). %% Format a pattern ad highlight errorous part in red. bad_pat("pattern " ++ P) -> - fmt("pattern ~!r~s",[P]); + fmt("pattern ~!r~ts",[P]); bad_pat("variable " ++ P) -> - fmt("variable ~!r~s",[P]); + fmt("variable ~!r~ts",[P]); bad_pat(P) -> - fmt("~!r~s",[P]). + fmt("~!r~ts",[P]). bad_arg(N, Args) -> @@ -368,7 +370,7 @@ good_arg(N, Args) -> colour_arg(N, C, Args) when is_integer(N) -> colour_arg([N], C, Args); colour_arg(Ns, C, Args) -> - {Args1, Rest} =seperate_args(Args), + {Args1, Rest} =separate_args(Args), Args2 = highlight(Ns, 1, C, Args1), join_args(Args2) ++ Rest. @@ -376,53 +378,53 @@ highlight([], _N, _C, Rest) -> Rest; highlight([N | Nr], N, g, [Arg | Rest]) -> - [fmt("~!g~s", [Arg]) | highlight(Nr, N+1, g, Rest)]; + [fmt("~!g~ts", [Arg]) | highlight(Nr, N+1, g, Rest)]; highlight([N | Nr], N, r, [Arg | Rest]) -> - [fmt("~!r~s", [Arg]) | highlight(Nr, N+1, r, Rest)]; + [fmt("~!r~ts", [Arg]) | highlight(Nr, N+1, r, Rest)]; highlight(Ns, N, C, [Arg | Rest]) -> [Arg | highlight(Ns, N + 1, C, Rest)]. %% Arugments to functions and constraints are passed as %% strings not as data, this function pulls them apart -%% to allow interacting with them seperately and not +%% to allow interacting with them separately and not %% as one bug chunk of data. -seperate_args([$( | S]) -> - seperate_args([], S, "", []). +separate_args([$( | S]) -> + separate_args([], S, "", []). %% We strip this space since dialyzer is inconsistant in adding or not adding %% it .... -seperate_args([], [$,, $\s | R], Arg, Args) -> - seperate_args([], R, [], [lists:reverse(Arg) | Args]); +separate_args([], [$,, $\s | R], Arg, Args) -> + separate_args([], R, [], [lists:reverse(Arg) | Args]); -seperate_args([], [$, | R], Arg, Args) -> - seperate_args([], R, [], [lists:reverse(Arg) | Args]); +separate_args([], [$, | R], Arg, Args) -> + separate_args([], R, [], [lists:reverse(Arg) | Args]); -seperate_args([], [$) | Rest], Arg, Args) -> +separate_args([], [$) | Rest], Arg, Args) -> {lists:reverse([lists:reverse(Arg) | Args]), Rest}; -seperate_args([C | D], [C | R], Arg, Args) -> - seperate_args(D, R, [C | Arg], Args); +separate_args([C | D], [C | R], Arg, Args) -> + separate_args(D, R, [C | Arg], Args); %% Brackets -seperate_args(D, [${ | R], Arg, Args) -> - seperate_args([$}|D], R, [${ | Arg], Args); +separate_args(D, [${ | R], Arg, Args) -> + separate_args([$}|D], R, [${ | Arg], Args); -seperate_args(D, [$( | R], Arg, Args) -> - seperate_args([$)|D], R, [$( | Arg], Args); +separate_args(D, [$( | R], Arg, Args) -> + separate_args([$)|D], R, [$( | Arg], Args); -seperate_args(D, [$[ | R], Arg, Args) -> - seperate_args([$]|D], R, [$[ | Arg], Args); +separate_args(D, [$[ | R], Arg, Args) -> + separate_args([$]|D], R, [$[ | Arg], Args); -seperate_args(D, [$< | R], Arg, Args) -> - seperate_args([$>|D], R, [$< | Arg], Args); +separate_args(D, [$< | R], Arg, Args) -> + separate_args([$>|D], R, [$< | Arg], Args); %% 'strings' -seperate_args(D, [$' | R], Arg, Args) -> - seperate_args([$'|D], R, [$' | Arg], Args); -seperate_args(D, [$" | R], Arg, Args) -> - seperate_args([$"|D], R, [$" | Arg], Args); +separate_args(D, [$' | R], Arg, Args) -> + separate_args([$'|D], R, [$' | Arg], Args); +separate_args(D, [$" | R], Arg, Args) -> + separate_args([$"|D], R, [$" | Arg], Args); -seperate_args(D, [C | R], Arg, Args) -> - seperate_args(D, R, [C | Arg], Args). +separate_args(D, [C | R], Arg, Args) -> + separate_args(D, R, [C | Arg], Args). join_args(Args) -> - [$(, string:join(Args, ", "), $)]. + [$(, rebar_string:join(Args, ", "), $)]. diff --git a/src/rebar_digraph.erl b/src/rebar_digraph.erl index 363253a..776d7b8 100644 --- a/src/rebar_digraph.erl +++ b/src/rebar_digraph.erl @@ -1,3 +1,5 @@ +%%% @doc build a digraph of applications in order to figure out dependency +%%% and compile order. -module(rebar_digraph). -export([compile_order/1 @@ -7,7 +9,9 @@ -include("rebar.hrl"). -%% Sort apps with topological sort to get proper build order +%% @doc Sort apps with topological sort to get proper build order +-spec compile_order([rebar_app_info:t()]) -> + {ok, [rebar_app_info:t()]} | {error, no_sort | {cycles, [[binary(),...]]}}. compile_order(Apps) -> Graph = digraph:new(), lists:foreach(fun(App) -> @@ -33,6 +37,11 @@ compile_order(Apps) -> true = digraph:delete(Graph), Order. +%% @private Add a package and its dependencies to an existing digraph +-spec add(digraph:graph(), {PkgName, [Dep]}) -> ok when + PkgName :: binary(), + Dep :: {Name, term()} | Name, + Name :: atom() | iodata(). add(Graph, {PkgName, Deps}) -> case digraph:vertex(Graph, PkgName) of false -> @@ -44,9 +53,9 @@ add(Graph, {PkgName, Deps}) -> lists:foreach(fun(DepName) -> Name1 = case DepName of {Name, _Vsn} -> - ec_cnv:to_binary(Name); + rebar_utils:to_binary(Name); Name -> - ec_cnv:to_binary(Name) + rebar_utils:to_binary(Name) end, V3 = case digraph:vertex(Graph, Name1) of false -> @@ -57,6 +66,8 @@ add(Graph, {PkgName, Deps}) -> digraph:add_edge(Graph, V, V3) end, Deps). +%% @doc based on a list of vertices and edges, build a digraph. +-spec restore_graph({[digraph:vertex()], [digraph:edge()]}) -> digraph:graph(). restore_graph({Vs, Es}) -> Graph = digraph:new(), lists:foreach(fun({V, LastUpdated}) -> @@ -67,6 +78,8 @@ restore_graph({Vs, Es}) -> end, Es), Graph. +%% @doc convert a given exception's payload into an io description. +-spec format_error(any()) -> iolist(). format_error(no_solution) -> io_lib:format("No solution for packages found.", []). @@ -74,22 +87,27 @@ format_error(no_solution) -> %% Internal Functions %%==================================================================== +%% @doc alias for `digraph_utils:subgraph/2'. subgraph(Graph, Vertices) -> digraph_utils:subgraph(Graph, Vertices). +%% @private from a list of app names, fetch the proper app info records +%% for them. -spec names_to_apps([atom()], [rebar_app_info:t()]) -> [rebar_app_info:t()]. names_to_apps(Names, Apps) -> [element(2, App) || App <- [find_app_by_name(Name, Apps) || Name <- Names], App =/= error]. +%% @private fetch the proper app info record for a given app name. -spec find_app_by_name(atom(), [rebar_app_info:t()]) -> {ok, rebar_app_info:t()} | error. find_app_by_name(Name, Apps) -> ec_lists:find(fun(App) -> rebar_app_info:name(App) =:= Name end, Apps). -%% The union of all entries in the applications list for an app and +%% @private The union of all entries in the applications list for an app and %% the deps listed in its rebar.config is all deps that may be needed %% for building the app. +-spec all_apps_deps(rebar_app_info:t()) -> [binary()]. all_apps_deps(App) -> Applications = lists:usort([atom_to_binary(X, utf8) || X <- rebar_app_info:applications(App)]), Deps = lists:usort(lists:map(fun({Name, _}) -> Name; (Name) -> Name end, rebar_app_info:deps(App))), diff --git a/src/rebar_dir.erl b/src/rebar_dir.erl index 1ec58d4..17bc48e 100644 --- a/src/rebar_dir.erl +++ b/src/rebar_dir.erl @@ -1,3 +1,4 @@ +%%% @doc utility functions for directory and path handling of all kind. -module(rebar_dir). -export([base_dir/1, @@ -22,19 +23,25 @@ processing_base_dir/2, make_relative_path/2, src_dirs/1, src_dirs/2, + src_dir_opts/2, recursive/2, extra_src_dirs/1, extra_src_dirs/2, all_src_dirs/1, all_src_dirs/3, - retarget_path/2]). + retarget_path/2, + format_source_file_name/2]). -include("rebar.hrl"). +%% @doc returns the directory root for build artifacts +%% for the current profile, such as `_build/default/'. -spec base_dir(rebar_state:t()) -> file:filename_all(). base_dir(State) -> profile_dir(rebar_state:opts(State), rebar_state:current_profiles(State)). +%% @doc returns the directory root for build artifacts for a given set +%% of profiles. -spec profile_dir(rebar_dict(), [atom()]) -> file:filename_all(). profile_dir(Opts, Profiles) -> - {BaseDir, ProfilesStrings} = case [ec_cnv:to_list(P) || P <- Profiles] of + {BaseDir, ProfilesStrings} = case [rebar_utils:to_list(P) || P <- Profiles] of ["global" | _] -> {?MODULE:global_cache_dir(Opts), [""]}; ["bootstrap", "default"] -> {rebar_opts:get(Opts, base_dir, ?DEFAULT_BASE_DIR), ["default"]}; ["default"] -> {rebar_opts:get(Opts, base_dir, ?DEFAULT_BASE_DIR), ["default"]}; @@ -42,28 +49,39 @@ profile_dir(Opts, Profiles) -> %% of profiles to match order passed to `as` ["default"|Rest] -> {rebar_opts:get(Opts, base_dir, ?DEFAULT_BASE_DIR), Rest} end, - ProfilesDir = string:join(ProfilesStrings, "+"), + ProfilesDir = rebar_string:join(ProfilesStrings, "+"), filename:join(BaseDir, ProfilesDir). +%% @doc returns the directory where dependencies should be placed +%% given the current profile. -spec deps_dir(rebar_state:t()) -> file:filename_all(). deps_dir(State) -> filename:join(base_dir(State), rebar_state:get(State, deps_dir, ?DEFAULT_DEPS_DIR)). +%% @doc returns the directory where a dependency should be placed +%% given the current profile, based on its app name. Expects to be passed +%% the result of `deps_dir/1' as a first argument. -spec deps_dir(file:filename_all(), file:filename_all()) -> file:filename_all(). deps_dir(DepsDir, App) -> filename:join(DepsDir, App). +%% @doc returns the absolute path for the project root (by default, +%% the current working directory for the currently running escript). root_dir(State) -> filename:absname(rebar_state:get(State, root_dir, ?DEFAULT_ROOT_DIR)). +%% @doc returns the expected location of the `_checkouts' directory. -spec checkouts_dir(rebar_state:t()) -> file:filename_all(). checkouts_dir(State) -> filename:join(root_dir(State), rebar_state:get(State, checkouts_dir, ?DEFAULT_CHECKOUTS_DIR)). +%% @doc returns the expected location of a given app in the checkouts +%% directory for the project. -spec checkouts_dir(rebar_state:t(), file:filename_all()) -> file:filename_all(). checkouts_dir(State, App) -> filename:join(checkouts_dir(State), App). +%% @doc Returns the directory where plugins are located. -spec plugins_dir(rebar_state:t()) -> file:filename_all(). plugins_dir(State) -> case lists:member(global, rebar_state:current_profiles(State)) of @@ -73,33 +91,50 @@ plugins_dir(State) -> filename:join(base_dir(State), rebar_state:get(State, plugins_dir, ?DEFAULT_PLUGINS_DIR)) end. +%% @doc returns the list of relative path where the project applications can +%% be located. -spec lib_dirs(rebar_state:t()) -> file:filename_all(). lib_dirs(State) -> rebar_state:get(State, project_app_dirs, ?DEFAULT_PROJECT_APP_DIRS). +%% @doc returns the user's home directory. +-spec home_dir() -> file:filename_all(). home_dir() -> {ok, [[Home]]} = init:get_argument(home), Home. +%% @doc returns the directory where the global configuration files for rebar3 +%% may be stored. +-spec global_config_dir(rebar_state:t()) -> file:filename_all(). global_config_dir(State) -> Home = home_dir(), rebar_state:get(State, global_rebar_dir, filename:join([Home, ".config", "rebar3"])). +%% @doc returns the path of the global rebar.config file +-spec global_config(rebar_state:t()) -> file:filename_all(). global_config(State) -> filename:join(global_config_dir(State), "rebar.config"). +%% @doc returns the default path of the global rebar.config file +-spec global_config() -> file:filename_all(). global_config() -> Home = home_dir(), filename:join([Home, ".config", "rebar3", "rebar.config"]). +%% @doc returns the location for the global cache directory -spec global_cache_dir(rebar_dict()) -> file:filename_all(). global_cache_dir(Opts) -> Home = home_dir(), rebar_opts:get(Opts, global_rebar_dir, filename:join([Home, ".cache", "rebar3"])). +%% @doc appends the cache directory to the path passed to this function. +-spec local_cache_dir(file:filename_all()) -> file:filename_all(). local_cache_dir(Dir) -> filename:join(Dir, ".rebar3"). +%% @doc returns the current working directory, with some specific +%% conversions and handling done to be cross-platform compatible. +-spec get_cwd() -> file:filename_all(). get_cwd() -> {ok, Dir} = file:get_cwd(), %% On windows cwd may return capital letter for drive, @@ -108,20 +143,33 @@ get_cwd() -> %% cwd as soon as it possible. filename:join([Dir]). +%% @doc returns the file location for the global template +%% configuration variables file. +-spec template_globals(rebar_state:t()) -> file:filename_all(). template_globals(State) -> filename:join([global_config_dir(State), "templates", "globals"]). +%% @doc returns the location for the global template directory +-spec template_dir(rebar_state:t()) -> file:filename_all(). template_dir(State) -> filename:join([global_config_dir(State), "templates"]). +%% @doc checks if the current working directory is the base directory +%% for the project. +-spec processing_base_dir(rebar_state:t()) -> boolean(). processing_base_dir(State) -> Cwd = get_cwd(), processing_base_dir(State, Cwd). +%% @doc checks if the passed in directory is the base directory for +%% the project. +-spec processing_base_dir(rebar_state:t(), file:filename()) -> boolean(). processing_base_dir(State, Dir) -> AbsDir = filename:absname(Dir), AbsDir =:= rebar_state:get(State, base_dir). +%% @doc make a path absolute +-spec make_absolute_path(file:filename()) -> file:filename(). make_absolute_path(Path) -> case filename:pathtype(Path) of absolute -> @@ -135,73 +183,151 @@ make_absolute_path(Path) -> filename:join([Dir, Path]) end. +%% @doc normalizing a path removes all of the `..' and the +%% `.' segments it may contain. +-spec make_normalized_path(file:filename()) -> file:filename(). make_normalized_path(Path) -> AbsPath = make_absolute_path(Path), Components = filename:split(AbsPath), make_normalized_path(Components, []). +%% @private drops path fragments for normalization +-spec make_normalized_path([string()], [string()]) -> file:filename(). make_normalized_path([], NormalizedPath) -> filename:join(lists:reverse(NormalizedPath)); make_normalized_path([H|T], NormalizedPath) -> case H of + "." when NormalizedPath == [], T == [] -> make_normalized_path(T, ["."]); "." -> make_normalized_path(T, NormalizedPath); - ".." -> make_normalized_path(T, tl(NormalizedPath)); + ".." when NormalizedPath == [] -> make_normalized_path(T, [".."]); + ".." when hd(NormalizedPath) =/= ".." -> make_normalized_path(T, tl(NormalizedPath)); _ -> make_normalized_path(T, [H|NormalizedPath]) end. +%% @doc take a source and a target path, and relativize the target path +%% onto the source. +%% +%% Example: +%% ``` +%% 1> rebar_dir:make_relative_path("a/b/c/d/file", "a/b/file"). +%% "c/d/file" +%% 2> rebar_dir:make_relative_path("a/b/file", "a/b/c/d/file"). +%% "../../file" +%% ''' +-spec make_relative_path(file:filename(), file:filename()) -> file:filename(). make_relative_path(Source, Target) -> AbsSource = make_normalized_path(Source), AbsTarget = make_normalized_path(Target), do_make_relative_path(filename:split(AbsSource), filename:split(AbsTarget)). +%% @private based on fragments of paths, replace the number of common +%% segments by `../' bits, and add the rest of the source alone after it +-spec do_make_relative_path([string()], [string()]) -> file:filename(). do_make_relative_path([H|T1], [H|T2]) -> do_make_relative_path(T1, T2); do_make_relative_path(Source, Target) -> Base = lists:duplicate(max(length(Target) - 1, 0), ".."), filename:join(Base ++ Source). +%%% @doc +%%% `src_dirs' and `extra_src_dirs' can be configured with options +%%% like this: +%%% ``` +%%% {src_dirs,[{"foo",[{recursive,false}]}]} +%%% {extra_src_dirs,[{"bar",[recursive]}]} (equivalent to {recursive,true}) +%%% ''' +%%% `src_dirs/1,2' and `extra_src_dirs/1,2' return only the list of +%%% directories for the `src_dirs' and `extra_src_dirs' options +%%% respectively, while `src_dirs_opts/2' returns the options list for +%%% the given directory, no matter if it is configured as `src_dirs' or +%%% `extra_src_dirs'. -spec src_dirs(rebar_dict()) -> list(file:filename_all()). src_dirs(Opts) -> src_dirs(Opts, []). +%% @doc same as `src_dirs/1', but allows to pass in a list of default options. -spec src_dirs(rebar_dict(), list(file:filename_all())) -> list(file:filename_all()). src_dirs(Opts, Default) -> - ErlOpts = rebar_opts:erl_opts(Opts), - Vs = proplists:get_all_values(src_dirs, ErlOpts), - case lists:append([rebar_opts:get(Opts, src_dirs, []) | Vs]) of - [] -> Default; - Dirs -> lists:usort(Dirs) - end. + src_dirs(src_dirs, Opts, Default). +%% @doc same as `src_dirs/1', but for the `extra_src_dirs' options -spec extra_src_dirs(rebar_dict()) -> list(file:filename_all()). extra_src_dirs(Opts) -> extra_src_dirs(Opts, []). +%% @doc same as `src_dirs/2', but for the `extra_src_dirs' options -spec extra_src_dirs(rebar_dict(), list(file:filename_all())) -> list(file:filename_all()). extra_src_dirs(Opts, Default) -> + src_dirs(extra_src_dirs, Opts, Default). + +%% @private agnostic version of src_dirs and extra_src_dirs. +src_dirs(Type, Opts, Default) -> + lists:usort([ + case D0 of + {D,_} -> normalize_relative_path(D); + _ -> normalize_relative_path(D0) + end || D0 <- raw_src_dirs(Type,Opts,Default)]). + +%% @private extracts the un-formatted src_dirs or extra_src_dirs +%% options as configured. +raw_src_dirs(Type, Opts, Default) -> ErlOpts = rebar_opts:erl_opts(Opts), - Vs = proplists:get_all_values(extra_src_dirs, ErlOpts), - case lists:append([rebar_opts:get(Opts, extra_src_dirs, []) | Vs]) of + Vs = proplists:get_all_values(Type, ErlOpts), + case lists:append([rebar_opts:get(Opts, Type, []) | Vs]) of [] -> Default; - Dirs -> lists:usort(Dirs) + Dirs -> Dirs end. +%% @private normalizes relative paths so that ./a/b/c/ => a/b/c +normalize_relative_path(Path) -> + make_normalized_path(filename:split(Path), []). + +%% @doc returns all the source directories (`src_dirs' and +%% `extra_src_dirs'). -spec all_src_dirs(rebar_dict()) -> list(file:filename_all()). all_src_dirs(Opts) -> all_src_dirs(Opts, [], []). +%% @doc returns all the source directories (`src_dirs' and +%% `extra_src_dirs') while being able to configure defaults for both. -spec all_src_dirs(rebar_dict(), list(file:filename_all()), list(file:filename_all())) -> list(file:filename_all()). all_src_dirs(Opts, SrcDefault, ExtraDefault) -> lists:usort(src_dirs(Opts, SrcDefault) ++ extra_src_dirs(Opts, ExtraDefault)). -%% given a path if that path is an ancestor of an app dir return the path relative to that -%% apps outdir. if the path is not an ancestor to any app dirs but is an ancestor of the -%% project root return the path relative to the project base_dir. if it is not an ancestor +%%% @doc +%%% Return the list of options for the given src directory +%%% If the same option is given multiple times for a directory in the +%%% config, the priority order is: first occurence of `src_dirs' +%%% followed by first occurence of `extra_src_dirs'. +-spec src_dir_opts(rebar_dict(), file:filename_all()) -> [{atom(),term()}]. +src_dir_opts(Opts, Dir) -> + RawSrcDirs = raw_src_dirs(src_dirs, Opts, []), + RawExtraSrcDirs = raw_src_dirs(extra_src_dirs, Opts, []), + AllOpts = [Opt || {D, Opt} <- RawSrcDirs++RawExtraSrcDirs, D==Dir], + lists:ukeysort(1, proplists:unfold(lists:append(AllOpts))). + +%%% @doc +%%% Return the value of the 'recursive' option for the given directory. +%%% If not given, the value of 'recursive' in the 'erlc_compiler' +%%% options is used, and finally the default is 'true'. +-spec recursive(rebar_dict(), file:filename_all()) -> boolean(). +recursive(Opts, Dir) -> + DirOpts = src_dir_opts(Opts, Dir), + Default = proplists:get_value(recursive, + rebar_opts:get(Opts, erlc_compiler, []), + true), + R = proplists:get_value(recursive, DirOpts, Default), + R. + +%% @doc given a path if that path is an ancestor of an app dir, return the path relative to that +%% apps outdir. If the path is not an ancestor to any app dirs but is an ancestor of the +%% project root, return the path relative to the project base_dir. If it is not an ancestor %% of either return it unmodified -spec retarget_path(rebar_state:t(), string()) -> string(). - retarget_path(State, Path) -> ProjectApps = rebar_state:project_apps(State), retarget_path(State, Path, ProjectApps). +%% @private worker for retarget_path/2 +%% @end %% not relative to any apps in project, check to see it's relative to %% project root retarget_path(State, Path, []) -> @@ -217,3 +343,39 @@ retarget_path(State, Path, [App|Rest]) -> {ok, NewPath} -> filename:join([rebar_app_info:out_dir(App), NewPath]); {error, badparent} -> retarget_path(State, Path, Rest) end. + +format_source_file_name(Path, Opts) -> + Type = case rebar_opts:get(Opts, compiler_source_format, + ?DEFAULT_COMPILER_SOURCE_FORMAT) of + V when V == absolute; V == relative; V == build -> + V; + Other -> + warn_source_format_once(Other) + end, + case Type of + absolute -> resolve_linked_source(Path); + build -> Path; + relative -> + Cwd = rebar_dir:get_cwd(), + rebar_dir:make_relative_path(resolve_linked_source(Path), Cwd) + end. + +%% @private displays a warning for the compiler source format option +%% only once +-spec warn_source_format_once(term()) -> ok. +warn_source_format_once(Format) -> + Warn = application:get_env(rebar, warn_source_format) =/= {ok, false}, + application:set_env(rebar, warn_source_format, false), + case Warn of + false -> + ok; + true -> + ?WARN("Invalid argument ~p for compiler_source_format - " + "assuming ~ts~n", [Format, ?DEFAULT_COMPILER_SOURCE_FORMAT]) + end. + +%% @private takes a filename and canonicalizes its path if it is a link. +-spec resolve_linked_source(file:filename()) -> file:filename(). +resolve_linked_source(Src) -> + {Dir, Base} = rebar_file_utils:split_dirname(Src), + filename:join(rebar_file_utils:resolve_link(Dir), Base). diff --git a/src/rebar_dist_utils.erl b/src/rebar_dist_utils.erl index f462826..5de858e 100644 --- a/src/rebar_dist_utils.erl +++ b/src/rebar_dist_utils.erl @@ -7,6 +7,9 @@ %%%%%%%%%%%%%%%%%% %%% PUBLIC API %%% %%%%%%%%%%%%%%%%%% + +%% @doc allows to pick whether to use a short or long name, and +%% starts the distributed mode for it. -spec either(Name::atom(), SName::atom(), Opts::[{setcookie,term()}]) -> atom(). either(undefined, undefined, _) -> 'nonode@nohost'; @@ -19,13 +22,19 @@ either(undefined, SName, Opts) -> either(_, _, _) -> ?ABORT("Cannot have both short and long node names defined", []). +%% @doc starts a node with a short name. +-spec short(SName::atom(), Opts::[{setcookie,term()}]) -> term(). short(Name, Opts) -> start(Name, shortnames, Opts). +%% @doc starts a node with a long name. +-spec long(Name::atom(), Opts::[{setcookie,term()}]) -> term(). long(Name, Opts) -> start(Name, longnames, Opts). --spec find_options(rebar_state:state()) -> {Long, Short, Opts} when +%% @doc utility function to extract all distribution options +%% from a rebar3 state tuple. +-spec find_options(rebar_state:t()) -> {Long, Short, Opts} when Long :: atom(), Short :: atom(), Opts :: [{setcookie,term()}]. @@ -42,14 +51,27 @@ find_options(State) -> %%% PRIVATE %%% %%%%%%%%%%%%%%% start(Name, Type, Opts) -> - check_epmd(net_kernel:start([Name, Type])), + case dist_up(net_kernel:start([Name, Type])) of + false -> + start_epmd(), + dist_up(net_kernel:start([Name, Type])) orelse warn_dist(); + true -> + ok + end, setup_cookie(Opts). -check_epmd({error,{{shutdown, {_,net_kernel,{'EXIT',nodistribution}}},_}}) -> - ?ERROR("Erlang Distribution failed, falling back to nonode@nohost. " - "Verify that epmd is running and try again.",[]); -check_epmd(_) -> - ok. +dist_up({error,{{shutdown,{_,net_kernel,{'EXIT',nodistribution}}},_}}) -> false; +dist_up(_) -> true. + +start_epmd() -> + %% Indirectly boot EPMD through calling Erlang so that we don't risk + %% attaching it to the current proc + ?CONSOLE("Attempting to start epmd...", []), + os:cmd("erl -sname a -eval 'halt(0).'"). + +warn_dist() -> + ?ERROR("Erlang Distribution failed, falling back to nonode@nohost.", []). + setup_cookie(Opts) -> case {node(), proplists:get_value(setcookie, Opts, nocookie)} of diff --git a/src/rebar_env.erl b/src/rebar_env.erl new file mode 100644 index 0000000..e9adafb --- /dev/null +++ b/src/rebar_env.erl @@ -0,0 +1,86 @@ +-module(rebar_env). + +-export([create_env/1, + create_env/2]). + +-include("rebar.hrl"). + +%% @doc The following environment variables are exported when running +%% a hook (absolute paths): +%% +%% REBAR_DEPS_DIR = rebar_dir:deps_dir/1 +%% REBAR_BUILD_DIR = rebar_dir:base_dir/1 +%% REBAR_ROOT_DIR = rebar_dir:root_dir/1 +%% REBAR_CHECKOUTS_DIR = rebar_dir:checkouts_dir/1 +%% REBAR_PLUGINS_DIR = rebar_dir:plugins_dir/1 +%% REBAR_GLOBAL_CONFIG_DIR = rebar_dir:global_config_dir/1 +%% REBAR_GLOBAL_CACHE_DIR = rebar_dir:global_cache_dir/1 +%% REBAR_TEMPLATE_DIR = rebar_dir:template_dir/1 +%% REBAR_APP_DIRS = rebar_dir:lib_dirs/1 +%% REBAR_SRC_DIRS = rebar_dir:src_dirs/1 +%% +%% autoconf compatible variables +%% (see: http://www.gnu.org/software/autoconf/manual/autoconf.html#Erlang-Libraries): +%% ERLANG_ERTS_VER = erlang:system_info(version) +%% ERLANG_ROOT_DIR = code:root_dir/0 +%% ERLANG_LIB_DIR_erl_interface = code:lib_dir(erl_interface) +%% ERLANG_LIB_VER_erl_interface = version part of path returned by code:lib_dir(erl_interface) +%% ERL = ERLANG_ROOT_DIR/bin/erl +%% ERLC = ERLANG_ROOT_DIR/bin/erl +%% + +-spec create_env(rebar_state:t()) -> proplists:proplist(). +create_env(State) -> + Opts = rebar_state:opts(State), + create_env(State, Opts). + +-spec create_env(rebar_state:t(), rebar_dict()) -> proplists:proplist(). +create_env(State, Opts) -> + BaseDir = rebar_dir:base_dir(State), + EnvVars = [ + {"REBAR_DEPS_DIR", filename:absname(rebar_dir:deps_dir(State))}, + {"REBAR_BUILD_DIR", filename:absname(rebar_dir:base_dir(State))}, + {"REBAR_ROOT_DIR", filename:absname(rebar_dir:root_dir(State))}, + {"REBAR_CHECKOUTS_DIR", filename:absname(rebar_dir:checkouts_dir(State))}, + {"REBAR_PLUGINS_DIR", filename:absname(rebar_dir:plugins_dir(State))}, + {"REBAR_GLOBAL_CONFIG_DIR", filename:absname(rebar_dir:global_config_dir(State))}, + {"REBAR_GLOBAL_CACHE_DIR", filename:absname(rebar_dir:global_cache_dir(Opts))}, + {"REBAR_TEMPLATE_DIR", filename:absname(rebar_dir:template_dir(State))}, + {"REBAR_APP_DIRS", join_dirs(BaseDir, rebar_dir:lib_dirs(State))}, + {"REBAR_SRC_DIRS", join_dirs(BaseDir, rebar_dir:all_src_dirs(Opts))}, + {"ERLANG_ERTS_VER", erlang:system_info(version)}, + {"ERLANG_ROOT_DIR", code:root_dir()}, + {"ERL", filename:join([code:root_dir(), "bin", "erl"])}, + {"ERLC", filename:join([code:root_dir(), "bin", "erlc"])}, + {"ERLANG_ARCH" , rebar_api:wordsize()}, + {"ERLANG_TARGET", rebar_api:get_arch()} + ], + EInterfaceVars = create_erl_interface_env(), + lists:append([EnvVars, EInterfaceVars]). + +-spec create_erl_interface_env() -> list(). +create_erl_interface_env() -> + case code:lib_dir(erl_interface) of + {error, bad_name} -> + ?WARN("erl_interface is missing. ERLANG_LIB_DIR_erl_interface and " + "ERLANG_LIB_VER_erl_interface will not be added to the environment.", []), + []; + Dir -> + [ + {"ERLANG_LIB_DIR_erl_interface", Dir}, + {"ERLANG_LIB_VER_erl_interface", re_version(Dir)} + ] + end. + +%% ==================================================================== +%% Internal functions +%% ==================================================================== + +join_dirs(BaseDir, Dirs) -> + rebar_string:join([filename:join(BaseDir, Dir) || Dir <- Dirs], ":"). + +re_version(Path) -> + case re:run(Path, "^.*-(?<VER>[^/-]*)$", [{capture,[1],list}, unicode]) of + nomatch -> ""; + {match, [Ver]} -> Ver + end. diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index 167f2bb..e52791c 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -47,12 +47,8 @@ -type compile_opts() :: [compile_opt()]. -type compile_opt() :: {recursive, boolean()}. --record(compile_opts, { - recursive = true -}). - -define(DEFAULT_OUTDIR, "ebin"). --define(RE_PREFIX, "^[^._]"). +-define(RE_PREFIX, "^(?!\\._)"). %% =================================================================== %% Public API @@ -88,34 +84,38 @@ %% 'old_inets'}]}. %% -%% @equiv compile(AppInfo, []). - +%% @equiv compile(AppInfo, []) -spec compile(rebar_app_info:t()) -> ok. compile(AppInfo) when element(1, AppInfo) == app_info_t -> compile(AppInfo, []). %% @doc compile an individual application. - -spec compile(rebar_app_info:t(), compile_opts()) -> ok. compile(AppInfo, CompileOpts) when element(1, AppInfo) == app_info_t -> - Dir = ec_cnv:to_list(rebar_app_info:out_dir(AppInfo)), + warn_deprecated(), + Dir = rebar_utils:to_list(rebar_app_info:out_dir(AppInfo)), RebarOpts = rebar_app_info:opts(AppInfo), + SrcOpts = [check_last_mod, + {recursive, dir_recursive(RebarOpts, "src", CompileOpts)}], + MibsOpts = [check_last_mod, + {recursive, dir_recursive(RebarOpts, "mibs", CompileOpts)}], + rebar_base_compiler:run(RebarOpts, check_files([filename:join(Dir, File) || File <- rebar_opts:get(RebarOpts, xrl_first_files, [])]), filename:join(Dir, "src"), ".xrl", filename:join(Dir, "src"), ".erl", - fun compile_xrl/3), + fun compile_xrl/3, SrcOpts), rebar_base_compiler:run(RebarOpts, check_files([filename:join(Dir, File) || File <- rebar_opts:get(RebarOpts, yrl_first_files, [])]), filename:join(Dir, "src"), ".yrl", filename:join(Dir, "src"), ".erl", - fun compile_yrl/3), + fun compile_yrl/3, SrcOpts), rebar_base_compiler:run(RebarOpts, check_files([filename:join(Dir, File) || File <- rebar_opts:get(RebarOpts, mib_first_files, [])]), filename:join(Dir, "mibs"), ".mib", filename:join([Dir, "priv", "mibs"]), ".bin", - compile_mib(AppInfo)), + compile_mib(AppInfo), MibsOpts), SrcDirs = lists:map(fun(SrcDir) -> filename:join(Dir, SrcDir) end, rebar_dir:src_dirs(RebarOpts, ["src"])), @@ -149,6 +149,7 @@ compile(RebarOpts, BaseDir, OutDir) -> compile(State, BaseDir, OutDir, CompileOpts) when element(1, State) == state_t -> compile(rebar_state:opts(State), BaseDir, OutDir, CompileOpts); compile(RebarOpts, BaseDir, OutDir, CompileOpts) -> + warn_deprecated(), SrcDirs = lists:map(fun(SrcDir) -> filename:join(BaseDir, SrcDir) end, rebar_dir:src_dirs(RebarOpts, ["src"])), compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, CompileOpts), @@ -162,16 +163,14 @@ compile(RebarOpts, BaseDir, OutDir, CompileOpts) -> end, lists:foreach(F, lists:map(fun(SrcDir) -> filename:join(BaseDir, SrcDir) end, ExtraDirs)). -%% @equiv compile_dirs(Context, BaseDir, [Dir], Dir, [{recursive, false}]). - +%% @equiv compile_dirs(Context, BaseDir, [Dir], Dir, [{recursive, false}]) -spec compile_dir(rebar_dict() | rebar_state:t(), file:name(), file:name()) -> ok. compile_dir(State, BaseDir, Dir) when element(1, State) == state_t -> compile_dir(rebar_state:opts(State), BaseDir, Dir, [{recursive, false}]); compile_dir(RebarOpts, BaseDir, Dir) -> compile_dir(RebarOpts, BaseDir, Dir, [{recursive, false}]). -%% @equiv compile_dirs(Context, BaseDir, [Dir], Dir, Opts). - +%% @equiv compile_dirs(Context, BaseDir, [Dir], Dir, Opts) -spec compile_dir(rebar_dict() | rebar_state:t(), file:name(), file:name(), compile_opts()) -> ok. compile_dir(State, BaseDir, Dir, Opts) when element(1, State) == state_t -> compile_dirs(rebar_state:opts(State), BaseDir, [Dir], Dir, Opts); @@ -179,7 +178,6 @@ compile_dir(RebarOpts, BaseDir, Dir, Opts) -> compile_dirs(RebarOpts, BaseDir, [Dir], Dir, Opts). %% @doc compile a list of directories with the given opts. - -spec compile_dirs(rebar_dict() | rebar_state:t(), file:filename(), [file:filename()], @@ -187,13 +185,10 @@ compile_dir(RebarOpts, BaseDir, Dir, Opts) -> compile_opts()) -> ok. compile_dirs(State, BaseDir, Dirs, OutDir, CompileOpts) when element(1, State) == state_t -> compile_dirs(rebar_state:opts(State), BaseDir, Dirs, OutDir, CompileOpts); -compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, Opts) -> - CompileOpts = parse_opts(Opts), - +compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, CompileOpts) -> ErlOpts = rebar_opts:erl_opts(RebarOpts), ?DEBUG("erlopts ~p", [ErlOpts]), - Recursive = CompileOpts#compile_opts.recursive, - AllErlFiles = gather_src(SrcDirs, Recursive), + AllErlFiles = gather_src(RebarOpts, BaseDir, SrcDirs, CompileOpts), ?DEBUG("files to compile ~p", [AllErlFiles]), %% Make sure that outdir is on the path @@ -202,7 +197,13 @@ compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, Opts) -> G = init_erlcinfo(include_abs_dirs(ErlOpts, BaseDir), AllErlFiles, BaseDir, OutDir), - NeededErlFiles = needed_files(G, ErlOpts, BaseDir, OutDir, AllErlFiles), + {ParseTransforms, Rest} = split_source_files(AllErlFiles, ErlOpts), + NeededErlFiles = case needed_files(G, ErlOpts, RebarOpts, BaseDir, OutDir, ParseTransforms) of + [] -> needed_files(G, ErlOpts, RebarOpts, BaseDir, OutDir, Rest); + %% at least one parse transform in the opts needs updating, so recompile all + _ -> AllErlFiles + end, + {ErlFirstFiles, ErlOptsFirst} = erl_first_files(RebarOpts, ErlOpts, BaseDir, NeededErlFiles), {DepErls, OtherErls} = lists:partition( fun(Source) -> digraph:in_degree(G, Source) > 0 end, @@ -218,7 +219,7 @@ compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, Opts) -> true -> ErlOptsFirst; false -> ErlOpts end, - internal_erl_compile(C, BaseDir, S, OutDir, ErlOpts1) + internal_erl_compile(C, BaseDir, S, OutDir, ErlOpts1, RebarOpts) end) after true = digraph:delete(SubGraph), @@ -227,7 +228,6 @@ compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, Opts) -> ok. %% @doc remove compiled artifacts from an AppDir. - -spec clean(rebar_app_info:t()) -> 'ok'. clean(AppInfo) -> AppDir = rebar_app_info:out_dir(AppInfo), @@ -240,8 +240,8 @@ clean(AppInfo) -> YrlFiles = rebar_utils:find_files(filename:join([AppDir, "src"]), ?RE_PREFIX".*\\.[x|y]rl\$"), rebar_file_utils:delete_each( - [ binary_to_list(iolist_to_binary(re:replace(F, "\\.[x|y]rl$", ".erl"))) - || F <- YrlFiles ]), + [rebar_utils:to_list(re:replace(F, "\\.[x|y]rl$", ".erl", [unicode])) + || F <- YrlFiles]), BinDirs = ["ebin"|rebar_dir:extra_src_dirs(rebar_app_info:opts(AppInfo))], ok = clean_dirs(AppDir, BinDirs), @@ -266,18 +266,29 @@ clean_dirs(AppDir, [Dir|Rest]) -> %% Internal functions %% =================================================================== -gather_src(Dirs, Recursive) -> - gather_src(Dirs, [], Recursive). +gather_src(Opts, BaseDir, Dirs, CompileOpts) -> + gather_src(Opts, filename:split(BaseDir), Dirs, [], CompileOpts). + +gather_src(_Opts, _BaseDirParts, [], Srcs, _CompileOpts) -> Srcs; +gather_src(Opts, BaseDirParts, [Dir|Rest], Srcs, CompileOpts) -> + DirParts = filename:split(Dir), + RelDir = case lists:prefix(BaseDirParts,DirParts) of + true -> + case lists:nthtail(length(BaseDirParts),DirParts) of + [] -> "."; + RestParts -> filename:join(RestParts) + end; + false -> Dir + end, + DirRecursive = dir_recursive(Opts, RelDir, CompileOpts), + gather_src(Opts, BaseDirParts, Rest, Srcs ++ rebar_utils:find_files(Dir, ?RE_PREFIX".*\\.erl\$", DirRecursive), CompileOpts). -gather_src([], Srcs, _Recursive) -> Srcs; -gather_src([Dir|Rest], Srcs, Recursive) -> - gather_src(Rest, Srcs ++ rebar_utils:find_files(Dir, ?RE_PREFIX".*\\.erl\$", Recursive), Recursive). - %% Get files which need to be compiled first, i.e. those specified in erl_first_files %% and parse_transform options. Also produce specific erl_opts for these first %% files, so that yet to be compiled parse transformations are excluded from it. erl_first_files(Opts, ErlOpts, Dir, NeededErlFiles) -> ErlFirstFilesConf = rebar_opts:get(Opts, erl_first_files, []), + valid_erl_first_conf(ErlFirstFilesConf), NeededSrcDirs = lists:usort(lists:map(fun filename:dirname/1, NeededErlFiles)), %% NOTE: order of files here is important! ErlFirstFiles = @@ -296,15 +307,29 @@ erl_first_files(Opts, ErlOpts, Dir, NeededErlFiles) -> end, ErlOpts), {ErlFirstFiles ++ ParseTransformsErls, ErlOptsFirst}. +split_source_files(SourceFiles, ErlOpts) -> + ParseTransforms = proplists:get_all_values(parse_transform, ErlOpts), + lists:partition(fun(Source) -> + lists:member(filename_to_atom(Source), ParseTransforms) + end, SourceFiles). + +filename_to_atom(F) -> list_to_atom(filename:rootname(filename:basename(F))). + %% Get subset of SourceFiles which need to be recompiled, respecting %% dependencies induced by given graph G. -needed_files(G, ErlOpts, Dir, OutDir, SourceFiles) -> +needed_files(G, ErlOpts, RebarOpts, Dir, OutDir, SourceFiles) -> lists:filter(fun(Source) -> TargetBase = target_base(OutDir, Source), Target = TargetBase ++ ".beam", + PrivIncludes = [{i, filename:join(Dir, Src)} + || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])], AllOpts = [{outdir, filename:dirname(Target)} ,{i, filename:join(Dir, "include")} - ,{i, Dir}] ++ ErlOpts, + ,{i, Dir}] ++ PrivIncludes ++ ErlOpts, + %% necessary for erlang:function_exported/3 to work as expected + %% called here for clarity as it's required by both opts_changed/2 + %% and erl_compiler_opts_set/0 + _ = code:ensure_loaded(compile), digraph:vertex(G, Source) > {Source, filelib:last_modified(Target)} orelse opts_changed(AllOpts, TargetBase) orelse erl_compiler_opts_set() @@ -318,18 +343,38 @@ maybe_rm_beam_and_edge(G, OutDir, Source) -> false; false -> Target = target_base(OutDir, Source) ++ ".beam", - ?DEBUG("Source ~s is gone, deleting previous beam file if it exists ~s", [Source, Target]), + ?DEBUG("Source ~ts is gone, deleting previous beam file if it exists ~ts", [Source, Target]), file:delete(Target), digraph:del_vertex(G, Source), true end. opts_changed(NewOpts, Target) -> + TotalOpts = case erlang:function_exported(compile, env_compiler_options, 0) of + true -> NewOpts ++ compile:env_compiler_options(); + false -> NewOpts + end, case compile_info(Target) of - {ok, Opts} -> lists:sort(Opts) =/= lists:sort(NewOpts); + {ok, Opts} -> lists:any(fun effects_code_generation/1, lists:usort(TotalOpts) -- lists:usort(Opts)); _ -> true end. +effects_code_generation(Option) -> + case Option of + beam -> false; + report_warnings -> false; + report_errors -> false; + return_errors-> false; + return_warnings-> false; + report -> false; + warnings_as_errors -> false; + binary -> false; + verbose -> false; + {cwd,_} -> false; + {outdir, _} -> false; + _ -> true + end. + compile_info(Target) -> case beam_lib:chunks(Target, [compile_info]) of {ok, {_mod, Chunks}} -> @@ -341,10 +386,12 @@ compile_info(Target) -> end. erl_compiler_opts_set() -> - case os:getenv("ERL_COMPILER_OPTIONS") of + EnvSet = case os:getenv("ERL_COMPILER_OPTIONS") of false -> false; _ -> true - end. + end, + %% return false if changed env opts would have been caught in opts_changed/2 + EnvSet andalso not erlang:function_exported(compile, env_compiler_options, 0). erlcinfo_file(Dir) -> filename:join(rebar_dir:local_cache_dir(Dir), ?ERLCINFO_FILE). @@ -358,7 +405,7 @@ init_erlcinfo(InclDirs, Erls, Dir, OutDir) -> try restore_erlcinfo(G, InclDirs, Dir) catch _:_ -> - ?WARN("Failed to restore ~s file. Discarding it.~n", [erlcinfo_file(Dir)]), + ?WARN("Failed to restore ~ts file. Discarding it.~n", [erlcinfo_file(Dir)]), file:delete(erlcinfo_file(Dir)) end, Dirs = source_and_include_dirs(InclDirs, Erls), @@ -504,12 +551,15 @@ expand_file_names(Files, Dirs) -> end, Files). -spec internal_erl_compile(rebar_dict(), file:filename(), file:filename(), - file:filename(), list()) -> ok | {ok, any()} | {error, any(), any()}. -internal_erl_compile(Opts, Dir, Module, OutDir, ErlOpts) -> + file:filename(), list(), rebar_dict()) -> + ok | {ok, any()} | {error, any(), any()}. +internal_erl_compile(Opts, Dir, Module, OutDir, ErlOpts, RebarOpts) -> Target = target_base(OutDir, Module) ++ ".beam", ok = filelib:ensure_dir(Target), - AllOpts = [{outdir, filename:dirname(Target)}] ++ ErlOpts ++ - [{i, filename:join(Dir, "include")}, {i, Dir}, return], + PrivIncludes = [{i, filename:join(Dir, Src)} + || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])], + AllOpts = [{outdir, filename:dirname(Target)}] ++ ErlOpts ++ PrivIncludes ++ + [{i, filename:join(Dir, "include")}, {i, Dir}, return], case compile:file(Module, AllOpts) of {ok, _Mod} -> ok; @@ -554,9 +604,11 @@ compile_mib(AppInfo) -> MibToHrlOpts = case proplists:get_value(verbosity, AllOpts, undefined) of undefined -> - #options{specific = []}; + #options{specific = [], + cwd = rebar_dir:get_cwd()}; Verbosity -> - #options{specific = [{verbosity, Verbosity}]} + #options{specific = [{verbosity, Verbosity}], + cwd = rebar_dir:get_cwd()} end, ok = snmpc:mib_to_hrl(Mib, Mib, MibToHrlOpts), rebar_file_utils:mv(HrlFilename, AppInclude), @@ -654,6 +706,8 @@ process_attr(include_lib, Form, Includes, Dir) -> [FileNode] = erl_syntax:attribute_arguments(Form), RawFile = erl_syntax:string_value(FileNode), maybe_expand_include_lib_path(RawFile, Dir) ++ Includes; +process_attr(behavior, Form, Includes, _Dir) -> + process_attr(behaviour, Form, Includes, _Dir); process_attr(behaviour, Form, Includes, _Dir) -> [FileNode] = erl_syntax:attribute_arguments(Form), File = module_to_erl(erl_syntax:atom_value(FileNode)), @@ -738,11 +792,50 @@ outdir(RebarOpts) -> proplists:get_value(outdir, ErlOpts, ?DEFAULT_OUTDIR). include_abs_dirs(ErlOpts, BaseDir) -> - InclDirs = ["include"|proplists:get_all_values(i, ErlOpts)], - lists:map(fun(Incl) -> filename:join([BaseDir, Incl]) end, InclDirs). + ErlOptIncludes = proplists:get_all_values(i, ErlOpts), + InclDirs = lists:map(fun(Incl) -> filename:absname(Incl) end, ErlOptIncludes), + [filename:join([BaseDir, "include"])|InclDirs]. + +dir_recursive(Opts, Dir, CompileOpts) when is_list(CompileOpts) -> + case proplists:get_value(recursive,CompileOpts) of + undefined -> rebar_dir:recursive(Opts, Dir); + Recursive -> Recursive + end. -parse_opts(Opts) -> parse_opts(Opts, #compile_opts{}). +valid_erl_first_conf(FileList) -> + Strs = filter_file_list(FileList), + case rebar_utils:is_list_of_strings(Strs) of + true -> true; + false -> ?ABORT("An invalid file list (~p) was provided as part of your erl_first_files directive", + [FileList]) + end. -parse_opts([], CompileOpts) -> CompileOpts; -parse_opts([{recursive, Recursive}|Rest], CompileOpts) when Recursive == true; Recursive == false -> - parse_opts(Rest, CompileOpts#compile_opts{recursive = Recursive}). +filter_file_list(FileList) -> + Atoms = lists:filter( fun(X) -> is_atom(X) end, FileList), + case Atoms of + [] -> + FileList; + _ -> + atoms_in_erl_first_files_warning(Atoms), + lists:filter( fun(X) -> not(is_atom(X)) end, FileList) + end. + +atoms_in_erl_first_files_warning(Atoms) -> + W = "You have provided atoms as file entries in erl_first_files; " + "erl_first_files only expects lists of filenames as strings. " + "The following modules (~p) may not work as expected and it is advised " + "that you change these entires to string format " + "(e.g., \"src/module.erl\") ", + ?WARN(W, [Atoms]). + +warn_deprecated() -> + case get({deprecate_warn, ?MODULE}) of + undefined -> + ?WARN("Calling deprecated ~p compiler module. This module has been " + "replaced by rebar_compiler and rebar_compiler_erl, but will " + "remain available.", [?MODULE]), + put({deprecate_warn, ?MODULE}, true), + ok; + _ -> + ok + end. diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index 47bfe1d..9c76e0e 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -7,104 +7,74 @@ %% ------------------------------------------------------------------- -module(rebar_fetch). --export([lock_source/3, - download_source/3, - needs_update/3]). +-export([lock_source/2, + download_source/2, + needs_update/2]). -export([format_error/1]). -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). --spec lock_source(file:filename_all(), rebar_resource:resource(), rebar_state:t()) -> - rebar_resource:resource() | {error, string()}. -lock_source(AppDir, Source, State) -> - Resources = rebar_state:resources(State), - Module = get_resource_type(Source, Resources), - Module:lock(AppDir, Source). +-spec lock_source(rebar_app_info:t(), rebar_state:t()) + -> rebar_resource_v2:source() | {error, string()}. +lock_source(AppInfo, State) -> + rebar_resource_v2:lock(AppInfo, State). --spec download_source(file:filename_all(), rebar_resource:resource(), rebar_state:t()) -> - true | {error, any()}. -download_source(AppDir, Source, State) -> - try download_source_(AppDir, Source, State) of - true -> - true; - Error -> - throw(?PRV_ERROR(Error)) +-spec download_source(rebar_app_info:t(), rebar_state:t()) + -> rebar_app_info:t() | {error, any()}. +download_source(AppInfo, State) -> + AppDir = rebar_app_info:dir(AppInfo), + try download_source_(AppInfo, State) of + ok -> + %% freshly downloaded, update the app info opts to reflect the new config + Config = rebar_config:consult(AppDir), + AppInfo1 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), Config), + case rebar_app_discover:find_app(AppInfo1, AppDir, all) of + {true, AppInfo2} -> + rebar_app_info:is_available(AppInfo2, true); + false -> + throw(?PRV_ERROR({dep_app_not_found, rebar_app_info:name(AppInfo1)})) + end; + {error, Reason} -> + throw(?PRV_ERROR(Reason)) catch - C:T -> - ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, erlang:get_stacktrace()]), - throw(?PRV_ERROR({fetch_fail, Source})) + throw:{no_resource, Type, Location} -> + throw(?PRV_ERROR({no_resource, Location, Type})); + ?WITH_STACKTRACE(C,T,S) + ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, S]), + throw(?PRV_ERROR({fetch_fail, rebar_app_info:source(AppInfo)})) end. -download_source_(AppDir, Source, State) -> - Resources = rebar_state:resources(State), - Module = get_resource_type(Source, Resources), +download_source_(AppInfo, State) -> + AppDir = rebar_app_info:dir(AppInfo), TmpDir = ec_file:insecure_mkdtemp(), - AppDir1 = ec_cnv:to_list(AppDir), - case Module:download(TmpDir, Source, State) of - {ok, _} -> + AppDir1 = rebar_utils:to_list(AppDir), + case rebar_resource_v2:download(TmpDir, AppInfo, State) of + ok -> ec_file:mkdir_p(AppDir1), code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), - ec_file:remove(filename:absname(AppDir1), [recursive]), + ok = rebar_file_utils:rm_rf(filename:absname(AppDir1)), ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]), - ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)), - true; + rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)); Error -> Error end. --spec needs_update(file:filename_all(), rebar_resource:resource(), rebar_state:t()) -> boolean() | {error, string()}. -needs_update(AppDir, Source, State) -> - Resources = rebar_state:resources(State), - Module = get_resource_type(Source, Resources), +-spec needs_update(rebar_app_info:t(), rebar_state:t()) + -> boolean() | {error, string()}. +needs_update(AppInfo, State) -> try - Module:needs_update(AppDir, Source) + rebar_resource_v2:needs_update(AppInfo, State) catch _:_ -> true end. -format_error({bad_download, CachePath}) -> - io_lib:format("Download of package does not match md5sum from server: ~s", [CachePath]); -format_error({unexpected_hash, CachePath, Expected, Found}) -> - io_lib:format("The checksum for package at ~s (~s) does not match the " - "checksum previously locked (~s). Either unlock or " - "upgrade the package, or make sure you fetched it from " - "the same index from which it was initially fetched.", - [CachePath, Found, Expected]); -format_error({failed_extract, CachePath}) -> - io_lib:format("Failed to extract package: ~s", [CachePath]); -format_error({bad_etag, Source}) -> - io_lib:format("MD5 Checksum comparison failed for: ~s", [Source]); format_error({fetch_fail, Name, Vsn}) -> - io_lib:format("Failed to fetch and copy dep: ~s-~s", [Name, Vsn]); + io_lib:format("Failed to fetch and copy dep: ~ts-~ts", [Name, Vsn]); format_error({fetch_fail, Source}) -> io_lib:format("Failed to fetch and copy dep: ~p", [Source]); -format_error({bad_checksum, File}) -> - io_lib:format("Checksum mismatch against tarball in ~s", [File]); -format_error({bad_registry_checksum, File}) -> - io_lib:format("Checksum mismatch against registry in ~s", [File]). - -get_resource_type({Type, Location}, Resources) -> - find_resource_module(Type, Location, Resources); -get_resource_type({Type, Location, _}, Resources) -> - find_resource_module(Type, Location, Resources); -get_resource_type({Type, _, _, Location}, Resources) -> - find_resource_module(Type, Location, Resources); -get_resource_type(_, _) -> - rebar_pkg_resource. - -find_resource_module(Type, Location, Resources) -> - case lists:keyfind(Type, 1, Resources) of - false -> - case code:which(Type) of - non_existing -> - {error, io_lib:format("Cannot handle dependency ~s.~n" - " No module for resource type ~p", [Location, Type])}; - _ -> - Type - end; - {Type, Module} -> - Module - end. +format_error({dep_app_not_found, AppName}) -> + io_lib:format("Dependency failure: source for ~ts does not contain a " + "recognizable project and can not be built", [AppName]). diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 104c047..a51a557 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -35,6 +35,7 @@ mv/2, delete_each/1, write_file_if_contents_differ/2, + write_file_if_contents_differ/3, system_tmpdir/0, system_tmpdir/1, reset_dir/1, @@ -42,7 +43,8 @@ path_from_ancestor/2, canonical_path/1, resolve_link/1, - split_dirname/1]). + split_dirname/1, + ensure_dir/1]). -include("rebar.hrl"). @@ -72,14 +74,20 @@ consult_config(State, Filename) -> [T] -> T; [] -> [] end, - SubConfigs = [consult_config(State, Entry ++ ".config") || - Entry <- Config, is_list(Entry) - ], - - [Config | lists:merge(SubConfigs)]. + JoinedConfig = lists:flatmap( + fun (SubConfig) when is_list(SubConfig) -> + case lists:suffix(".config", SubConfig) of + %% since consult_config returns a list in a list we take the head here + false -> hd(consult_config(State, SubConfig ++ ".config")); + true -> hd(consult_config(State, SubConfig)) + end; + (Entry) -> [Entry] + end, Config), + %% Backwards compatibility + [JoinedConfig]. format_error({bad_term_file, AppFile, Reason}) -> - io_lib:format("Error reading file ~s: ~s", [AppFile, file:format_error(Reason)]). + io_lib:format("Error reading file ~ts: ~ts", [AppFile, file:format_error(Reason)]). symlink_or_copy(Source, Target) -> Link = case os:type() of @@ -100,7 +108,7 @@ symlink_or_copy(Source, Target) -> T = unicode:characters_to_list(Target), case filelib:is_dir(S) of true -> - win32_symlink(S, T); + win32_symlink_or_copy(S, T); false -> cp_r([S], T) end; @@ -114,20 +122,48 @@ symlink_or_copy(Source, Target) -> end end. -win32_symlink(Source, Target) -> +%% @private Compatibility function for windows +win32_symlink_or_copy(Source, Target) -> Res = rebar_utils:sh( - ?FMT("cmd /c mklink /j \"~s\" \"~s\"", + ?FMT("cmd /c mklink /j \"~ts\" \"~ts\"", [rebar_utils:escape_double_quotes(filename:nativename(Target)), rebar_utils:escape_double_quotes(filename:nativename(Source))]), [{use_stdout, false}, return_on_error]), - case win32_ok(Res) of + case win32_mklink_ok(Res, Target) of true -> ok; - false -> - {error, lists:flatten( - io_lib:format("Failed to symlink ~s to ~s~n", - [Source, Target]))} + false -> cp_r_win32(Source, drop_last_dir_from_path(Target)) end. +%% @private specifically pattern match against the output +%% of the windows 'mklink' shell call; different values from +%% what win32_ok/1 handles +win32_mklink_ok({ok, _}, _) -> + true; +win32_mklink_ok({error,{1,"Local NTFS volumes are required to complete the operation.\n"}}, _) -> + false; +win32_mklink_ok({error,{1,"Cannot create a file when that file already exists.\n"}}, Target) -> + % File or dir is already in place; find if it is already a symlink (true) or + % if it is a directory (copy-required; false) + is_symlink(Target); +win32_mklink_ok(_, _) -> + false. + +%% @private +is_symlink(Filename) -> + {ok, Info} = file:read_link_info(Filename), + Info#file_info.type == symlink. + +%% @private +%% drops the last 'node' of the filename, presumably the last dir such as 'src' +%% this is because cp_r_win32/2 automatically adds the dir name, to appease +%% robocopy and be more uniform with POSIX +drop_last_dir_from_path([]) -> + []; +drop_last_dir_from_path(Path) -> + case lists:droplast(filename:split(Path)) of + [] -> []; + Dirs -> filename:join(Dirs) + end. %% @doc Remove files and directories. %% Target is a single filename, directoryname or wildcard expression. @@ -136,7 +172,7 @@ rm_rf(Target) -> case os:type() of {unix, _} -> EscTarget = rebar_utils:escape_chars(Target), - {ok, []} = rebar_utils:sh(?FMT("rm -rf ~s", [EscTarget]), + {ok, []} = rebar_utils:sh(?FMT("rm -rf ~ts", [EscTarget]), [{use_stdout, false}, abort_on_error]), ok; {win32, _} -> @@ -155,8 +191,12 @@ cp_r(Sources, Dest) -> case os:type() of {unix, _} -> EscSources = [rebar_utils:escape_chars(Src) || Src <- Sources], - SourceStr = string:join(EscSources, " "), - {ok, []} = rebar_utils:sh(?FMT("cp -Rp ~s \"~s\"", + SourceStr = rebar_string:join(EscSources, " "), + % ensure destination exists before copying files into it + {ok, []} = rebar_utils:sh(?FMT("mkdir -p ~ts", + [rebar_utils:escape_chars(Dest)]), + [{use_stdout, false}, abort_on_error]), + {ok, []} = rebar_utils:sh(?FMT("cp -Rp ~ts \"~ts\"", [SourceStr, rebar_utils:escape_double_quotes(Dest)]), [{use_stdout, false}, abort_on_error]), ok; @@ -171,36 +211,122 @@ mv(Source, Dest) -> {unix, _} -> EscSource = rebar_utils:escape_chars(Source), EscDest = rebar_utils:escape_chars(Dest), - {ok, []} = rebar_utils:sh(?FMT("mv ~s ~s", [EscSource, EscDest]), - [{use_stdout, false}, abort_on_error]), - ok; + case rebar_utils:sh(?FMT("mv ~ts ~ts", [EscSource, EscDest]), + [{use_stdout, false}, abort_on_error]) of + {ok, []} -> + ok; + {ok, Warning} -> + ?WARN("mv: ~p", [Warning]), + ok + end; {win32, _} -> - Cmd = case filelib:is_dir(Source) of - true -> - ?FMT("robocopy /move /e \"~s\" \"~s\" 1> nul", - [rebar_utils:escape_double_quotes(filename:nativename(Source)), - rebar_utils:escape_double_quotes(filename:nativename(Dest))]); - false -> - ?FMT("robocopy /move /e \"~s\" \"~s\" \"~s\" 1> nul", - [rebar_utils:escape_double_quotes(filename:nativename(filename:dirname(Source))), - rebar_utils:escape_double_quotes(filename:nativename(Dest)), - rebar_utils:escape_double_quotes(filename:basename(Source))]) - end, - Res = rebar_utils:sh(Cmd, - [{use_stdout, false}, return_on_error]), - case win32_ok(Res) of - true -> ok; + case filelib:is_dir(Source) of + true -> + SrcDir = filename:nativename(Source), + DestDir = case filelib:is_dir(Dest) of + true -> + %% to simulate unix/posix mv, we have to replicate + %% the same directory movement by moving the whole + %% top-level directory, not just the insides + SrcName = filename:basename(Source), + filename:nativename(filename:join(Dest, SrcName)); + false -> + filename:nativename(Dest) + end, + robocopy_dir(SrcDir, DestDir); false -> - {error, lists:flatten( - io_lib:format("Failed to move ~s to ~s~n", - [Source, Dest]))} + SrcDir = filename:nativename(filename:dirname(Source)), + SrcName = filename:basename(Source), + DestDir = filename:nativename(filename:dirname(Dest)), + DestName = filename:basename(Dest), + IsDestDir = filelib:is_dir(Dest), + if IsDestDir -> + %% if basename and target name are different because + %% we move to a directory, then just move there. + %% Similarly, if they are the same but we're going to + %% a directory, let's just do that directly. + FullDestDir = filename:nativename(Dest), + robocopy_file(SrcDir, FullDestDir, SrcName) + ; SrcName =:= DestName -> + %% if basename and target name are the same and both are files, + %% we do a regular move with robocopy without rename. + robocopy_file(SrcDir, DestDir, DestName) + ; SrcName =/= DestName-> + robocopy_mv_and_rename(Source, Dest, SrcDir, SrcName, DestDir, DestName) + end + end end. +robocopy_mv_and_rename(Source, Dest, SrcDir, SrcName, DestDir, DestName) -> + %% If we're moving a file and the origin and + %% destination names are different: + %% - mktmp + %% - robocopy source_dir tmp_dir srcname + %% - rename srcname destname (to avoid clobbering) + %% - robocopy tmp_dir dest_dir destname + %% - remove tmp_dir + case ec_file:insecure_mkdtemp() of + {error, _Reason} -> + {error, lists:flatten( + io_lib:format("Failed to move ~ts to ~ts (tmpdir failed)~n", + [Source, Dest]))}; + TmpPath -> + case robocopy_file(SrcDir, TmpPath, SrcName) of + {error, Reason} -> + {error, Reason}; + ok -> + TmpSrc = filename:join(TmpPath, SrcName), + TmpDst = filename:join(TmpPath, DestName), + case file:rename(TmpSrc, TmpDst) of + {error, _} -> + {error, lists:flatten( + io_lib:format("Failed to move ~ts to ~ts (via rename)~n", + [Source, Dest]))}; + ok -> + case robocopy_file(TmpPath, DestDir, DestName) of + Err = {error, _} -> Err; + OK -> rm_rf(TmpPath), OK + end + end + end + end. + +robocopy_file(SrcPath, DestPath, FileName) -> + Cmd = ?FMT("robocopy /move /e \"~ts\" \"~ts\" \"~ts\"", + [rebar_utils:escape_double_quotes(SrcPath), + rebar_utils:escape_double_quotes(DestPath), + rebar_utils:escape_double_quotes(FileName)]), + Res = rebar_utils:sh(Cmd, [{use_stdout, false}, return_on_error]), + case win32_ok(Res) of + false -> + {error, lists:flatten( + io_lib:format("Failed to move ~ts to ~ts~n", + [filename:join(SrcPath, FileName), + filename:join(DestPath, FileName)]))}; + true -> + ok + end. + +robocopy_dir(Source, Dest) -> + Cmd = ?FMT("robocopy /move /e \"~ts\" \"~ts\"", + [rebar_utils:escape_double_quotes(Source), + rebar_utils:escape_double_quotes(Dest)]), + Res = rebar_utils:sh(Cmd, + [{use_stdout, false}, return_on_error]), + case win32_ok(Res) of + true -> ok; + false -> + {error, lists:flatten( + io_lib:format("Failed to move ~ts to ~ts~n", + [Source, Dest]))} + end. + win32_ok({ok, _}) -> true; win32_ok({error, {Rc, _}}) when Rc<9; Rc=:=16 -> true; win32_ok(_) -> false. + delete_each([]) -> ok; delete_each([File | Rest]) -> @@ -210,12 +336,23 @@ delete_each([File | Rest]) -> {error, enoent} -> delete_each(Rest); {error, Reason} -> - ?ERROR("Failed to delete file ~s: ~p\n", [File, Reason]), + ?ERROR("Failed to delete file ~ts: ~p\n", [File, Reason]), ?FAIL end. +%% @doc backwards compat layer to pre-utf8 support write_file_if_contents_differ(Filename, Bytes) -> - ToWrite = iolist_to_binary(Bytes), + write_file_if_contents_differ(Filename, Bytes, raw). + +%% @doc let the user pick the encoding required; there are no good +%% heuristics for data encoding +write_file_if_contents_differ(Filename, Bytes, raw) -> + write_file_if_contents_differ_(Filename, iolist_to_binary(Bytes)); +write_file_if_contents_differ(Filename, Bytes, utf8) -> + write_file_if_contents_differ_(Filename, unicode:characters_to_binary(Bytes, utf8)). + +%% @private compare raw strings and check contents +write_file_if_contents_differ_(Filename, ToWrite) -> case file:read_file(Filename) of {ok, ToWrite} -> ok; @@ -227,10 +364,10 @@ write_file_if_contents_differ(Filename, Bytes) -> %% returns an os appropriate tmpdir given a path -spec system_tmpdir() -> file:filename(). +system_tmpdir() -> system_tmpdir([]). + -spec system_tmpdir(PathComponents) -> file:filename() when PathComponents :: [file:name()]. - -system_tmpdir() -> system_tmpdir([]). system_tmpdir(PathComponents) -> Tmp = case erlang:system_info(system_architecture) of "win32" -> @@ -250,7 +387,7 @@ reset_dir(Path) -> %% delete the directory if it exists _ = ec_file:remove(Path, [recursive]), %% recreate the directory - filelib:ensure_dir(filename:join([Path, "dummy.beam"])). + ensure_dir(Path). %% Linux touch but using erlang functions to work in bot *nix os and @@ -290,9 +427,8 @@ canonical_path([_|Acc], [".."|Rest]) -> canonical_path(Acc, Rest); canonical_path([], [".."|Rest]) -> canonical_path([], Rest); canonical_path(Acc, [Component|Rest]) -> canonical_path([Component|Acc], Rest). -%% returns canonical target of path if path is a link, otherwise returns path +%% @doc returns canonical target of path if path is a link, otherwise returns path -spec resolve_link(string()) -> string(). - resolve_link(Path) -> case file:read_link(Path) of {ok, Target} -> @@ -300,25 +436,28 @@ resolve_link(Path) -> {error, _} -> Path end. -%% splits a path into dirname and basename +%% @doc splits a path into dirname and basename -spec split_dirname(string()) -> {string(), string()}. - split_dirname(Path) -> {filename:dirname(Path), filename:basename(Path)}. +-spec ensure_dir(filelib:dirname_all()) -> ok | {error, file:posix()}. +ensure_dir(Path) -> + filelib:ensure_dir(filename:join(Path, "fake_file")). + %% =================================================================== %% Internal functions %% =================================================================== delete_each_dir_win32([]) -> ok; delete_each_dir_win32([Dir | Rest]) -> - {ok, []} = rebar_utils:sh(?FMT("rd /q /s \"~s\"", + {ok, []} = rebar_utils:sh(?FMT("rd /q /s \"~ts\"", [rebar_utils:escape_double_quotes(filename:nativename(Dir))]), [{use_stdout, false}, return_on_error]), delete_each_dir_win32(Rest). xcopy_win32(Source,Dest)-> - %% "xcopy \"~s\" \"~s\" /q /y /e 2> nul", Changed to robocopy to + %% "xcopy \"~ts\" \"~ts\" /q /y /e 2> nul", Changed to robocopy to %% handle long names. May have issues with older windows. Cmd = case filelib:is_dir(Source) of true -> @@ -328,11 +467,11 @@ xcopy_win32(Source,Dest)-> %% must manually add the last fragment of a directory to the `Dest` %% in order to properly replicate POSIX platforms NewDest = filename:join([Dest, filename:basename(Source)]), - ?FMT("robocopy \"~s\" \"~s\" /e /is 1> nul", + ?FMT("robocopy \"~ts\" \"~ts\" /e 1> nul", [rebar_utils:escape_double_quotes(filename:nativename(Source)), rebar_utils:escape_double_quotes(filename:nativename(NewDest))]); false -> - ?FMT("robocopy \"~s\" \"~s\" \"~s\" /e /is 1> nul", + ?FMT("robocopy \"~ts\" \"~ts\" \"~ts\" /e 1> nul", [rebar_utils:escape_double_quotes(filename:nativename(filename:dirname(Source))), rebar_utils:escape_double_quotes(filename:nativename(Dest)), rebar_utils:escape_double_quotes(filename:basename(Source))]) @@ -343,7 +482,7 @@ xcopy_win32(Source,Dest)-> true -> ok; false -> {error, lists:flatten( - io_lib:format("Failed to copy ~s to ~s~n", + io_lib:format("Failed to copy ~ts to ~ts~n", [Source, Dest]))} end. @@ -371,7 +510,7 @@ cp_r_win32({true, SourceDir}, {false, DestDir}) -> false -> %% Specifying a target directory that doesn't currently exist. %% So let's attempt to create this directory - case filelib:ensure_dir(filename:join(DestDir, "dummy")) of + case ensure_dir(DestDir) of ok -> ok = xcopy_win32(SourceDir, DestDir); {error, Reason} -> diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl index acb9ec0..0ca6627 100644 --- a/src/rebar_git_resource.erl +++ b/src/rebar_git_resource.erl @@ -2,22 +2,32 @@ %% ex: ts=4 sw=4 et -module(rebar_git_resource). --behaviour(rebar_resource). +-behaviour(rebar_resource_v2). --export([lock/2 - ,download/3 - ,needs_update/2 - ,make_vsn/1]). +-export([init/2, + lock/2, + download/4, + needs_update/2, + make_vsn/2]). -include("rebar.hrl"). %% Regex used for parsing scp style remote url -define(SCP_PATTERN, "\\A(?<username>[^@]+)@(?<host>[^:]+):(?<path>.+)\\z"). -lock(AppDir, {git, Url, _}) -> - lock(AppDir, {git, Url}); -lock(AppDir, {git, Url}) -> - AbortMsg = lists:flatten(io_lib:format("Locking of git dependency failed in ~s", [AppDir])), +-spec init(atom(), rebar_state:t()) -> {ok, rebar_resource_v2:resource()}. +init(Type, _State) -> + Resource = rebar_resource_v2:new(Type, ?MODULE, #{}), + {ok, Resource}. + +lock(AppInfo, _) -> + check_type_support(), + lock_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). + +lock_(AppDir, {git, Url, _}) -> + lock_(AppDir, {git, Url}); +lock_(AppDir, {git, Url}) -> + AbortMsg = lists:flatten(io_lib:format("Locking of git dependency failed in ~ts", [AppDir])), Dir = rebar_utils:escape_double_quotes(AppDir), {ok, VsnString} = case os:type() of @@ -28,55 +38,58 @@ lock(AppDir, {git, Url}) -> rebar_utils:sh("git --git-dir=\"" ++ Dir ++ "/.git\" rev-parse --verify HEAD", [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]) end, - Ref = string:strip(VsnString, both, $\n), + Ref = rebar_string:trim(VsnString, both, "\n"), {git, Url, {ref, Ref}}. %% Return true if either the git url or tag/branch/ref is not the same as the currently %% checked out git repo for the dep -needs_update(Dir, {git, Url, {tag, Tag}}) -> +needs_update(AppInfo, _) -> + check_type_support(), + needs_update_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). + +needs_update_(Dir, {git, Url, {tag, Tag}}) -> {ok, Current} = rebar_utils:sh(?FMT("git describe --tags --exact-match", []), [{cd, Dir}]), - Current1 = string:strip(string:strip(Current, both, $\n), both, $\r), - - ?DEBUG("Comparing git tag ~s with ~s", [Tag, Current1]), + Current1 = rebar_string:trim(rebar_string:trim(Current, both, "\n"), + both, "\r"), + ?DEBUG("Comparing git tag ~ts with ~ts", [Tag, Current1]), not ((Current1 =:= Tag) andalso compare_url(Dir, Url)); -needs_update(Dir, {git, Url, {branch, Branch}}) -> +needs_update_(Dir, {git, Url, {branch, Branch}}) -> %% Fetch remote so we can check if the branch has changed SafeBranch = rebar_utils:escape_chars(Branch), - {ok, _} = rebar_utils:sh(?FMT("git fetch origin ~s", [SafeBranch]), + {ok, _} = rebar_utils:sh(?FMT("git fetch origin ~ts", [SafeBranch]), [{cd, Dir}]), %% Check for new commits to origin/Branch - {ok, Current} = rebar_utils:sh(?FMT("git log HEAD..origin/~s --oneline", [SafeBranch]), + {ok, Current} = rebar_utils:sh(?FMT("git log HEAD..origin/~ts --oneline", [SafeBranch]), [{cd, Dir}]), - ?DEBUG("Checking git branch ~s for updates", [Branch]), + ?DEBUG("Checking git branch ~ts for updates", [Branch]), not ((Current =:= []) andalso compare_url(Dir, Url)); -needs_update(Dir, {git, Url, "master"}) -> - needs_update(Dir, {git, Url, {branch, "master"}}); -needs_update(Dir, {git, _, Ref}) -> - {ok, Current} = rebar_utils:sh(?FMT("git rev-parse -q HEAD", []), +needs_update_(Dir, {git, Url, "master"}) -> + needs_update_(Dir, {git, Url, {branch, "master"}}); +needs_update_(Dir, {git, _, Ref}) -> + {ok, Current} = rebar_utils:sh(?FMT("git rev-parse --short=7 -q HEAD", []), [{cd, Dir}]), - Current1 = string:strip(string:strip(Current, both, $\n), both, $\r), - + Current1 = rebar_string:trim(rebar_string:trim(Current, both, "\n"), + both, "\r"), Ref2 = case Ref of {ref, Ref1} -> Length = length(Current1), - if - Length >= 7 -> - lists:sublist(Ref1, Length); - true -> - Ref1 + case Length >= 7 of + true -> lists:sublist(Ref1, Length); + false -> Ref1 end; - Ref1 -> - Ref1 + _ -> + Ref end, - ?DEBUG("Comparing git ref ~s with ~s", [Ref1, Current1]), + ?DEBUG("Comparing git ref ~ts with ~ts", [Ref2, Current1]), (Current1 =/= Ref2). compare_url(Dir, Url) -> {ok, CurrentUrl} = rebar_utils:sh(?FMT("git config --get remote.origin.url", []), [{cd, Dir}]), - CurrentUrl1 = string:strip(string:strip(CurrentUrl, both, $\n), both, $\r), + CurrentUrl1 = rebar_string:trim(rebar_string:trim(CurrentUrl, both, "\n"), + both, "\r"), {ok, ParsedUrl} = parse_git_url(Url), {ok, ParsedCurrentUrl} = parse_git_url(CurrentUrl1), ?DEBUG("Comparing git url ~p with ~p", [ParsedUrl, ParsedCurrentUrl]), @@ -84,7 +97,7 @@ compare_url(Dir, Url) -> parse_git_url(Url) -> %% Checks for standard scp style git remote - case re:run(Url, ?SCP_PATTERN, [{capture, [host, path], list}]) of + case re:run(Url, ?SCP_PATTERN, [{capture, [host, path], list}, unicode]) of {match, [Host, Path]} -> {ok, {Host, filename:rootname(Path, ".git")}}; nomatch -> @@ -99,44 +112,124 @@ parse_git_url(not_scp, Url) -> {error, Reason} end. -download(Dir, {git, Url}, State) -> +download(TmpDir, AppInfo, State, _) -> + check_type_support(), + case download_(TmpDir, rebar_app_info:source(AppInfo), State) of + {ok, _} -> + ok; + {error, Reason} -> + {error, Reason}; + Error -> + {error, Error} + end. + +download_(Dir, {git, Url}, State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), - download(Dir, {git, Url, {branch, "master"}}, State); -download(Dir, {git, Url, ""}, State) -> + download_(Dir, {git, Url, {branch, "master"}}, State); +download_(Dir, {git, Url, ""}, State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), - download(Dir, {git, Url, {branch, "master"}}, State); -download(Dir, {git, Url, {branch, Branch}}, _State) -> + download_(Dir, {git, Url, {branch, "master"}}, State); +download_(Dir, {git, Url, {branch, Branch}}, _State) -> ok = filelib:ensure_dir(Dir), - rebar_utils:sh(?FMT("git clone ~s ~s -b ~s --single-branch", - [rebar_utils:escape_chars(Url), - rebar_utils:escape_chars(filename:basename(Dir)), - rebar_utils:escape_chars(Branch)]), - [{cd, filename:dirname(Dir)}]); -download(Dir, {git, Url, {tag, Tag}}, _State) -> + maybe_warn_local_url(Url), + git_clone(branch, git_vsn(), Url, Dir, Branch); +download_(Dir, {git, Url, {tag, Tag}}, _State) -> ok = filelib:ensure_dir(Dir), - rebar_utils:sh(?FMT("git clone ~s ~s -b ~s --single-branch", - [rebar_utils:escape_chars(Url), - rebar_utils:escape_chars(filename:basename(Dir)), - rebar_utils:escape_chars(Tag)]), - [{cd, filename:dirname(Dir)}]); -download(Dir, {git, Url, {ref, Ref}}, _State) -> + maybe_warn_local_url(Url), + git_clone(tag, git_vsn(), Url, Dir, Tag); +download_(Dir, {git, Url, {ref, Ref}}, _State) -> ok = filelib:ensure_dir(Dir), - rebar_utils:sh(?FMT("git clone -n ~s ~s", - [rebar_utils:escape_chars(Url), - rebar_utils:escape_chars(filename:basename(Dir))]), - [{cd, filename:dirname(Dir)}]), - rebar_utils:sh(?FMT("git checkout -q ~s", [Ref]), [{cd, Dir}]); -download(Dir, {git, Url, Rev}, _State) -> + maybe_warn_local_url(Url), + git_clone(ref, git_vsn(), Url, Dir, Ref); +download_(Dir, {git, Url, Rev}, _State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), ok = filelib:ensure_dir(Dir), - rebar_utils:sh(?FMT("git clone -n ~s ~s", - [rebar_utils:escape_chars(Url), + maybe_warn_local_url(Url), + git_clone(rev, git_vsn(), Url, Dir, Rev). + +maybe_warn_local_url(Url) -> + WarnStr = "Local git resources (~ts) are unsupported and may have odd behaviour. " + "Use remote git resources, or a plugin for local dependencies.", + case parse_git_url(Url) of + {error, no_scheme} -> ?WARN(WarnStr, [Url]); + {error, {no_default_port, _, _}} -> ?WARN(WarnStr, [Url]); + {error, {malformed_url, _, _}} -> ?WARN(WarnStr, [Url]); + _ -> ok + end. + +%% Use different git clone commands depending on git --version +git_clone(branch,Vsn,Url,Dir,Branch) when Vsn >= {1,7,10}; Vsn =:= undefined -> + rebar_utils:sh(?FMT("git clone ~ts ~ts ~ts -b ~ts --single-branch", + [git_clone_options(), + rebar_utils:escape_chars(Url), + rebar_utils:escape_chars(filename:basename(Dir)), + rebar_utils:escape_chars(Branch)]), + [{cd, filename:dirname(Dir)}]); +git_clone(branch,_Vsn,Url,Dir,Branch) -> + rebar_utils:sh(?FMT("git clone ~ts ~ts ~ts -b ~ts", + [git_clone_options(), + rebar_utils:escape_chars(Url), + rebar_utils:escape_chars(filename:basename(Dir)), + rebar_utils:escape_chars(Branch)]), + [{cd, filename:dirname(Dir)}]); +git_clone(tag,Vsn,Url,Dir,Tag) when Vsn >= {1,7,10}; Vsn =:= undefined -> + rebar_utils:sh(?FMT("git clone ~ts ~ts ~ts -b ~ts --single-branch", + [git_clone_options(), + rebar_utils:escape_chars(Url), + rebar_utils:escape_chars(filename:basename(Dir)), + rebar_utils:escape_chars(Tag)]), + [{cd, filename:dirname(Dir)}]); +git_clone(tag,_Vsn,Url,Dir,Tag) -> + rebar_utils:sh(?FMT("git clone ~ts ~ts ~ts -b ~ts", + [git_clone_options(), + rebar_utils:escape_chars(Url), + rebar_utils:escape_chars(filename:basename(Dir)), + rebar_utils:escape_chars(Tag)]), + [{cd, filename:dirname(Dir)}]); +git_clone(ref,_Vsn,Url,Dir,Ref) -> + rebar_utils:sh(?FMT("git clone ~ts -n ~ts ~ts", + [git_clone_options(), + rebar_utils:escape_chars(Url), + rebar_utils:escape_chars(filename:basename(Dir))]), + [{cd, filename:dirname(Dir)}]), + rebar_utils:sh(?FMT("git checkout -q ~ts", [Ref]), [{cd, Dir}]); +git_clone(rev,_Vsn,Url,Dir,Rev) -> + rebar_utils:sh(?FMT("git clone ~ts -n ~ts ~ts", + [git_clone_options(), + rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]), - rebar_utils:sh(?FMT("git checkout -q ~s", [rebar_utils:escape_chars(Rev)]), + rebar_utils:sh(?FMT("git checkout -q ~ts", [rebar_utils:escape_chars(Rev)]), [{cd, Dir}]). -make_vsn(Dir) -> +git_vsn() -> + case application:get_env(rebar, git_vsn) of + {ok, Vsn} -> Vsn; + undefined -> + Vsn = git_vsn_fetch(), + application:set_env(rebar, git_vsn, Vsn), + Vsn + end. + +git_vsn_fetch() -> + case rebar_utils:sh("git --version",[]) of + {ok, VsnStr} -> + case re:run(VsnStr, "git version\\h+(\\d)\\.(\\d)\\.(\\d).*", [{capture,[1,2,3],list}, unicode]) of + {match,[Maj,Min,Patch]} -> + {list_to_integer(Maj), + list_to_integer(Min), + list_to_integer(Patch)}; + nomatch -> + undefined + end; + {error, _} -> + undefined + end. + +make_vsn(AppInfo, _) -> + make_vsn_(rebar_app_info:dir(AppInfo)). + +make_vsn_(Dir) -> case collect_default_refcount(Dir) of Vsn={plain, _} -> Vsn; @@ -154,10 +247,10 @@ collect_default_refcount(Dir) -> return_on_error, {cd, Dir}]) of {error, _} -> - ?WARN("Getting log of git dependency failed in ~s. Falling back to version 0.0.0", [rebar_dir:get_cwd()]), + ?WARN("Getting log of git dependency failed in ~ts. Falling back to version 0.0.0", [rebar_dir:get_cwd()]), {plain, "0.0.0"}; {ok, String} -> - RawRef = string:strip(String, both, $\n), + RawRef = rebar_string:trim(String, both, "\n"), {Tag, TagVsn} = parse_tags(Dir), {ok, RawCount} = @@ -178,21 +271,20 @@ collect_default_refcount(Dir) -> build_vsn_string(Vsn, RawRef, Count) -> %% Cleanup the tag and the Ref information. Basically leading 'v's and %% whitespace needs to go away. - RefTag = [".ref", re:replace(RawRef, "\\s", "", [global])], + RefTag = [".ref", re:replace(RawRef, "\\s", "", [global, unicode])], %% Create the valid [semver](http://semver.org) version from the tag case Count of 0 -> - erlang:binary_to_list(erlang:iolist_to_binary(Vsn)); + rebar_utils:to_list(Vsn); _ -> - erlang:binary_to_list(erlang:iolist_to_binary([Vsn, "+build.", - integer_to_list(Count), RefTag])) + rebar_utils:to_list([Vsn, "+build.", integer_to_list(Count), RefTag]) end. get_patch_count(Dir, RawRef) -> AbortMsg = "Getting rev-list of git dep failed in " ++ Dir, - Ref = re:replace(RawRef, "\\s", "", [global]), - Cmd = io_lib:format("git rev-list ~s..HEAD", + Ref = re:replace(RawRef, "\\s", "", [global, unicode]), + Cmd = io_lib:format("git rev-list ~ts..HEAD", [rebar_utils:escape_chars(Ref)]), {ok, PatchLines} = rebar_utils:sh(Cmd, [{use_stdout, false}, @@ -203,12 +295,12 @@ get_patch_count(Dir, RawRef) -> parse_tags(Dir) -> %% Don't abort on error, we want the bad return to be turned into 0.0.0 - case rebar_utils:sh("git log --oneline --no-walk --tags --decorate", + case rebar_utils:sh("git -c color.ui=false log --oneline --no-walk --tags --decorate", [{use_stdout, false}, return_on_error, {cd, Dir}]) of {error, _} -> {undefined, "0.0.0"}; {ok, Line} -> - case re:run(Line, "(\\(|\\s)(HEAD[^,]*,\\s)tag:\\s(v?([^,\\)]+))", [{capture, [3, 4], list}]) of + case re:run(Line, "(\\(|\\s)(HEAD[^,]*,\\s)tag:\\s(v?([^,\\)]+))", [{capture, [3, 4], list}, unicode]) of {match,[Tag, Vsn]} -> {Tag, Vsn}; nomatch -> @@ -216,8 +308,37 @@ parse_tags(Dir) -> [{use_stdout, false}, return_on_error, {cd, Dir}]) of {error, _} -> {undefined, "0.0.0"}; + %% strip the v prefix if it exists like is done in the above match + {ok, [$v | LatestVsn]} -> + {undefined, rebar_string:trim(LatestVsn, both, "\n")}; {ok, LatestVsn} -> - {undefined, string:strip(LatestVsn, both, $\n)} + {undefined, rebar_string:trim(LatestVsn,both, "\n")} end end end. + +git_clone_options() -> + Option = case os:getenv("REBAR_GIT_CLONE_OPTIONS") of + false -> "" ; %% env var not set + Opt -> %% env var set to empty or others + Opt + end, + + ?DEBUG("Git clone Option = ~p",[Option]), + Option. + +check_type_support() -> + case get({is_supported, ?MODULE}) of + true -> + ok; + _ -> + case rebar_utils:sh("git --version", [{return_on_error, true}, + {use_stdout, false}]) of + {error, _} -> + ?ABORT("git not installed", []); + _ -> + put({is_supported, ?MODULE}, true), + ok + end + end. + diff --git a/src/rebar_hex_repos.erl b/src/rebar_hex_repos.erl new file mode 100644 index 0000000..ebee191 --- /dev/null +++ b/src/rebar_hex_repos.erl @@ -0,0 +1,142 @@ +-module(rebar_hex_repos). + +-export([from_state/2, + get_repo_config/2, + auth_config/1, + update_auth_config/2, + format_error/1]). + +-ifdef(TEST). +%% exported for test purposes +-export([repos/1, merge_repos/1]). +-endif. + +-include("rebar.hrl"). +-include_lib("providers/include/providers.hrl"). + +-export_type([repo/0]). + +-type repo() :: #{name => unicode:unicode_binary(), + api_url => binary(), + api_key => binary(), + repo_url => binary(), + repo_public_key => binary(), + repo_verify => binary()}. + +from_state(BaseConfig, State) -> + HexConfig = rebar_state:get(State, hex, []), + Repos = repos(HexConfig), + %% auth is stored in a separate config file since the plugin generates and modifies it + Auth = ?MODULE:auth_config(State), + %% add base config entries that are specific to use by rebar3 and not overridable + Repos1 = merge_with_base_and_auth(Repos, BaseConfig, Auth), + %% merge organizations parent repo options into each oraganization repo + update_organizations(Repos1). + +-spec get_repo_config(unicode:unicode_binary(), rebar_state:t() | [repo()]) + -> {ok, repo()} | error. +get_repo_config(RepoName, Repos) when is_list(Repos) -> + case ec_lists:find(fun(#{name := N}) -> N =:= RepoName end, Repos) of + error -> + throw(?PRV_ERROR({repo_not_found, RepoName})); + {ok, RepoConfig} -> + {ok, RepoConfig} + end; +get_repo_config(RepoName, State) -> + Resources = rebar_state:resources(State), + #{repos := Repos} = rebar_resource_v2:find_resource_state(pkg, Resources), + get_repo_config(RepoName, Repos). + +merge_with_base_and_auth(Repos, BaseConfig, Auth) -> + [maps:merge(maps:get(maps:get(name, Repo), Auth, #{}), + maps:merge(Repo, BaseConfig)) || Repo <- Repos]. + +%% A user's list of repos are merged by name while keeping the order +%% intact. The order is based on the first use of a repo by name in the +%% list. The default repo is appended to the user's list. +repos(HexConfig) -> + HexDefaultConfig = default_repo(), + case [R || R <- HexConfig, element(1, R) =:= repos] of + [] -> + [HexDefaultConfig]; + %% we only care if the first element is a replace entry + [{repos, replace, Repos} | _]-> + merge_repos(Repos); + Repos -> + RepoList = repo_list(Repos), + merge_repos(RepoList ++ [HexDefaultConfig]) + end. + +-spec merge_repos([repo()]) -> [repo()]. +merge_repos(Repos) -> + lists:foldl(fun(R=#{name := Name}, ReposAcc) -> + %% private organizations include the parent repo before a : + case rebar_string:split(Name, <<":">>) of + [Repo, Org] -> + update_repo_list(R#{name => Name, + organization => Org, + parent => Repo}, ReposAcc); + _ -> + update_repo_list(R, ReposAcc) + end + end, [], Repos). + +update_organizations(Repos) -> + lists:map(fun(Repo=#{organization := Organization, + parent := ParentName}) -> + {ok, Parent} = get_repo_config(ParentName, Repos), + ParentRepoUrl = rebar_utils:to_list(maps:get(repo_url, Parent)), + {ok, RepoUrl} = + rebar_utils:url_append_path(ParentRepoUrl, + filename:join("repos", rebar_utils:to_list(Organization))), + %% still let the organization config override this constructed repo url + maps:merge(Parent#{repo_url => rebar_utils:to_binary(RepoUrl)}, Repo); + (Repo) -> + Repo + end, Repos). + +update_repo_list(R=#{name := N}, [H=#{name := HN} | Rest]) when N =:= HN -> + [maps:merge(R, H) | Rest]; +update_repo_list(R, [H | Rest]) -> + [H | update_repo_list(R, Rest)]; +update_repo_list(R, []) -> + [R]. + +default_repo() -> + HexDefaultConfig = hex_core:default_config(), + HexDefaultConfig#{name => ?PUBLIC_HEX_REPO}. + +repo_list([]) -> + []; +repo_list([{repos, Repos} | T]) -> + Repos ++ repo_list(T); +repo_list([{repos, replace, Repos} | T]) -> + Repos ++ repo_list(T). + +format_error({repo_not_found, RepoName}) -> + io_lib:format("The repo ~ts was not found in the configuration.", [RepoName]). + +%% auth functions + +%% authentication is in a separate config file because the hex plugin updates it + +-spec auth_config_file(rebar_state:t()) -> file:filename_all(). +auth_config_file(State) -> + filename:join(rebar_dir:global_config_dir(State), ?HEX_AUTH_FILE). + +-spec auth_config(rebar_state:t()) -> map(). +auth_config(State) -> + case file:consult(auth_config_file(State)) of + {ok, [Config]} -> + Config; + _ -> + #{} + end. + +-spec update_auth_config(map(), rebar_state:t()) -> ok. +update_auth_config(Updates, State) -> + Config = auth_config(State), + AuthConfigFile = auth_config_file(State), + ok = filelib:ensure_dir(AuthConfigFile), + NewConfig = iolist_to_binary([io_lib:print(maps:merge(Config, Updates)) | ".\n"]), + ok = file:write_file(AuthConfigFile, NewConfig). diff --git a/src/rebar_hg_resource.erl b/src/rebar_hg_resource.erl index 7d03eda..8139d04 100644 --- a/src/rebar_hg_resource.erl +++ b/src/rebar_hg_resource.erl @@ -2,39 +2,52 @@ %% ex: ts=4 sw=4 et -module(rebar_hg_resource). --behaviour(rebar_resource). +-behaviour(rebar_resource_v2). --export([lock/2 - ,download/3 - ,needs_update/2 - ,make_vsn/1]). +-export([init/2, + lock/2, + download/4, + needs_update/2, + make_vsn/2]). -include("rebar.hrl"). -lock(AppDir, {hg, Url, _}) -> - lock(AppDir, {hg, Url}); -lock(AppDir, {hg, Url}) -> +-spec init(atom(), rebar_state:t()) -> {ok, rebar_resource_v2:resource()}. +init(Type, _State) -> + Resource = rebar_resource_v2:new(Type, ?MODULE, #{}), + {ok, Resource}. + +lock(AppInfo, _) -> + check_type_support(), + lock_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). + +lock_(AppDir, {hg, Url, _}) -> + lock_(AppDir, {hg, Url}); +lock_(AppDir, {hg, Url}) -> Ref = get_ref(AppDir), {hg, Url, {ref, Ref}}. %% Return `true' if either the hg url or tag/branch/ref is not the same as %% the currently checked out repo for the dep -needs_update(Dir, {hg, Url, {tag, Tag}}) -> +needs_update(AppInfo, _) -> + needs_update_(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). + +needs_update_(Dir, {hg, Url, {tag, Tag}}) -> Ref = get_ref(Dir), {ClosestTag, Distance} = get_tag_distance(Dir, Ref), - ?DEBUG("Comparing hg tag ~s with ref ~s (closest tag is ~s at distance ~s)", + ?DEBUG("Comparing hg tag ~ts with ref ~ts (closest tag is ~ts at distance ~ts)", [Tag, Ref, ClosestTag, Distance]), not ((Distance =:= "0") andalso (Tag =:= ClosestTag) andalso compare_url(Dir, Url)); -needs_update(Dir, {hg, Url, {branch, Branch}}) -> +needs_update_(Dir, {hg, Url, {branch, Branch}}) -> Ref = get_ref(Dir), BRef = get_branch_ref(Dir, Branch), not ((Ref =:= BRef) andalso compare_url(Dir, Url)); -needs_update(Dir, {hg, Url, "default"}) -> +needs_update_(Dir, {hg, Url, "default"}) -> Ref = get_ref(Dir), BRef = get_branch_ref(Dir, "default"), not ((Ref =:= BRef) andalso compare_url(Dir, Url)); -needs_update(Dir, {hg, Url, Ref}) -> +needs_update_(Dir, {hg, Url, Ref}) -> LocalRef = get_ref(Dir), TargetRef = case Ref of {ref, Ref1} -> @@ -45,54 +58,73 @@ needs_update(Dir, {hg, Url, Ref}) -> Ref1 -> Ref1 end, - ?DEBUG("Comparing hg ref ~s with ~s", [Ref1, LocalRef]), + ?DEBUG("Comparing hg ref ~ts with ~ts", [Ref1, LocalRef]), not ((LocalRef =:= TargetRef) andalso compare_url(Dir, Url)). -download(Dir, {hg, Url}, State) -> +download(TmpDir, AppInfo, State, _) -> + check_type_support(), + case download_(TmpDir, rebar_app_info:source(AppInfo), State) of + {ok, _} -> + ok; + {error, Reason} -> + {error, Reason}; + Error -> + {error, Error} + end. + +download_(Dir, {hg, Url}, State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), - download(Dir, {hg, Url, {branch, "default"}}, State); -download(Dir, {hg, Url, ""}, State) -> + download_(Dir, {hg, Url, {branch, "default"}}, State); +download_(Dir, {hg, Url, ""}, State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), - download(Dir, {hg, Url, {branch, "default"}}, State); -download(Dir, {hg, Url, {branch, Branch}}, _State) -> + download_(Dir, {hg, Url, {branch, "default"}}, State); +download_(Dir, {hg, Url, {branch, Branch}}, _State) -> ok = filelib:ensure_dir(Dir), - rebar_utils:sh(?FMT("hg clone -q -b ~s ~s ~s", + maybe_warn_local_url(Url), + rebar_utils:sh(?FMT("hg clone -q -b ~ts ~ts ~ts", [rebar_utils:escape_chars(Branch), rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]); -download(Dir, {hg, Url, {tag, Tag}}, _State) -> +download_(Dir, {hg, Url, {tag, Tag}}, _State) -> ok = filelib:ensure_dir(Dir), - rebar_utils:sh(?FMT("hg clone -q -u ~s ~s ~s", + maybe_warn_local_url(Url), + rebar_utils:sh(?FMT("hg clone -q -u ~ts ~ts ~ts", [rebar_utils:escape_chars(Tag), rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]); -download(Dir, {hg, Url, {ref, Ref}}, _State) -> +download_(Dir, {hg, Url, {ref, Ref}}, _State) -> ok = filelib:ensure_dir(Dir), - rebar_utils:sh(?FMT("hg clone -q -r ~s ~s ~s", + maybe_warn_local_url(Url), + rebar_utils:sh(?FMT("hg clone -q -r ~ts ~ts ~ts", [rebar_utils:escape_chars(Ref), rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]); -download(Dir, {hg, Url, Rev}, _State) -> +download_(Dir, {hg, Url, Rev}, _State) -> ok = filelib:ensure_dir(Dir), - rebar_utils:sh(?FMT("hg clone -q -r ~s ~s ~s", + maybe_warn_local_url(Url), + rebar_utils:sh(?FMT("hg clone -q -r ~ts ~ts ~ts", [rebar_utils:escape_chars(Rev), rebar_utils:escape_chars(Url), rebar_utils:escape_chars(filename:basename(Dir))]), [{cd, filename:dirname(Dir)}]). -make_vsn(Dir) -> +make_vsn(AppInfo, _) -> + check_type_support(), + make_vsn_(rebar_app_info:dir(AppInfo)). + +make_vsn_(Dir) -> BaseHg = "hg -R \"" ++ rebar_utils:escape_double_quotes(Dir) ++ "\" ", Ref = get_ref(Dir), Cmd = BaseHg ++ "log --template \"{latesttag}+build.{latesttagdistance}.rev.{node|short}\"" " --rev " ++ Ref, - AbortMsg = io_lib:format("Version resolution of hg dependency failed in ~s", [Dir]), + AbortMsg = io_lib:format("Version resolution of hg dependency failed in ~ts", [Dir]), {ok, VsnString} = rebar_utils:sh(Cmd, [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]), - RawVsn = string:strip(VsnString, both, $\n), + RawVsn = rebar_string:trim(VsnString, both, "\n"), Vsn = case RawVsn of "null+" ++ Rest -> "0.0.0+" ++ Rest; @@ -103,43 +135,70 @@ make_vsn(Dir) -> %%% Internal functions compare_url(Dir, Url) -> - CurrentUrl = string:strip(os:cmd("hg -R \"" ++ rebar_utils:escape_double_quotes(Dir) ++"\" paths default"), both, $\n), - CurrentUrl1 = string:strip(CurrentUrl, both, $\r), + CurrentUrl = rebar_string:trim(os:cmd("hg -R \"" ++ rebar_utils:escape_double_quotes(Dir) ++"\" paths default"), both, "\n"), + CurrentUrl1 = rebar_string:trim(CurrentUrl, both, "\r"), parse_hg_url(CurrentUrl1) =:= parse_hg_url(Url). get_ref(Dir) -> - AbortMsg = io_lib:format("Get ref of hg dependency failed in ~s", [Dir]), + AbortMsg = io_lib:format("Get ref of hg dependency failed in ~ts", [Dir]), {ok, RefString} = rebar_utils:sh("hg -R \"" ++ rebar_utils:escape_double_quotes(Dir) ++ "\" --debug id -i", [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]), - string:strip(RefString, both, $\n). + rebar_string:trim(RefString, both, "\n"). get_tag_distance(Dir, Ref) -> - AbortMsg = io_lib:format("Get tag distance of hg dependency failed in ~s", [Dir]), + AbortMsg = io_lib:format("Get tag distance of hg dependency failed in ~ts", [Dir]), {ok, LogString} = rebar_utils:sh("hg -R \"" ++ rebar_utils:escape_double_quotes(Dir) ++ "\" " "log --template \"{latesttag}-{latesttagdistance}\n\" " "--rev " ++ rebar_utils:escape_chars(Ref), [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]), - Log = string:strip(LogString, - both, $\n), - [Tag, Distance] = re:split(Log, "-([0-9]+)$", [{parts,0}, {return, list}]), + Log = rebar_string:trim(LogString, + both, "\n"), + [Tag, Distance] = re:split(Log, "-([0-9]+)$", + [{parts,0}, {return,list}, unicode]), {Tag, Distance}. get_branch_ref(Dir, Branch) -> - AbortMsg = io_lib:format("Get branch ref of hg dependency failed in ~s", [Dir]), + AbortMsg = io_lib:format("Get branch ref of hg dependency failed in ~ts", [Dir]), {ok, BranchRefString} = rebar_utils:sh("hg -R \"" ++ rebar_utils:escape_double_quotes(Dir) ++ "\" log --template \"{node}\n\" --rev " ++ rebar_utils:escape_chars(Branch), [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]), - string:strip(BranchRefString, both, $\n). + rebar_string:trim(BranchRefString, both, "\n"). + + +maybe_warn_local_url(Url) -> + try + _ = parse_hg_url(Url), + ok + catch + _:_ -> + ?WARN("URL format (~ts) unsupported.", []) + end. parse_hg_url("ssh://" ++ HostPath) -> - [Host | Path] = string:tokens(HostPath, "/"), + [Host | Path] = rebar_string:lexemes(HostPath, "/"), {Host, filename:rootname(filename:join(Path), ".hg")}; parse_hg_url("http://" ++ HostPath) -> - [Host | Path] = string:tokens(HostPath, "/"), + [Host | Path] = rebar_string:lexemes(HostPath, "/"), {Host, filename:rootname(filename:join(Path), ".hg")}; parse_hg_url("https://" ++ HostPath) -> - [Host | Path] = string:tokens(HostPath, "/"), + [Host | Path] = rebar_string:lexemes(HostPath, "/"), {Host, filename:rootname(filename:join(Path), ".hg")}. + +check_type_support() -> + case get({is_supported, ?MODULE}) of + true -> + ok; + false -> + case rebar_utils:sh("hg --version", [{return_on_error, true}, + {use_stdout, false}]) of + {error, _} -> + ?ABORT("hg not installed", []); + _ -> + put({is_supported, ?MODULE}, true), + ok + end + end. + diff --git a/src/rebar_hooks.erl b/src/rebar_hooks.erl index d6a0e2b..358458e 100644 --- a/src/rebar_hooks.erl +++ b/src/rebar_hooks.erl @@ -42,8 +42,7 @@ run_provider_hooks_(Dir, Type, Command, Providers, TypeHooks, State) -> [] -> State; HookProviders -> - PluginDepsPaths = lists:usort(rebar_state:code_paths(State, all_plugin_deps)), - code:add_pathsa(PluginDepsPaths), + rebar_paths:set_paths([plugins], State), Providers1 = rebar_state:providers(State), State1 = rebar_state:providers(rebar_state:dir(State, Dir), Providers++Providers1), case rebar_core:do(HookProviders, State1) of @@ -51,39 +50,16 @@ run_provider_hooks_(Dir, Type, Command, Providers, TypeHooks, State) -> ?DEBUG(format_error({bad_provider, Type, Command, ProviderName}), []), throw(?PRV_ERROR({bad_provider, Type, Command, ProviderName})); {ok, State2} -> - rebar_utils:remove_from_code_path(PluginDepsPaths), + rebar_paths:set_paths([deps], State2), State2 end end. format_error({bad_provider, Type, Command, {Name, Namespace}}) -> - io_lib:format("Unable to run ~s hooks for '~p', command '~p' in namespace '~p' not found.", [Type, Command, Namespace, Name]); + io_lib:format("Unable to run ~ts hooks for '~p', command '~p' in namespace '~p' not found.", [Type, Command, Namespace, Name]); format_error({bad_provider, Type, Command, Name}) -> - io_lib:format("Unable to run ~s hooks for '~p', command '~p' not found.", [Type, Command, Name]). + io_lib:format("Unable to run ~ts hooks for '~p', command '~p' not found.", [Type, Command, Name]). -%% @doc The following environment variables are exported when running -%% a hook (absolute paths): -%% -%% REBAR_DEPS_DIR = rebar_dir:deps_dir/1 -%% REBAR_BUILD_DIR = rebar_dir:base_dir/1 -%% REBAR_ROOT_DIR = rebar_dir:root_dir/1 -%% REBAR_CHECKOUTS_DIR = rebar_dir:checkouts_dir/1 -%% REBAR_PLUGINS_DIR = rebar_dir:plugins_dir/1 -%% REBAR_GLOBAL_CONFIG_DIR = rebar_dir:global_config_dir/1 -%% REBAR_GLOBAL_CACHE_DIR = rebar_dir:global_cache_dir/1 -%% REBAR_TEMPLATE_DIR = rebar_dir:template_dir/1 -%% REBAR_APP_DIRS = rebar_dir:lib_dirs/1 -%% REBAR_SRC_DIRS = rebar_dir:src_dirs/1 -%% -%% autoconf compatible variables -%% (see: http://www.gnu.org/software/autoconf/manual/autoconf.html#Erlang-Libraries): -%% ERLANG_ERTS_VER = erlang:system_info(version) -%% ERLANG_ROOT_DIR = code:root_dir/0 -%% ERLANG_LIB_DIR_erl_interface = code:lib_dir(erl_interface) -%% ERLANG_LIB_VER_erl_interface = version part of path returned by code:lib_dir(erl_interface) -%% ERL = ERLANG_ROOT_DIR/bin/erl -%% ERLC = ERLANG_ROOT_DIR/bin/erl -%% run_hooks(Dir, pre, Command, Opts, State) -> run_hooks(Dir, pre_hooks, Command, Opts, State); run_hooks(Dir, post, Command, Opts, State) -> @@ -94,7 +70,7 @@ run_hooks(Dir, Type, Command, Opts, State) -> ?DEBUG("run_hooks(~p, ~p, ~p) -> no hooks defined\n", [Dir, Type, Command]), ok; Hooks -> - Env = create_env(State, Opts), + Env = rebar_env:create_env(State, Opts), lists:foreach(fun({_, C, _}=Hook) when C =:= Command -> apply_hook(Dir, Env, Hook); ({C, _}=Hook) when C =:= Command -> @@ -114,36 +90,3 @@ apply_hook(Dir, Env, {Arch, Command, Hook}) -> apply_hook(Dir, Env, {Command, Hook}) -> Msg = lists:flatten(io_lib:format("Hook for ~p failed!~n", [Command])), rebar_utils:sh(Hook, [use_stdout, {cd, Dir}, {env, Env}, {abort_on_error, Msg}]). - -create_env(State, Opts) -> - BaseDir = rebar_dir:base_dir(State), - [ - {"REBAR_DEPS_DIR", filename:absname(rebar_dir:deps_dir(State))}, - {"REBAR_BUILD_DIR", filename:absname(rebar_dir:base_dir(State))}, - {"REBAR_ROOT_DIR", filename:absname(rebar_dir:root_dir(State))}, - {"REBAR_CHECKOUTS_DIR", filename:absname(rebar_dir:checkouts_dir(State))}, - {"REBAR_PLUGINS_DIR", filename:absname(rebar_dir:plugins_dir(State))}, - {"REBAR_GLOBAL_CONFIG_DIR", filename:absname(rebar_dir:global_config_dir(State))}, - {"REBAR_GLOBAL_CACHE_DIR", filename:absname(rebar_dir:global_cache_dir(Opts))}, - {"REBAR_TEMPLATE_DIR", filename:absname(rebar_dir:template_dir(State))}, - {"REBAR_APP_DIRS", join_dirs(BaseDir, rebar_dir:lib_dirs(State))}, - {"REBAR_SRC_DIRS", join_dirs(BaseDir, rebar_dir:all_src_dirs(Opts))}, - {"ERLANG_ERTS_VER", erlang:system_info(version)}, - {"ERLANG_ROOT_DIR", code:root_dir()}, - {"ERLANG_LIB_DIR_erl_interface", code:lib_dir(erl_interface)}, - {"ERLANG_LIB_VER_erl_interface", re_version(code:lib_dir(erl_interface))}, - {"ERL", filename:join([code:root_dir(), "bin", "erl"])}, - {"ERLC", filename:join([code:root_dir(), "bin", "erlc"])}, - {"ERLANG_ARCH" , rebar_api:wordsize()}, - {"ERLANG_TARGET", rebar_api:get_arch()} - - ]. - -join_dirs(BaseDir, Dirs) -> - string:join([ filename:join(BaseDir, Dir) || Dir <- Dirs ], ":"). - -re_version(Path) -> - case re:run(Path, "^.*-(?<VER>[^/-]*)$", [{capture, [1], list}]) of - nomatch -> ""; - {match, [Ver]} -> Ver - end. diff --git a/src/rebar_log.erl b/src/rebar_log.erl index b1a70c2..7fc2312 100644 --- a/src/rebar_log.erl +++ b/src/rebar_log.erl @@ -57,6 +57,8 @@ intensity() -> high; "low" -> low; + "none" -> + none; _ -> ?DFLT_INTENSITY end, @@ -91,11 +93,18 @@ get_level() -> end. log(Level = error, Str, Args) -> - {ok, LogState} = application:get_env(rebar, log), - ec_cmd_log:Level(LogState, lists:flatten(cf:format("~!^~s~n", [Str])), Args); + case application:get_env(rebar, log) of + {ok, LogState} -> + NewStr = lists:flatten(cf:format("~!^~ts~n", [Str])), + ec_cmd_log:Level( LogState, NewStr, Args); + undefined -> % fallback + io:format(standard_error, Str++"~n", Args) + end; log(Level, Str, Args) -> - {ok, LogState} = application:get_env(rebar, log), - ec_cmd_log:Level(LogState, Str++"~n", Args). + case application:get_env(rebar, log) of + {ok, LogState} -> ec_cmd_log:Level(LogState, Str++"~n", Args); + undefined -> io:format(Str++"~n", Args) + end. crashdump(Str, Args) -> crashdump("rebar3.crashdump", Str, Args). diff --git a/src/rebar_opts.erl b/src/rebar_opts.erl index b02a504..8195a77 100644 --- a/src/rebar_opts.erl +++ b/src/rebar_opts.erl @@ -35,46 +35,80 @@ erl_opts(Opts) -> Defines = [{d, list_to_atom(D)} || D <- ?MODULE:get(Opts, defines, [])], AllOpts = Defines ++ RawErlOpts, - case proplists:is_defined(no_debug_info, AllOpts) of - true -> - [O || O <- AllOpts, O =/= no_debug_info]; - false -> - [debug_info|AllOpts] - end. + lists:reverse(filter_debug_info(lists:reverse(AllOpts))). + +filter_debug_info([]) -> + %% Default == ON + [debug_info]; +filter_debug_info([debug_info|_] = L) -> + %% drop no_debug_info and {debug_info_key, _} since those would + %% conflict with a plain debug_info + [debug_info | + lists:filter(fun(K) -> + K =/= no_debug_info andalso K =/= debug_info andalso + not (is_tuple(K) andalso element(1,K) =:= debug_info_key) + end, L)]; +filter_debug_info([{debug_info, _} = H | T]) -> + %% custom debug_info field; keep and filter the rest except + %% without no_debug_info. Still have to filter for regular or crypto + %% debug_info. + [H | filter_debug_info(lists:filter(fun(K) -> K =/= no_debug_info end, T))]; +filter_debug_info([{debug_info_key, _}=H | T]) -> + %% Drop no_debug_info and regular debug_info + [H | lists:filter(fun(K) -> + K =/= no_debug_info andalso K =/= debug_info andalso + not (is_tuple(K) andalso element(1,K) =:= debug_info_key) + end, T)]; +filter_debug_info([no_debug_info|T]) -> + %% Drop all debug info + lists:filter(fun(debug_info) -> false + ; ({debug_info, _}) -> false + ; ({debug_info_key, _}) -> false + ; (no_debug_info) -> false + ; (_Other) -> true + end, T); +filter_debug_info([H|T]) -> + [H|filter_debug_info(T)]. apply_overrides(Opts, Name, Overrides) -> %% Inefficient. We want the order we get here though. Opts1 = lists:foldl(fun({override, O}, OptsAcc) -> - lists:foldl(fun({deps, Value}, OptsAcc1) -> - set(OptsAcc1, {deps,default}, Value); - ({Key, Value}, OptsAcc1) -> - set(OptsAcc1, Key, Value) - end, OptsAcc, O); + override_opt(O, OptsAcc); (_, OptsAcc) -> OptsAcc - end, Opts, Overrides), - - Opts2 = lists:foldl(fun({override, N, O}, OptsAcc) when N =:= Name -> - lists:foldl(fun({deps, Value}, OptsAcc1) -> - set(OptsAcc1, {deps,default}, Value); - ({Key, Value}, OptsAcc1) -> - set(OptsAcc1, Key, Value) - end, OptsAcc, O); + end, Opts, Overrides), + + Opts2 = lists:foldl(fun({add, O}, OptsAcc) -> + add_opt(O, OptsAcc); + (_, OptsAcc) -> + OptsAcc + end, Opts1, Overrides), + + Opts3 = lists:foldl(fun({del, O}, OptsAcc) -> + del_opt(O, OptsAcc); (_, OptsAcc) -> OptsAcc - end, Opts1, Overrides), - - lists:foldl(fun({add, N, O}, OptsAcc) when N =:= Name -> - lists:foldl(fun({deps, Value}, OptsAcc1) -> - OldValue = ?MODULE:get(OptsAcc1, {deps,default}, []), - set(OptsAcc1, {deps,default}, Value++OldValue); - ({Key, Value}, OptsAcc1) -> - OldValue = ?MODULE:get(OptsAcc1, Key, []), - set(OptsAcc1, Key, Value++OldValue) - end, OptsAcc, O); - (_, OptsAcc) -> - OptsAcc - end, Opts2, Overrides). + end, Opts2, Overrides), + + Opts4 = lists:foldl(fun({override, N, O}, OptsAcc) when N =:= Name -> + override_opt(O, OptsAcc); + (_, OptsAcc) -> + OptsAcc + end, Opts3, Overrides), + + Opts5 = lists:foldl(fun({add, N, O}, OptsAcc) when N =:= Name -> + add_opt(O, OptsAcc); + (_, OptsAcc) -> + OptsAcc + end, Opts4, Overrides), + + Opts6 = lists:foldl(fun({del, N, O}, OptsAcc) when N =:= Name -> + del_opt(O, OptsAcc); + (_, OptsAcc) -> + OptsAcc + end, Opts5, Overrides), + + Opts6. add_to_profile(Opts, Profile, KVs) when is_atom(Profile), is_list(KVs) -> Profiles = ?MODULE:get(Opts, profiles, []), @@ -101,42 +135,107 @@ merge_opts(Profile, NewOpts, OldOpts) -> end. merge_opts(NewOpts, OldOpts) -> - dict:merge(fun(deps, _NewValue, OldValue) -> - OldValue; - ({deps, _}, NewValue, _OldValue) -> - NewValue; - (plugins, NewValue, _OldValue) -> - NewValue; - ({plugins, _}, NewValue, _OldValue) -> - NewValue; - (profiles, NewValue, OldValue) -> - dict:to_list(merge_opts(dict:from_list(NewValue), dict:from_list(OldValue))); - (mib_first_files, Value, Value) -> - Value; - (mib_first_files, NewValue, OldValue) -> - OldValue ++ NewValue; - (relx, NewValue, OldValue) -> - rebar_utils:tup_umerge(OldValue, NewValue); - (_Key, NewValue, OldValue) when is_list(NewValue) -> - case io_lib:printable_list(NewValue) of - true when NewValue =:= [] -> - case io_lib:printable_list(OldValue) of - true -> - NewValue; - false -> - OldValue - end; - true -> - NewValue; - false -> - rebar_utils:tup_umerge(NewValue, OldValue) - end; - (_Key, NewValue, _OldValue) -> - NewValue - end, NewOpts, OldOpts). + dict:merge(fun merge_opt/3, NewOpts, OldOpts). %% Internal functions +add_opt(Opts1, Opts2) -> + lists:foldl(fun({deps, Value}, OptsAcc) -> + OldValue = ?MODULE:get(OptsAcc, {deps,default}, []), + set(OptsAcc, {deps,default}, Value++OldValue); + ({Key, Value}, OptsAcc) -> + OldValue = ?MODULE:get(OptsAcc, Key, []), + set(OptsAcc, Key, Value++OldValue) + end, Opts2, Opts1). + +del_opt(Opts1, Opts2) -> + lists:foldl(fun({deps, Value}, OptsAcc) -> + OldValue = ?MODULE:get(OptsAcc, {deps,default}, []), + set(OptsAcc, {deps,default}, OldValue--Value); + ({Key, Value}, OptsAcc) -> + OldValue = ?MODULE:get(OptsAcc, Key, []), + set(OptsAcc, Key, OldValue--Value) + end, Opts2, Opts1). + +override_opt(Opts1, Opts2) -> + lists:foldl(fun({deps, Value}, OptsAcc) -> + set(OptsAcc, {deps,default}, Value); + ({Key, Value}, OptsAcc) -> + set(OptsAcc, Key, Value) + end, Opts2, Opts1). + +%% +%% Function for dict:merge/3 (in merge_opts/2) to merge options by priority. +%% +merge_opt(deps, _NewValue, OldValue) -> + OldValue; +merge_opt({deps, _}, NewValue, _OldValue) -> + NewValue; +merge_opt(plugins, NewValue, _OldValue) -> + NewValue; +merge_opt({plugins, _}, NewValue, _OldValue) -> + NewValue; +merge_opt(profiles, NewValue, OldValue) -> + %% Merge up sparse pairs of {Profile, Opts} into a joined up + %% {Profile, OptsNew, OptsOld} list. + ToMerge = normalise_profile_pairs(lists:sort(NewValue), + lists:sort(OldValue)), + [{K,dict:to_list(merge_opts(dict:from_list(New), dict:from_list(Old)))} + || {K,New,Old} <- ToMerge]; +merge_opt(erl_first_files, Value, Value) -> + Value; +merge_opt(erl_first_files, NewValue, OldValue) -> + OldValue ++ NewValue; +merge_opt(mib_first_files, Value, Value) -> + Value; +merge_opt(mib_first_files, NewValue, OldValue) -> + OldValue ++ NewValue; +merge_opt(relx, NewValue, OldValue) -> + Partition = fun(C) -> is_tuple(C) andalso element(1, C) =:= overlay end, + {NewOverlays, NewOther} = lists:partition(Partition, NewValue), + {OldOverlays, OldOther} = lists:partition(Partition, OldValue), + rebar_utils:tup_umerge(NewOverlays, OldOverlays) + ++ rebar_utils:tup_umerge(OldOther, NewOther); +merge_opt(Key, NewValue, OldValue) + when Key == erl_opts; Key == eunit_compile_opts; Key == ct_compile_opts -> + merge_erl_opts(lists:reverse(OldValue), NewValue); +merge_opt(_Key, NewValue, OldValue) when is_list(NewValue) -> + case io_lib:printable_list(NewValue) of + true when NewValue =:= [] -> + case io_lib:printable_list(OldValue) of + true -> + NewValue; + false -> + OldValue + end; + true -> + NewValue; + false -> + rebar_utils:tup_umerge(NewValue, OldValue) + end; +merge_opt(_Key, NewValue, _OldValue) -> + NewValue. + +%% +%% Merge Erlang compiler options such that the result +%% a) Doesn't contain duplicates. +%% b) Resulting options are ordered by increasing precedence as expected by +%% the compiler. +%% The first parameter is the lower precedence options, in reverse order, to +%% be merged with the higher-precedence options in the second parameter. +%% +merge_erl_opts([Opt | Opts], []) -> + merge_erl_opts(Opts, [Opt]); +merge_erl_opts([Opt | Opts], Merged) -> + case lists:member(Opt, Merged) of + true -> + merge_erl_opts(Opts, Merged); + _ -> + merge_erl_opts(Opts, [Opt | Merged]) + end; +merge_erl_opts([], Merged) -> + Merged. + %% %% Filter a list of erl_opts platform_define options such that only %% those which match the provided architecture regex are returned. @@ -159,3 +258,26 @@ filter_defines([{platform_define, ArchRegex, Key, Value} | Rest], Acc) -> end; filter_defines([Opt | Rest], Acc) -> filter_defines(Rest, [Opt | Acc]). + +%% @private takes two lists of profile tuples and merges them +%% into one list of 3-tuples containing the values of either +%% profiles. +%% Any missing profile in one of the keys is replaced by an +%% empty one. +-spec normalise_profile_pairs([Profile], [Profile]) -> [Pair] when + Profile :: {Name, Opts}, + Pair :: {Name, Opts, Opts}, + Name :: atom(), + Opts :: [term()]. +normalise_profile_pairs([], []) -> + []; +normalise_profile_pairs([{P,V}|Ps], []) -> + [{P,V,[]} | normalise_profile_pairs(Ps, [])]; +normalise_profile_pairs([], [{P,V}|Ps]) -> + [{P,[],V} | normalise_profile_pairs([], Ps)]; +normalise_profile_pairs([{P,VA}|PAs], [{P,VB}|PBs]) -> + [{P,VA,VB} | normalise_profile_pairs(PAs, PBs)]; +normalise_profile_pairs([{PA,VA}|PAs], [{PB,VB}|PBs]) when PA < PB -> + [{PA,VA,[]} | normalise_profile_pairs(PAs, [{PB, VB}|PBs])]; +normalise_profile_pairs([{PA,VA}|PAs], [{PB,VB}|PBs]) when PA > PB -> + [{PB,[],VB} | normalise_profile_pairs([{PA,VA}|PAs], PBs)]. diff --git a/src/rebar_otp_app.erl b/src/rebar_otp_app.erl index ddaa44b..952271b 100644 --- a/src/rebar_otp_app.erl +++ b/src/rebar_otp_app.erl @@ -58,11 +58,12 @@ compile(State, App) -> validate_app(State, App1). format_error({missing_app_file, Filename}) -> - io_lib:format("App file is missing: ~s", [Filename]); -format_error({file_read, File, Reason}) -> - io_lib:format("Failed to read required file ~s for processing: ~s", [File, file:format_error(Reason)]); + io_lib:format("App file is missing: ~ts", [Filename]); +format_error({file_read, AppName, File, Reason}) -> + io_lib:format("Failed to read required ~ts file for processing the application '~ts': ~ts", + [File, AppName, file:format_error(Reason)]); format_error({invalid_name, File, AppName}) -> - io_lib:format("Invalid ~s: name of application (~p) must match filename.", [File, AppName]). + io_lib:format("Invalid ~ts: name of application (~p) must match filename.", [File, AppName]). %% =================================================================== %% Internal functions @@ -79,7 +80,7 @@ validate_app(State, App) -> Error end; {error, Reason} -> - ?PRV_ERROR({file_read, AppFile, Reason}) + ?PRV_ERROR({file_read, rebar_app_info:name(App), ".app", Reason}) end. validate_app_modules(State, App, AppData) -> @@ -110,25 +111,28 @@ preprocess(State, AppInfo, AppSrcFile) -> A1 = apply_app_vars(AppVars, AppData), %% AppSrcFile may contain instructions for generating a vsn number - Vsn = app_vsn(AppData, AppSrcFile, State), + Vsn = app_vsn(AppInfo, AppData, AppSrcFile, State), A2 = lists:keystore(vsn, 1, A1, {vsn, Vsn}), %% systools:make_relup/4 fails with {missing_param, registered} %% without a 'registered' value. A3 = ensure_registered(A2), + %% some tools complain if a description is not present. + A4 = ensure_description(A3), + %% Build the final spec as a string - Spec = io_lib:format("~p.\n", [{application, AppName, A3}]), + Spec = io_lib:format("~p.\n", [{application, AppName, A4}]), %% Setup file .app filename and write new contents EbinDir = rebar_app_info:ebin_dir(AppInfo), - filelib:ensure_dir(filename:join(EbinDir, "dummy.beam")), + rebar_file_utils:ensure_dir(EbinDir), AppFile = rebar_app_utils:app_src_to_app(OutDir, AppSrcFile), - ok = rebar_file_utils:write_file_if_contents_differ(AppFile, Spec), + ok = rebar_file_utils:write_file_if_contents_differ(AppFile, Spec, utf8), AppFile; {error, Reason} -> - throw(?PRV_ERROR({file_read, AppSrcFile, Reason})) + throw(?PRV_ERROR({file_read, rebar_app_info:name(AppInfo), ".app.src", Reason})) end. load_app_vars(State) -> @@ -195,6 +199,15 @@ ensure_registered(AppData) -> AppData end. +ensure_description(AppData) -> + case lists:keyfind(description, 1, AppData) of + false -> + %% Required for releases to work. + [{description, ""} | AppData]; + {description, _} -> + AppData + end. + %% In the case of *.app.src we want to give the user the ability to %% dynamically script the application resource file (think dynamic version %% string, etc.), in a way similar to what can be done with the rebar @@ -214,15 +227,13 @@ consult_app_file(Filename) -> end end. -app_vsn(AppData, AppFile, State) -> - AppDir = filename:dirname(filename:dirname(AppFile)), - Resources = rebar_state:resources(State), - rebar_utils:vcs_vsn(get_value(vsn, AppData, AppFile), AppDir, Resources). +app_vsn(AppInfo, AppData, AppFile, State) -> + rebar_utils:vcs_vsn(AppInfo, get_value(vsn, AppData, AppFile), State). get_value(Key, AppInfo, AppFile) -> case proplists:get_value(Key, AppInfo) of undefined -> - ?ABORT("Failed to get app value '~p' from '~s'~n", [Key, AppFile]); + ?ABORT("Failed to get app value '~p' from '~ts'~n", [Key, AppFile]); Value -> Value end. diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 8b4611b..757eb86 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -1,17 +1,18 @@ -module(rebar_packages). --export([packages/1 - ,close_packages/0 - ,load_and_verify_version/1 - ,deps/3 +-export([get/2 + ,get_all_names/1 ,registry_dir/1 - ,package_dir/1 - ,registry_checksum/2 - ,find_highest_matching/6 - ,find_highest_matching/4 - ,find_all/3 + ,package_dir/2 + ,find_highest_matching/5 ,verify_table/1 - ,format_error/1]). + ,format_error/1 + ,update_package/3 + ,resolve_version/5]). + +-ifdef(TEST). +-export([new_package_table/0, find_highest_matching_/5, cmp_/4, cmpl_/4, valid_vsn/1]). +-endif. -export_type([package/0]). @@ -22,114 +23,131 @@ -type vsn() :: binary(). -type package() :: pkg_name() | {pkg_name(), vsn()}. --spec packages(rebar_state:t()) -> ets:tid(). -packages(State) -> - catch ets:delete(?PACKAGE_TABLE), - case load_and_verify_version(State) of - true -> - ok; - false -> - ?DEBUG("Error loading package index.", []), - handle_bad_index(State) +format_error({missing_package, Name, Vsn}) -> + io_lib:format("Package not found in any repo: ~ts ~ts", [rebar_utils:to_binary(Name), + rebar_utils:to_binary(Vsn)]); +format_error({missing_package, Pkg}) -> + io_lib:format("Package not found in any repo: ~p", [Pkg]). + +-spec get(rebar_hex_repos:repo(), binary()) -> {ok, map()} | {error, term()}. +get(Config, Name) -> + try hex_api_package:get(Config, Name) of + {ok, {200, _Headers, PkgInfo}} -> + {ok, PkgInfo}; + {ok, {404, _, _}} -> + {error, not_found}; + Error -> + ?DEBUG("Hex api request failed: ~p", [Error]), + {error, unknown} + catch + error:{badmatch, {error, {failed_connect, _}}} -> + {error, failed_to_connect}; + _:Exception -> + ?DEBUG("hex_api_package:get failed: ~p", [Exception]), + {error, unknown} end. -handle_bad_index(State) -> - ?ERROR("Bad packages index. Trying to fix by updating the registry.", []), - {ok, State1} = rebar_prv_update:do(State), - case load_and_verify_version(State1) of - true -> - ok; - false -> - %% Still unable to load after an update, create an empty registry - ets:new(?PACKAGE_TABLE, [named_table, public]) + +-spec get_all_names(rebar_state:t()) -> [binary()]. +get_all_names(State) -> + verify_table(State), + lists:usort(ets:select(?PACKAGE_TABLE, [{#package{key={'$1', '_', '_'}, + _='_'}, + [], ['$1']}])). + +-spec get_package_versions(unicode:unicode_binary(), ec_semver:semver(), + unicode:unicode_binary(), + ets:tid(), rebar_state:t()) -> [vsn()]. +get_package_versions(Dep, {_, AlphaInfo}, Repo, Table, State) -> + ?MODULE:verify_table(State), + AllowPreRelease = rebar_state:get(State, deps_allow_prerelease, false) + orelse AlphaInfo =/= {[],[]}, + ets:select(Table, [{#package{key={Dep, {'$1', '$2'}, Repo}, + _='_'}, + [{'==', '$2', {{[],[]}}} || not AllowPreRelease], [{{'$1', '$2'}}]}]). + +-spec get_package(unicode:unicode_binary(), unicode:unicode_binary(), + binary() | undefined | '_', + [unicode:unicode_binary()] | ['_'], ets:tab(), rebar_state:t()) + -> {ok, #package{}} | not_found. +get_package(Dep, Vsn, undefined, Repos, Table, State) -> + get_package(Dep, Vsn, '_', Repos, Table, State); +get_package(Dep, Vsn, Hash, Repos, Table, State) -> + ?MODULE:verify_table(State), + case ets:select(Table, [{#package{key={Dep, ec_semver:parse(Vsn), Repo}, + checksum=Hash, + _='_'}, [], ['$_']} || Repo <- Repos]) of + %% have to allow multiple matches in the list for cases that Repo is `_` + [Package | _] -> + {ok, Package}; + _ -> + not_found end. -close_packages() -> - catch ets:delete(?PACKAGE_TABLE). +new_package_table() -> + ?PACKAGE_TABLE = ets:new(?PACKAGE_TABLE, [named_table, public, ordered_set, {keypos, 2}]), + ets:insert(?PACKAGE_TABLE, {?PACKAGE_INDEX_VERSION, package_index_version}). load_and_verify_version(State) -> {ok, RegistryDir} = registry_dir(State), case ets:file2tab(filename:join(RegistryDir, ?INDEX_FILE)) of {ok, _} -> - case ets:lookup_element(?PACKAGE_TABLE, package_index_version, 2) of + case ets:lookup_element(?PACKAGE_TABLE, package_index_version, 1) of ?PACKAGE_INDEX_VERSION -> true; - _ -> + V -> + %% no reason to confuse the user since we just start fresh and they + %% shouldn't notice, so log as a debug message only + ?DEBUG("Package index version mismatch. Current version ~p, this rebar3 expecting ~p", + [V, ?PACKAGE_INDEX_VERSION]), (catch ets:delete(?PACKAGE_TABLE)), - rebar_prv_update:hex_to_index(State) + new_package_table() end; - _ -> - rebar_prv_update:hex_to_index(State) + _ -> + new_package_table() end. -deps(Name, Vsn, State) -> - try - deps_(Name, Vsn, State) - catch - _:_ -> - handle_missing_package({Name, Vsn}, State, fun(State1) -> deps_(Name, Vsn, State1) end) - end. - -deps_(Name, Vsn, State) -> - ?MODULE:verify_table(State), - ets:lookup_element(?PACKAGE_TABLE, {ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}, 2). - -handle_missing_package(Dep, State, Fun) -> - case Dep of - {Name, Vsn} -> - ?INFO("Package ~s-~s not found. Fetching registry updates and trying again...", [Name, Vsn]); - _ -> - ?INFO("Package ~p not found. Fetching registry updates and trying again...", [Dep]) - end, +handle_missing_package(PkgKey, Repo, State, Fun) -> + Name = + case PkgKey of + {N, Vsn, _Repo} -> + ?DEBUG("Package ~ts-~ts not found. Fetching registry updates for " + "package and trying again...", [N, Vsn]), + N; + _ -> + ?DEBUG("Package ~p not found. Fetching registry updates for " + "package and trying again...", [PkgKey]), + PkgKey + end, - {ok, State1} = rebar_prv_update:do(State), - try - Fun(State1) + update_package(Name, Repo, State), + try + Fun(State) catch _:_ -> %% Even after an update the package is still missing, time to error out - throw(?PRV_ERROR({missing_package, Dep})) + throw(?PRV_ERROR({missing_package, PkgKey})) end. registry_dir(State) -> CacheDir = rebar_dir:global_cache_dir(rebar_state:opts(State)), - case rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN) of - ?DEFAULT_CDN -> - RegistryDir = filename:join([CacheDir, "hex", "default"]), - ok = filelib:ensure_dir(filename:join(RegistryDir, "placeholder")), - {ok, RegistryDir}; - CDN -> - case rebar_utils:url_append_path(CDN, ?REMOTE_PACKAGE_DIR) of - {ok, Parsed} -> - {ok, {_, _, Host, _, Path, _}} = http_uri:parse(Parsed), - CDNHostPath = lists:reverse(string:tokens(Host, ".")), - CDNPath = tl(filename:split(Path)), - RegistryDir = filename:join([CacheDir, "hex"] ++ CDNHostPath ++ CDNPath), - ok = filelib:ensure_dir(filename:join(RegistryDir, "placeholder")), - {ok, RegistryDir}; - _ -> - {uri_parse_error, CDN} - end - end. + RegistryDir = filename:join([CacheDir, "hex"]), + case filelib:ensure_dir(filename:join(RegistryDir, "placeholder")) of + ok -> ok; + {error, Posix} when Posix == eaccess; Posix == enoent -> + ?ABORT("Could not write to ~p. Please ensure the path is writeable.", + [RegistryDir]) + end, + {ok, RegistryDir}. -package_dir(State) -> - case registry_dir(State) of - {ok, RegistryDir} -> - PackageDir = filename:join([RegistryDir, "packages"]), - ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")), - {ok, PackageDir}; - Error -> - Error - end. +-spec package_dir(rebar_hex_repos:repo(), rebar_state:t()) -> {ok, file:filename_all()}. +package_dir(Repo, State) -> + {ok, RegistryDir} = registry_dir(State), + RepoName = maps:get(name, Repo), + PackageDir = filename:join([RegistryDir, rebar_utils:to_list(RepoName), "packages"]), + ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")), + {ok, PackageDir}. -registry_checksum({pkg, Name, Vsn, _Hash}, State) -> - try - ?MODULE:verify_table(State), - ets:lookup_element(?PACKAGE_TABLE, {Name, Vsn}, 3) - catch - _:_ -> - throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)})) - end. %% Hex supports use of ~> to specify the version required for a dependency. %% Since rebar3 requires exact versions to choose from we find the highest @@ -146,80 +164,271 @@ registry_checksum({pkg, Name, Vsn, _Hash}, State) -> %% `~> 2.1.3-dev` | `>= 2.1.3-dev and < 2.2.0` %% `~> 2.0` | `>= 2.0.0 and < 3.0.0` %% `~> 2.1` | `>= 2.1.0 and < 3.0.0` -find_highest_matching(Dep, Constraint, Table, State) -> - find_highest_matching(undefined, undefined, Dep, Constraint, Table, State). - -find_highest_matching(Pkg, PkgVsn, Dep, Constraint, Table, State) -> - try find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State) of +find_highest_matching(Dep, Constraint, Repo, Table, State) -> + try find_highest_matching_(Dep, Constraint, Repo, Table, State) of none -> - handle_missing_package(Dep, State, + handle_missing_package(Dep, Repo, State, fun(State1) -> - find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State1) + find_highest_matching_(Dep, Constraint, Repo, Table, State1) end); Result -> Result catch _:_ -> - handle_missing_package(Dep, State, + handle_missing_package(Dep, Repo, State, fun(State1) -> - find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State1) + find_highest_matching_(Dep, Constraint, Repo, Table, State1) end) end. -find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State) -> - try find_all(Dep, Table, State) of - {ok, [Vsn]} -> - handle_single_vsn(Pkg, PkgVsn, Dep, Vsn, Constraint); - {ok, [HeadVsn | VsnTail]} -> - {ok, handle_vsns(Constraint, HeadVsn, VsnTail)} - catch - error:badarg -> - none - end. - -find_all(Dep, Table, State) -> - ?MODULE:verify_table(State), - try ets:lookup_element(Table, Dep, 2) of - [Vsns] when is_list(Vsns)-> - {ok, Vsns}; +find_highest_matching_(Dep, Constraint, #{name := Repo}, Table, State) -> + try get_package_versions(Dep, Constraint, Repo, Table, State) of + [Vsn] -> + handle_single_vsn(Vsn, Constraint); Vsns -> - {ok, Vsns} + case handle_vsns(Constraint, Vsns) of + none -> + none; + FoundVsn -> + {ok, FoundVsn} + end catch error:badarg -> none end. -handle_vsns(Constraint, HeadVsn, VsnTail) -> +handle_vsns(Constraint, Vsns) -> lists:foldl(fun(Version, Highest) -> case ec_semver:pes(Version, Constraint) andalso - ec_semver:gt(Version, Highest) of + (Highest =:= none orelse ec_semver:gt(Version, Highest)) of true -> Version; false -> Highest end - end, HeadVsn, VsnTail). + end, none, Vsns). -handle_single_vsn(Pkg, PkgVsn, Dep, Vsn, Constraint) -> +handle_single_vsn(Vsn, Constraint) -> case ec_semver:pes(Vsn, Constraint) of true -> {ok, Vsn}; false -> - case {Pkg, PkgVsn} of - {undefined, undefined} -> - ?WARN("Only existing version of ~s is ~s which does not match constraint ~~> ~s. " - "Using anyway, but it is not guaranteed to work.", [Dep, Vsn, Constraint]); - _ -> - ?WARN("[~s:~s] Only existing version of ~s is ~s which does not match constraint ~~> ~s. " - "Using anyway, but it is not guaranteed to work.", [Pkg, PkgVsn, Dep, Vsn, Constraint]) - end, - {ok, Vsn} + none end. -format_error({missing_package, {Name, Vsn}}) -> - io_lib:format("Package not found in registry: ~s-~s.", [ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)]); -format_error({missing_package, Dep}) -> - io_lib:format("Package not found in registry: ~p.", [Dep]). - verify_table(State) -> ets:info(?PACKAGE_TABLE, named_table) =:= true orelse load_and_verify_version(State). + +parse_deps(Deps) -> + [{maps:get(app, D, Name), {pkg, Name, Constraint, undefined}} + || D=#{package := Name, + requirement := Constraint} <- Deps]. + +parse_checksum(<<Checksum:256/big-unsigned>>) -> + list_to_binary( + rebar_string:uppercase( + lists:flatten(io_lib:format("~64.16.0b", [Checksum])))); +parse_checksum(Checksum) -> + Checksum. + +update_package(Name, RepoConfig=#{name := Repo}, State) -> + ?MODULE:verify_table(State), + try hex_repo:get_package(RepoConfig#{repo_key => maps:get(read_key, RepoConfig, <<>>)}, Name) of + {ok, {200, _Headers, #{releases := Releases}}} -> + _ = insert_releases(Name, Releases, Repo, ?PACKAGE_TABLE), + {ok, RegistryDir} = rebar_packages:registry_dir(State), + PackageIndex = filename:join(RegistryDir, ?INDEX_FILE), + ok = ets:tab2file(?PACKAGE_TABLE, PackageIndex); + {ok, {403, _Headers, <<>>}} -> + not_found; + {ok, {404, _Headers, _}} -> + not_found; + Error -> + ?DEBUG("Hex get_package request failed: ~p", [Error]), + %% TODO: add better log message. hex_core should export a format_error + ?WARN("Failed to update package from repo ~ts", [Repo]), + fail + catch + _:Exception -> + ?DEBUG("hex_repo:get_package failed for package ~p: ~p", [Name, Exception]), + fail + end. + +insert_releases(Name, Releases, Repo, Table) -> + [true = ets:insert(Table, + #package{key={Name, ec_semver:parse(Version), Repo}, + checksum=parse_checksum(Checksum), + retired=maps:get(retired, Release, false), + dependencies=parse_deps(Dependencies)}) + || Release=#{checksum := Checksum, + version := Version, + dependencies := Dependencies} <- Releases]. + +-spec resolve_version(unicode:unicode_binary(), unicode:unicode_binary() | undefined, + binary() | undefined, + ets:tab(), rebar_state:t()) + -> {error, {invalid_vsn, unicode:unicode_binary()}} | + not_found | + {ok, #package{}, map()}. +%% if checksum is defined search for any matching repo matching pkg-vsn and checksum +resolve_version(Dep, DepVsn, Hash, HexRegistry, State) when is_binary(Hash) -> + Resources = rebar_state:resources(State), + #{repos := RepoConfigs} = rebar_resource_v2:find_resource_state(pkg, Resources), + RepoNames = [RepoName || #{name := RepoName} <- RepoConfigs], + + %% allow retired packages when we have a checksum + case get_package(Dep, DepVsn, Hash, RepoNames, HexRegistry, State) of + {ok, Package=#package{key={_, _, RepoName}}} -> + {ok, RepoConfig} = rebar_hex_repos:get_repo_config(RepoName, RepoConfigs), + {ok, Package, RepoConfig}; + _ -> + Fun = fun(Repo) -> + case resolve_version_(Dep, DepVsn, Repo, HexRegistry, State) of + none -> + not_found; + {ok, Vsn} -> + get_package(Dep, Vsn, Hash, [Repo], HexRegistry, State) + end + end, + handle_missing_no_exception(Fun, Dep, State) + end; +resolve_version(Dep, undefined, Hash, HexRegistry, State) -> + Fun = fun(Repo) -> + case highest_matching(Dep, {0,{[],[]}}, Repo, HexRegistry, State) of + none -> + not_found; + {ok, Vsn} -> + get_package(Dep, Vsn, Hash, [Repo], HexRegistry, State) + end + end, + handle_missing_no_exception(Fun, Dep, State); +resolve_version(Dep, DepVsn, Hash, HexRegistry, State) -> + case valid_vsn(DepVsn) of + false -> + {error, {invalid_vsn, DepVsn}}; + _ -> + Fun = fun(Repo) -> + case resolve_version_(Dep, DepVsn, Repo, HexRegistry, State) of + none -> + not_found; + {ok, Vsn} -> + get_package(Dep, Vsn, Hash, [Repo], HexRegistry, State) + end + end, + handle_missing_no_exception(Fun, Dep, State) + end. + +check_all_repos(Fun, RepoConfigs) -> + ec_lists:search(fun(#{name := R}) -> + Fun(R) + end, RepoConfigs). + +handle_missing_no_exception(Fun, Dep, State) -> + Resources = rebar_state:resources(State), + #{repos := RepoConfigs} = rebar_resource_v2:find_resource_state(pkg, Resources), + + %% first check all repos in order for a local match + %% if none is found then we step through checking after updating the repo registry + case check_all_repos(Fun, RepoConfigs) of + not_found -> + ec_lists:search(fun(Config=#{name := R}) -> + case ?MODULE:update_package(Dep, Config, State) of + ok -> + Fun(R); + _ -> + not_found + end + end, RepoConfigs); + Result -> + Result + end. + +resolve_version_(Dep, DepVsn, Repo, HexRegistry, State) -> + case DepVsn of + <<"~>", Vsn/binary>> -> + highest_matching(Dep, rm_ws(Vsn), Repo, HexRegistry, State); + <<">=", Vsn/binary>> -> + cmp(Dep, rm_ws(Vsn), Repo, HexRegistry, State, fun ec_semver:gte/2); + <<">", Vsn/binary>> -> + cmp(Dep, rm_ws(Vsn), Repo, HexRegistry, State, fun ec_semver:gt/2); + <<"<=", Vsn/binary>> -> + cmpl(Dep, rm_ws(Vsn), Repo, HexRegistry, State, fun ec_semver:lte/2); + <<"<", Vsn/binary>> -> + cmpl(Dep, rm_ws(Vsn), Repo, HexRegistry, State, fun ec_semver:lt/2); + <<"==", Vsn/binary>> -> + {ok, Vsn}; + Vsn -> + {ok, Vsn} + end. + +rm_ws(<<" ", R/binary>>) -> + ec_semver:parse(rm_ws(R)); +rm_ws(R) -> + ec_semver:parse(R). + +valid_vsn(Vsn) -> + %% Regepx from https://github.com/sindresorhus/semver-regex/blob/master/index.js + SemVerRegExp = "v?(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)(\\.(0|[1-9][0-9]*))?" + "(-[0-9a-z-]+(\\.[0-9a-z-]+)*)?(\\+[0-9a-z-]+(\\.[0-9a-z-]+)*)?", + SupportedVersions = "^(>=?|<=?|~>|==)?\\s*" ++ SemVerRegExp ++ "$", + re:run(Vsn, SupportedVersions, [unicode]) =/= nomatch. + +highest_matching(Dep, Vsn, Repo, HexRegistry, State) -> + find_highest_matching_(Dep, Vsn, #{name => Repo}, HexRegistry, State). + +cmp(Dep, Vsn, Repo, HexRegistry, State, CmpFun) -> + case get_package_versions(Dep, Vsn, Repo, HexRegistry, State) of + [] -> + none; + Vsns -> + cmp_(undefined, Vsn, Vsns, CmpFun) + end. + +cmp_(undefined, MinVsn, [], _CmpFun) -> + {ok, MinVsn}; +cmp_(HighestDepVsn, _MinVsn, [], _CmpFun) -> + {ok, HighestDepVsn}; + +cmp_(BestMatch, MinVsn, [Vsn | R], CmpFun) -> + case CmpFun(Vsn, MinVsn) of + true -> + cmp_(Vsn, Vsn, R, CmpFun); + false -> + cmp_(BestMatch, MinVsn, R, CmpFun) + end. + +%% We need to treat this differently since we want a version that is LOWER but +%% the higest possible one. +cmpl(Dep, Vsn, Repo, HexRegistry, State, CmpFun) -> + case get_package_versions(Dep, Vsn, Repo, HexRegistry, State) of + [] -> + none; + Vsns -> + cmpl_(undefined, Vsn, Vsns, CmpFun) + end. + +cmpl_(undefined, MaxVsn, [], _CmpFun) -> + {ok, MaxVsn}; +cmpl_(HighestDepVsn, _MaxVsn, [], _CmpFun) -> + {ok, HighestDepVsn}; + +cmpl_(undefined, MaxVsn, [Vsn | R], CmpFun) -> + case CmpFun(Vsn, MaxVsn) of + true -> + cmpl_(Vsn, MaxVsn, R, CmpFun); + false -> + cmpl_(undefined, MaxVsn, R, CmpFun) + end; + +cmpl_(BestMatch, MaxVsn, [Vsn | R], CmpFun) -> + case CmpFun(Vsn, MaxVsn) of + true -> + case ec_semver:gte(Vsn, BestMatch) of + true -> + cmpl_(Vsn, MaxVsn, R, CmpFun); + false -> + cmpl_(BestMatch, MaxVsn, R, CmpFun) + end; + false -> + cmpl_(BestMatch, MaxVsn, R, CmpFun) + end. diff --git a/src/rebar_paths.erl b/src/rebar_paths.erl new file mode 100644 index 0000000..160f9fa --- /dev/null +++ b/src/rebar_paths.erl @@ -0,0 +1,211 @@ +-module(rebar_paths). +-include("rebar.hrl"). + +-type target() :: deps | plugins. +-type targets() :: [target(), ...]. +-export_type([target/0, targets/0]). +-export([set_paths/2, unset_paths/2]). +-export([clashing_apps/2]). + +-ifdef(TEST). +-export([misloaded_modules/2]). +-endif. + +-spec set_paths(targets(), rebar_state:t()) -> ok. +set_paths(UserTargets, State) -> + Targets = normalize_targets(UserTargets), + GroupPaths = path_groups(Targets, State), + Paths = lists:append(lists:reverse([P || {_, P} <- GroupPaths])), + code:add_pathsa(Paths), + AppGroups = app_groups(Targets, State), + purge_and_load(AppGroups, sets:new()), + ok. + +-spec unset_paths(targets(), rebar_state:t()) -> ok. +unset_paths(UserTargets, State) -> + Targets = normalize_targets(UserTargets), + GroupPaths = path_groups(Targets, State), + Paths = lists:append([P || {_, P} <- GroupPaths]), + [code:del_path(P) || P <- Paths], + purge(Paths, code:all_loaded()), + ok. + +-spec clashing_apps(targets(), rebar_state:t()) -> [{target(), [binary()]}]. +clashing_apps(Targets, State) -> + AppGroups = app_groups(Targets, State), + AppNames = [{G, sets:from_list( + [rebar_app_info:name(App) || App <- Apps] + )} || {G, Apps} <- AppGroups], + clashing_app_names(sets:new(), AppNames, []). + +%%%%%%%%%%%%%%% +%%% PRIVATE %%% +%%%%%%%%%%%%%%% + +%% The paths are to be set in the reverse order; i.e. the default +%% path is always last when possible (minimize cases where a build +%% tool version clashes with an app's), and put the highest priorities +%% first. +-spec normalize_targets(targets()) -> targets(). +normalize_targets(List) -> + %% Plan for the eventuality of getting values piped in + %% from future versions of rebar3, possibly from plugins and so on, + %% which means we'd risk failing kind of violently. We only support + %% deps and plugins + TmpList = lists:foldl( + fun(deps, [deps | _] = Acc) -> Acc; + (plugins, [plugins | _] = Acc) -> Acc; + (deps, Acc) -> [deps | Acc -- [deps]]; + (plugins, Acc) -> [plugins | Acc -- [plugins]]; + (_, Acc) -> Acc + end, + [], + List + ), + lists:reverse(TmpList). + +purge_and_load([], _) -> + ok; +purge_and_load([{_Group, Apps}|Rest], Seen) -> + %% We have: a list of all applications in the current priority group, + %% a list of all loaded modules with their active path, and a list of + %% seen applications. + %% + %% We do the following: + %% 1. identify the apps that have not been solved yet + %% 2. find the paths for all apps in the current group + %% 3. unload and reload apps that may have changed paths in order + %% to get updated module lists and specs + %% (we ignore started apps and apps that have not run for this) + %% This part turns out to be the bottleneck of this module, so + %% to speed it up, using clash detection proves useful: + %% only reload apps that clashed since others are unlikely to + %% conflict in significant ways + %% 4. create a list of modules to check from that app list—only loaded + %% modules make sense to check. + %% 5. check the modules to match their currently loaded paths with + %% the path set from the apps in the current group; modules + %% that differ must be purged; others can stay + + %% 1) + AppNames = [AppName || App <- Apps, + AppName <- [rebar_app_info:name(App)], + not sets:is_element(AppName, Seen)], + GoodApps = [App || AppName <- AppNames, + App <- Apps, + rebar_app_info:name(App) =:= AppName], + %% 2) + %% (no need for extra_src_dirs since those get put into ebin; + %% also no need for OTP libs; we want to allow overtaking them) + GoodAppPaths = [rebar_app_info:ebin_dir(App) || App <- GoodApps], + %% 3) + [begin + AtomApp = binary_to_atom(AppName, utf8), + %% blind load/unload won't interrupt an already-running app, + %% preventing odd errors, maybe! + case application:unload(AtomApp) of + ok -> application:load(AtomApp); + _ -> ok + end + end || AppName <- AppNames, + %% Shouldn't unload ourselves; rebar runs without ever + %% being started and unloading breaks logging! + AppName =/= <<"rebar">>], + %% 4) + CandidateMods = lists:append( + %% Start by asking the currently loaded app (if loaded) + %% since it would be the primary source of conflicting modules + [case application:get_key(AppName, modules) of + {ok, Mods} -> + Mods; + undefined -> + %% if not found, parse the app file on disk, in case + %% the app's modules are used without it being loaded; + %% invalidate the cache in case we're proceeding during + %% compilation steps by setting the app details to `[]', which + %% is its empty value; the details will then be reloaded + %% from disk when found + case rebar_app_info:app_details(rebar_app_info:app_details(App, [])) of + [] -> []; + Details -> proplists:get_value(modules, Details, []) + end + end || App <- GoodApps, + AppName <- [binary_to_atom(rebar_app_info:name(App), utf8)]] + ), + ModPaths = [{Mod,Path} || Mod <- CandidateMods, + erlang:function_exported(Mod, module_info, 0), + {file, Path} <- [code:is_loaded(Mod)]], + + %% 5) + Mods = misloaded_modules(GoodAppPaths, ModPaths), + [purge_mod(Mod) || Mod <- Mods], + + purge_and_load(Rest, sets:union(Seen, sets:from_list(AppNames))). + +purge(Paths, ModPaths) -> + SortedPaths = lists:sort(Paths), + lists:map(fun purge_mod/1, + [Mod || {Mod, Path} <- ModPaths, + is_list(Path), % not 'preloaded' or mocked + any_prefix(Path, SortedPaths)] + ). + +misloaded_modules(GoodAppPaths, ModPaths) -> + %% Identify paths that are invalid; i.e. app paths that cover an + %% app in the desired group, but are not in the desired group. + lists:usort( + [Mod || {Mod, Path} <- ModPaths, + is_list(Path), % not 'preloaded' or mocked + not any_prefix(Path, GoodAppPaths)] + ). + +any_prefix(Path, Paths) -> + lists:any(fun(P) -> lists:prefix(P, Path) end, Paths). + +%% assume paths currently set are good; only unload a module so next call +%% uses the correctly set paths +purge_mod(Mod) -> + code:soft_purge(Mod) andalso code:delete(Mod). + + +%% This is a tricky O(n²) check since we want to +%% know whether an app clashes with any of the top priority groups. +%% +%% For example, let's say we have `[deps, plugins]', then we want +%% to find the plugins that clash with deps: +%% +%% `[{deps, [ClashingPlugins]}, {plugins, []}]' +%% +%% In case we'd ever have alternative or additional types, we can +%% find all clashes from other 'groups'. +clashing_app_names(_, [], Acc) -> + lists:reverse(Acc); +clashing_app_names(PrevNames, [{G,AppNames} | Rest], Acc) -> + CurrentNames = sets:subtract(AppNames, PrevNames), + NextNames = sets:subtract(sets:union([A || {_, A} <- Rest]), PrevNames), + Clashes = sets:intersection(CurrentNames, NextNames), + NewAcc = [{G, sets:to_list(Clashes)} | Acc], + clashing_app_names(sets:union(PrevNames, CurrentNames), Rest, NewAcc). + +path_groups(Targets, State) -> + [{Target, get_paths(Target, State)} || Target <- Targets]. + +app_groups(Targets, State) -> + [{Target, get_apps(Target, State)} || Target <- Targets]. + +get_paths(deps, State) -> + rebar_state:code_paths(State, all_deps); +get_paths(plugins, State) -> + rebar_state:code_paths(State, all_plugin_deps). + +get_apps(deps, State) -> + %% The code paths for deps also include the top level apps + %% and the extras, which we don't have here; we have to + %% add the apps by hand + case rebar_state:project_apps(State) of + undefined -> []; + List -> List + end ++ + rebar_state:all_deps(State); +get_apps(plugins, State) -> + rebar_state:all_plugin_deps(State). diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 5817817..823b7fc 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -2,208 +2,280 @@ %% ex: ts=4 sw=4 et -module(rebar_pkg_resource). --behaviour(rebar_resource). +-behaviour(rebar_resource_v2). --export([lock/2 - ,download/3 - ,needs_update/2 - ,make_vsn/1]). +-export([init/2, + lock/2, + download/4, + download/5, + needs_update/2, + make_vsn/2, + format_error/1]). --export([request/2 - ,etag/1 - ,ssl_opts/1]). +-ifdef(TEST). +%% exported for test purposes +-export([store_etag_in_cache/2]). +-endif. -include("rebar.hrl"). --include_lib("public_key/include/OTP-PUB-KEY.hrl"). +-include_lib("providers/include/providers.hrl"). -lock(_AppDir, Source) -> - Source. +-type package() :: {pkg, binary(), binary(), binary(), rebar_hex_repos:repo()}. -needs_update(Dir, {pkg, _Name, Vsn, _Hash}) -> - [AppInfo] = rebar_app_discover:find_apps([Dir], all), - case rebar_app_info:original_vsn(AppInfo) =:= ec_cnv:to_list(Vsn) of +%%============================================================================== +%% Public API +%%============================================================================== + +-spec init(atom(), rebar_state:t()) -> {ok, rebar_resource_v2:resource()}. +init(Type, State) -> + {ok, Vsn} = application:get_key(rebar, vsn), + BaseConfig = #{http_adapter => hex_http_httpc, + http_user_agent_fragment => + <<"(rebar3/", (list_to_binary(Vsn))/binary, ") (httpc)">>, + http_adapter_config => #{profile => rebar}}, + Repos = rebar_hex_repos:from_state(BaseConfig, State), + Resource = rebar_resource_v2:new(Type, ?MODULE, #{repos => Repos, + base_config => BaseConfig}), + {ok, Resource}. + + + +-spec lock(AppInfo, ResourceState) -> Res when + AppInfo :: rebar_app_info:t(), + ResourceState :: rebar_resource_v2:resource_state(), + Res :: {atom(), string(), any(), binary()}. +lock(AppInfo, _) -> + {pkg, Name, Vsn, Hash, _RepoConfig} = rebar_app_info:source(AppInfo), + {pkg, Name, Vsn, Hash}. + +%%------------------------------------------------------------------------------ +%% @doc +%% Return true if the stored version of the pkg is older than the current +%% version. +%% @end +%%------------------------------------------------------------------------------ +-spec needs_update(AppInfo, ResourceState) -> Res when + AppInfo :: rebar_app_info:t(), + ResourceState :: rebar_resource_v2:resource_state(), + Res :: boolean(). +needs_update(AppInfo, _) -> + {pkg, _Name, Vsn, _Hash, _} = rebar_app_info:source(AppInfo), + case rebar_utils:to_binary(rebar_app_info:original_vsn(AppInfo)) =:= rebar_utils:to_binary(Vsn) of true -> false; false -> true end. -download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State) -> - CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), - {ok, PackageDir} = rebar_packages:package_dir(State), - Package = binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>), - CachePath = filename:join(PackageDir, Package), - case rebar_utils:url_append_path(CDN, filename:join(?REMOTE_PACKAGE_DIR, Package)) of - {ok, Url} -> - cached_download(TmpDir, CachePath, Pkg, Url, etag(CachePath), State); - _ -> - {fetch_fail, Name, Vsn} +%%------------------------------------------------------------------------------ +%% @doc +%% Download the given pkg. +%% @end +%%------------------------------------------------------------------------------ +-spec download(TmpDir, AppInfo, State, ResourceState) -> Res when + TmpDir :: file:name(), + AppInfo :: rebar_app_info:t(), + ResourceState :: rebar_resource_v2:resource_state(), + State :: rebar_state:t(), + Res :: ok | {error,_}. +download(TmpDir, AppInfo, State, ResourceState) -> + case download(TmpDir, rebar_app_info:source(AppInfo), State, ResourceState, true) of + ok -> + ok; + Error -> + {error, Error} end. -cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, Url, ETag, State) -> - case request(Url, ETag) of - {ok, cached} -> - ?INFO("Version cached at ~s is up to date, reusing it", [CachePath]), - serve_from_cache(TmpDir, CachePath, Pkg, State); - {ok, Body, NewETag} -> - ?INFO("Downloaded package, caching at ~s", [CachePath]), - serve_from_download(TmpDir, CachePath, Pkg, NewETag, Body, State); - error when ETag =/= false -> - ?INFO("Download error, using cached file at ~s", [CachePath]), - serve_from_cache(TmpDir, CachePath, Pkg, State); - error -> - {fetch_fail, Name, Vsn} - end. - -serve_from_cache(TmpDir, CachePath, Pkg, State) -> - {Files, Contents, Version, Meta} = extract(TmpDir, CachePath), - case checksums(Pkg, Files, Contents, Version, Meta, State) of - {Chk, Chk, Chk, Chk} -> - ok = erl_tar:extract({binary, Contents}, [{cwd, TmpDir}, compressed]), - {ok, true}; - {_Hash, Chk, Chk, Chk} -> - ?DEBUG("Expected hash ~p does not match checksums ~p", [_Hash, Chk]), - {unexpected_hash, CachePath, _Hash, Chk}; - {Chk, _Bin, Chk, Chk} -> - ?DEBUG("Checksums: registry: ~p, pkg: ~p", [Chk, _Bin]), - {failed_extract, CachePath}; - {Chk, Chk, _Reg, Chk} -> - ?DEBUG("Checksums: registry: ~p, pkg: ~p", [_Reg, Chk]), - {bad_registry_checksum, CachePath}; - {_Hash, _Bin, _Reg, _Tar} -> - ?DEBUG("Checksums: expected: ~p, registry: ~p, pkg: ~p, meta: ~p", [_Hash, _Reg, _Bin, _Tar]), - {bad_checksum, CachePath} - end. - -serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State) -> - ?DEBUG("Writing ~p to cache at ~s", [Package, CachePath]), - file:write_file(CachePath, Binary), - case etag(CachePath) of - ETag -> - serve_from_cache(TmpDir, CachePath, Package, State); - FileETag -> - ?DEBUG("Downloaded file ~s ETag ~s doesn't match returned ETag ~s", [CachePath, ETag, FileETag]), - {bad_download, CachePath} +%%------------------------------------------------------------------------------ +%% @doc +%% Download the given pkg. The etag belonging to the pkg file will be updated +%% only if the UpdateEtag is true and the ETag returned from the hexpm server +%% is different. +%% @end +%%------------------------------------------------------------------------------ +-spec download(TmpDir, Pkg, State, ResourceState, UpdateETag) -> Res when + TmpDir :: file:name(), + Pkg :: package(), + State :: rebar_state:t(), + ResourceState:: rebar_resource_v2:resource_state(), + UpdateETag :: boolean(), + Res :: ok | {error,_} | {unexpected_hash, string(), integer(), integer()} | + {fetch_fail, binary(), binary()}. +download(TmpDir, Pkg={pkg, Name, Vsn, _Hash, Repo}, State, _ResourceState, UpdateETag) -> + {ok, PackageDir} = rebar_packages:package_dir(Repo, State), + Package = binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>), + ETagFile = binary_to_list(<<Name/binary, "-", Vsn/binary, ".etag">>), + CachePath = filename:join(PackageDir, Package), + ETagPath = filename:join(PackageDir, ETagFile), + case cached_download(TmpDir, CachePath, Pkg, etag(CachePath, ETagPath), ETagPath, UpdateETag) of + {bad_registry_checksum, Expected, Found} -> + %% checksum comparison failed. in case this is from a modified cached package + %% overwrite the etag if it exists so it is not relied on again + store_etag_in_cache(ETagPath, <<>>), + ?PRV_ERROR({bad_registry_checksum, Name, Vsn, Expected, Found}); + Result -> + Result end. - -extract(TmpDir, CachePath) -> - ec_file:mkdir_p(TmpDir), - {ok, Files} = erl_tar:extract(CachePath, [memory]), - {"contents.tar.gz", Contents} = lists:keyfind("contents.tar.gz", 1, Files), - {"VERSION", Version} = lists:keyfind("VERSION", 1, Files), - {"metadata.config", Meta} = lists:keyfind("metadata.config", 1, Files), - {Files, Contents, Version, Meta}. - -checksums(Pkg={pkg, _Name, _Vsn, Hash}, Files, Contents, Version, Meta, State) -> - Blob = <<Version/binary, Meta/binary, Contents/binary>>, - <<X:256/big-unsigned>> = crypto:hash(sha256, Blob), - BinChecksum = list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))), - RegistryChecksum = rebar_packages:registry_checksum(Pkg, State), - {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files), - {Hash, BinChecksum, RegistryChecksum, TarChecksum}. - -make_vsn(_) -> +%%------------------------------------------------------------------------------ +%% @doc +%% Implementation of rebar_resource make_vsn callback. +%% Returns {error, string()} as this operation is not supported for pkg sources. +%% @end +%%------------------------------------------------------------------------------ +-spec make_vsn(AppInfo, ResourceState) -> Res when + AppInfo :: rebar_app_info:t(), + ResourceState :: rebar_resource_v2:resource_state(), + Res :: {'error', string()}. +make_vsn(_, _) -> {error, "Replacing version of type pkg not supported."}. -request(Url, ETag) -> - case httpc:request(get, {Url, [{"if-none-match", ETag} || ETag =/= false]++[{"User-Agent", rebar_utils:user_agent()}]}, - [{ssl, ssl_opts(Url)}, {relaxed, true}], - [{body_format, binary}], - rebar) of - {ok, {{_Version, 200, _Reason}, Headers, Body}} -> - ?DEBUG("Successfully downloaded ~s", [Url]), - {"etag", ETag1} = lists:keyfind("etag", 1, Headers), - {ok, Body, string:strip(ETag1, both, $")}; - {ok, {{_Version, 304, _Reason}, _Headers, _Body}} -> - ?DEBUG("Cached copy of ~s still valid", [Url]), +format_error({bad_registry_checksum, Name, Vsn, Expected, Found}) -> + io_lib:format("The checksum for package at ~ts-~ts (~ts) does not match the " + "checksum expected from the registry (~ts). " + "Run `rebar3 do unlock ~ts, update` and then try again.", + [Name, Vsn, Found, Expected, Name]). + +%%------------------------------------------------------------------------------ +%% @doc +%% Download the pkg belonging to the given address. If the etag of the pkg +%% is the same what we stored in the etag file previously return {ok, cached}, +%% if the file has changed (so the etag is not the same anymore) return +%% {ok, Contents, NewEtag}, otherwise if some error occured return error. +%% @end +%%------------------------------------------------------------------------------ +-spec request(rebar_hex_repos:repo(), binary(), binary(), false | binary()) + -> {ok, cached} | {ok, binary(), binary()} | error. +request(Config, Name, Version, ETag) -> + Config1 = Config#{http_etag => ETag}, + try hex_repo:get_tarball(Config1, Name, Version) of + {ok, {200, #{<<"etag">> := ETag1}, Tarball}} -> + {ok, Tarball, ETag1}; + {ok, {304, _Headers, _}} -> {ok, cached}; - {ok, {{_Version, Code, _Reason}, _Headers, _Body}} -> - ?DEBUG("Request to ~p failed: status code ~p", [Url, Code]), + {ok, {Code, _Headers, _Body}} -> + ?DEBUG("Request for package ~s-~s failed: status code ~p", [Name, Version, Code]), error; {error, Reason} -> - ?DEBUG("Request to ~p failed: ~p", [Url, Reason]), + ?DEBUG("Request for package ~s-~s failed: ~p", [Name, Version, Reason]), + error + catch + _:Exception -> + ?DEBUG("hex_repo:get_tarball failed: ~p", [Exception]), error end. -etag(Path) -> - case file:read_file(Path) of - {ok, Binary} -> - <<X:128/big-unsigned-integer>> = crypto:hash(md5, Binary), - string:to_lower(lists:flatten(io_lib:format("~32.16.0b", [X]))); +%%------------------------------------------------------------------------------ +%% @doc +%% Read the etag belonging to the pkg file from the cache directory. The etag +%% is stored in a separate file when the etag belonging to the package is +%% returned from the hexpm server. The name is package-vsn.etag. +%% @end +%%------------------------------------------------------------------------------ +-spec etag(PackagePath, ETagPath) -> Res when + PackagePath :: file:name(), + ETagPath :: file:name(), + Res :: binary(). +etag(PackagePath, ETagPath) -> + case file:read_file(ETagPath) of + {ok, Bin} -> + %% just in case a user deleted a cached package but not its etag + %% verify the package is also there, and if not, ignore the etag + case filelib:is_file(PackagePath) of + true -> + Bin; + false -> + <<>> + end; {error, _} -> - false + <<>> end. -ssl_opts(Url) -> - case get_ssl_config() of - ssl_verify_enabled -> - ssl_opts(ssl_verify_enabled, Url); - ssl_verify_disabled -> - [{verify, verify_none}] - end. - -ssl_opts(ssl_verify_enabled, Url) -> - case check_ssl_version() of - true -> - {ok, {_, _, Hostname, _, _, _}} = http_uri:parse(ec_cnv:to_list(Url)), - VerifyFun = {fun ssl_verify_hostname:verify_fun/3, [{check_hostname, Hostname}]}, - CACerts = certifi:cacerts(), - [{verify, verify_peer}, {depth, 2}, {cacerts, CACerts} - ,{partial_chain, fun partial_chain/1}, {verify_fun, VerifyFun}]; - false -> - ?WARN("Insecure HTTPS request (peer verification disabled), please update to OTP 17.4 or later", []), - [{verify, verify_none}] - end. +%%------------------------------------------------------------------------------ +%% @doc +%% Store the given etag in the .cache folder. The name is pakckage-vsn.etag. +%% @end +%%------------------------------------------------------------------------------ +-spec store_etag_in_cache(File, ETag) -> Res when + File :: file:name(), + ETag :: binary(), + Res :: ok. +store_etag_in_cache(Path, ETag) -> + _ = file:write_file(Path, ETag). -partial_chain(Certs) -> - Certs1 = [{Cert, public_key:pkix_decode_cert(Cert, otp)} || Cert <- Certs], - CACerts = certifi:cacerts(), - CACerts1 = [public_key:pkix_decode_cert(Cert, otp) || Cert <- CACerts], - - case ec_lists:find(fun({_, Cert}) -> - check_cert(CACerts1, Cert) - end, Certs1) of - {ok, Trusted} -> - {trusted_ca, element(1, Trusted)}; - _ -> - unknown_ca +%%%============================================================================= +%%% Private functions +%%%============================================================================= +-spec cached_download(TmpDir, CachePath, Pkg, ETag, ETagPath, UpdateETag) -> Res when + TmpDir :: file:name(), + CachePath :: file:name(), + Pkg :: package(), + ETag :: binary(), + ETagPath :: file:name(), + UpdateETag :: boolean(), + Res :: ok | {unexpected_hash, integer(), integer()} | {fetch_fail, binary(), binary()}. +cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash, RepoConfig}, ETag, + ETagPath, UpdateETag) -> + case request(RepoConfig, Name, Vsn, ETag) of + {ok, cached} -> + ?INFO("Version cached at ~ts is up to date, reusing it", [CachePath]), + serve_from_cache(TmpDir, CachePath, Pkg); + {ok, Body, NewETag} -> + ?INFO("Downloaded package, caching at ~ts", [CachePath]), + maybe_store_etag_in_cache(UpdateETag, ETagPath, NewETag), + serve_from_download(TmpDir, CachePath, Pkg, Body); + error when ETag =/= <<>> -> + store_etag_in_cache(ETagPath, ETag), + ?INFO("Download error, using cached file at ~ts", [CachePath]), + serve_from_cache(TmpDir, CachePath, Pkg); + error -> + {fetch_fail, Name, Vsn} end. -extract_public_key_info(Cert) -> - ((Cert#'OTPCertificate'.tbsCertificate)#'OTPTBSCertificate'.subjectPublicKeyInfo). - -check_cert(CACerts, Cert) -> - lists:any(fun(CACert) -> - extract_public_key_info(CACert) == extract_public_key_info(Cert) - end, CACerts). +-spec serve_from_cache(TmpDir, CachePath, Pkg) -> Res when + TmpDir :: file:name(), + CachePath :: file:name(), + Pkg :: package(), + Res :: ok | {error,_} | {bad_registry_checksum, integer(), integer()}. +serve_from_cache(TmpDir, CachePath, Pkg) -> + {ok, Binary} = file:read_file(CachePath), + serve_from_memory(TmpDir, Binary, Pkg). -check_ssl_version() -> - case application:get_key(ssl, vsn) of - {ok, Vsn} -> - parse_vsn(Vsn) >= {5, 3, 6}; - _ -> - false +-spec serve_from_memory(TmpDir, Tarball, Package) -> Res when + TmpDir :: file:name(), + Tarball :: binary(), + Package :: package(), + Res :: ok | {error,_} | {bad_registry_checksum, integer(), integer()}. +serve_from_memory(TmpDir, Binary, {pkg, _Name, _Vsn, Hash, _RepoConfig}) -> + RegistryChecksum = list_to_integer(binary_to_list(Hash), 16), + case hex_tarball:unpack(Binary, TmpDir) of + {ok, #{checksum := <<Checksum:256/big-unsigned>>}} when RegistryChecksum =/= Checksum -> + ?DEBUG("Expected hash ~64.16.0B does not match checksum of fetched package ~64.16.0B", + [RegistryChecksum, Checksum]), + {bad_registry_checksum, RegistryChecksum, Checksum}; + {ok, #{checksum := <<RegistryChecksum:256/big-unsigned>>}} -> + ok; + {error, Reason} -> + {error, {hex_tarball, Reason}} end. -get_ssl_config() -> - GlobalConfigFile = rebar_dir:global_config(), - Config = rebar_config:consult_file(GlobalConfigFile), - case proplists:get_value(ssl_verify, Config, []) of - false -> - ssl_verify_disabled; - _ -> - ssl_verify_enabled - end. +-spec serve_from_download(TmpDir, CachePath, Package, Binary) -> Res when + TmpDir :: file:name(), + CachePath :: file:name(), + Package :: package(), + Binary :: binary(), + Res :: ok | {error,_}. +serve_from_download(TmpDir, CachePath, Package, Binary) -> + ?DEBUG("Writing ~p to cache at ~ts", [Package, CachePath]), + file:write_file(CachePath, Binary), + serve_from_memory(TmpDir, Binary, Package). -parse_vsn(Vsn) -> - version_pad(string:tokens(Vsn, ".-")). - -version_pad([Major]) -> - {list_to_integer(Major), 0, 0}; -version_pad([Major, Minor]) -> - {list_to_integer(Major), list_to_integer(Minor), 0}; -version_pad([Major, Minor, Patch]) -> - {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}; -version_pad([Major, Minor, Patch | _]) -> - {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}. +-spec maybe_store_etag_in_cache(UpdateETag, Path, ETag) -> Res when + UpdateETag :: boolean(), + Path :: file:name(), + ETag :: binary(), + Res :: ok. +maybe_store_etag_in_cache(false = _UpdateETag, _Path, _ETag) -> + ok; +maybe_store_etag_in_cache(true = _UpdateETag, Path, ETag) -> + store_etag_in_cache(Path, ETag). diff --git a/src/rebar_plugins.erl b/src/rebar_plugins.erl index 68ba6da..2a78c6e 100644 --- a/src/rebar_plugins.erl +++ b/src/rebar_plugins.erl @@ -39,13 +39,18 @@ project_apps_install(State) -> Profiles = rebar_state:current_profiles(State), ProjectApps = rebar_state:project_apps(State), lists:foldl(fun(Profile, StateAcc) -> - Plugins = rebar_state:get(State, {plugins, Profile}, []), - StateAcc1 = handle_plugins(Profile, Plugins, StateAcc), + StateAcc1 = case Profile of + default -> + %% default profile top level plugins + %% are installed in run_aux + StateAcc; + _ -> + Plugins = rebar_state:get(State, {plugins, Profile}, []), + handle_plugins(Profile, Plugins, StateAcc) + end, lists:foldl(fun(AppInfo, StateAcc2) -> - C = rebar_config:consult(rebar_app_info:dir(AppInfo)), - AppInfo0 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), C), - Plugins2 = rebar_app_info:get(AppInfo0, {plugins, Profile}, []), + Plugins2 = rebar_app_info:get(AppInfo, {plugins, Profile}, []), handle_plugins(Profile, Plugins2, StateAcc2) end, StateAcc1, ProjectApps) end, State, Profiles). @@ -62,12 +67,25 @@ install(State, AppInfo) -> State2 = lists:foldl(fun(Profile, StateAcc) -> Plugins = rebar_app_info:get(AppInfo, {plugins, Profile}, []), - handle_plugins(Profile, Plugins, StateAcc) + Plugins1 = filter_existing_plugins(Plugins, StateAcc), + handle_plugins(Profile, Plugins1, StateAcc) end, State1, Profiles), %% Reset the overrides after processing the dep rebar_state:set(State2, overrides, StateOverrides). +filter_existing_plugins(Plugins, State) -> + PluginNames = lists:zip(Plugins, rebar_state:deps_names(Plugins)), + AllPlugins = rebar_state:all_plugin_deps(State), + lists:filtermap(fun({Plugin, PluginName}) -> + case rebar_app_utils:find(PluginName, AllPlugins) of + {ok, _} -> + false; + _ -> + {true, Plugin} + end + end, PluginNames). + handle_plugins(Profile, Plugins, State) -> handle_plugins(Profile, Plugins, State, false). @@ -76,7 +94,6 @@ handle_plugins(Profile, Plugins, State, Upgrade) -> Locks = rebar_state:lock(State), DepsDir = rebar_state:get(State, deps_dir, ?DEFAULT_DEPS_DIR), State1 = rebar_state:set(State, deps_dir, ?DEFAULT_PLUGINS_DIR), - %% Install each plugin individually so if one fails to install it doesn't effect the others {_PluginProviders, State2} = lists:foldl(fun(Plugin, {PluginAcc, StateAcc}) -> @@ -96,8 +113,8 @@ handle_plugin(Profile, Plugin, State, Upgrade) -> ToBuild = rebar_prv_install_deps:cull_compile(Sorted, []), %% Add already built plugin deps to the code path - CodePaths = [rebar_app_info:ebin_dir(A) || A <- Apps -- ToBuild], - code:add_pathsa(CodePaths), + PreBuiltPaths = [rebar_app_info:ebin_dir(A) || A <- Apps] -- ToBuild, + code:add_pathsa(PreBuiltPaths), %% Build plugin and its deps [build_plugin(AppInfo, Apps, State2) || AppInfo <- ToBuild], @@ -105,23 +122,21 @@ handle_plugin(Profile, Plugin, State, Upgrade) -> %% Add newly built deps and plugin to code path State3 = rebar_state:update_all_plugin_deps(State2, Apps), NewCodePaths = [rebar_app_info:ebin_dir(A) || A <- ToBuild], - code:add_pathsa(CodePaths), %% Store plugin code paths so we can remove them when compiling project apps - State4 = rebar_state:update_code_paths(State3, all_plugin_deps, CodePaths++NewCodePaths), + State4 = rebar_state:update_code_paths(State3, all_plugin_deps, PreBuiltPaths++NewCodePaths), + rebar_paths:set_paths([plugins], State4), {plugin_providers(Plugin), State4} catch - C:T -> - ?DEBUG("~p ~p ~p", [C, T, erlang:get_stacktrace()]), + ?WITH_STACKTRACE(C,T,S) + ?DEBUG("~p ~p ~p", [C, T, S]), ?WARN("Plugin ~p not available. It will not be used.", [Plugin]), {[], State} end. build_plugin(AppInfo, Apps, State) -> Providers = rebar_state:providers(State), - %Providers1 = rebar_state:providers(rebar_app_info:state(AppInfo)), - %rebar_app_info:state_or_new(State, AppInfo) S = rebar_state:all_deps(State, Apps), S1 = rebar_state:set(S, deps_dir, ?DEFAULT_PLUGINS_DIR), rebar_prv_compile:compile(S1, Providers, AppInfo). diff --git a/src/rebar_prv_alias.erl b/src/rebar_prv_alias.erl new file mode 100644 index 0000000..ce56f29 --- /dev/null +++ b/src/rebar_prv_alias.erl @@ -0,0 +1,138 @@ +%%% @doc Meta-provider that dynamically compiles providers +%%% to run aliased commands. +%%% +%%% This is hackish and out-there, but this module has graduated +%%% from a plugin at https://github.com/tsloughter/rebar_alias after +%%% years of stability. Only some error checks were added +-module(rebar_prv_alias). + +-export([init/1]). +-include("rebar.hrl"). + +%% =================================================================== +%% Public API +%% =================================================================== +-spec init(rebar_state:t()) -> {ok, rebar_state:t()}. +init(State) -> + Aliases = rebar_state:get(State, alias, []), + lists:foldl(fun({Alias, Cmds}, {ok, StateAcc}) -> + case validate_provider(Alias, Cmds, State) of + true -> init_alias(Alias, Cmds, StateAcc); + false -> {ok, State} + end + end, {ok, State}, Aliases). + +init_alias(Alias, Cmds, State) -> + Module = list_to_atom("rebar_prv_alias_" ++ atom_to_list(Alias)), + + MF = module(Module), + EF = exports(), + FF = do_func(Cmds), + + {ok, _, Bin} = compile:forms([MF, EF, FF]), + code:load_binary(Module, "none", Bin), + + Provider = providers:create([ + {name, Alias}, + {module, Module}, + {bare, true}, + {deps, []}, + {example, example(Alias)}, + {opts, []}, + {short_desc, desc(Cmds)}, + {desc, desc(Cmds)} + ]), + {ok, rebar_state:add_provider(State, Provider)}. + +validate_provider(Alias, Cmds, State) -> + %% This would be caught and prevented anyway, but the warning + %% is friendlier + case providers:get_provider(Alias, rebar_state:providers(State)) of + not_found -> + %% check for circular deps in the alias. + case not proplists:is_defined(Alias, Cmds) of + true -> true; + false -> + ?WARN("Alias ~p contains itself and would never " + "terminate. It will be ignored.", + [Alias]), + false + end; + _ -> + ?WARN("Alias ~p is already the name of a command in " + "the default namespace and will be ignored.", + [Alias]), + false + end. + + +example(Alias) -> + "rebar3 " ++ atom_to_list(Alias). + +desc(Cmds) -> + "Equivalent to running: rebar3 do " + ++ rebar_string:join(lists:map(fun to_desc/1, Cmds), ","). + +to_desc({Cmd, Args}) when is_list(Args) -> + atom_to_list(Cmd) ++ " " ++ Args; +to_desc({Namespace, Cmd}) -> + atom_to_list(Namespace) ++ " " ++ atom_to_list(Cmd); +to_desc({Namespace, Cmd, Args}) -> + atom_to_list(Namespace) ++ " " ++ atom_to_list(Cmd) ++ " " ++ Args; +to_desc(Cmd) -> + atom_to_list(Cmd). + +module(Name) -> + {attribute, 1, module, Name}. + +exports() -> + {attribute, 1, export, [{do, 1}]}. + +do_func(Cmds) -> + {function, 1, do, 1, + [{clause, 1, + [{var, 1, 'State'}], + [], + [{call, 1, + {remote, 1, {atom, 1, rebar_prv_do}, {atom, 1, do_tasks}}, + [make_args(Cmds), {var, 1, 'State'}]}]}]}. + +make_args(Cmds) -> + make_list( + lists:map(fun make_tuple/1, + lists:map(fun make_arg/1, Cmds))). + +make_arg({Namespace, Command, Args}) when is_atom(Namespace), is_atom(Command) -> + {make_atom(Namespace), + make_atom(Command), + make_list([make_string(A) || A <- split_args(Args)])}; +make_arg({Namespace, Command}) when is_atom(Namespace), is_atom(Command) -> + {make_atom(Namespace), make_atom(Command)}; +make_arg({Cmd, Args}) -> + {make_string(Cmd), make_list([make_string(A) || A <- split_args(Args)])}; +make_arg(Cmd) -> + {make_string(Cmd), make_list([])}. + +make_tuple(Tuple) -> + {tuple, 1, tuple_to_list(Tuple)}. + +make_list(List) -> + lists:foldr( + fun(Elem, Acc) -> {cons, 1, Elem, Acc} end, + {nil, 1}, + List). + +make_string(Atom) when is_atom(Atom) -> + make_string(atom_to_list(Atom)); +make_string(String) when is_list(String) -> + {string, 1, String}. + +make_atom(Atom) when is_atom(Atom) -> + {atom, 1, Atom}. + +%% In case someone used the long option format, the option needs to get +%% separated from its value. +split_args(Args) -> + rebar_string:lexemes( + lists:map(fun($=) -> 32; (C) -> C end, Args), + " "). diff --git a/src/rebar_prv_app_discovery.erl b/src/rebar_prv_app_discovery.erl index 1954214..f5bab49 100644 --- a/src/rebar_prv_app_discovery.erl +++ b/src/rebar_prv_app_discovery.erl @@ -49,19 +49,19 @@ do(State) -> -spec format_error(any()) -> iolist(). format_error({multiple_app_files, Files}) -> - io_lib:format("Multiple app files found in one app dir: ~s", [string:join(Files, " and ")]); + io_lib:format("Multiple app files found in one app dir: ~ts", [rebar_string:join(Files, " and ")]); format_error({invalid_app_file, File, Reason}) -> case Reason of {Line, erl_parse, Description} -> - io_lib:format("Invalid app file ~s at line ~b: ~p", + io_lib:format("Invalid app file ~ts at line ~b: ~p", [File, Line, lists:flatten(Description)]); _ -> - io_lib:format("Invalid app file ~s: ~p", [File, Reason]) + io_lib:format("Invalid app file ~ts: ~p", [File, Reason]) end; %% Provide a slightly more informative error message for consult of app file failure format_error({rebar_file_utils, {bad_term_file, AppFile, Reason}}) -> - io_lib:format("Error in app file ~s: ~s", [rebar_dir:make_relative_path(AppFile, - rebar_dir:get_cwd()), - file:format_error(Reason)]); + io_lib:format("Error in app file ~ts: ~ts", [rebar_dir:make_relative_path(AppFile, + rebar_dir:get_cwd()), + file:format_error(Reason)]); format_error(Reason) -> io_lib:format("~p", [Reason]). diff --git a/src/rebar_prv_as.erl b/src/rebar_prv_as.erl index b4f7ac4..562ce99 100644 --- a/src/rebar_prv_as.erl +++ b/src/rebar_prv_as.erl @@ -33,9 +33,11 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> {Profiles, Tasks} = args_to_profiles_and_tasks(rebar_state:command_args(State)), - case Profiles of - [] -> + case {Profiles, Tasks} of + {[], _} -> {error, "At least one profile must be specified when using `as`"}; + {_, []} -> + {error, "At least one task must be specified when using `as`"}; _ -> warn_on_empty_profile(Profiles, State), State1 = rebar_state:apply_profiles(State, [list_to_atom(X) || X <- Profiles]), @@ -62,7 +64,7 @@ args_to_profiles_and_tasks(Args) -> first_profile([]) -> {[], []}; first_profile([ProfileList|Rest]) -> - case re:split(ProfileList, ",", [{return, list}, {parts, 2}]) of + case re:split(ProfileList, ",", [{return, list}, {parts, 2}, unicode]) of %% `foo, bar` [P, ""] -> profiles(Rest, [P]); %% `foo,bar` @@ -73,7 +75,7 @@ first_profile([ProfileList|Rest]) -> profiles([], Acc) -> {lists:reverse(Acc), rebar_utils:args_to_tasks([])}; profiles([ProfileList|Rest], Acc) -> - case re:split(ProfileList, ",", [{return, list}, {parts, 2}]) of + case re:split(ProfileList, ",", [{return, list}, {parts, 2}, unicode]) of %% `foo, bar` [P, ""] -> profiles(Rest, [P|Acc]); %% `foo,bar` @@ -99,5 +101,5 @@ warn_on_empty_profile(Profiles, State) -> ProjectApps = rebar_state:project_apps(State), DefinedProfiles = rebar_state:get(State, profiles, []) ++ lists:flatten([rebar_app_info:get(AppInfo, profiles, []) || AppInfo <- ProjectApps]), - [?WARN("No entry for profile ~s in config.", [Profile]) || + [?WARN("No entry for profile ~ts in config.", [Profile]) || Profile <- Profiles, not(lists:keymember(list_to_atom(Profile), 1, DefinedProfiles))]. diff --git a/src/rebar_prv_bare_compile.erl b/src/rebar_prv_bare_compile.erl index 201620a..5d3e977 100644 --- a/src/rebar_prv_bare_compile.erl +++ b/src/rebar_prv_bare_compile.erl @@ -29,7 +29,8 @@ init(State) -> {example, ""}, {short_desc, ""}, {desc, ""}, - {opts, [{paths, $p, "paths", string, "Wildcard path of ebin directories to add to code path"}]}])), + {opts, [{paths, $p, "paths", string, "Wildcard paths of ebin directories to add to code path, separated by a colon"}, + {separator, $s, "separator", string, "In case of multiple return paths, the separator character to use to join them."}]}])), {ok, State1}. -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. @@ -39,12 +40,15 @@ do(State) -> %% Add code paths from --paths to the beginning of the code path {RawOpts, _} = rebar_state:command_parsed_args(State), Paths = proplists:get_value(paths, RawOpts), - CodePaths = filelib:wildcard(Paths), - code:add_pathsa(CodePaths), + Sep = proplists:get_value(separator, RawOpts, " "), + [ code:add_pathsa(filelib:wildcard(PathWildcard)) + || PathWildcard <- rebar_string:lexemes(Paths, Sep) ], [AppInfo] = rebar_state:project_apps(State), AppInfo1 = rebar_app_info:out_dir(AppInfo, rebar_dir:get_cwd()), - rebar_prv_compile:compile(State, AppInfo1), + + %% run compile in the default namespace + rebar_prv_compile:compile(rebar_state:namespace(State, default), AppInfo1), rebar_utils:cleanup_code_path(OrigPath), diff --git a/src/rebar_prv_clean.erl b/src/rebar_prv_clean.erl index 8f31fdd..3c8a0c3 100644 --- a/src/rebar_prv_clean.erl +++ b/src/rebar_prv_clean.erl @@ -12,7 +12,7 @@ -include("rebar.hrl"). -define(PROVIDER, clean). --define(DEPS, [app_discovery]). +-define(DEPS, [app_discovery, install_deps]). %% =================================================================== %% Public API @@ -44,7 +44,8 @@ do(State) -> case All of true -> DepsDir = rebar_dir:deps_dir(State1), - AllApps = rebar_app_discover:find_apps([filename:join(DepsDir, "*")], all), + DepsDirs = filelib:wildcard(filename:join(DepsDir, "*")), + AllApps = rebar_app_discover:find_apps(DepsDirs, all), clean_apps(State1, Providers, AllApps); false -> ProjectApps = rebar_state:project_apps(State1), @@ -66,11 +67,12 @@ format_error(Reason) -> %% =================================================================== clean_apps(State, Providers, Apps) -> + Compilers = rebar_state:compilers(State), [begin - ?INFO("Cleaning out ~s...", [rebar_app_info:name(AppInfo)]), + ?INFO("Cleaning out ~ts...", [rebar_app_info:name(AppInfo)]), AppDir = rebar_app_info:dir(AppInfo), AppInfo1 = rebar_hooks:run_all_hooks(AppDir, pre, ?PROVIDER, Providers, AppInfo, State), - rebar_erlc_compiler:clean(AppInfo1), + rebar_compiler:clean(Compilers, AppInfo1), rebar_hooks:run_all_hooks(AppDir, post, ?PROVIDER, Providers, AppInfo1, State) end || AppInfo <- Apps]. diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index fbd0e89..3d3bd8a 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -8,8 +8,11 @@ -export([init/1, do/1, format_error/1]). -%% exported for test purposes, consider private --export([compile/2, prepare_tests/1, translate_paths/2]). + +-ifdef(TEST). +%% exported for test purposes +-export([compile/2, prepare_tests/1, translate_paths/2, maybe_write_coverdata/1]). +-endif. -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). @@ -41,14 +44,21 @@ do(State) -> Tests = prepare_tests(State), case compile(State, Tests) of %% successfully compiled apps - {ok, S} -> do(S, Tests); + {ok, S} -> + {RawOpts, _} = rebar_state:command_parsed_args(S), + case proplists:get_value(compile_only, RawOpts, false) of + true -> + {ok, S}; + false -> + do(S, Tests) + end; %% this should look like a compiler error, not a ct error Error -> Error end. do(State, Tests) -> ?INFO("Running Common Test suites...", []), - rebar_utils:update_code(rebar_state:code_paths(State, all_deps), [soft_purge]), + rebar_paths:set_paths([deps, plugins], State), %% Run ct provider prehooks Providers = rebar_state:providers(State), @@ -63,14 +73,14 @@ do(State, Tests) -> ok -> %% Run ct provider post hooks for all project apps and top level project hooks rebar_hooks:run_project_and_app_hooks(Cwd, post, ?PROVIDER, Providers, State), - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), + rebar_paths:set_paths([plugins, deps], State), {ok, State}; Error -> - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), + rebar_paths:set_paths([plugins, deps], State), Error end; Error -> - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), + rebar_paths:set_paths([plugins, deps], State), Error end. @@ -93,14 +103,16 @@ format_error({error, Reason}) -> format_error({error_running_tests, Reason}) -> format_error({error, Reason}); format_error({failures_running_tests, {Failed, AutoSkipped}}) -> - io_lib:format("Failures occured running tests: ~b", [Failed+AutoSkipped]); + io_lib:format("Failures occurred running tests: ~b", [Failed+AutoSkipped]); format_error({badconfig, {Msg, {Value, Key}}}) -> io_lib:format(Msg, [Value, Key]); format_error({badconfig, Msg}) -> io_lib:format(Msg, []); format_error({multiple_errors, Errors}) -> io_lib:format(lists:concat(["Error running tests:"] ++ - lists:map(fun(Error) -> "~n " ++ Error end, Errors)), []). + lists:map(fun(Error) -> "~n " ++ Error end, Errors)), []); +format_error({error_reading_testspec, Reason}) -> + io_lib:format("Error reading testspec: ~p", [Reason]). %% =================================================================== %% Internal functions @@ -126,7 +138,7 @@ cmdopts(State) -> {RawOpts, _} = rebar_state:command_parsed_args(State), %% filter out opts common_test doesn't know about and convert %% to ct acceptable forms - transform_opts(RawOpts, []). + transform_retry(transform_opts(RawOpts, []), State). transform_opts([], Acc) -> lists:reverse(Acc); transform_opts([{dir, Dirs}|Rest], Acc) -> @@ -139,6 +151,8 @@ transform_opts([{testcase, Cases}|Rest], Acc) -> transform_opts(Rest, [{testcase, split_string(Cases)}|Acc]); transform_opts([{config, Configs}|Rest], Acc) -> transform_opts(Rest, [{config, split_string(Configs)}|Acc]); +transform_opts([{spec, Specs}|Rest], Acc) -> + transform_opts(Rest, [{spec, split_string(Specs)}|Acc]); transform_opts([{include, Includes}|Rest], Acc) -> transform_opts(Rest, [{include, split_string(Includes)}|Acc]); transform_opts([{logopts, LogOpts}|Rest], Acc) -> @@ -161,8 +175,20 @@ transform_opts([{verbose, _}|Rest], Acc) -> transform_opts([Opt|Rest], Acc) -> transform_opts(Rest, [Opt|Acc]). +%% @private only retry if specified and if no other spec +%% is given. +transform_retry(Opts, State) -> + case proplists:get_value(retry, Opts, false) andalso + not is_any_defined([spec,dir,suite], Opts) of + false -> + Opts; + true -> + Path = filename:join([rebar_dir:base_dir(State), "logs", "retry.spec"]), + filelib:is_file(Path) andalso [{spec, Path}|Opts] + end. + split_string(String) -> - string:tokens(String, [$,]). + rebar_string:lexemes(String, [$,]). cfgopts(State) -> case rebar_state:get(State, ct_opts, []) of @@ -174,9 +200,6 @@ cfgopts(State) -> end. ensure_opts([], Acc) -> lists:reverse(Acc); -ensure_opts([{test_spec, _}|Rest], Acc) -> - ?WARN("Test specs not supported. See http://www.rebar3.org/docs/running-tests#common-test", []), - ensure_opts(Rest, Acc); ensure_opts([{cover, _}|Rest], Acc) -> ?WARN("Cover specs not supported. See http://www.rebar3.org/docs/running-tests#common-test", []), ensure_opts(Rest, Acc); @@ -204,16 +227,20 @@ add_hooks(Opts, State) -> case {readable(State), lists:keyfind(ct_hooks, 1, Opts)} of {false, _} -> Opts; - {true, false} -> - [{ct_hooks, [cth_readable_failonly, cth_readable_shell]} | Opts]; - {true, {ct_hooks, Hooks}} -> + {Other, false} -> + [{ct_hooks, [cth_readable_failonly, readable_shell_type(Other), cth_retry]} | Opts]; + {Other, {ct_hooks, Hooks}} -> %% Make sure hooks are there once only. - ReadableHooks = [cth_readable_failonly, cth_readable_shell], - NewHooks = (Hooks -- ReadableHooks) ++ ReadableHooks, + ReadableHooks = [cth_readable_failonly, readable_shell_type(Other), cth_retry], + AllReadableHooks = [cth_readable_failonly, cth_retry, + cth_readable_shell, cth_readable_compact_shell], + NewHooks = (Hooks -- AllReadableHooks) ++ ReadableHooks, lists:keyreplace(ct_hooks, 1, Opts, {ct_hooks, NewHooks}) end. -select_tests(_, _, {error, _} = Error, _) -> Error; +readable_shell_type(true) -> cth_readable_shell; +readable_shell_type(compact) -> cth_readable_compact_shell. + select_tests(_, _, _, {error, _} = Error) -> Error; select_tests(State, ProjectApps, CmdOpts, CfgOpts) -> %% set application env if sys_config argument is provided @@ -221,23 +248,48 @@ select_tests(State, ProjectApps, CmdOpts, CfgOpts) -> Configs = lists:flatmap(fun(Filename) -> rebar_file_utils:consult_config(State, Filename) end, SysConfigs), - [application:load(Application) || Config <- SysConfigs, {Application, _} <- Config], + %% NB: load the applications (from user directories too) to support OTP < 17 + %% to our best ability. + rebar_paths:set_paths([deps, plugins], State), + [application:load(Application) || Config <- Configs, {Application, _} <- Config], rebar_utils:reread_config(Configs), - Merged = lists:ukeymerge(1, - lists:ukeysort(1, CmdOpts), - lists:ukeysort(1, CfgOpts)), - %% make sure `dir` and/or `suite` from command line go in as - %% a pair overriding both `dir` and `suite` from config if - %% they exist - Opts = case {proplists:get_value(suite, CmdOpts), proplists:get_value(dir, CmdOpts)} of - {undefined, undefined} -> Merged; - {_Suite, undefined} -> lists:keydelete(dir, 1, Merged); - {undefined, _Dir} -> lists:keydelete(suite, 1, Merged); - {_Suite, _Dir} -> Merged - end, + Opts = merge_opts(CmdOpts,CfgOpts), discover_tests(State, ProjectApps, Opts). +%% Merge the option lists from command line and rebar.config: +%% +%% - Options set on the command line will replace the same options if +%% set in rebar.config. +%% +%% - Special care is taken with options that select which tests to +%% run - ANY such option on the command line will replace ALL such +%% options in the config. +%% +%% Note that if 'spec' is given, common_test will ignore all 'dir', +%% 'suite', 'group' and 'case', so there is no need to explicitly +%% remove any options from the command line. +%% +%% All faulty combinations of options are also handled by +%% common_test and are not taken into account here. +merge_opts(CmdOpts0, CfgOpts0) -> + TestSelectOpts = [spec,dir,suite,group,testcase], + CmdOpts = lists:ukeysort(1, CmdOpts0), + CfgOpts1 = lists:ukeysort(1, CfgOpts0), + CfgOpts = case is_any_defined(TestSelectOpts,CmdOpts) of + false -> + CfgOpts1; + true -> + [Opt || Opt={K,_} <- CfgOpts1, + not lists:member(K,TestSelectOpts)] + end, + lists:ukeymerge(1, CmdOpts, CfgOpts). + +is_any_defined([Key|Keys],Opts) -> + proplists:is_defined(Key,Opts) orelse is_any_defined(Keys,Opts); +is_any_defined([],_Opts) -> + false. + sys_config_list(CmdOpts, CfgOpts) -> CmdSysConfigs = split_string(proplists:get_value(sys_config, CmdOpts, "")), case proplists:get_value(sys_config, CfgOpts, []) of @@ -250,11 +302,10 @@ sys_config_list(CmdOpts, CfgOpts) -> end. discover_tests(State, ProjectApps, Opts) -> - case {proplists:get_value(suite, Opts), proplists:get_value(dir, Opts)} of - %% no dirs or suites defined, try using `$APP/test` and `$ROOT/test` - %% as suites - {undefined, undefined} -> {ok, [default_tests(State, ProjectApps)|Opts]}; - {_, _} -> {ok, Opts} + case is_any_defined([spec,dir,suite],Opts) of + %% no tests defined, try using `$APP/test` and `$ROOT/test` as dirs + false -> {ok, [default_tests(State, ProjectApps)|Opts]}; + true -> {ok, Opts} end. default_tests(State, ProjectApps) -> @@ -289,14 +340,9 @@ compile(State, {ok, _} = Tests) -> compile(_State, Error) -> Error. do_compile(State) -> - case rebar_prv_compile:do(State) of - %% successfully compiled apps - {ok, S} -> - ok = maybe_cover_compile(S), - {ok, S}; - %% this should look like a compiler error, not an eunit error - Error -> Error - end. + {ok, S} = rebar_prv_compile:do(State), + ok = maybe_cover_compile(S), + {ok, S}. inject_ct_state(State, {ok, Tests}) -> Apps = rebar_state:project_apps(State), @@ -304,8 +350,7 @@ inject_ct_state(State, {ok, Tests}) -> {ok, {NewState, ModdedApps}} -> test_dirs(NewState, ModdedApps, Tests); {error, _} = Error -> Error - end; -inject_ct_state(_State, Error) -> Error. + end. inject_ct_state(State, Tests, [App|Rest], Acc) -> case inject(rebar_app_info:opts(App), State, Tests) of @@ -383,31 +428,50 @@ append(A, B) -> A ++ B. add_transforms(CTOpts, State) when is_list(CTOpts) -> case readable(State) of - true -> - ReadableTransform = [{parse_transform, cth_readable_transform}], - (CTOpts -- ReadableTransform) ++ ReadableTransform; false -> - CTOpts + CTOpts; + Other when Other == true; Other == compact -> + ReadableTransform = [{parse_transform, cth_readable_transform}], + (CTOpts -- ReadableTransform) ++ ReadableTransform end; add_transforms({error, _} = Error, _State) -> Error. readable(State) -> {RawOpts, _} = rebar_state:command_parsed_args(State), case proplists:get_value(readable, RawOpts) of - true -> true; - false -> false; - undefined -> rebar_state:get(State, ct_readable, true) + "true" -> true; + "false" -> false; + "compact" -> compact; + undefined -> rebar_state:get(State, ct_readable, compact) end. test_dirs(State, Apps, Opts) -> - case {proplists:get_value(suite, Opts), proplists:get_value(dir, Opts)} of - {Suites, undefined} -> set_compile_dirs(State, Apps, {suite, Suites}); - {undefined, Dirs} -> set_compile_dirs(State, Apps, {dir, Dirs}); - {Suites, Dir} when is_integer(hd(Dir)) -> - set_compile_dirs(State, Apps, join(Suites, Dir)); - {Suites, [Dir]} when is_integer(hd(Dir)) -> - set_compile_dirs(State, Apps, join(Suites, Dir)); - {_Suites, _Dirs} -> {error, "Only a single directory may be specified when specifying suites"} + case proplists:get_value(spec, Opts) of + undefined -> + case {proplists:get_value(suite, Opts), proplists:get_value(dir, Opts)} of + {Suites, undefined} -> set_compile_dirs(State, Apps, {suite, Suites}); + {undefined, Dirs} -> set_compile_dirs(State, Apps, {dir, Dirs}); + {Suites, Dir} when is_integer(hd(Dir)) -> + set_compile_dirs(State, Apps, join(Suites, Dir)); + {Suites, [Dir]} when is_integer(hd(Dir)) -> + set_compile_dirs(State, Apps, join(Suites, Dir)); + {_Suites, _Dirs} -> {error, "Only a single directory may be specified when specifying suites"} + end; + Spec when is_integer(hd(Spec)) -> + spec_test_dirs(State, Apps, [Spec]); + Specs -> + spec_test_dirs(State, Apps, Specs) + end. + +spec_test_dirs(State, Apps, Specs0) -> + case get_dirs_from_specs(Specs0) of + {ok,{Specs,SuiteDirs}} -> + {State1,Apps1} = set_compile_dirs1(State, Apps, {dir, SuiteDirs}), + {State2,Apps2} = set_compile_dirs1(State1, Apps1, {spec, Specs}), + [maybe_copy_spec(State2,Apps2,S) || S <- Specs], + {ok, rebar_state:project_apps(State2, Apps2)}; + Error -> + Error end. join(Suite, Dir) when is_integer(hd(Suite)) -> @@ -415,27 +479,28 @@ join(Suite, Dir) when is_integer(hd(Suite)) -> join(Suites, Dir) -> {suite, lists:map(fun(S) -> filename:join([Dir, S]) end, Suites)}. -set_compile_dirs(State, Apps, {dir, Dir}) when is_integer(hd(Dir)) -> +set_compile_dirs(State, Apps, What) -> + {NewState,NewApps} = set_compile_dirs1(State, Apps, What), + {ok, rebar_state:project_apps(NewState, NewApps)}. + +set_compile_dirs1(State, Apps, {dir, Dir}) when is_integer(hd(Dir)) -> %% single directory %% insert `Dir` into an app if relative, or the base state if not %% app relative but relative to the root or not at all if outside %% project scope - {NewState, NewApps} = maybe_inject_test_dir(State, [], Apps, Dir), - {ok, rebar_state:project_apps(NewState, NewApps)}; -set_compile_dirs(State, Apps, {dir, Dirs}) -> + maybe_inject_test_dir(State, [], Apps, Dir); +set_compile_dirs1(State, Apps, {dir, Dirs}) -> %% multiple directories F = fun(Dir, {S, A}) -> maybe_inject_test_dir(S, [], A, Dir) end, - {NewState, NewApps} = lists:foldl(F, {State, Apps}, Dirs), - {ok, rebar_state:project_apps(NewState, NewApps)}; -set_compile_dirs(State, Apps, {suite, Suites}) -> - %% suites with dir component - Dirs = find_suite_dirs(Suites), + lists:foldl(F, {State, Apps}, Dirs); +set_compile_dirs1(State, Apps, {Type, Files}) when Type==spec; Type==suite -> + %% specs or suites with dir component + Dirs = find_file_dirs(Files), F = fun(Dir, {S, A}) -> maybe_inject_test_dir(S, [], A, Dir) end, - {NewState, NewApps} = lists:foldl(F, {State, Apps}, Dirs), - {ok, rebar_state:project_apps(NewState, NewApps)}. + lists:foldl(F, {State, Apps}, Dirs). -find_suite_dirs(Suites) -> - AllDirs = lists:map(fun(S) -> filename:dirname(filename:absname(S)) end, Suites), +find_file_dirs(Files) -> + AllDirs = lists:map(fun(F) -> filename:dirname(filename:absname(F)) end, Files), %% eliminate duplicates lists:usort(AllDirs). @@ -483,52 +548,79 @@ copy_bare_suites(From, To) -> ok = rebar_file_utils:cp_r(SrcFiles, To), rebar_file_utils:cp_r(DataDirs, To). +maybe_copy_spec(State, [App|Apps], Spec) -> + case rebar_file_utils:path_from_ancestor(filename:dirname(Spec), rebar_app_info:dir(App)) of + {ok, []} -> + ok = rebar_file_utils:cp_r([Spec],rebar_app_info:out_dir(App)); + {ok,_} -> + ok; + {error,badparent} -> + maybe_copy_spec(State, Apps, Spec) + end; +maybe_copy_spec(State, [], Spec) -> + case rebar_file_utils:path_from_ancestor(filename:dirname(Spec), rebar_state:dir(State)) of + {ok, []} -> + ExtrasDir = filename:join([rebar_dir:base_dir(State), "extras"]), + ok = rebar_file_utils:cp_r([Spec],ExtrasDir); + _R -> + ok + end. + inject_test_dir(Opts, Dir) -> %% append specified test targets to app defined `extra_src_dirs` ExtraSrcDirs = rebar_opts:get(Opts, extra_src_dirs, []), rebar_opts:set(Opts, extra_src_dirs, ExtraSrcDirs ++ [Dir]). +get_dirs_from_specs(Specs) -> + case get_tests_from_specs(Specs) of + {ok,Tests} -> + {SpecLists,NodeRunSkipLists} = lists:unzip(Tests), + SpecList = lists:append(SpecLists), + NodeRunSkipList = lists:append(NodeRunSkipLists), + RunList = lists:append([R || {_,R,_} <- NodeRunSkipList]), + DirList = [element(1,R) || R <- RunList], + {ok,{SpecList,DirList}}; + {error,Reason} -> + {error,{?MODULE,{error_reading_testspec,Reason}}} + end. + +get_tests_from_specs(Specs) -> + _ = ct_testspec:module_info(), % make sure ct_testspec is loaded + case erlang:function_exported(ct_testspec,get_tests,1) of + true -> + ct_testspec:get_tests(Specs); + false -> + case ct_testspec:collect_tests_from_file(Specs,true) of + Tests when is_list(Tests) -> + {ok,[{S,ct_testspec:prepare_tests(R)} || {S,R} <- Tests]}; + Error -> + Error + end + end. + translate_paths(State, Opts) -> - case {proplists:get_value(suite, Opts), proplists:get_value(dir, Opts)} of - {_Suites, undefined} -> translate_suites(State, Opts, []); - {undefined, _Dirs} -> translate_dirs(State, Opts, []); - %% both dirs and suites are defined, only translate dir paths - _ -> translate_dirs(State, Opts, []) + case proplists:get_value(spec, Opts) of + undefined -> + case {proplists:get_value(suite, Opts), proplists:get_value(dir, Opts)} of + {_Suites, undefined} -> translate_paths(State, suite, Opts, []); + {undefined, _Dirs} -> translate_paths(State, dir, Opts, []); + %% both dirs and suites are defined, only translate dir paths + _ -> translate_paths(State, dir, Opts, []) + end; + _Specs -> + translate_paths(State, spec, Opts, []) end. -translate_dirs(_State, [], Acc) -> lists:reverse(Acc); -translate_dirs(State, [{dir, Dir}|Rest], Acc) when is_integer(hd(Dir)) -> - %% single dir - Apps = rebar_state:project_apps(State), - translate_dirs(State, Rest, [{dir, translate(State, Apps, Dir)}|Acc]); -translate_dirs(State, [{dir, Dirs}|Rest], Acc) -> - %% multiple dirs - Apps = rebar_state:project_apps(State), - NewDirs = {dir, lists:map(fun(Dir) -> translate(State, Apps, Dir) end, Dirs)}, - translate_dirs(State, Rest, [NewDirs|Acc]); -translate_dirs(State, [Test|Rest], Acc) -> - translate_dirs(State, Rest, [Test|Acc]). - -translate_suites(_State, [], Acc) -> lists:reverse(Acc); -translate_suites(State, [{suite, Suite}|Rest], Acc) when is_integer(hd(Suite)) -> - %% single suite +translate_paths(_State, _Type, [], Acc) -> lists:reverse(Acc); +translate_paths(State, Type, [{Type, Val}|Rest], Acc) when is_integer(hd(Val)) -> + %% single file or dir + translate_paths(State, Type, [{Type, [Val]}|Rest], Acc); +translate_paths(State, Type, [{Type, Files}|Rest], Acc) -> Apps = rebar_state:project_apps(State), - translate_suites(State, Rest, [{suite, translate_suite(State, Apps, Suite)}|Acc]); -translate_suites(State, [{suite, Suites}|Rest], Acc) -> - %% multiple suites - Apps = rebar_state:project_apps(State), - NewSuites = {suite, lists:map(fun(Suite) -> translate_suite(State, Apps, Suite) end, Suites)}, - translate_suites(State, Rest, [NewSuites|Acc]); -translate_suites(State, [Test|Rest], Acc) -> - translate_suites(State, Rest, [Test|Acc]). - -translate_suite(State, Apps, Suite) -> - Dirname = filename:dirname(Suite), - Basename = filename:basename(Suite), - case Dirname of - "." -> Suite; - _ -> filename:join([translate(State, Apps, Dirname), Basename]) - end. + New = {Type, lists:map(fun(File) -> translate(State, Apps, File) end, Files)}, + translate_paths(State, Type, Rest, [New|Acc]); +translate_paths(State, Type, [Test|Rest], Acc) -> + translate_paths(State, Type, Rest, [Test|Acc]). translate(State, [App|Rest], Path) -> case rebar_file_utils:path_from_ancestor(Path, rebar_app_info:dir(App)) of @@ -584,7 +676,11 @@ handle_results(_) -> sum_results({Passed, Failed, {UserSkipped, AutoSkipped}}, {Passed2, Failed2, {UserSkipped2, AutoSkipped2}}) -> {Passed+Passed2, Failed+Failed2, - {UserSkipped+UserSkipped2, AutoSkipped+AutoSkipped2}}. + {UserSkipped+UserSkipped2, AutoSkipped+AutoSkipped2}}; +sum_results(_, {error, Reason}) -> + {error, Reason}; +sum_results(Unknown, _) -> + {error, Unknown}. handle_quiet_results(_, {error, _} = Result) -> handle_results(Result); @@ -607,7 +703,10 @@ format_result({Passed, 0, {0, 0}}) -> format_result({Passed, Failed, Skipped}) -> Format = [format_failed(Failed), format_skipped(Skipped), format_passed(Passed)], - ?CONSOLE("~s", [Format]). + ?CONSOLE("~ts", [Format]); +format_result(_Unknown) -> + %% Happens when CT itself encounters a bug + ok. format_failed(0) -> []; @@ -636,20 +735,24 @@ maybe_write_coverdata(State) -> true -> rebar_state:set(State, cover_enabled, true); false -> State end, - rebar_prv_cover:maybe_write_coverdata(State1, ?PROVIDER). + Name = proplists:get_value(cover_export_name, RawOpts, ?PROVIDER), + rebar_prv_cover:maybe_write_coverdata(State1, Name). ct_opts(_State) -> - [{dir, undefined, "dir", string, help(dir)}, %% comma-seperated list - {suite, undefined, "suite", string, help(suite)}, %% comma-seperated list - {group, undefined, "group", string, help(group)}, %% comma-seperated list - {testcase, undefined, "case", string, help(testcase)}, %% comma-seperated list + [{dir, undefined, "dir", string, help(dir)}, %% comma-separated list + {suite, undefined, "suite", string, help(suite)}, %% comma-separated list + {group, undefined, "group", string, help(group)}, %% comma-separated list + {testcase, undefined, "case", string, help(testcase)}, %% comma-separated list {label, undefined, "label", string, help(label)}, %% String - {config, undefined, "config", string, help(config)}, %% comma-seperated list + {config, undefined, "config", string, help(config)}, %% comma-separated list + {spec, undefined, "spec", string, help(spec)}, %% comma-separated list + {join_specs, undefined, "join_specs", boolean, help(join_specs)}, {allow_user_terms, undefined, "allow_user_terms", boolean, help(allow_user_terms)}, %% Bool {logdir, undefined, "logdir", string, help(logdir)}, %% dir - {logopts, undefined, "logopts", string, help(logopts)}, %% comma seperated list + {logopts, undefined, "logopts", string, help(logopts)}, %% comma-separated list {verbosity, undefined, "verbosity", integer, help(verbosity)}, %% Integer {cover, $c, "cover", {boolean, false}, help(cover)}, + {cover_export_name, undefined, "cover_export_name", string, help(cover_export_name)}, {repeat, undefined, "repeat", integer, help(repeat)}, %% integer {duration, undefined, "duration", string, help(duration)}, % format: HHMMSS {until, undefined, "until", string, help(until)}, %% format: YYMoMoDD[HHMMSS] @@ -663,14 +766,18 @@ ct_opts(_State) -> {scale_timetraps, undefined, "scale_timetraps", boolean, help(scale_timetraps)}, {create_priv_dir, undefined, "create_priv_dir", string, help(create_priv_dir)}, {include, undefined, "include", string, help(include)}, - {readable, undefined, "readable", boolean, help(readable)}, + {readable, undefined, "readable", string, help(readable)}, {verbose, $v, "verbose", boolean, help(verbose)}, {name, undefined, "name", atom, help(name)}, {sname, undefined, "sname", atom, help(sname)}, {setcookie, undefined, "setcookie", atom, help(setcookie)}, - {sys_config, undefined, "sys_config", string, help(sys_config)} %% comma-seperated list + {sys_config, undefined, "sys_config", string, help(sys_config)}, %% comma-separated list + {compile_only, undefined, "compile_only", boolean, help(compile_only)}, + {retry, undefined, "retry", boolean, help(retry)} ]. +help(compile_only) -> + "Compile modules in the project with the test configuration but do not run the tests"; help(dir) -> "List of additional directories containing test suites"; help(suite) -> @@ -683,6 +790,10 @@ help(label) -> "Test label"; help(config) -> "List of config files"; +help(spec) -> + "List of test specifications"; +help(join_specs) -> + "Merge all test specifications and perform a single test run"; help(sys_config) -> "List of application config files"; help(allow_user_terms) -> @@ -695,6 +806,8 @@ help(verbosity) -> "Verbosity"; help(cover) -> "Generate cover data"; +help(cover_export_name) -> + "Base name of the coverdata file to write"; help(repeat) -> "How often to repeat tests"; help(duration) -> @@ -722,7 +835,7 @@ help(create_priv_dir) -> help(include) -> "Directories containing additional include files"; help(readable) -> - "Shows test case names and only displays logs to shell on failures"; + "Shows test case names and only displays logs to shell on failures (true | compact | false)"; help(verbose) -> "Verbose output"; help(name) -> @@ -731,5 +844,7 @@ help(sname) -> "Gives a short name to the node"; help(setcookie) -> "Sets the cookie if the node is distributed"; +help(retry) -> + "Experimental feature. If any specification for previously failing test is found, runs them."; help(_) -> "". diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index effc763..cf2bcf2 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -30,34 +30,49 @@ init(State) -> {example, "rebar3 compile"}, {short_desc, "Compile apps .app.src and .erl files."}, {desc, "Compile apps .app.src and .erl files."}, - {opts, []}])), + {opts, [{deps_only, $d, "deps_only", undefined, + "Only compile dependencies, no project apps will be built."}]}])), {ok, State1}. -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - DepsPaths = rebar_state:code_paths(State, all_deps), - PluginDepsPaths = rebar_state:code_paths(State, all_plugin_deps), - rebar_utils:remove_from_code_path(PluginDepsPaths), - code:add_pathsa(DepsPaths), + IsDepsOnly = is_deps_only(State), + rebar_paths:set_paths([deps], State), - ProjectApps = rebar_state:project_apps(State), Providers = rebar_state:providers(State), Deps = rebar_state:deps_to_build(State), - Cwd = rebar_state:dir(State), + copy_and_build_apps(State, Providers, Deps), + + State1 = case IsDepsOnly of + true -> + State; + false -> + handle_project_apps(Providers, State) + end, + + rebar_paths:set_paths([plugins], State1), + + {ok, State1}. + +is_deps_only(State) -> + {Args, _} = rebar_state:command_parsed_args(State), + proplists:get_value(deps_only, Args, false). - build_apps(State, Providers, Deps), +handle_project_apps(Providers, State) -> + Cwd = rebar_state:dir(State), + ProjectApps = rebar_state:project_apps(State), {ok, ProjectApps1} = rebar_digraph:compile_order(ProjectApps), %% Run top level hooks *before* project apps compiled but *after* deps are rebar_hooks:run_all_hooks(Cwd, pre, ?PROVIDER, Providers, State), - ProjectApps2 = build_apps(State, Providers, ProjectApps1), + ProjectApps2 = copy_and_build_project_apps(State, Providers, ProjectApps1), State2 = rebar_state:project_apps(State, ProjectApps2), %% projects with structures like /apps/foo,/apps/bar,/test build_extra_dirs(State, ProjectApps2), - State3 = update_code_paths(State2, ProjectApps2, DepsPaths), + State3 = update_code_paths(State2, ProjectApps2), rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State2), case rebar_state:has_all_artifacts(State3) of @@ -66,18 +81,23 @@ do(State) -> true -> true end, - rebar_utils:cleanup_code_path(rebar_state:code_paths(State3, default) - ++ rebar_state:code_paths(State, all_plugin_deps)), - {ok, State3}. + State3. + -spec format_error(any()) -> iolist(). format_error({missing_artifact, File}) -> - io_lib:format("Missing artifact ~s", [File]); + io_lib:format("Missing artifact ~ts", [File]); +format_error({bad_project_builder, Name, Type, Module}) -> + io_lib:format("Error building application ~s:~n Required project builder ~s function " + "~s:build/1 not found", [Name, Type, Module]); +format_error({unknown_project_type, Name, Type}) -> + io_lib:format("Error building application ~s:~n " + "No project builder is configured for type ~s", [Name, Type]); format_error(Reason) -> io_lib:format("~p", [Reason]). -build_apps(State, Providers, Apps) -> +copy_and_build_apps(State, Providers, Apps) -> [build_app(State, Providers, AppInfo) || AppInfo <- Apps]. build_app(State, Providers, AppInfo) -> @@ -86,6 +106,19 @@ build_app(State, Providers, AppInfo) -> copy_app_dirs(AppInfo, AppDir, OutDir), compile(State, Providers, AppInfo). +copy_and_build_project_apps(State, Providers, Apps) -> + %% Top-level apps, because of profile usage and specific orderings (i.e. + %% may require an include file from a profile-specific app for an extra_dirs + %% entry that only exists in a test context), need to be + %% copied and added to the path at once, and not just in compile order. + [copy_app_dirs(AppInfo, + rebar_app_info:dir(AppInfo), + rebar_app_info:out_dir(AppInfo)) + || AppInfo <- Apps], + code:add_pathsa([rebar_app_info:ebin_dir(AppInfo) || AppInfo <- Apps]), + [compile(State, Providers, AppInfo) || AppInfo <- Apps]. + + build_extra_dirs(State, Apps) -> BaseDir = rebar_state:dir(State), F = fun(App) -> rebar_app_info:dir(App) == BaseDir end, @@ -104,26 +137,47 @@ build_extra_dir(State, Dir) -> true -> BaseDir = filename:join([rebar_dir:base_dir(State), "extras"]), OutDir = filename:join([BaseDir, Dir]), - filelib:ensure_dir(filename:join([OutDir, "dummy.beam"])), + rebar_file_utils:ensure_dir(OutDir), copy(rebar_state:dir(State), BaseDir, Dir), - rebar_erlc_compiler:compile_dir(State, BaseDir, OutDir); - false -> ok + + Compilers = rebar_state:compilers(State), + FakeApp = rebar_app_info:new(), + FakeApp1 = rebar_app_info:out_dir(FakeApp, BaseDir), + FakeApp2 = rebar_app_info:ebin_dir(FakeApp1, OutDir), + Opts = rebar_state:opts(State), + FakeApp3 = rebar_app_info:opts(FakeApp2, Opts), + FakeApp4 = rebar_app_info:set(FakeApp3, src_dirs, [OutDir]), + rebar_compiler:compile_all(Compilers, FakeApp4); + false -> + ok end. compile(State, AppInfo) -> compile(State, rebar_state:providers(State), AppInfo). compile(State, Providers, AppInfo) -> - ?INFO("Compiling ~s", [rebar_app_info:name(AppInfo)]), + ?INFO("Compiling ~ts", [rebar_app_info:name(AppInfo)]), AppDir = rebar_app_info:dir(AppInfo), AppInfo1 = rebar_hooks:run_all_hooks(AppDir, pre, ?PROVIDER, Providers, AppInfo, State), AppInfo2 = rebar_hooks:run_all_hooks(AppDir, pre, ?ERLC_HOOK, Providers, AppInfo1, State), - rebar_erlc_compiler:compile(AppInfo2), + + build_app(AppInfo2, State), + AppInfo3 = rebar_hooks:run_all_hooks(AppDir, post, ?ERLC_HOOK, Providers, AppInfo2, State), AppInfo4 = rebar_hooks:run_all_hooks(AppDir, pre, ?APP_HOOK, Providers, AppInfo3, State), - case rebar_otp_app:compile(State, AppInfo4) of + + %% Load plugins back for make_vsn calls in custom resources. + %% The rebar_otp_app compilation step is safe regarding the + %% overall path management, so we can just load all plugins back + %% in memory. + rebar_paths:set_paths([plugins], State), + AppFileCompileResult = rebar_otp_app:compile(State, AppInfo4), + %% Clean up after ourselves, leave things as they were with deps first + rebar_paths:set_paths([deps], State), + + case AppFileCompileResult of {ok, AppInfo5} -> AppInfo6 = rebar_hooks:run_all_hooks(AppDir, post, ?APP_HOOK, Providers, AppInfo5, State), AppInfo7 = rebar_hooks:run_all_hooks(AppDir, post, ?PROVIDER, Providers, AppInfo6, State), @@ -137,9 +191,33 @@ compile(State, Providers, AppInfo) -> %% Internal functions %% =================================================================== -update_code_paths(State, ProjectApps, DepsPaths) -> +build_app(AppInfo, State) -> + case rebar_app_info:project_type(AppInfo) of + Type when Type =:= rebar3 ; Type =:= undefined -> + Compilers = rebar_state:compilers(State), + rebar_paths:set_paths([deps], State), + rebar_compiler:compile_all(Compilers, AppInfo); + Type -> + ProjectBuilders = rebar_state:project_builders(State), + case lists:keyfind(Type, 1, ProjectBuilders) of + {_, Module} -> + %% load plugins since thats where project builders would be + rebar_paths:set_paths([deps, plugins], State), + Res = Module:build(AppInfo), + rebar_paths:set_paths([deps], State), + case Res of + ok -> ok; + {error, Reason} -> throw({error, {Module, Reason}}) + end; + _ -> + throw(?PRV_ERROR({unknown_project_type, rebar_app_info:name(AppInfo), Type})) + end + end. + +update_code_paths(State, ProjectApps) -> ProjAppsPaths = paths_for_apps(ProjectApps), ExtrasPaths = paths_for_extras(State, ProjectApps), + DepsPaths = rebar_state:code_paths(State, all_deps), rebar_state:code_paths(State, all_deps, DepsPaths ++ ProjAppsPaths ++ ExtrasPaths). paths_for_apps(Apps) -> paths_for_apps(Apps, []). @@ -173,8 +251,8 @@ has_all_artifacts(AppInfo1) -> end. copy_app_dirs(AppInfo, OldAppDir, AppDir) -> - case ec_cnv:to_binary(filename:absname(OldAppDir)) =/= - ec_cnv:to_binary(filename:absname(AppDir)) of + case rebar_utils:to_binary(filename:absname(OldAppDir)) =/= + rebar_utils:to_binary(filename:absname(AppDir)) of true -> EbinDir = filename:join([OldAppDir, "ebin"]), %% copy all files from ebin if it exists @@ -201,7 +279,11 @@ copy_app_dirs(AppInfo, OldAppDir, AppDir) -> end, {SrcDirs, ExtraDirs} = resolve_src_dirs(rebar_app_info:opts(AppInfo)), %% link to src_dirs to be adjacent to ebin is needed for R15 use of cover/xref - [symlink_or_copy(OldAppDir, AppDir, Dir) || Dir <- ["priv", "include"] ++ SrcDirs], + %% priv/ and include/ are symlinked unconditionally to allow hooks + %% to write to them _after_ compilation has taken place when the + %% initial directory did not, and still work + [symlink_or_copy(OldAppDir, AppDir, Dir) || Dir <- ["priv", "include"]], + [symlink_or_copy_existing(OldAppDir, AppDir, Dir) || Dir <- SrcDirs], %% copy all extra_src_dirs as they build into themselves and linking means they %% are shared across profiles [copy(OldAppDir, AppDir, Dir) || Dir <- ExtraDirs]; @@ -214,6 +296,14 @@ symlink_or_copy(OldAppDir, AppDir, Dir) -> Target = filename:join([AppDir, Dir]), rebar_file_utils:symlink_or_copy(Source, Target). +symlink_or_copy_existing(OldAppDir, AppDir, Dir) -> + Source = filename:join([OldAppDir, Dir]), + Target = filename:join([AppDir, Dir]), + case ec_file:is_dir(Source) of + true -> rebar_file_utils:symlink_or_copy(Source, Target); + false -> ok + end. + copy(OldAppDir, AppDir, Dir) -> Source = filename:join([OldAppDir, Dir]), Target = filename:join([AppDir, Dir]), diff --git a/src/rebar_prv_cover.erl b/src/rebar_prv_cover.erl index 464967b..a279293 100644 --- a/src/rebar_prv_cover.erl +++ b/src/rebar_prv_cover.erl @@ -12,10 +12,11 @@ maybe_write_coverdata/2, format_error/1]). +-include_lib("providers/include/providers.hrl"). -include("rebar.hrl"). -define(PROVIDER, cover). --define(DEPS, [app_discovery]). +-define(DEPS, [lock]). %% =================================================================== %% Public API @@ -62,6 +63,9 @@ maybe_write_coverdata(State, Task) -> end. -spec format_error(any()) -> iolist(). +format_error({min_coverage_failed, {PassRate, Total}}) -> + io_lib:format("Requiring ~p% coverage to pass. Only ~p% obtained", + [PassRate, Total]); format_error(Reason) -> io_lib:format("~p", [Reason]). @@ -84,6 +88,15 @@ reset(State) -> {ok, State}. analyze(State) -> + %% modules have to be compiled and then cover compiled + %% in order for cover data to be reloaded + %% this maybe breaks if modules have been deleted + %% since code coverage was collected? + {ok, S} = rebar_prv_compile:do(State), + ok = cover_compile(S, apps), + do_analyze(State). + +do_analyze(State) -> ?INFO("Performing cover analysis...", []), %% figure out what coverdata we have CoverDir = cover_dir(State), @@ -93,13 +106,13 @@ analyze(State) -> %% redirect cover output true = redirect_cover_output(State, CoverPid), %% analyze! - ok = case analyze(State, CoverFiles) of - [] -> ok; + case analyze(State, CoverFiles) of + [] -> {ok, State}; Analysis -> print_analysis(Analysis, verbose(State)), - write_index(State, Analysis) - end, - {ok, State}. + write_index(State, Analysis), + maybe_fail_coverage(Analysis, State) + end. get_all_coverdata(CoverDir) -> ok = filelib:ensure_dir(filename:join([CoverDir, "dummy.log"])), @@ -187,10 +200,7 @@ mod_to_filename(TaskDir, M) -> process(Coverage) -> process(Coverage, {0, 0}). -process([], {0, 0}) -> - "0%"; -process([], {Cov, Not}) -> - integer_to_list(trunc((Cov / (Cov + Not)) * 100)) ++ "%"; +process([], Acc) -> Acc; %% line 0 is a line added by eunit and never executed so ignore it process([{{_, 0}, _}|Rest], Acc) -> process(Rest, Acc); process([{_, {Cov, Not}}|Rest], {Covered, NotCovered}) -> @@ -199,56 +209,56 @@ process([{_, {Cov, Not}}|Rest], {Covered, NotCovered}) -> print_analysis(_, false) -> ok; print_analysis(Analysis, true) -> {_, CoverFiles, Stats} = lists:keyfind("aggregate", 1, Analysis), - ConsoleStats = [ {atom_to_list(M), C} || {M, C, _} <- Stats ], - Table = format_table(ConsoleStats, CoverFiles), + Table = format_table(Stats, CoverFiles), io:format("~ts", [Table]). format_table(Stats, CoverFiles) -> - MaxLength = max(lists:foldl(fun max_length/2, 0, Stats), 20), + MaxLength = lists:max([20 | lists:map(fun({M, _, _}) -> mod_length(M) end, Stats)]), Header = header(MaxLength), - Seperator = seperator(MaxLength), + Separator = separator(MaxLength), TotalLabel = format("total", MaxLength), - TotalCov = format(calculate_total(Stats), 8), - [io_lib:format("~ts~n~ts~n~ts~n", [Seperator, Header, Seperator]), - lists:map(fun({Mod, Coverage}) -> + TotalCov = format(calculate_total_string(Stats), 8), + [io_lib:format("~ts~n~ts~n~ts~n", [Separator, Header, Separator]), + lists:map(fun({Mod, Coverage, _}) -> Name = format(Mod, MaxLength), - Cov = format(Coverage, 8), + Cov = format(percentage_string(Coverage), 8), io_lib:format(" | ~ts | ~ts |~n", [Name, Cov]) end, Stats), - io_lib:format("~ts~n", [Seperator]), + io_lib:format("~ts~n", [Separator]), io_lib:format(" | ~ts | ~ts |~n", [TotalLabel, TotalCov]), - io_lib:format("~ts~n", [Seperator]), + io_lib:format("~ts~n", [Separator]), io_lib:format(" coverage calculated from:~n", []), lists:map(fun(File) -> io_lib:format(" ~ts~n", [File]) end, CoverFiles)]. -max_length({ModName, _}, Min) -> - Length = length(lists:flatten(ModName)), - case Length > Min of - true -> Length; - false -> Min - end. +mod_length(Mod) when is_atom(Mod) -> mod_length(atom_to_list(Mod)); +mod_length(Mod) -> length(Mod). header(Width) -> [" | ", format("module", Width), " | ", format("coverage", 8), " |"]. -seperator(Width) -> +separator(Width) -> [" |--", io_lib:format("~*c", [Width, $-]), "--|------------|"]. format(String, Width) -> io_lib:format("~*.ts", [Width, String]). -calculate_total(Stats) when length(Stats) =:= 0 -> - "0%"; +calculate_total_string(Stats) -> + integer_to_list(calculate_total(Stats))++"%". + calculate_total(Stats) -> - TotalStats = length(Stats), - TotalCovInt = round(lists:foldl( - fun({_Mod, Coverage, _File}, Acc) -> - Acc + (list_to_integer(string:strip(Coverage, right, $%)) / TotalStats); - ({_Mod, Coverage}, Acc) -> - Acc + (list_to_integer(string:strip(Coverage, right, $%)) / TotalStats) - end, 0, Stats)), - integer_to_list(TotalCovInt) ++ "%". + percentage(lists:foldl( + fun({_Mod, {Cov, Not}, _File}, {CovAcc, NotAcc}) -> + {CovAcc + Cov, NotAcc + Not} + end, + {0, 0}, + Stats + )). + +percentage_string(Data) -> integer_to_list(percentage(Data))++"%". + +percentage({_, 0}) -> 100; +percentage({Cov, Not}) -> trunc((Cov / (Cov + Not)) * 100). write_index(State, Coverage) -> CoverDir = cover_dir(State), @@ -269,7 +279,7 @@ write_index(State, Coverage) -> write_index_section(_F, []) -> ok; write_index_section(F, [{Section, DataFile, Mods}|Rest]) -> %% Write the report - ok = file:write(F, ?FMT("<h1>~s summary</h1>\n", [Section])), + ok = file:write(F, ?FMT("<h1>~ts summary</h1>\n", [Section])), ok = file:write(F, "coverage calculated from:\n<ul>"), ok = lists:foreach(fun(D) -> ok = file:write(F, io_lib:format("<li>~ts</li>", [D])) end, DataFile), @@ -278,14 +288,25 @@ write_index_section(F, [{Section, DataFile, Mods}|Rest]) -> FmtLink = fun({Mod, Cov, Report}) -> ?FMT("<tr><td><a href='~ts'>~ts</a></td><td>~ts</td>\n", - [strip_coverdir(Report), Mod, Cov]) + [strip_coverdir(Report), Mod, percentage_string(Cov)]) end, lists:foreach(fun(M) -> ok = file:write(F, FmtLink(M)) end, Mods), ok = file:write(F, ?FMT("<tr><td><strong>Total</strong></td><td>~ts</td>\n", - [calculate_total(Mods)])), + [calculate_total_string(Mods)])), ok = file:write(F, "</table>\n"), write_index_section(F, Rest). +maybe_fail_coverage(Analysis, State) -> + {_, _CoverFiles, Stats} = lists:keyfind("aggregate", 1, Analysis), + Total = calculate_total(Stats), + PassRate = min_coverage(State), + ?DEBUG("Comparing ~p to pass rate ~p", [Total, PassRate]), + if Total >= PassRate -> + {ok, State} + ; Total < PassRate -> + ?PRV_ERROR({min_coverage_failed, {PassRate, Total}}) + end. + %% fix for r15b which doesn't put the correct path in the `source` section %% of `module_info(compile)` strip_coverdir([]) -> ""; @@ -294,45 +315,66 @@ strip_coverdir(File) -> 2))). cover_compile(State, apps) -> - Apps = filter_checkouts(rebar_state:project_apps(State)), + ExclApps = [rebar_utils:to_binary(A) || A <- rebar_state:get(State, cover_excl_apps, [])], + Apps = filter_checkouts_and_excluded(rebar_state:project_apps(State), ExclApps), AppDirs = app_dirs(Apps), cover_compile(State, lists:filter(fun(D) -> ec_file:is_dir(D) end, AppDirs)); cover_compile(State, Dirs) -> - rebar_utils:update_code(rebar_state:code_paths(State, all_deps), [soft_purge]), + rebar_paths:set_paths([deps], State), %% start the cover server if necessary {ok, CoverPid} = start_cover(), %% redirect cover output true = redirect_cover_output(State, CoverPid), + ExclMods = rebar_state:get(State, cover_excl_mods, []), lists:foreach(fun(Dir) -> - ?DEBUG("cover compiling ~p", [Dir]), - case catch(cover:compile_beam_directory(Dir)) of + case file:list_dir(Dir) of + {ok, Files} -> + ?DEBUG("cover compiling ~p", [Dir]), + [cover_compile_file(filename:join(Dir, File)) + || File <- Files, + filename:extension(File) == ".beam", + not is_ignored(Dir, File, ExclMods)], + ok; {error, eacces} -> ?WARN("Directory ~p not readable, modules will not be included in coverage", [Dir]); {error, enoent} -> ?WARN("Directory ~p not found", [Dir]); - {'EXIT', {Reason, _}} -> - ?WARN("Cover compilation for directory ~p failed: ~p", [Dir, Reason]); - Results -> - %% print any warnings about modules that failed to cover compile - lists:foreach(fun print_cover_warnings/1, lists:flatten(Results)) + {error, Reason} -> + ?WARN("Directory ~p error ~p", [Dir, Reason]) end end, Dirs), - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), ok. +is_ignored(Dir, File, ExclMods) -> + Ignored = lists:any(fun(Excl) -> + File =:= atom_to_list(Excl) ++ ".beam" + end, + ExclMods), + Ignored andalso ?DEBUG("cover ignoring ~p ~p", [Dir, File]), + Ignored. + +cover_compile_file(FileName) -> + case catch(cover:compile_beam(FileName)) of + {error, Reason} -> + ?WARN("Cover compilation failed: ~p", [Reason]); + {ok, _} -> + ok + end. + app_dirs(Apps) -> lists:foldl(fun app_ebin_dirs/2, [], Apps). app_ebin_dirs(App, Acc) -> [rebar_app_info:ebin_dir(App)|Acc]. -filter_checkouts(Apps) -> filter_checkouts(Apps, []). +filter_checkouts_and_excluded(Apps, ExclApps) -> + filter_checkouts_and_excluded(Apps, ExclApps, []). -filter_checkouts([], Acc) -> lists:reverse(Acc); -filter_checkouts([App|Rest], Acc) -> - case rebar_app_info:is_checkout(App) of - true -> filter_checkouts(Rest, Acc); - false -> filter_checkouts(Rest, [App|Acc]) +filter_checkouts_and_excluded([], _ExclApps, Acc) -> lists:reverse(Acc); +filter_checkouts_and_excluded([App|Rest], ExclApps, Acc) -> + case rebar_app_info:is_checkout(App) orelse lists:member(rebar_app_info:name(App), ExclApps) of + true -> filter_checkouts_and_excluded(Rest, ExclApps, Acc); + false -> filter_checkouts_and_excluded(Rest, ExclApps, [App|Acc]) end. start_cover() -> @@ -349,16 +391,14 @@ redirect_cover_output(State, CoverPid) -> [append]), group_leader(F, CoverPid). -print_cover_warnings({ok, _}) -> ok; -print_cover_warnings({error, Error}) -> - ?WARN("Cover compilation failed: ~p", [Error]). - -write_coverdata(State, Task) -> +write_coverdata(State, Name) -> DataDir = cover_dir(State), ok = filelib:ensure_dir(filename:join([DataDir, "dummy.log"])), - ExportFile = filename:join([DataDir, atom_to_list(Task) ++ ".coverdata"]), + ExportFile = filename:join([DataDir, rebar_utils:to_list(Name) ++ ".coverdata"]), case cover:export(ExportFile) of ok -> + %% dump accumulated coverdata after writing + ok = cover:reset(), ?DEBUG("Cover data written to ~p.", [ExportFile]); {error, Reason} -> ?WARN("Cover data export failed: ~p", [Reason]) @@ -380,12 +420,23 @@ verbose(State) -> {Verbose, _} -> Verbose end. +min_coverage(State) -> + Command = proplists:get_value(min_coverage, command_line_opts(State), undefined), + Config = proplists:get_value(min_coverage, config_opts(State), undefined), + case {Command, Config} of + {undefined, undefined} -> 0; + {undefined, Rate} -> Rate; + {Rate, _} -> Rate + end. + cover_dir(State) -> filename:join([rebar_dir:base_dir(State), "cover"]). cover_opts(_State) -> [{reset, $r, "reset", boolean, help(reset)}, - {verbose, $v, "verbose", boolean, help(verbose)}]. + {verbose, $v, "verbose", boolean, help(verbose)}, + {min_coverage, $m, "min_coverage", integer, help(min_coverage)}]. help(reset) -> "Reset all coverdata."; -help(verbose) -> "Print coverage analysis.". +help(verbose) -> "Print coverage analysis."; +help(min_coverage) -> "Mandate a coverage percentage required to succeed (0..100)". diff --git a/src/rebar_prv_deps.erl b/src/rebar_prv_deps.erl index c865276..577a859 100644 --- a/src/rebar_prv_deps.erl +++ b/src/rebar_prv_deps.erl @@ -55,7 +55,7 @@ merge(Deps, SourceDeps) -> normalize(Name) when is_binary(Name) -> Name; normalize(Name) when is_atom(Name) -> - ec_cnv:to_binary(Name); + atom_to_binary(Name, unicode); normalize(Dep) when is_tuple(Dep) -> Name = element(1, Dep), setelement(1, Dep, normalize(Name)). @@ -87,31 +87,33 @@ display_deps(State, Deps) -> %% packages display_dep(_State, {Name, Vsn}) when is_list(Vsn) -> - ?CONSOLE("~s* (package ~s)", [ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)]); + ?CONSOLE("~ts* (package ~ts)", [rebar_utils:to_binary(Name), rebar_utils:to_binary(Vsn)]); display_dep(_State, Name) when is_binary(Name) -> - ?CONSOLE("~s* (package)", [Name]); + ?CONSOLE("~ts* (package)", [Name]); display_dep(_State, {Name, Source}) when is_tuple(Source) -> - ?CONSOLE("~s* (~s source)", [ec_cnv:to_binary(Name), type(Source)]); + ?CONSOLE("~ts* (~ts source)", [rebar_utils:to_binary(Name), type(Source)]); display_dep(_State, {Name, _Vsn, Source}) when is_tuple(Source) -> - ?CONSOLE("~s* (~s source)", [ec_cnv:to_binary(Name), type(Source)]); + ?CONSOLE("~ts* (~ts source)", [rebar_utils:to_binary(Name), type(Source)]); display_dep(_State, {Name, _Vsn, Source, _Opts}) when is_tuple(Source) -> - ?CONSOLE("~s* (~s source)", [ec_cnv:to_binary(Name), type(Source)]); + ?CONSOLE("~ts* (~ts source)", [rebar_utils:to_binary(Name), type(Source)]); %% Locked -display_dep(State, {Name, Source={pkg, _, Vsn, _}, Level}) when is_integer(Level) -> +display_dep(State, {Name, _Source={pkg, _, Vsn}, Level}) when is_integer(Level) -> DepsDir = rebar_dir:deps_dir(State), - AppDir = filename:join([DepsDir, ec_cnv:to_binary(Name)]), - NeedsUpdate = case rebar_fetch:needs_update(AppDir, Source, State) of + AppDir = filename:join([DepsDir, rebar_utils:to_binary(Name)]), + {ok, AppInfo} = rebar_app_info:discover(AppDir), + NeedsUpdate = case rebar_fetch:needs_update(AppInfo, State) of true -> "*"; false -> "" end, - ?CONSOLE("~s~s (locked package ~s)", [Name, NeedsUpdate, Vsn]); + ?CONSOLE("~ts~ts (locked package ~ts)", [Name, NeedsUpdate, Vsn]); display_dep(State, {Name, Source, Level}) when is_tuple(Source), is_integer(Level) -> DepsDir = rebar_dir:deps_dir(State), - AppDir = filename:join([DepsDir, ec_cnv:to_binary(Name)]), - NeedsUpdate = case rebar_fetch:needs_update(AppDir, Source, State) of + AppDir = filename:join([DepsDir, rebar_utils:to_binary(Name)]), + {ok, AppInfo} = rebar_app_info:discover(AppDir), + NeedsUpdate = case rebar_fetch:needs_update(AppInfo, State) of true -> "*"; false -> "" end, - ?CONSOLE("~s~s (locked ~s source)", [Name, NeedsUpdate, type(Source)]). + ?CONSOLE("~ts~ts (locked ~ts source)", [Name, NeedsUpdate, type(Source)]). type(Source) when is_tuple(Source) -> element(1, Source). diff --git a/src/rebar_prv_deps_tree.erl b/src/rebar_prv_deps_tree.erl index c0c8bab..d7b49c5 100644 --- a/src/rebar_prv_deps_tree.erl +++ b/src/rebar_prv_deps_tree.erl @@ -39,27 +39,23 @@ format_error(Reason) -> %% Internal functions print_deps_tree(SrcDeps, Verbose, State) -> - Resources = rebar_state:resources(State), D = lists:foldl(fun(App, Dict) -> Name = rebar_app_info:name(App), Vsn = rebar_app_info:original_vsn(App), - AppDir = rebar_app_info:dir(App), - Vsn1 = rebar_utils:vcs_vsn(Vsn, AppDir, Resources), + Vsn1 = rebar_utils:vcs_vsn(App, Vsn, State), Source = rebar_app_info:source(App), Parent = rebar_app_info:parent(App), dict:append_list(Parent, [{Name, Vsn1, Source}], Dict) end, dict:new(), SrcDeps), ProjectAppNames = [{rebar_app_info:name(App) - ,rebar_utils:vcs_vsn(rebar_app_info:original_vsn(App), rebar_app_info:dir(App), Resources) + ,rebar_utils:vcs_vsn(App, rebar_app_info:original_vsn(App), State) ,project} || App <- rebar_state:project_apps(State)], - io:setopts([{encoding, unicode}]), case dict:find(root, D) of {ok, Children} -> print_children("", lists:keysort(1, Children++ProjectAppNames), D, Verbose); error -> print_children("", lists:keysort(1, ProjectAppNames), D, Verbose) - end, - io:setopts([{encoding, latin1}]). + end. print_children(_, [], _, _) -> ok; @@ -90,7 +86,7 @@ type(Source, Verbose) when is_tuple(Source) -> {pkg, _} -> "hex package"; {Other, false} -> - io_lib:format("~s repo", [Other]); + io_lib:format("~ts repo", [Other]); {_, true} -> - io_lib:format("~s", [element(2, Source)]) + io_lib:format("~ts", [element(2, Source)]) end. diff --git a/src/rebar_prv_dialyzer.erl b/src/rebar_prv_dialyzer.erl index 82d2d07..585051c 100644 --- a/src/rebar_prv_dialyzer.erl +++ b/src/rebar_prv_dialyzer.erl @@ -47,26 +47,33 @@ desc() -> "`plt_apps` - the strategy for determining the applications which included " "in the PLT file, `top_level_deps` to include just the direct dependencies " "or `all_deps` to include all nested dependencies*\n" - "`plt_extra_apps` - a list of applications to include in the PLT file**\n" + "`plt_extra_apps` - a list of extra applications to include in the PLT " + "file\n" + "`plt_extra_mods` - a list of extra modules to includes in the PLT file\n" "`plt_location` - the location of the PLT file, `local` to store in the " "profile's base directory (default) or a custom directory.\n" - "`plt_prefix` - the prefix to the PLT file, defaults to \"rebar3\"***\n" + "`plt_prefix` - the prefix to the PLT file, defaults to \"rebar3\"**\n" "`base_plt_apps` - a list of applications to include in the base " - "PLT file****\n" + "PLT file***\n" + "`base_plt_mods` - a list of modules to include in the base " + "PLT file***\n" "`base_plt_location` - the location of base PLT file, `global` to store in " - "$HOME/.cache/rebar3 (default) or a custom directory****\n" + "$HOME/.cache/rebar3 (default) or a custom directory***\n" "`base_plt_prefix` - the prefix to the base PLT file, defaults to " - "\"rebar3\"*** ****\n" + "\"rebar3\"** ***\n" + "`exclude_apps` - a list of applications to exclude from PLT files and " + "success typing analysis, `plt_extra_mods` and `base_plt_mods` can add " + "modules from excluded applications\n" + "`exclude_mods` - a list of modules to exclude from PLT files and " + "success typing analysis\n" "\n" "For example, to warn on unmatched returns: \n" "{dialyzer, [{warnings, [unmatched_returns]}]}.\n" "\n" "*The direct dependent applications are listed in `applications` and " "`included_applications` of their .app files.\n" - "**The applications in `base_plt_apps` will be added to the " - "list. \n" - "***PLT files are named \"<prefix>_<otp_release>_plt\".\n" - "****The base PLT is a PLT containing the core applications often required " + "**PLT files are named \"<prefix>_<otp_release>_plt\".\n" + "***The base PLT is a PLT containing the core applications often required " "for a project's PLT. One base PLT is created per OTP version and " "stored in `base_plt_location`. A base PLT is used to build project PLTs." "\n". @@ -78,7 +85,8 @@ short_desc() -> do(State) -> maybe_fix_env(), ?INFO("Dialyzer starting, this may take a while...", []), - code:add_pathsa(rebar_state:code_paths(State, all_deps)), + rebar_paths:unset_paths([plugins], State), % no plugins in analysis + rebar_paths:set_paths([deps], State), Plt = get_plt(State), try @@ -90,10 +98,14 @@ do(State) -> ?PRV_ERROR({dialyzer_warnings, Warnings}); throw:{unknown_application, _} = Error -> ?PRV_ERROR(Error); + throw:{unknown_module, _} = Error -> + ?PRV_ERROR(Error); + throw:{duplicate_module, _, _, _} = Error -> + ?PRV_ERROR(Error); throw:{output_file_error, _, _} = Error -> ?PRV_ERROR(Error) after - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)) + rebar_paths:set_paths([plugins,deps], State) end. %% This is used to workaround dialyzer quirk discussed here @@ -105,14 +117,18 @@ maybe_fix_env() -> -spec format_error(any()) -> iolist(). format_error({error_processing_apps, Error}) -> - io_lib:format("Error in dialyzing apps: ~s", [Error]); + io_lib:format("Error in dialyzing apps: ~ts", [Error]); format_error({dialyzer_warnings, Warnings}) -> - io_lib:format("Warnings occured running dialyzer: ~b", [Warnings]); + io_lib:format("Warnings occurred running dialyzer: ~b", [Warnings]); format_error({unknown_application, App}) -> - io_lib:format("Could not find application: ~s", [App]); + io_lib:format("Could not find application: ~ts", [App]); +format_error({unknown_module, Mod}) -> + io_lib:format("Could not find module: ~ts", [Mod]); +format_error({duplicate_module, Mod, File1, File2}) -> + io_lib:format("Duplicates of module ~ts: ~ts ~ts", [Mod, File1, File2]); format_error({output_file_error, File, Error}) -> Error1 = file:format_error(Error), - io_lib:format("Failed to write to ~s: ~s", [File, Error1]); + io_lib:format("Failed to write to ~ts: ~ts", [File, Error1]); format_error(Reason) -> io_lib:format("~p", [Reason]). @@ -140,7 +156,7 @@ do(State, Plt) -> 0 -> {ok, State2}; TotalWarnings -> - ?INFO("Warnings written to ~s", [Output]), + ?INFO("Warnings written to ~ts", [Output]), throw({dialyzer_warnings, TotalWarnings}) end. @@ -178,45 +194,45 @@ do_update_proj_plt(State, Plt, Output) -> end. proj_plt_files(State) -> - BasePltApps = get_config(State, base_plt_apps, default_plt_apps()), - PltApps = get_config(State, plt_extra_apps, []), + BasePltApps = base_plt_apps(State), + PltApps = get_config(State, plt_extra_apps, []) ++ BasePltApps, + BasePltMods = get_config(State, base_plt_mods, []), + PltMods = get_config(State, plt_extra_mods, []) ++ BasePltMods, + Apps = proj_apps(State), + DepApps = proj_deps(State), + get_files(State, DepApps ++ PltApps, Apps -- PltApps, PltMods, []). + +proj_apps(State) -> + [ec_cnv:to_atom(rebar_app_info:name(App)) || + App <- rebar_state:project_apps(State)]. + +proj_deps(State) -> Apps = rebar_state:project_apps(State), DepApps = lists:flatmap(fun rebar_app_info:applications/1, Apps), - DepApps1 = - case get_config(State, plt_apps, top_level_deps) of - top_level_deps -> DepApps; - all_deps -> collect_nested_dependent_apps(DepApps) - end, - get_plt_files(BasePltApps ++ PltApps ++ DepApps1, Apps). - -default_plt_apps() -> - [erts, - crypto, - kernel, - stdlib]. - -get_plt_files(DepApps, Apps) -> + case get_config(State, plt_apps, top_level_deps) of + top_level_deps -> DepApps; + all_deps -> collect_nested_dependent_apps(DepApps) + end. + +get_files(State, Apps, SkipApps, Mods, SkipMods) -> ?INFO("Resolving files...", []), - get_plt_files(DepApps, Apps, [], []). + ExcludeApps = get_config(State, exclude_apps, []), + Files = apps_files(Apps, ExcludeApps ++ SkipApps, dict:new()), + ExcludeMods = get_config(State, exclude_mods, []), + Files2 = mods_files(Mods, ExcludeMods ++ SkipMods, Files), + dict:fold(fun(_, File, Acc) -> [File | Acc] end, [], Files2). -get_plt_files([], _, _, Files) -> +apps_files([], _, Files) -> Files; -get_plt_files([AppName | DepApps], Apps, PltApps, Files) -> - case lists:member(AppName, PltApps) orelse app_member(AppName, Apps) of +apps_files([AppName | DepApps], SkipApps, Files) -> + case lists:member(AppName, SkipApps) of true -> - get_plt_files(DepApps, Apps, PltApps, Files); + apps_files(DepApps, SkipApps, Files); false -> - Files2 = app_files(AppName), - ?DEBUG("~s files: ~p", [AppName, Files2]), - get_plt_files(DepApps, Apps, [AppName | PltApps], Files2 ++ Files) - end. - -app_member(AppName, Apps) -> - case rebar_app_utils:find(ec_cnv:to_binary(AppName), Apps) of - {ok, _App} -> - true; - error -> - false + AppFiles = app_files(AppName), + ?DEBUG("~ts modules: ~p", [AppName, dict:fetch_keys(AppFiles)]), + Files2 = merge_files(Files, AppFiles), + apps_files(DepApps, [AppName | SkipApps], Files2) end. app_files(AppName) -> @@ -244,9 +260,41 @@ check_ebin(EbinDir) -> end. ebin_files(EbinDir) -> - Wildcard = "*" ++ code:objfile_extension(), - [filename:join(EbinDir, File) || - File <- filelib:wildcard(Wildcard, EbinDir)]. + Ext = code:objfile_extension(), + Wildcard = "*" ++ Ext, + Files = filelib:wildcard(Wildcard, EbinDir), + Store = fun(File, Mods) -> + Mod = list_to_atom(filename:basename(File, Ext)), + Absname = filename:join(EbinDir, File), + dict:store(Mod, Absname, Mods) + end, + lists:foldl(Store, dict:new(), Files). + +merge_files(Files1, Files2) -> + Duplicate = fun(Mod, File1, File2) -> + throw({duplicate_module, Mod, File1, File2}) + end, + dict:merge(Duplicate, Files1, Files2). + +mods_files(Mods, SkipMods, Files) -> + Keep = fun(File) -> File end, + Ensure = fun(Mod, Acc) -> + case lists:member(Mod, SkipMods) of + true -> + Acc; + false -> + dict:update(Mod, Keep, mod_file(Mod), Acc) + end + end, + Files2 = lists:foldl(Ensure, Files, Mods), + lists:foldl(fun dict:erase/2, Files2, SkipMods). + +mod_file(Mod) -> + File = atom_to_list(Mod) ++ code:objfile_extension(), + case code:where_is_file(File) of + non_existing -> throw({unknown_module, Mod}); + Absname -> Absname + end. read_plt(_State, Plt) -> Vsn = dialyzer_version(), @@ -260,6 +308,8 @@ read_plt(_State, Plt) -> Result; {error, no_such_file} -> error; + {error, not_valid} -> + error; {error, read_error} -> Error = io_lib:format("Could not read the PLT file ~p", [Plt]), throw({dialyzer_error, Error}) @@ -353,9 +403,12 @@ get_base_plt(State) -> end. base_plt_files(State) -> - BasePltApps = get_config(State, base_plt_apps, default_plt_apps()), - Apps = rebar_state:project_apps(State), - get_plt_files(BasePltApps, Apps). + BasePltApps = base_plt_apps(State), + BasePltMods = get_config(State, base_plt_mods, []), + get_files(State, BasePltApps, [], BasePltMods, []). + +base_plt_apps(State) -> + get_config(State, base_plt_apps, [erts, crypto, kernel, stdlib]). update_base_plt(State, BasePlt, Output, BaseFiles) -> case read_plt(State, BasePlt) of @@ -392,9 +445,8 @@ succ_typings(State, Plt, Output) -> false -> {0, State}; _ -> - Apps = rebar_state:project_apps(State), ?INFO("Doing success typing analysis...", []), - Files = apps_to_files(Apps), + Files = proj_files(State), succ_typings(State, Plt, Output, Files) end. @@ -410,14 +462,13 @@ succ_typings(State, Plt, Output, Files) -> {init_plt, Plt}], run_dialyzer(State, Opts, Output). -apps_to_files(Apps) -> - ?INFO("Resolving files...", []), - [File || App <- Apps, - File <- app_to_files(App)]. - -app_to_files(App) -> - AppName = ec_cnv:to_atom(rebar_app_info:name(App)), - app_files(AppName). +proj_files(State) -> + Apps = proj_apps(State), + BasePltApps = get_config(State, base_plt_apps, []), + PltApps = get_config(State, plt_extra_apps, []) ++ BasePltApps, + BasePltMods = get_config(State, base_plt_mods, []), + PltMods = get_config(State, plt_extra_mods, []) ++ BasePltMods, + get_files(State, Apps, PltApps, [], PltMods). run_dialyzer(State, Opts, Output) -> %% dialyzer may return callgraph warnings when get_warnings is false @@ -428,7 +479,8 @@ run_dialyzer(State, Opts, Output) -> {check_plt, false} | Opts], ?DEBUG("Running dialyzer with options: ~p~n", [Opts2]), - Warnings = format_warnings(Output, dialyzer:run(Opts2)), + Warnings = format_warnings(rebar_state:opts(State), + Output, dialyzer:run(Opts2)), {Warnings, State}; false -> Opts2 = [{warnings, no_warnings()}, @@ -447,14 +499,14 @@ legacy_warnings(Warnings) -> Warnings end. -format_warnings(Output, Warnings) -> - Warnings1 = rebar_dialyzer_format:format_warnings(Warnings), +format_warnings(Opts, Output, Warnings) -> + Warnings1 = rebar_dialyzer_format:format_warnings(Opts, Warnings), console_warnings(Warnings1), file_warnings(Output, Warnings), length(Warnings). console_warnings(Warnings) -> - _ = [?CONSOLE("~s", [Warning]) || Warning <- Warnings], + _ = [?CONSOLE("~ts", [Warning]) || Warning <- Warnings], ok. file_warnings(_, []) -> @@ -514,7 +566,7 @@ collect_nested_dependent_apps(App, Seen) -> dialyzer_version() -> _ = application:load(dialyzer), {ok, Vsn} = application:get_key(dialyzer, vsn), - case string:tokens(Vsn, ".") of + case rebar_string:lexemes(Vsn, ".") of [Major, Minor] -> version_tuple(Major, Minor, "0"); [Major, Minor, Patch | _] -> diff --git a/src/rebar_prv_do.erl b/src/rebar_prv_do.erl index f850135..5f7aa12 100644 --- a/src/rebar_prv_do.erl +++ b/src/rebar_prv_do.erl @@ -44,13 +44,31 @@ do(State) -> do_tasks(Tasks, State) end. +-spec do_tasks(list(Task), State) -> Res when + Task :: {string(), string()} | + {string(), atom()} | + {atom(), atom(), string()}, + State :: rebar_state:t(), + Res :: {ok, rebar_state:t()} | + {error, term()}. do_tasks([], State) -> {ok, State}; -do_tasks([{TaskStr, Args}|Tail], State) -> +do_tasks([{TaskStr, Args} | Tail], State) when is_list(Args) -> Task = list_to_atom(TaskStr), State1 = rebar_state:set(State, task, Task), State2 = rebar_state:command_args(State1, Args), Namespace = rebar_state:namespace(State2), + do_task(TaskStr, Args, Tail, State, Namespace); +do_tasks([{Namespace, Task} | Tail], State) -> + do_task(atom_to_list(Task), [], Tail, State, Namespace); +do_tasks([{Namespace, Task, Args} | Tail], State) + when is_atom(Namespace), is_atom(Task) -> + do_task(atom_to_list(Task), Args, Tail, State, Namespace). + +do_task(TaskStr, Args, Tail, State, Namespace) -> + Task = list_to_atom(TaskStr), + State1 = rebar_state:set(State, task, Task), + State2 = rebar_state:command_args(State1, Args), case Namespace of default -> %% The first task we hit might be a namespace! @@ -65,7 +83,8 @@ do_tasks([{TaskStr, Args}|Tail], State) -> _ -> %% We're already in a non-default namespace, check the %% task directly. - case rebar_core:process_command(State2, Task) of + State3 = rebar_state:namespace(State2, Namespace), + case rebar_core:process_command(State3, Task) of {ok, FinalState} when Tail =:= [] -> {ok, FinalState}; {ok, _} -> @@ -75,7 +94,6 @@ do_tasks([{TaskStr, Args}|Tail], State) -> end end. - -spec format_error(any()) -> iolist(). format_error(Reason) -> io_lib:format("~p", [Reason]). diff --git a/src/rebar_prv_edoc.erl b/src/rebar_prv_edoc.erl index 6cefe14..c78296a 100644 --- a/src/rebar_prv_edoc.erl +++ b/src/rebar_prv_edoc.erl @@ -7,6 +7,7 @@ format_error/1]). -include("rebar.hrl"). +-include_lib("providers/include/providers.hrl"). -define(PROVIDER, edoc). -define(DEPS, [compile]). @@ -28,30 +29,64 @@ init(State) -> {profiles, [docs]}])), {ok, State1}. --spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. +-spec do(rebar_state:t()) -> + {ok, rebar_state:t()} | {error, string()} | {error, {module(), any()}}. do(State) -> - code:add_pathsa(rebar_state:code_paths(State, all_deps)), + rebar_paths:set_paths([deps, plugins], State), ProjectApps = rebar_state:project_apps(State), Providers = rebar_state:providers(State), - EDocOpts = rebar_state:get(State, edoc_opts, []), + EdocOpts = rebar_state:get(State, edoc_opts, []), + ShouldAccPaths = not has_configured_paths(EdocOpts), Cwd = rebar_state:dir(State), rebar_hooks:run_all_hooks(Cwd, pre, ?PROVIDER, Providers, State), - lists:foreach(fun(AppInfo) -> - rebar_hooks:run_all_hooks(Cwd, pre, ?PROVIDER, Providers, AppInfo, State), - AppName = ec_cnv:to_list(rebar_app_info:name(AppInfo)), - ?INFO("Running edoc for ~s", [AppName]), - AppDir = rebar_app_info:dir(AppInfo), - ok = edoc:application(list_to_atom(AppName), AppDir, EDocOpts), - rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, AppInfo, State) - end, ProjectApps), + Res = try + lists:foldl(fun(AppInfo, EdocOptsAcc) -> + rebar_hooks:run_all_hooks(Cwd, pre, ?PROVIDER, Providers, AppInfo, State), + AppName = rebar_utils:to_list(rebar_app_info:name(AppInfo)), + ?INFO("Running edoc for ~ts", [AppName]), + AppDir = rebar_app_info:dir(AppInfo), + AppRes = (catch edoc:application(list_to_atom(AppName), AppDir, EdocOptsAcc)), + rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, AppInfo, State), + case {AppRes, ShouldAccPaths} of + {ok, true} -> + %% edoc wants / on all OSes + add_to_paths(EdocOptsAcc, AppDir++"/doc"); + {ok, false} -> + EdocOptsAcc; + {{'EXIT', error}, _} -> + %% EDoc is not very descriptive + %% in terms of failures + throw({app_failed, AppName}) + end + end, EdocOpts, ProjectApps) + catch + {app_failed, AppName} -> + {app_failed, AppName} + end, rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State), - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), - {ok, State}. + rebar_paths:set_paths([plugins, deps], State), + case Res of + {app_failed, App} -> + ?PRV_ERROR({app_failed, App}); + _ -> + {ok, State} + end. -spec format_error(any()) -> iolist(). +format_error({app_failed, AppName}) -> + io_lib:format("Failed to generate documentation for app '~ts'", [AppName]); format_error(Reason) -> io_lib:format("~p", [Reason]). %% =================================================================== %% Internal functions %% =================================================================== +has_configured_paths(EdocOpts) -> + proplists:get_value(dir, EdocOpts) =/= undefined. + +add_to_paths([], Path) -> + [{doc_path, [Path]}]; +add_to_paths([{doc_path, Paths}|T], Path) -> + [{doc_path, [Path | Paths]} | T]; +add_to_paths([H|T], Path) -> + [H | add_to_paths(T, Path)]. diff --git a/src/rebar_prv_escriptize.erl b/src/rebar_prv_escriptize.erl index d8704f6..fceb65e 100644 --- a/src/rebar_prv_escriptize.erl +++ b/src/rebar_prv_escriptize.erl @@ -61,8 +61,11 @@ desc() -> "the project's and its dependencies' BEAM files.". do(State) -> + Providers = rebar_state:providers(State), + Cwd = rebar_state:dir(State), + rebar_hooks:run_project_and_app_hooks(Cwd, pre, ?PROVIDER, Providers, State), ?INFO("Building escript...", []), - case rebar_state:get(State, escript_main_app, undefined) of + Res = case rebar_state:get(State, escript_main_app, undefined) of undefined -> case rebar_state:project_apps(State) of [App] -> @@ -72,18 +75,24 @@ do(State) -> end; Name -> AllApps = rebar_state:all_deps(State)++rebar_state:project_apps(State), - {ok, AppInfo} = rebar_app_utils:find(ec_cnv:to_binary(Name), AllApps), - escriptize(State, AppInfo) - end. + case rebar_app_utils:find(rebar_utils:to_binary(Name), AllApps) of + {ok, AppInfo} -> + escriptize(State, AppInfo); + _ -> + ?PRV_ERROR({bad_name, Name}) + end + end, + rebar_hooks:run_project_and_app_hooks(Cwd, post, ?PROVIDER, Providers, State), + Res. escriptize(State0, App) -> AppName = rebar_app_info:name(App), - AppNameStr = ec_cnv:to_list(AppName), + AppNameStr = rebar_utils:to_list(AppName), %% Get the output filename for the escript -- this may include dirs Filename = filename:join([rebar_dir:base_dir(State0), "bin", rebar_state:get(State0, escript_name, AppName)]), - ?DEBUG("Creating escript file ~s", [Filename]), + ?DEBUG("Creating escript file ~ts", [Filename]), ok = filelib:ensure_dir(Filename), State = rebar_state:escript_path(State0, Filename), @@ -105,9 +114,9 @@ escriptize(State0, App) -> EbinFiles = usort(load_files(EbinPrefix, "*", "ebin")), ExtraFiles = usort(InclBeams ++ InclExtra), - Files = get_nonempty(EbinFiles ++ ExtraFiles), + Files = get_nonempty(EbinFiles ++ (ExtraFiles -- EbinFiles)), % drop dupes - DefaultEmuArgs = ?FMT("%%! -escript main ~s -pz ~s/~s/ebin\n", + DefaultEmuArgs = ?FMT("%%! -escript main ~ts -pz ~ts/~ts/ebin\n", [AppNameStr, AppNameStr, AppNameStr]), EscriptSections = [ {shebang, @@ -121,9 +130,15 @@ escriptize(State0, App) -> throw(?PRV_ERROR({escript_creation_failed, AppName, EscriptError})) end, - %% Finally, update executable perms for our script - {ok, #file_info{mode = Mode}} = file:read_file_info(Filename), - ok = file:change_mode(Filename, Mode bor 8#00111), + %% Finally, update executable perms for our script on *nix or write out + %% script files on win32 + case os:type() of + {unix, _} -> + {ok, #file_info{mode = Mode}} = file:read_file_info(Filename), + ok = file:change_mode(Filename, Mode bor 8#00111); + {win32, _} -> + write_windows_script(Filename) + end, {ok, State}. -spec format_error(any()) -> iolist(). @@ -148,7 +163,7 @@ get_apps_beams(Apps, AllApps) -> get_apps_beams([], _, Acc) -> Acc; get_apps_beams([App | Rest], AllApps, Acc) -> - case rebar_app_utils:find(ec_cnv:to_binary(App), AllApps) of + case rebar_app_utils:find(rebar_utils:to_binary(App), AllApps) of {ok, App1} -> OutDir = filename:absname(rebar_app_info:ebin_dir(App1)), Beams = get_app_beams(App, OutDir), @@ -179,7 +194,8 @@ load_files(Wildcard, Dir) -> load_files(Prefix, Wildcard, Dir) -> [read_file(Prefix, Filename, Dir) - || Filename <- filelib:wildcard(Wildcard, Dir)]. + || Filename <- filelib:wildcard(Wildcard, Dir), + not filelib:is_dir(filename:join(Dir, Filename))]. read_file(Prefix, Filename, Dir) -> Filename1 = case Prefix of @@ -220,7 +236,7 @@ get_nonempty(Files) -> [{FName,FBin} || {FName,FBin} <- Files, FBin =/= <<>>]. find_deps(AppNames, AllApps) -> - BinAppNames = [ec_cnv:to_binary(Name) || Name <- AppNames], + BinAppNames = [rebar_utils:to_binary(Name) || Name <- AppNames], [ec_cnv:to_atom(Name) || Name <- find_deps_of_deps(BinAppNames, AllApps, BinAppNames)]. @@ -230,9 +246,11 @@ find_deps_of_deps([Name|Names], Apps, Acc) -> ?DEBUG("processing ~p", [Name]), {ok, App} = rebar_app_utils:find(Name, Apps), DepNames = proplists:get_value(applications, rebar_app_info:app_details(App), []), - BinDepNames = [ec_cnv:to_binary(Dep) || Dep <- DepNames, + BinDepNames = [rebar_utils:to_binary(Dep) || Dep <- DepNames, %% ignore system libs; shouldn't include them. - not lists:prefix(code:root_dir(), code:lib_dir(Dep))] + DepDir <- [code:lib_dir(Dep)], + DepDir =:= {error, bad_name} orelse % those are all local + not lists:prefix(code:root_dir(), DepDir)] -- ([Name|Names]++Acc), % avoid already seen deps ?DEBUG("new deps of ~p found to be ~p", [Name, BinDepNames]), find_deps_of_deps(BinDepNames ++ Names, Apps, BinDepNames ++ Acc). @@ -247,3 +265,12 @@ def(Rm, State, Key, Default) -> rm_newline(String) -> [C || C <- String, C =/= $\n]. + +write_windows_script(Target) -> + CmdPath = unicode:characters_to_list(Target) ++ ".cmd", + CmdScript= + "@echo off\r\n" + "setlocal\r\n" + "set rebarscript=%~f0\r\n" + "escript.exe \"%rebarscript:.cmd=%\" %*\r\n", + ok = file:write_file(CmdPath, CmdScript). diff --git a/src/rebar_prv_eunit.erl b/src/rebar_prv_eunit.erl index 942fd10..f120926 100644 --- a/src/rebar_prv_eunit.erl +++ b/src/rebar_prv_eunit.erl @@ -18,6 +18,8 @@ %% we need to modify app_info state before compile -define(DEPS, [lock]). +-define(DEFAULT_TEST_REGEX, "^(?!\\._).*\\.erl\$"). + %% =================================================================== %% Public API %% =================================================================== @@ -52,7 +54,7 @@ do(State, Tests) -> ?INFO("Performing EUnit tests...", []), setup_name(State), - rebar_utils:update_code(rebar_state:code_paths(State, all_deps), [soft_purge]), + rebar_paths:set_paths([deps, plugins], State), %% Run eunit provider prehooks Providers = rebar_state:providers(State), @@ -65,14 +67,14 @@ do(State, Tests) -> {ok, State1} -> %% Run eunit provider posthooks rebar_hooks:run_project_and_app_hooks(Cwd, post, ?PROVIDER, Providers, State1), - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), + rebar_paths:set_paths([plugins, deps], State), {ok, State1}; Error -> - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), + rebar_paths:set_paths([plugins, deps], State), Error end; Error -> - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), + rebar_paths:set_paths([plugins, deps], State), Error end. @@ -81,13 +83,16 @@ run_tests(State, Tests) -> EUnitOpts = resolve_eunit_opts(State), ?DEBUG("eunit_tests ~p", [T]), ?DEBUG("eunit_opts ~p", [EUnitOpts]), - Result = eunit:test(T, EUnitOpts), - ok = maybe_write_coverdata(State), - case handle_results(Result) of - {error, Reason} -> - ?PRV_ERROR(Reason); - ok -> - {ok, State} + try eunit:test(T, EUnitOpts) of + Result -> + ok = maybe_write_coverdata(State), + case handle_results(Result) of + {error, Reason} -> + ?PRV_ERROR(Reason); + ok -> + {ok, State} + end + catch error:badarg -> ?PRV_ERROR({error, badarg}) end. -spec format_error(any()) -> iolist(). @@ -136,7 +141,8 @@ resolve(Flag, RawOpts) -> resolve(Flag, Flag, RawOpts). resolve(Flag, EUnitKey, RawOpts) -> case proplists:get_value(Flag, RawOpts) of undefined -> []; - Args -> lists:map(fun(Arg) -> normalize(EUnitKey, Arg) end, string:tokens(Args, [$,])) + Args -> lists:map(fun(Arg) -> normalize(EUnitKey, Arg) end, + rebar_string:lexemes(Args, [$,])) end. normalize(Key, Value) when Key == dir; Key == file -> {Key, Value}; @@ -151,7 +157,6 @@ cfg_tests(State) -> ?PRV_ERROR({badconfig, {"Value `~p' of option `~p' must be a list", {Wrong, eunit_tests}}}) end. -select_tests(_State, _ProjectApps, {error, _} = Error, _) -> Error; select_tests(_State, _ProjectApps, _, {error, _} = Error) -> Error; select_tests(State, ProjectApps, [], []) -> {ok, default_tests(State, ProjectApps)}; select_tests(_State, _ProjectApps, [], Tests) -> {ok, Tests}; @@ -174,34 +179,38 @@ set_apps([App|Rest], Acc) -> set_modules(Apps, State) -> set_modules(Apps, State, {[], []}). set_modules([], State, {AppAcc, TestAcc}) -> - TestSrc = gather_src([filename:join([rebar_state:dir(State), "test"])]), + Regex = rebar_state:get(State, eunit_test_regex, ?DEFAULT_TEST_REGEX), + BareTestDir = [filename:join([rebar_state:dir(State), "test"])], + TestSrc = gather_src(BareTestDir, Regex), dedupe_tests({AppAcc, TestAcc ++ TestSrc}); set_modules([App|Rest], State, {AppAcc, TestAcc}) -> F = fun(Dir) -> filename:join([rebar_app_info:dir(App), Dir]) end, AppDirs = lists:map(F, rebar_dir:src_dirs(rebar_app_info:opts(App), ["src"])), - AppSrc = gather_src(AppDirs), + Regex = rebar_state:get(State, eunit_test_regex, ?DEFAULT_TEST_REGEX), + AppSrc = gather_src(AppDirs, Regex), TestDirs = [filename:join([rebar_app_info:dir(App), "test"])], - TestSrc = gather_src(TestDirs), + TestSrc = gather_src(TestDirs, Regex), set_modules(Rest, State, {AppSrc ++ AppAcc, TestSrc ++ TestAcc}). -gather_src(Dirs) -> gather_src(Dirs, []). +gather_src(Dirs, Regex) -> gather_src(Dirs, Regex, []). -gather_src([], Srcs) -> Srcs; -gather_src([Dir|Rest], Srcs) -> - gather_src(Rest, Srcs ++ rebar_utils:find_files(Dir, "^[^._].*\\.erl\$", true)). +gather_src([], _Regex, Srcs) -> Srcs; +gather_src([Dir|Rest], Regex, Srcs) -> + gather_src(Rest, Regex, Srcs ++ rebar_utils:find_files(Dir, Regex, true)). dedupe_tests({AppMods, TestMods}) -> + UniqueTestMods = lists:usort(TestMods) -- AppMods, %% for each modules in TestMods create a test if there is not a module %% in AppMods that will trigger it - F = fun(Mod) -> - M = filename:basename(Mod, ".erl"), - MatchesTest = fun(Dir) -> filename:basename(Dir, ".erl") ++ "_tests" == M end, + F = fun(TestMod) -> + M = filename:rootname(filename:basename(TestMod)), + MatchesTest = fun(AppMod) -> filename:rootname(filename:basename(AppMod)) ++ "_tests" == M end, case lists:any(MatchesTest, AppMods) of false -> {true, {module, list_to_atom(M)}}; true -> false end end, - lists:usort(rebar_utils:filtermap(F, TestMods)). + rebar_utils:filtermap(F, UniqueTestMods). inject_eunit_state(State, {ok, Tests}) -> Apps = rebar_state:project_apps(State), @@ -306,19 +315,14 @@ maybe_inject_test_dir(State, AppAcc, [], Dir) -> inject_test_dir(Opts, Dir) -> %% append specified test targets to app defined `extra_src_dirs` - ExtraSrcDirs = rebar_dir:extra_src_dirs(Opts), + ExtraSrcDirs = rebar_opts:get(Opts, extra_src_dirs, []), rebar_opts:set(Opts, extra_src_dirs, ExtraSrcDirs ++ [Dir]). compile({error, _} = Error) -> Error; compile(State) -> - case rebar_prv_compile:do(State) of - %% successfully compiled apps - {ok, S} -> - ok = maybe_cover_compile(S), - {ok, S}; - %% this should look like a compiler error, not an eunit error - Error -> Error - end. + {ok, S} = rebar_prv_compile:do(State), + ok = maybe_cover_compile(S), + {ok, S}. validate_tests(State, {ok, Tests}) -> gather_tests(fun(Elem) -> validate(State, Elem) end, Tests, []); @@ -448,7 +452,7 @@ translate(State, [], {dir, Dir}) -> translate(State, [], {file, FilePath}) -> Dir = filename:dirname(FilePath), File = filename:basename(FilePath), - case rebar_file_utils:path_from_ancestor(Dir, rebar_app_info:dir(State)) of + case rebar_file_utils:path_from_ancestor(Dir, rebar_state:dir(State)) of {ok, Path} -> {file, filename:join([rebar_dir:base_dir(State), "extras", Path, File])}; %% not relative, leave as is {error, badparent} -> {file, FilePath} @@ -468,7 +472,8 @@ maybe_write_coverdata(State) -> true -> rebar_state:set(State, cover_enabled, true); false -> State end, - rebar_prv_cover:maybe_write_coverdata(State1, ?PROVIDER). + Name = proplists:get_value(cover_export_name, RawOpts, ?PROVIDER), + rebar_prv_cover:maybe_write_coverdata(State1, Name). handle_results(ok) -> ok; handle_results(error) -> @@ -480,6 +485,7 @@ eunit_opts(_State) -> [{app, undefined, "app", string, help(app)}, {application, undefined, "application", string, help(app)}, {cover, $c, "cover", boolean, help(cover)}, + {cover_export_name, undefined, "cover_export_name", string, help(cover_export_name)}, {dir, $d, "dir", string, help(dir)}, {file, $f, "file", string, help(file)}, {module, $m, "module", string, help(module)}, @@ -491,6 +497,7 @@ eunit_opts(_State) -> help(app) -> "Comma separated list of application test suites to run. Equivalent to `[{application, App}]`."; help(cover) -> "Generate cover data. Defaults to false."; +help(cover_export_name) -> "Base name of the coverdata file to write"; help(dir) -> "Comma separated list of dirs to load tests from. Equivalent to `[{dir, Dir}]`."; help(file) -> "Comma separated list of files to load tests from. Equivalent to `[{file, File}]`."; help(module) -> "Comma separated list of modules to load tests from. Equivalent to `[{module, Module}]`."; diff --git a/src/rebar_prv_get_deps.erl b/src/rebar_prv_get_deps.erl new file mode 100644 index 0000000..020e50b --- /dev/null +++ b/src/rebar_prv_get_deps.erl @@ -0,0 +1,37 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et + +-module(rebar_prv_get_deps). + +-behaviour(provider). + +-export([init/1, + do/1, + format_error/1]). + +-define(PROVIDER, 'get-deps'). +-define(DEPS, [lock]). + +%% =================================================================== +%% Public API +%% =================================================================== + +-spec init(rebar_state:t()) -> {ok, rebar_state:t()}. +init(State) -> + Provider = providers:create([{name, ?PROVIDER}, + {module, ?MODULE}, + {deps, ?DEPS}, + {bare, true}, + {example, "rebar3 get-deps"}, + {short_desc, "Fetch dependencies."}, + {desc, "Fetch project dependencies."}, + {opts, []}, + {profiles, []}]), + {ok, rebar_state:add_provider(State, Provider)}. + +-spec do(rebar_state:t()) -> {ok, rebar_state:t()}. +do(State) -> {ok, State}. + +-spec format_error(any()) -> iolist(). +format_error(Reason) -> + io_lib:format("~p", [Reason]).
\ No newline at end of file diff --git a/src/rebar_prv_help.erl b/src/rebar_prv_help.erl index c028264..f34c755 100644 --- a/src/rebar_prv_help.erl +++ b/src/rebar_prv_help.erl @@ -41,7 +41,10 @@ do(State) -> [Name] -> % default namespace task_help(default, list_to_atom(Name), State); [Namespace, Name] -> - task_help(list_to_atom(Namespace), list_to_atom(Name), State) + task_help(list_to_atom(Namespace), list_to_atom(Name), State); + _ -> + {error, "Too many arguments given. " ++ + "Usage: rebar3 help [<namespace>] <task>"} end. -spec format_error(any()) -> iolist(). @@ -54,7 +57,7 @@ format_error(Reason) -> help(State) -> ?CONSOLE("Rebar3 is a tool for working with Erlang projects.~n~n", []), OptSpecList = rebar3:global_option_spec_list(), - getopt:usage(OptSpecList, "rebar", "", []), + getopt:usage(OptSpecList, "rebar3", "", []), ?CONSOLE("~nSeveral tasks are available:~n", []), providers:help(rebar_state:providers(State)), diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index a8a7ea0..068c4c8 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -101,39 +101,48 @@ do_(State) -> {error, Reason} end. +%% @doc convert a given exception's payload into an io description. -spec format_error(any()) -> iolist(). format_error({dep_app_not_found, AppDir, AppName}) -> - io_lib:format("Dependency failure: Application ~s not found at the top level of directory ~s", [AppName, AppDir]); + io_lib:format("Dependency failure: Application ~ts not found at the top level of directory ~ts", [AppName, AppDir]); format_error({load_registry_fail, Dep}) -> - io_lib:format("Error loading registry to resolve version of ~s. Try fixing by running 'rebar3 update'", [Dep]); + io_lib:format("Error loading registry to resolve version of ~ts. Try fixing by running 'rebar3 update'", [Dep]); format_error({bad_constraint, Name, Constraint}) -> - io_lib:format("Unable to parse version for package ~s: ~s", [Name, Constraint]); + io_lib:format("Unable to parse version for package ~ts: ~ts", [Name, Constraint]); format_error({parse_dep, Dep}) -> io_lib:format("Failed parsing dep ~p", [Dep]); format_error({not_rebar_package, Package, Version}) -> - io_lib:format("Package not buildable with rebar3: ~s-~s", [Package, Version]); + io_lib:format("Package not buildable with rebar3: ~ts-~ts", [Package, Version]); format_error({missing_package, Package, Version}) -> - io_lib:format("Package not found in registry: ~s-~s", [Package, Version]); + io_lib:format("Package not found in registry: ~ts-~ts", [Package, Version]); format_error({missing_package, Package}) -> - io_lib:format("Package not found in registry: ~s", [Package]); + io_lib:format("Package not found in registry: ~ts", [Package]); format_error({cycles, Cycles}) -> Prints = [["applications: ", - [io_lib:format("~s ", [Dep]) || Dep <- Cycle], - "depend on each other~n"] + [io_lib:format("~ts ", [Dep]) || Dep <- Cycle], + "depend on each other\n"] || Cycle <- Cycles], - ["Dependency cycle(s) detected:~n", Prints]; + ["Dependency cycle(s) detected:\n", Prints]; format_error(Reason) -> io_lib:format("~p", [Reason]). -%% Allows other providers to install deps in a given profile +%% @doc Allows other providers to install deps in a given profile %% manually, outside of what is provided by rebar3's deps tuple. +-spec handle_deps_as_profile(Profile, State, Deps, Upgrade) -> {Apps, State} when + Profile :: atom(), + State :: rebar_state:t(), + Deps :: [tuple() | atom() | binary()], % TODO: meta to source() | lock() + Upgrade :: boolean(), + Apps :: [rebar_app_info:t()]. handle_deps_as_profile(Profile, State, Deps, Upgrade) -> Locks = [], Level = 0, DepsDir = profile_dep_dir(State, Profile), Deps1 = rebar_app_utils:parse_deps(DepsDir, Deps, State, Locks, Level), ProfileLevelDeps = [{Profile, Deps1, Level}], - handle_profile_level(ProfileLevelDeps, [], sets:new(), Upgrade, Locks, State). + RootSeen = sets:from_list([rebar_app_info:name(AppInfo) + || AppInfo <- rebar_state:project_apps(State)]), + handle_profile_level(ProfileLevelDeps, [], RootSeen, RootSeen, Upgrade, Locks, State). %% =================================================================== %% Internal functions @@ -146,7 +155,9 @@ deps_per_profile(Profiles, Upgrade, State) -> Deps = lists:foldl(fun(Profile, DepAcc) -> [parsed_profile_deps(State, Profile, Level) | DepAcc] end, [], Profiles), - handle_profile_level(Deps, [], sets:new(), Upgrade, Locks, State). + RootSeen = sets:from_list([rebar_app_info:name(AppInfo) + || AppInfo <- rebar_state:project_apps(State)]), + handle_profile_level(Deps, [], RootSeen, RootSeen, Upgrade, Locks, State). parsed_profile_deps(State, Profile, Level) -> ParsedDeps = rebar_state:get(State, {parsed_deps, Profile}, []), @@ -155,17 +166,27 @@ parsed_profile_deps(State, Profile, Level) -> %% Level-order traversal of all dependencies, across profiles. %% If profiles x,y,z are present, then the traversal will go: %% x0, y0, z0, x1, y1, z1, ..., xN, yN, zN. -handle_profile_level([], Apps, _Seen, _Upgrade, _Locks, State) -> +%% +%% There are two 'seen' sets: one for the top-level apps (`RootSeen') and +%% one for all dependencies (`Seen'). The former is used to know when +%% to skip the resolving of dependencies altogether (since they're already +%% top-level apps), while the latter is used to prevent reprocessing +%% deps more than one. +handle_profile_level([], Apps, _RootSeen, _Seen, _Upgrade, _Locks, State) -> {Apps, State}; -handle_profile_level([{Profile, Deps, Level} | Rest], Apps, Seen, Upgrade, Locks, State) -> +handle_profile_level([{Profile, Deps, Level} | Rest], Apps, RootSeen, Seen, Upgrade, Locks, State) -> + Deps0 = [rebar_app_utils:expand_deps_sources(Dep, State) + || Dep <- Deps, + %% skip top-level apps being double-declared + not sets:is_element(rebar_app_info:name(Dep), RootSeen)], {Deps1, Apps1, State1, Seen1} = - update_deps(Profile, Level, Deps, Apps + update_deps(Profile, Level, Deps0, Apps ,State, Upgrade, Seen, Locks), Deps2 = case Deps1 of [] -> Rest; _ -> Rest ++ [{Profile, Deps1, Level+1}] end, - handle_profile_level(Deps2, Apps1, sets:union(Seen, Seen1), Upgrade, Locks, State1). + handle_profile_level(Deps2, Apps1, RootSeen, sets:union(Seen, Seen1), Upgrade, Locks, State1). find_cycles(Apps) -> case rebar_digraph:compile_order(Apps) of @@ -238,9 +259,21 @@ update_seen_dep(AppInfo, _Profile, _Level, Deps, Apps, State, Upgrade, Seen, Loc %% If seen from lock file or user requested an upgrade %% don't print warning about skipping case lists:keymember(Name, 1, Locks) of - false when Upgrade -> ok; - false when not Upgrade -> warn_skip_deps(AppInfo, State); - true -> ok + false when Upgrade -> + ok; + false when not Upgrade -> + {ok, SeenApp} = rebar_app_utils:find(Name, Apps), + Source = rebar_app_info:source(AppInfo), + case rebar_app_info:source(SeenApp) of + Source -> + %% dep is the same version and checksum as the one we already saw. + %% meaning there is no conflict, so don't warn about it. + skip; + _ -> + warn_skip_deps(Name, Source, State) + end; + true -> + ok end, {Deps, Apps, State, Seen}. @@ -256,10 +289,8 @@ update_unseen_dep(AppInfo, Profile, Level, Deps, Apps, State, Upgrade, Seen, Loc -spec handle_dep(rebar_state:t(), atom(), file:filename_all(), rebar_app_info:t(), list(), integer()) -> {rebar_app_info:t(), [rebar_app_info:t()], rebar_state:t()}. handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) -> Name = rebar_app_info:name(AppInfo), - C = rebar_config:consult(rebar_app_info:dir(AppInfo)), - AppInfo0 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), C), - AppInfo1 = rebar_app_info:apply_overrides(rebar_app_info:get(AppInfo, overrides, []), AppInfo0), + AppInfo1 = rebar_app_info:apply_overrides(rebar_app_info:get(AppInfo, overrides, []), AppInfo), AppInfo2 = rebar_app_info:apply_profiles(AppInfo1, [default, prod]), Plugins = rebar_app_info:get(AppInfo2, plugins, []), @@ -276,34 +307,33 @@ handle_dep(State, Profile, DepsDir, AppInfo, Locks, Level) -> AppInfo4 = rebar_app_info:deps(AppInfo3, rebar_state:deps_names(Deps)), %% Keep all overrides from the global config and this dep when parsing its deps - Overrides = rebar_app_info:get(AppInfo0, overrides, []), + Overrides = rebar_app_info:get(AppInfo, overrides, []), Deps1 = rebar_app_utils:parse_deps(Name, DepsDir, Deps, rebar_state:set(State, overrides, Overrides) ,Locks, Level+1), {AppInfo4, Deps1, State1}. -spec maybe_fetch(rebar_app_info:t(), atom(), boolean(), - sets:set(binary()), rebar_state:t()) -> {boolean(), rebar_app_info:t()}. + sets:set(binary()), rebar_state:t()) -> {ok, rebar_app_info:t()}. maybe_fetch(AppInfo, Profile, Upgrade, Seen, State) -> - AppDir = ec_cnv:to_list(rebar_app_info:dir(AppInfo)), + AppDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)), %% Don't fetch dep if it exists in the _checkouts dir case rebar_app_info:is_checkout(AppInfo) of true -> - {false, AppInfo}; + {ok, AppInfo}; false -> - case rebar_app_discover:find_app(AppInfo, AppDir, all) of + case rebar_app_info:is_available(AppInfo) of false -> - true = fetch_app(AppInfo, AppDir, State), - maybe_symlink_default(State, Profile, AppDir, AppInfo), - {true, rebar_app_info:valid(update_app_info(AppDir, AppInfo), false)}; - {true, AppInfo1} -> - case sets:is_element(rebar_app_info:name(AppInfo1), Seen) of + AppInfo1 = fetch_app(AppInfo, State), + maybe_symlink_default(State, Profile, AppDir, AppInfo1), + {ok, rebar_app_info:is_available(rebar_app_info:valid(AppInfo1, false), true)}; + true -> + case sets:is_element(rebar_app_info:name(AppInfo), Seen) of true -> - {false, AppInfo1}; + {ok, AppInfo}; false -> - maybe_symlink_default(State, Profile, AppDir, AppInfo1), - MaybeUpgrade = maybe_upgrade(AppInfo, AppDir, Upgrade, State), - AppInfo2 = update_app_info(AppDir, AppInfo1), - {MaybeUpgrade, AppInfo2} + maybe_symlink_default(State, Profile, AppDir, AppInfo), + AppInfo1 = maybe_upgrade(AppInfo, AppDir, Upgrade, State), + {ok, AppInfo1} end end end. @@ -339,7 +369,7 @@ symlink_dep(State, From, To) -> ok -> RelativeFrom = make_relative_to_root(State, From), RelativeTo = make_relative_to_root(State, To), - ?INFO("Linking ~s to ~s", [RelativeFrom, RelativeTo]), + ?INFO("Linking ~ts to ~ts", [RelativeFrom, RelativeTo]), ok; exists -> ok @@ -351,55 +381,45 @@ make_relative_to_root(State, Path) when is_list(Path) -> Root = rebar_dir:root_dir(State), rebar_dir:make_relative_path(Path, Root). -fetch_app(AppInfo, AppDir, State) -> - ?INFO("Fetching ~s (~p)", [rebar_app_info:name(AppInfo), - format_source(rebar_app_info:source(AppInfo))]), - Source = rebar_app_info:source(AppInfo), - true = rebar_fetch:download_source(AppDir, Source, State). - -format_source({pkg, Name, Vsn, _Hash}) -> {pkg, Name, Vsn}; -format_source(Source) -> Source. - -%% This is called after the dep has been downloaded and unpacked, if it hadn't been already. -%% So this is the first time for newly downloaded apps that its .app/.app.src data can -%% be read in an parsed. -update_app_info(AppDir, AppInfo) -> - case rebar_app_discover:find_app(AppInfo, AppDir, all) of - {true, AppInfo1} -> - AppInfo1; - false -> - throw(?PRV_ERROR({dep_app_not_found, AppDir, rebar_app_info:name(AppInfo)})) - end. +fetch_app(AppInfo, State) -> + ?INFO("Fetching ~ts (~p)", [rebar_app_info:name(AppInfo), + rebar_resource_v2:format_source(rebar_app_info:source(AppInfo))]), + rebar_fetch:download_source(AppInfo, State). -maybe_upgrade(AppInfo, AppDir, Upgrade, State) -> - Source = rebar_app_info:source(AppInfo), +maybe_upgrade(AppInfo, _AppDir, Upgrade, State) -> case Upgrade orelse rebar_app_info:is_lock(AppInfo) of true -> - case rebar_fetch:needs_update(AppDir, Source, State) of + case rebar_fetch:needs_update(AppInfo, State) of true -> - ?INFO("Upgrading ~s (~p)", [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)]), - true = rebar_fetch:download_source(AppDir, Source, State); + ?INFO("Upgrading ~ts (~p)", [rebar_app_info:name(AppInfo), + rebar_resource_v2:format_source(rebar_app_info:source(AppInfo))]), + rebar_fetch:download_source(AppInfo, State); false -> case Upgrade of true -> - ?INFO("No upgrade needed for ~s", [rebar_app_info:name(AppInfo)]), - false; + ?INFO("No upgrade needed for ~ts", [rebar_app_info:name(AppInfo)]), + AppInfo; false -> - false + AppInfo end end; false -> - false + AppInfo end. -warn_skip_deps(AppInfo, State) -> - Msg = "Skipping ~s (from ~p) as an app of the same name " +warn_skip_deps(Name, Source, State) -> + Msg = "Skipping ~ts (from ~p) as an app of the same name " "has already been fetched", - Args = [rebar_app_info:name(AppInfo), - rebar_app_info:source(AppInfo)], + Args = [Name, + rebar_resource_v2:format_source(Source)], case rebar_state:get(State, deps_error_on_conflict, false) of - false -> ?WARN(Msg, Args); - true -> ?ERROR(Msg, Args), ?FAIL + false -> + case rebar_state:get(State, deps_warning_on_conflict, true) of + true -> ?WARN(Msg, Args); + false -> ok + end; + true -> + ?ERROR(Msg, Args), ?FAIL end. not_needs_compile(App) -> diff --git a/src/rebar_prv_local_install.erl b/src/rebar_prv_local_install.erl index 1b58859..cd6a204 100644 --- a/src/rebar_prv_local_install.erl +++ b/src/rebar_prv_local_install.erl @@ -12,6 +12,7 @@ -export([extract_escript/2]). -include("rebar.hrl"). +-include_lib("providers/include/providers.hrl"). -include_lib("kernel/include/file.hrl"). -define(PROVIDER, install). @@ -54,13 +55,16 @@ do(State) -> end. -spec format_error(any()) -> iolist(). +format_error({non_writeable, Dir}) -> + io_lib:format("Could not write to ~p. Please ensure the path is writeable.", + [Dir]); format_error(Reason) -> io_lib:format("~p", [Reason]). bin_contents(OutputDir) -> <<"#!/usr/bin/env sh -erl -pz ", (ec_cnv:to_binary(OutputDir))/binary,"/*/ebin +sbtu +A0 -noshell -boot start_clean -s rebar3 main $REBAR3_ERL_ARGS -extra \"$@\" +erl -pz ", (rebar_utils:to_binary(OutputDir))/binary,"/*/ebin +sbtu +A1 -noshell -boot start_clean -s rebar3 main $REBAR3_ERL_ARGS -extra \"$@\" ">>. extract_escript(State, ScriptPath) -> @@ -71,25 +75,24 @@ extract_escript(State, ScriptPath) -> %% And add a rebar3 bin script to ~/.cache/rebar3/bin Opts = rebar_state:opts(State), OutputDir = filename:join(rebar_dir:global_cache_dir(Opts), "lib"), - filelib:ensure_dir(filename:join(OutputDir, "empty")), - - ?INFO("Extracting rebar3 libs to ~s...", [OutputDir]), + case filelib:ensure_dir(filename:join(OutputDir, "empty")) of + ok -> + ok; + {error, Posix} when Posix == eaccess; Posix == enoent -> + throw(?PRV_ERROR({non_writeable, OutputDir})) + end, + + ?INFO("Extracting rebar3 libs to ~ts...", [OutputDir]), zip:extract(Archive, [{cwd, OutputDir}]), BinDir = filename:join(rebar_dir:global_cache_dir(Opts), "bin"), BinFile = filename:join(BinDir, "rebar3"), filelib:ensure_dir(BinFile), - {ok, #file_info{mode = _, - uid = Uid, - gid = Gid}} = file:read_file_info(ScriptPath), - - ?INFO("Writing rebar3 run script ~s...", [BinFile]), + ?INFO("Writing rebar3 run script ~ts...", [BinFile]), file:write_file(BinFile, bin_contents(OutputDir)), - ok = file:write_file_info(BinFile, #file_info{mode=33277, - uid=Uid, - gid=Gid}), + ok = file:write_file_info(BinFile, #file_info{mode=33277}), - ?INFO("Add to $PATH for use: export PATH=$PATH:~s", [BinDir]), + ?INFO("Add to $PATH for use: export PATH=~ts:$PATH", [BinDir]), {ok, State}. diff --git a/src/rebar_prv_local_upgrade.erl b/src/rebar_prv_local_upgrade.erl index aa9ee44..1931d65 100644 --- a/src/rebar_prv_local_upgrade.erl +++ b/src/rebar_prv_local_upgrade.erl @@ -72,15 +72,15 @@ get_md5(Rebar3Path) -> {ok, Rebar3File} = file:read_file(Rebar3Path), Digest = crypto:hash(md5, Rebar3File), DigestHex = lists:flatten([io_lib:format("~2.16.0B", [X]) || X <- binary_to_list(Digest)]), - string:to_lower(DigestHex). + rebar_string:lowercase(DigestHex). maybe_fetch_rebar3(Rebar3Md5) -> TmpDir = ec_file:insecure_mkdtemp(), TmpFile = filename:join(TmpDir, "rebar3"), - case rebar_pkg_resource:request("https://s3.amazonaws.com/rebar3/rebar3", Rebar3Md5) of + case request("https://s3.amazonaws.com/rebar3/rebar3", Rebar3Md5) of {ok, Binary, ETag} -> file:write_file(TmpFile, Binary), - case rebar_pkg_resource:etag(TmpFile) of + case etag(TmpFile) of ETag -> {saved, TmpFile}; _ -> @@ -92,3 +92,38 @@ maybe_fetch_rebar3(Rebar3Md5) -> ?CONSOLE("No upgrade available", []), up_to_date end. + +etag(Path) -> + case file:read_file(Path) of + {ok, Binary} -> + <<X:128/big-unsigned-integer>> = crypto:hash(md5, Binary), + rebar_string:lowercase(lists:flatten(io_lib:format("~32.16.0b", [X]))); + {error, _} -> + false + end. + +-spec request(Url, ETag) -> Res when + Url :: string(), + ETag :: false | string(), + Res :: 'error' | {ok, cached} | {ok, any(), string()}. +request(Url, ETag) -> + HttpOptions = [{ssl, rebar_utils:ssl_opts(Url)}, + {relaxed, true} | rebar_utils:get_proxy_auth()], + case httpc:request(get, {Url, [{"if-none-match", "\"" ++ ETag ++ "\""} + || ETag =/= false] ++ + [{"User-Agent", rebar_utils:user_agent()}]}, + HttpOptions, [{body_format, binary}], rebar) of + {ok, {{_Version, 200, _Reason}, Headers, Body}} -> + ?DEBUG("Successfully downloaded ~ts", [Url]), + {"etag", ETag1} = lists:keyfind("etag", 1, Headers), + {ok, Body, rebar_string:trim(ETag1, both, [$"])}; + {ok, {{_Version, 304, _Reason}, _Headers, _Body}} -> + ?DEBUG("Cached copy of ~ts still valid", [Url]), + {ok, cached}; + {ok, {{_Version, Code, _Reason}, _Headers, _Body}} -> + ?DEBUG("Request to ~p failed: status code ~p", [Url, Code]), + error; + {error, Reason} -> + ?DEBUG("Request to ~p failed: ~p", [Url, Reason]), + error + end. diff --git a/src/rebar_prv_lock.erl b/src/rebar_prv_lock.erl index cbe8dfe..570c03f 100644 --- a/src/rebar_prv_lock.erl +++ b/src/rebar_prv_lock.erl @@ -54,12 +54,9 @@ format_error(Reason) -> build_locks(State) -> AllDeps = rebar_state:lock(State), [begin - Dir = rebar_app_info:dir(Dep), - Source = rebar_app_info:source(Dep), - %% If source is tuple it is a source dep %% e.g. {git, "git://github.com/ninenines/cowboy.git", "master"} - {rebar_app_info:name(Dep) - ,rebar_fetch:lock_source(Dir, Source, State) - ,rebar_app_info:dep_level(Dep)} + {rebar_app_info:name(Dep), + rebar_fetch:lock_source(Dep, State), + rebar_app_info:dep_level(Dep)} end || Dep <- AllDeps, not(rebar_app_info:is_checkout(Dep))]. diff --git a/src/rebar_prv_new.erl b/src/rebar_prv_new.erl index 064315e..c6a1e9b 100644 --- a/src/rebar_prv_new.erl +++ b/src/rebar_prv_new.erl @@ -60,7 +60,7 @@ do(State) -> -spec format_error(any()) -> iolist(). format_error({consult, File, Reason}) -> - io_lib:format("Error consulting file at ~s for reason ~p", [File, Reason]); + io_lib:format("Error consulting file at ~ts for reason ~p", [File, Reason]); format_error(Reason) -> io_lib:format("~p", [Reason]). @@ -70,7 +70,7 @@ format_error(Reason) -> list_templates(State) -> lists:foldl(fun({error, {consult, File, Reason}}, Acc) -> - ?WARN("Error consulting template file ~s for reason ~p", + ?WARN("Error consulting template file ~ts for reason ~p", [File, Reason]), Acc ; (Tpl, Acc) -> @@ -82,7 +82,9 @@ info() -> "Create rebar3 project based on template and vars.~n" "~n" "Valid command line options:~n" - " <template> [var=foo,...]~n", []). + " <template> [var=foo,...]~n" + "~n" + "See available templates with: `rebar3 new help`~n", []). strip_flags([]) -> []; strip_flags(["-"++_|Opts]) -> strip_flags(Opts); @@ -116,31 +118,34 @@ show_short_templates(List) -> lists:map(fun show_short_template/1, lists:sort(List)). show_short_template({Name, Type, _Location, Description, _Vars}) -> - io:format("~s (~s): ~s~n", + io:format("~ts (~ts): ~ts~n", [Name, format_type(Type), format_description(Description)]). show_template({Name, Type, Location, Description, Vars}) -> - io:format("~s:~n" - "\t~s~n" - "\tDescription: ~s~n" - "\tVariables:~n~s~n", + io:format("~ts:~n" + "\t~ts~n" + "\tDescription: ~ts~n" + "\tVariables:~n~ts~n", [Name, format_type(Type, Location), format_description(Description), format_vars(Vars)]). format_type(escript) -> "built-in"; +format_type(builtin) -> "built-in"; format_type(plugin) -> "plugin"; format_type(file) -> "custom". format_type(escript, _) -> "built-in template"; +format_type(builtin, _) -> + "built-in template"; format_type(plugin, Loc) -> - io_lib:format("plugin template (~s)", [Loc]); + io_lib:format("plugin template (~ts)", [Loc]); format_type(file, Loc) -> - io_lib:format("custom template (~s)", [Loc]). + io_lib:format("custom template (~ts)", [Loc]). format_description(Description) -> case Description of @@ -153,4 +158,4 @@ format_vars(Vars) -> [format_var(Var) || Var <- Vars]. format_var({Var, Default}) -> io_lib:format("\t\t~p=~p~n",[Var, Default]); format_var({Var, Default, Doc}) -> - io_lib:format("\t\t~p=~p (~s)~n", [Var, Default, Doc]). + io_lib:format("\t\t~p=~p (~ts)~n", [Var, Default, Doc]). diff --git a/src/rebar_prv_packages.erl b/src/rebar_prv_packages.erl index 7217ab8..3e54cdc 100644 --- a/src/rebar_prv_packages.erl +++ b/src/rebar_prv_packages.erl @@ -15,53 +15,75 @@ -spec init(rebar_state:t()) -> {ok, rebar_state:t()}. init(State) -> - State1 = rebar_state:add_provider(State, providers:create([{name, ?PROVIDER}, - {module, ?MODULE}, - {bare, true}, - {deps, ?DEPS}, - {example, "rebar3 pkgs"}, - {short_desc, "List available packages."}, - {desc, info("List available packages")}, - {opts, []}])), + State1 = rebar_state:add_provider(State, + providers:create([{name, ?PROVIDER}, + {module, ?MODULE}, + {bare, true}, + {deps, ?DEPS}, + {example, "rebar3 pkgs elli"}, + {short_desc, "List information for a package."}, + {desc, info("List information for a package")}, + {opts, [{package, undefined, undefined, string, + "Package to fetch information for."}]}])), {ok, State1}. -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - rebar_packages:packages(State), - case rebar_state:command_args(State) of - [Name] -> - print_packages(get_packages(iolist_to_binary(Name))); - _ -> - print_packages(sort_packages()) - end, - {ok, State}. + {Args, _} = rebar_state:command_parsed_args(State), + case proplists:get_value(package, Args, undefined) of + undefined -> + ?PRV_ERROR(no_package_arg); + Name -> + Resources = rebar_state:resources(State), + #{repos := Repos} = rebar_resource_v2:find_resource_state(pkg, Resources), + Results = get_package(rebar_utils:to_binary(Name), Repos), + case lists:all(fun({_, {error, not_found}}) -> true; (_) -> false end, Results) of + true -> + ?PRV_ERROR({not_found, Name}); + false -> + [print_packages(Result) || Result <- Results], + {ok, State} + end + end. --spec format_error(any()) -> iolist(). -format_error(load_registry_fail) -> - "Failed to load package regsitry. Try running 'rebar3 update' to fix". +-spec get_package(binary(), [map()]) -> [{binary(), {ok, map()} | {error, term()}}]. +get_package(Name, Repos) -> + lists:foldl(fun(RepoConfig, Acc) -> + [{maps:get(name, RepoConfig), rebar_packages:get(RepoConfig, Name)} | Acc] + end, [], Repos). -print_packages(Pkgs) -> - orddict:map(fun(Name, Vsns) -> - SortedVsns = lists:sort(fun(A, B) -> - ec_semver:lte(ec_semver:parse(A) - ,ec_semver:parse(B)) - end, Vsns), - VsnStr = join(SortedVsns, <<", ">>), - ?CONSOLE("~s:~n Versions: ~s~n", [Name, VsnStr]) - end, Pkgs). -sort_packages() -> - ets:foldl(fun({package_index_version, _}, Acc) -> - Acc; - ({Pkg, Vsns}, Acc) -> - orddict:store(Pkg, Vsns, Acc); - (_, Acc) -> - Acc - end, orddict:new(), ?PACKAGE_TABLE). +-spec format_error(any()) -> iolist(). +format_error(no_package_arg) -> + "Missing package argument to `rebar3 pkgs` command."; +format_error({not_found, Name}) -> + io_lib:format("Package ~ts not found in any repo.", [Name]); +format_error(unknown) -> + "Something went wrong with fetching package metadata.". -get_packages(Name) -> - ets:lookup(?PACKAGE_TABLE, Name). +print_packages({RepoName, {error, not_found}}) -> + ?CONSOLE("~ts: Package not found in this repo.~n", [RepoName]); +print_packages({RepoName, {error, _}}) -> + ?CONSOLE("~ts: Error fetching from this repo.~n", [RepoName]); +print_packages({RepoName, {ok, #{<<"name">> := Name, + <<"meta">> := Meta, + <<"releases">> := Releases}}}) -> + Description = maps:get(<<"description">>, Meta, ""), + Licenses = join(maps:get(<<"licenses">>, Meta, []), <<", ">>), + Links = join_map(maps:get(<<"links">>, Meta, []), <<"\n ">>), + Maintainers = join(maps:get(<<"maintainers">>, Meta, []), <<", ">>), + Versions = [V || #{<<"version">> := V} <- Releases], + VsnStr = join(Versions, <<", ">>), + ?CONSOLE("~ts:~n" + " Name: ~ts~n" + " Description: ~ts~n" + " Licenses: ~ts~n" + " Maintainers: ~ts~n" + " Links:~n ~ts~n" + " Versions: ~ts~n", [RepoName, Name, Description, Licenses, Maintainers, Links, VsnStr]); +print_packages(_) -> + ok. -spec join([binary()], binary()) -> binary(). join([Bin], _Sep) -> @@ -69,6 +91,14 @@ join([Bin], _Sep) -> join([Bin | T], Sep) -> <<Bin/binary, Sep/binary, (join(T, Sep))/binary>>. +-spec join_map(map(), binary()) -> binary(). +join_map(Map, Sep) -> + join_tuple_list(maps:to_list(Map), Sep). + +join_tuple_list([{K, V}], _Sep) -> + <<K/binary, ": ", V/binary>>; +join_tuple_list([{K, V} | T], Sep) -> + <<K/binary, ": ", V/binary, Sep/binary, (join_tuple_list(T, Sep))/binary>>. info(Description) -> - io_lib:format("~s.~n", [Description]). + io_lib:format("~ts.~n", [Description]). diff --git a/src/rebar_prv_path.erl b/src/rebar_prv_path.erl index 4259eec..5374b0c 100644 --- a/src/rebar_prv_path.erl +++ b/src/rebar_prv_path.erl @@ -27,7 +27,7 @@ init(State) -> {example, "rebar3 path"}, {short_desc, "Print paths to build dirs in current profile."}, {desc, "Print paths to build dirs in current profile."}, - {opts, eunit_opts(State)}])), + {opts, path_opts(State)}])), {ok, State1}. @@ -49,7 +49,7 @@ format_error(Reason) -> filter_apps(RawOpts, State) -> RawApps = proplists:get_all_values(app, RawOpts), - Apps = lists:foldl(fun(String, Acc) -> string:tokens(String, ",") ++ Acc end, [], RawApps), + Apps = lists:foldl(fun(String, Acc) -> rebar_string:lexemes(String, ",") ++ Acc end, [], RawApps), case Apps of [] -> ProjectDeps = project_deps(State), @@ -75,23 +75,23 @@ paths([{src, true}|Rest], Apps, State, Acc) -> paths([{rel, true}|Rest], Apps, State, Acc) -> paths(Rest, Apps, State, [rel_dir(State)|Acc]). -base_dir(State) -> io_lib:format("~s", [rebar_dir:base_dir(State)]). -bin_dir(State) -> io_lib:format("~s/bin", [rebar_dir:base_dir(State)]). -lib_dir(State) -> io_lib:format("~s", [rebar_dir:deps_dir(State)]). -rel_dir(State) -> io_lib:format("~s/rel", [rebar_dir:base_dir(State)]). +base_dir(State) -> io_lib:format("~ts", [rebar_dir:base_dir(State)]). +bin_dir(State) -> io_lib:format("~ts/bin", [rebar_dir:base_dir(State)]). +lib_dir(State) -> io_lib:format("~ts", [rebar_dir:deps_dir(State)]). +rel_dir(State) -> io_lib:format("~ts/rel", [rebar_dir:base_dir(State)]). ebin_dirs(Apps, State) -> - lists:map(fun(App) -> io_lib:format("~s/~s/ebin", [rebar_dir:deps_dir(State), App]) end, Apps). + lists:map(fun(App) -> io_lib:format("~ts/~ts/ebin", [rebar_dir:deps_dir(State), App]) end, Apps). priv_dirs(Apps, State) -> - lists:map(fun(App) -> io_lib:format("~s/~s/priv", [rebar_dir:deps_dir(State), App]) end, Apps). + lists:map(fun(App) -> io_lib:format("~ts/~ts/priv", [rebar_dir:deps_dir(State), App]) end, Apps). src_dirs(Apps, State) -> - lists:map(fun(App) -> io_lib:format("~s/~s/src", [rebar_dir:deps_dir(State), App]) end, Apps). + lists:map(fun(App) -> io_lib:format("~ts/~ts/src", [rebar_dir:deps_dir(State), App]) end, Apps). print_paths_if_exist(Paths, State) -> {RawOpts, _} = rebar_state:command_parsed_args(State), Sep = proplists:get_value(separator, RawOpts, " "), RealPaths = lists:filter(fun(P) -> ec_file:is_dir(P) end, Paths), - io:format("~s", [string:join(RealPaths, Sep)]). + io:format("~ts", [rebar_string:join(RealPaths, Sep)]). project_deps(State) -> Profiles = rebar_state:current_profiles(State), @@ -107,7 +107,7 @@ normalize(AppName) when is_list(AppName) -> AppName; normalize(AppName) when is_atom(AppName) -> atom_to_list(AppName); normalize(AppName) when is_binary(AppName) -> binary_to_list(AppName). -eunit_opts(_State) -> +path_opts(_State) -> [{app, undefined, "app", string, help(app)}, {base, undefined, "base", boolean, help(base)}, {bin, undefined, "bin", boolean, help(bin)}, @@ -118,7 +118,7 @@ eunit_opts(_State) -> {src, undefined, "src", boolean, help(src)}, {rel, undefined, "rel", boolean, help(rel)}]. -help(app) -> "Comma seperated list of applications to return paths for."; +help(app) -> "Comma separated list of applications to return paths for."; help(base) -> "Return the `base' path of the current profile."; help(bin) -> "Return the `bin' path of the current profile."; help(ebin) -> "Return all `ebin' paths of the current profile's applications."; diff --git a/src/rebar_prv_plugins.erl b/src/rebar_prv_plugins.erl index 7e6b88e..fba5da0 100644 --- a/src/rebar_prv_plugins.erl +++ b/src/rebar_prv_plugins.erl @@ -34,15 +34,19 @@ do(State) -> GlobalConfigFile = rebar_dir:global_config(), GlobalConfig = rebar_state:new(rebar_config:consult_file(GlobalConfigFile)), GlobalPlugins = rebar_state:get(GlobalConfig, plugins, []), + GlobalSrcDirs = rebar_state:get(GlobalConfig, src_dirs, ["src"]), GlobalPluginsDir = filename:join([rebar_dir:global_cache_dir(rebar_state:opts(State)), "plugins", "*"]), - GlobalApps = rebar_app_discover:find_apps([GlobalPluginsDir], all), + GlobalApps = rebar_app_discover:find_apps([GlobalPluginsDir], GlobalSrcDirs, all, State), display_plugins("Global plugins", GlobalApps, GlobalPlugins), + RebarOpts = rebar_state:opts(State), + SrcDirs = rebar_dir:src_dirs(RebarOpts, ["src"]), Plugins = rebar_state:get(State, plugins, []), - PluginsDir = filename:join(rebar_dir:plugins_dir(State), "*"), - CheckoutsDir = filename:join(rebar_dir:checkouts_dir(State), "*"), - Apps = rebar_app_discover:find_apps([CheckoutsDir, PluginsDir], all), - display_plugins("Local plugins", Apps, Plugins), + ProjectPlugins = rebar_state:get(State, project_plugins, []), + PluginsDirs = filelib:wildcard(filename:join(rebar_dir:plugins_dir(State), "*")), + CheckoutsDirs = filelib:wildcard(filename:join(rebar_dir:checkouts_dir(State), "*")), + Apps = rebar_app_discover:find_apps(CheckoutsDirs++PluginsDirs, SrcDirs, all, State), + display_plugins("Local plugins", Apps, Plugins ++ ProjectPlugins), {ok, State}. -spec format_error(any()) -> iolist(). @@ -52,19 +56,19 @@ format_error(Reason) -> display_plugins(_Header, _Apps, []) -> ok; display_plugins(Header, Apps, Plugins) -> - ?CONSOLE("--- ~s ---", [Header]), + ?CONSOLE("--- ~ts ---", [Header]), display_plugins(Apps, Plugins), ?CONSOLE("", []). display_plugins(Apps, Plugins) -> lists:foreach(fun(Plugin) -> - Name = if is_atom(Plugin) -> ec_cnv:to_binary(Plugin); - is_tuple(Plugin) -> ec_cnv:to_binary(element(1, Plugin)) + Name = if is_atom(Plugin) -> atom_to_binary(Plugin, unicode); + is_tuple(Plugin) -> rebar_utils:to_binary(element(1, Plugin)) end, case rebar_app_utils:find(Name, Apps) of {ok, _App} -> - ?CONSOLE("~s", [Name]); + ?CONSOLE("~ts", [Name]); error -> - ?DEBUG("Unable to find plugin ~s", [Name]) + ?DEBUG("Unable to find plugin ~ts", [Name]) end end, Plugins). diff --git a/src/rebar_prv_plugins_upgrade.erl b/src/rebar_prv_plugins_upgrade.erl index 03521c7..7420c83 100644 --- a/src/rebar_prv_plugins_upgrade.erl +++ b/src/rebar_prv_plugins_upgrade.erl @@ -44,7 +44,7 @@ do(State) -> format_error(no_plugin_arg) -> io_lib:format("Must give an installed plugin to upgrade as an argument", []); format_error({not_found, Plugin}) -> - io_lib:format("Plugin to upgrade not found: ~s", [Plugin]); + io_lib:format("Plugin to upgrade not found: ~ts", [Plugin]); format_error(Reason) -> io_lib:format("~p", [Reason]). diff --git a/src/rebar_prv_report.erl b/src/rebar_prv_report.erl index d6c8b60..73e9624 100644 --- a/src/rebar_prv_report.erl +++ b/src/rebar_prv_report.erl @@ -44,7 +44,7 @@ do(State) -> {ok, Vsn} = application:get_key(rebar, vsn), {ok, Apps} = application:get_key(rebar, applications), [application:load(App) || App <- Apps], - Vsns = [io_lib:format("~p: ~s~n", [App, AVsn]) + Vsns = [io_lib:format("~p: ~ts~n", [App, AVsn]) || App <- lists:sort(Apps), {ok, AVsn} <- [application:get_key(App, vsn)]], %% Show OS and versions @@ -59,10 +59,10 @@ do(State) -> %% ?CONSOLE( "Rebar3 report~n" - " version ~s~n" - " generated at ~s~n" + " version ~ts~n" + " generated at ~ts~n" "=================~n" - "Please submit this along with your issue at ~s " + "Please submit this along with your issue at ~ts " "(and feel free to edit out private information, if any)~n" "-----------------~n" "Task: ~ts~n" @@ -75,11 +75,11 @@ do(State) -> "Library directory: ~ts~n" "-----------------~n" "Loaded Applications:~n" - "~s~n" + "~ts~n" "-----------------~n" "Escript path: ~ts~n" "Providers:~n" - " ~s", + " ~ts", [Vsn, time_to_string(UTC), ?ISSUES_URL, Command, Task, OS, ERTS, Root, Lib, @@ -100,4 +100,4 @@ time_to_string({{Y,M,D},{H,Min,S}}) -> [Y,M,D,H,Min,S])). parse_task(Str) -> - hd(re:split(Str, " ")). + hd(re:split(Str, " ", [unicode])). diff --git a/src/rebar_prv_repos.erl b/src/rebar_prv_repos.erl new file mode 100644 index 0000000..0515910 --- /dev/null +++ b/src/rebar_prv_repos.erl @@ -0,0 +1,47 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et + +-module(rebar_prv_repos). + +-behaviour(provider). + +-export([init/1, + do/1, + format_error/1]). + +-include("rebar.hrl"). + +-define(PROVIDER, repos). +-define(DEPS, []). + +%% =================================================================== +%% Public API +%% =================================================================== + +-spec init(rebar_state:t()) -> {ok, rebar_state:t()}. +init(State) -> + Provider = providers:create( + [{name, ?PROVIDER}, + {module, ?MODULE}, + {bare, false}, + {deps, ?DEPS}, + {example, "rebar3 repos"}, + {short_desc, "Print current package repository configuration"}, + {desc, "Display repository configuration for debugging purpose"}, + {opts, []}]), + State1 = rebar_state:add_provider(State, Provider), + {ok, State1}. + +-spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. +do(State) -> + Resources = rebar_state:resources(State), + #{repos := Repos} = rebar_resource_v2:find_resource_state(pkg, Resources), + + ?CONSOLE("Repos:", []), + %%TODO: do some formatting + ?CONSOLE("~p", [Repos]), + {ok, State}. + +-spec format_error(any()) -> iolist(). +format_error(Reason) -> + io_lib:format("~p", [Reason]). diff --git a/src/rebar_prv_shell.erl b/src/rebar_prv_shell.erl index b7febf8..760f0d8 100644 --- a/src/rebar_prv_shell.erl +++ b/src/rebar_prv_shell.erl @@ -40,6 +40,8 @@ -define(PROVIDER, shell). -define(DEPS, [compile]). +-dialyzer({nowarn_function, rewrite_leaders/2}). + %% =================================================================== %% Public API %% =================================================================== @@ -75,7 +77,13 @@ init(State) -> "A list of apps to boot before starting the " "shell. (E.g. --apps app1,app2,app3) Defaults " "to rebar.config {shell, [{apps, Apps}]} or " - "relx apps if not specified."}]} + "relx apps if not specified."}, + {start_clean, undefined, "start-clean", boolean, + "Cancel any applications in the 'apps' list " + "or release."}, + {user_drv_args, undefined, "user_drv_args", string, + "Arguments passed to user_drv start function for " + "creating custom shells."}]} ]) ), {ok, State1}. @@ -99,7 +107,9 @@ format_error(Reason) -> shell(State) -> setup_name(State), setup_paths(State), - setup_shell(), + ShellArgs = debug_get_value(shell_args, rebar_state:get(State, shell, []), undefined, + "Found user_drv args from command line option."), + setup_shell(ShellArgs), maybe_run_script(State), %% apps must be started after the change in shell because otherwise %% their application masters never gets the new group leader (held in @@ -117,13 +127,13 @@ shell(State) -> info() -> "Start a shell with project and deps preloaded similar to~n'erl -pa ebin -pa deps/*/ebin'.~n". -setup_shell() -> +setup_shell(ShellArgs) -> OldUser = kill_old_user(), %% Test for support here - NewUser = try erlang:open_port({spawn,'tty_sl -c -e'}, []) of + NewUser = try erlang:open_port({spawn,"tty_sl -c -e"}, []) of Port when is_port(Port) -> true = port_close(Port), - setup_new_shell() + setup_new_shell(ShellArgs) catch error:_ -> setup_old_shell() @@ -153,11 +163,16 @@ wait_for_port_death(N, P) -> wait_for_port_death(N-10, P) end. -setup_new_shell() -> +setup_new_shell(ShellArgs) -> %% terminate the current user supervision structure, if any _ = supervisor:terminate_child(kernel_sup, user), %% start a new shell (this also starts a new user under the correct group) - _ = user_drv:start(), + case ShellArgs of + undefined -> + _ = user_drv:start(); + _ -> + _ = user_drv:start(ShellArgs) + end, %% wait until user_drv and user have been registered (max 3 seconds) ok = wait_until_user_started(3000), whereis(user). @@ -191,21 +206,28 @@ rewrite_leaders(OldUser, NewUser) -> lists:member(proplists:get_value(group_leader, erlang:process_info(Pid)), OldMasters)], try - %% enable error_logger's tty output - error_logger:swap_handler(tty), - %% disable the simple error_logger (which may have been added multiple - %% times). removes at most the error_logger added by init and the - %% error_logger added by the tty handler - remove_error_handler(3), - %% reset the tty handler once more for remote shells - error_logger:swap_handler(tty) + case erlang:function_exported(logger, module_info, 0) of + false -> + %% Old style logger had a lock-up issue and other problems related + %% to group leader handling. + %% enable error_logger's tty output + error_logger:swap_handler(tty), + %% disable the simple error_logger (which may have been added + %% multiple times). removes at most the error_logger added by + %% init and the error_logger added by the tty handler + remove_error_handler(3), + %% reset the tty handler once more for remote shells + error_logger:swap_handler(tty); + true -> + %% This is no longer a problem with the logger interface + ok + end catch - E:R -> % may fail with custom loggers - ?DEBUG("Logger changes failed for ~p:~p (~p)", [E,R,erlang:get_stacktrace()]), + ?WITH_STACKTRACE(E,R,S) % may fail with custom loggers + ?DEBUG("Logger changes failed for ~p:~p (~p)", [E,R,S]), hope_for_best end. - setup_paths(State) -> %% Add deps to path code:add_pathsa(rebar_state:code_paths(State, all_deps)), @@ -225,9 +247,9 @@ maybe_run_script(State) -> File = filename:absname(RelFile), try run_script_file(File) catch - C:E -> + ?WITH_STACKTRACE(C,E,S) ?ABORT("Couldn't run shell escript ~p - ~p:~p~nStack: ~p", - [File, C, E, erlang:get_stacktrace()]) + [File, C, E, S]) end end. @@ -261,11 +283,11 @@ maybe_boot_apps(State) -> case find_apps_to_boot(State) of undefined -> %% try to read in sys.config file - ok = reread_config(State); + ok = reread_config([], State); Apps -> %% load apps, then check config, then boot them. load_apps(Apps), - ok = reread_config(State), + ok = reread_config(Apps, State), boot_apps(Apps) end. @@ -294,10 +316,15 @@ find_apps_option(State) -> {Opts, _} = rebar_state:command_parsed_args(State), case debug_get_value(apps, Opts, no_value, "Found shell apps from command line option.") of - no_value -> no_value; + no_value -> + case debug_get_value(start_clean, Opts, false, + "Found start-clean argument to disable apps") of + false -> no_value; + true -> [] + end; AppsStr -> [ list_to_atom(AppStr) - || AppStr <- string:tokens(AppsStr, " ,:") ] + || AppStr <- rebar_string:lexemes(AppsStr, " ,:") ] end. -spec find_apps_rebar(rebar_state:t()) -> no_value | list(). @@ -312,6 +339,9 @@ find_apps_relx(State) -> {_, _, Apps} -> ?DEBUG("Found shell apps from relx.", []), Apps; + {_, _, Apps, _} -> + ?DEBUG("Found shell apps from relx.", []), + Apps; false -> no_value end. @@ -327,19 +357,44 @@ load_apps(Apps) -> not lists:keymember(App, 1, application:loaded_applications())], ok. -reread_config(State) -> +reread_config(AppsToStart, State) -> case find_config(State) of no_config -> ok; ConfigList -> - _ = rebar_utils:reread_config(ConfigList), + %% This allows people who use applications that are also + %% depended on by rebar3 or its plugins to change their + %% configuration at runtime based on the configuration files. + %% + %% To do this, we stop apps that are already started before + %% reloading their configuration. + %% + %% We make an exception for apps that: + %% - are not already running + %% - would not be restarted (and hence would break some + %% compatibility with rebar3) + %% - are not in the config files and would see no config + %% changes + %% - are not in a blacklist, where changing their config + %% would be risky to the shell or the rebar3 agent + %% functionality (i.e. changing inets may break proxy + %% settings, stopping `kernel' would break everything) + Running = [App || {App, _, _} <- application:which_applications()], + BlackList = [inets, stdlib, kernel, rebar], + _ = [application:stop(App) + || Config <- ConfigList, + {App, _} <- Config, + lists:member(App, Running), + lists:member(App, AppsToStart), + not lists:member(App, BlackList)], + _ = rebar_utils:reread_config(ConfigList, [update_logger]), ok end. boot_apps(Apps) -> ?WARN("The rebar3 shell is a development tool; to deploy " "applications in production, consider using releases " - "(http://www.rebar3.org/v3.0/docs/releases)", []), + "(http://www.rebar3.org/docs/releases)", []), Normalized = normalize_boot_apps(Apps), Res = [application:ensure_all_started(App) || App <- Normalized], _ = [?INFO("Booted ~p", [App]) @@ -350,17 +405,24 @@ boot_apps(Apps) -> ok. normalize_load_apps([]) -> []; +normalize_load_apps([{_App, none} | T]) -> normalize_load_apps(T); normalize_load_apps([{App, _} | T]) -> [App | normalize_load_apps(T)]; normalize_load_apps([{App, _Vsn, load} | T]) -> [App | normalize_load_apps(T)]; +normalize_load_apps([{_App, _Vsn, none} | T]) -> normalize_load_apps(T); +normalize_load_apps([{App, _Vsn, Operator} | T]) when is_atom(Operator) -> + [App | normalize_load_apps(T)]; normalize_load_apps([App | T]) when is_atom(App) -> [App | normalize_load_apps(T)]. normalize_boot_apps([]) -> []; normalize_boot_apps([{_App, load} | T]) -> normalize_boot_apps(T); normalize_boot_apps([{_App, _Vsn, load} | T]) -> normalize_boot_apps(T); +normalize_boot_apps([{_App, none} | T]) -> normalize_boot_apps(T); +normalize_boot_apps([{_App, _Vsn, none} | T]) -> normalize_boot_apps(T); +normalize_boot_apps([{App, _Vsn, Operator} | T]) when is_atom(Operator) -> + [App | normalize_boot_apps(T)]; normalize_boot_apps([{App, _Vsn} | T]) -> [App | normalize_boot_apps(T)]; normalize_boot_apps([App | T]) when is_atom(App) -> [App | normalize_boot_apps(T)]. - remove_error_handler(0) -> ?WARN("Unable to remove simple error_logger handler", []); remove_error_handler(N) -> diff --git a/src/rebar_prv_unlock.erl b/src/rebar_prv_unlock.erl index 7ff0d89..6fe8bd8 100644 --- a/src/rebar_prv_unlock.erl +++ b/src/rebar_prv_unlock.erl @@ -48,12 +48,8 @@ do(State) -> ?PRV_ERROR({file,Reason}); {ok, _} -> Locks = rebar_config:consult_lock_file(LockFile), - case handle_unlocks(State, Locks, LockFile) of - ok -> - {ok, State}; - {error, Reason} -> - ?PRV_ERROR({file,Reason}) - end + {ok, NewLocks} = handle_unlocks(State, Locks, LockFile), + {ok, rebar_state:set(State, {locks, default}, NewLocks)} end. -spec format_error(any()) -> iolist(). @@ -66,18 +62,21 @@ format_error(Reason) -> handle_unlocks(State, Locks, LockFile) -> {Args, _} = rebar_state:command_parsed_args(State), - Names = parse_names(ec_cnv:to_binary(proplists:get_value(package, Args, <<"">>))), + Names = parse_names(rebar_utils:to_binary(proplists:get_value(package, Args, <<"">>))), case [Lock || Lock = {Name, _, _} <- Locks, not lists:member(Name, Names)] of [] -> - file:delete(LockFile); + file:delete(LockFile), + {ok, []}; _ when Names =:= [] -> % implicitly all locks - file:delete(LockFile); + file:delete(LockFile), + {ok, []}; NewLocks -> - rebar_config:write_lock_file(LockFile, NewLocks) + rebar_config:write_lock_file(LockFile, NewLocks), + {ok, NewLocks} end. parse_names(Bin) -> - case lists:usort(re:split(Bin, <<" *, *">>, [trim])) of + case lists:usort(re:split(Bin, <<" *, *">>, [trim, unicode])) of [<<"">>] -> []; % nothing submitted Other -> Other end. diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index 54b135e..4c820c5 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -9,12 +9,6 @@ do/1, format_error/1]). --export([hex_to_index/1]). - --ifdef(TEST). --export([cmp_/6, cmpl_/6, valid_vsn/1]). --endif. - -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). @@ -39,43 +33,13 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - try - case rebar_packages:registry_dir(State) of - {ok, RegistryDir} -> - filelib:ensure_dir(filename:join(RegistryDir, "dummy")), - HexFile = filename:join(RegistryDir, "registry"), - ?INFO("Updating package registry...", []), - TmpDir = ec_file:insecure_mkdtemp(), - TmpFile = filename:join(TmpDir, "packages.gz"), - - CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), - case rebar_utils:url_append_path(CDN, ?REMOTE_REGISTRY_FILE) of - {ok, Url} -> - ?DEBUG("Fetching registry from ~p", [Url]), - case httpc:request(get, {Url, [{"User-Agent", rebar_utils:user_agent()}]}, - [], [{stream, TmpFile}, {sync, true}], - rebar) of - {ok, saved_to_file} -> - {ok, Data} = file:read_file(TmpFile), - Unzipped = zlib:gunzip(Data), - ok = file:write_file(HexFile, Unzipped), - ?INFO("Writing registry to ~s", [HexFile]), - hex_to_index(State), - {ok, State}; - _ -> - ?PRV_ERROR(package_index_download) - end; - _ -> - ?PRV_ERROR({package_parse_cdn, CDN}) - end; - {uri_parse_error, CDN} -> - ?PRV_ERROR({package_parse_cdn, CDN}) - end - catch - _E:C -> - ?DEBUG("Error creating package index: ~p ~p", [C, erlang:get_stacktrace()]), - throw(?PRV_ERROR(package_index_write)) - end. + Names = rebar_packages:get_all_names(State), + Resources = rebar_state:resources(State), + #{repos := RepoConfigs} = rebar_resource_v2:find_resource_state(pkg, Resources), + [[update_package(Name, RepoConfig, State) + || Name <- Names] + || RepoConfig <- RepoConfigs], + {ok, State}. -spec format_error(any()) -> iolist(). format_error({package_parse_cdn, Uri}) -> @@ -85,170 +49,11 @@ format_error(package_index_download) -> format_error(package_index_write) -> "Failed to write package index.". -is_supported(<<"make">>) -> true; -is_supported(<<"rebar">>) -> true; -is_supported(<<"rebar3">>) -> true; -is_supported(_) -> false. - -hex_to_index(State) -> - {ok, RegistryDir} = rebar_packages:registry_dir(State), - HexFile = filename:join(RegistryDir, "registry"), - try ets:file2tab(HexFile) of - {ok, Registry} -> - try - PackageIndex = filename:join(RegistryDir, "packages.idx"), - ?INFO("Generating package index...", []), - (catch ets:delete(?PACKAGE_TABLE)), - ets:new(?PACKAGE_TABLE, [named_table, public]), - ets:foldl(fun({{Pkg, PkgVsn}, [Deps, Checksum, BuildTools | _]}, _) when is_list(BuildTools) -> - case lists:any(fun is_supported/1, BuildTools) of - true -> - DepsList = update_deps_list(Pkg, PkgVsn, Deps, Registry, State), - ets:insert(?PACKAGE_TABLE, {{Pkg, PkgVsn}, DepsList, Checksum}); - false -> - true - end; - (_, _) -> - true - end, true, Registry), - - ets:foldl(fun({Pkg, [[]]}, _) when is_binary(Pkg) -> - true; - ({Pkg, [Vsns=[_Vsn | _Rest]]}, _) when is_binary(Pkg) -> - %% Verify the package is of the right build tool by checking if the first - %% version exists in the table from the foldl above - case [V || V <- Vsns, ets:member(?PACKAGE_TABLE, {Pkg, V})] of - [] -> - true; - Vsns1 -> - ets:insert(?PACKAGE_TABLE, {Pkg, Vsns1}) - end; - (_, _) -> - true - end, true, Registry), - ets:insert(?PACKAGE_TABLE, {package_index_version, ?PACKAGE_INDEX_VERSION}), - ?INFO("Writing index to ~s", [PackageIndex]), - ets:tab2file(?PACKAGE_TABLE, PackageIndex), - true - after - catch ets:delete(Registry) - end; - {error, Reason} -> - ?DEBUG("Error loading package registry: ~p", [Reason]), - false - catch - _:_ -> - fail - end. - -update_deps_list(Pkg, PkgVsn, Deps, HexRegistry, State) -> - lists:foldl(fun([Dep, DepVsn, false, _AppName | _], DepsListAcc) -> - Dep1 = {Pkg, PkgVsn, Dep}, - case {valid_vsn(DepVsn), DepVsn} of - %% Those are all not perfectly implemented! - %% and doubled since spaces seem not to be - %% enforced - {false, Vsn} -> - ?WARN("[~s:~s], Bad dependency version for ~s: ~s.", - [Pkg, PkgVsn, Dep, Vsn]), - DepsListAcc; - {_, <<"~>", Vsn/binary>>} -> - highest_matching(Dep1, rm_ws(Vsn), HexRegistry, - State, DepsListAcc); - {_, <<">=", Vsn/binary>>} -> - cmp(Dep1, rm_ws(Vsn), HexRegistry, State, - DepsListAcc, fun ec_semver:gte/2); - {_, <<">", Vsn/binary>>} -> - cmp(Dep1, rm_ws(Vsn), HexRegistry, State, - DepsListAcc, fun ec_semver:gt/2); - {_, <<"<=", Vsn/binary>>} -> - cmpl(Dep1, rm_ws(Vsn), HexRegistry, State, - DepsListAcc, fun ec_semver:lte/2); - {_, <<"<", Vsn/binary>>} -> - cmpl(Dep1, rm_ws(Vsn), HexRegistry, State, - DepsListAcc, fun ec_semver:lt/2); - {_, <<"==", Vsn/binary>>} -> - [{Dep, Vsn} | DepsListAcc]; - {_, Vsn} -> - [{Dep, Vsn} | DepsListAcc] - end; - ([_Dep, _DepVsn, true, _AppName | _], DepsListAcc) -> - DepsListAcc - end, [], Deps). - -rm_ws(<<" ", R/binary>>) -> - rm_ws(R); -rm_ws(R) -> - R. - -valid_vsn(Vsn) -> - %% Regepx from https://github.com/sindresorhus/semver-regex/blob/master/index.js - SemVerRegExp = "v?(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)(\\.(0|[1-9][0-9]*))?" - "(-[0-9a-z-]+(\\.[0-9a-z-]+)*)?(\\+[0-9a-z-]+(\\.[0-9a-z-]+)*)?", - SupportedVersions = "^(>=?|<=?|~>|==)?\\s*" ++ SemVerRegExp ++ "$", - re:run(Vsn, SupportedVersions) =/= nomatch. - -highest_matching({Pkg, PkgVsn, Dep}, Vsn, HexRegistry, State, DepsListAcc) -> - case rebar_packages:find_highest_matching(Pkg, PkgVsn, Dep, Vsn, HexRegistry, State) of - {ok, HighestDepVsn} -> - [{Dep, HighestDepVsn} | DepsListAcc]; - none -> - ?WARN("[~s:~s] Missing registry entry for package ~s. Try to fix with `rebar3 update`", - [Pkg, PkgVsn, Dep]), - DepsListAcc - end. - -cmp({_Pkg, _PkgVsn, Dep} = Dep1, Vsn, HexRegistry, State, DepsListAcc, CmpFun) -> - {ok, Vsns} = rebar_packages:find_all(Dep, HexRegistry, State), - cmp_(undefined, Vsn, Vsns, DepsListAcc, Dep1, CmpFun). - - -cmp_(undefined, _MinVsn, [], DepsListAcc, {Pkg, PkgVsn, Dep}, _CmpFun) -> - ?WARN("[~s:~s] Missing registry entry for package ~s. Try to fix with `rebar3 update`", - [Pkg, PkgVsn, Dep]), - DepsListAcc; -cmp_(HighestDepVsn, _MinVsn, [], DepsListAcc, {_Pkg, _PkgVsn, Dep}, _CmpFun) -> - [{Dep, HighestDepVsn} | DepsListAcc]; - -cmp_(BestMatch, MinVsn, [Vsn | R], DepsListAcc, Dep, CmpFun) -> - case CmpFun(Vsn, MinVsn) of - true -> - cmp_(Vsn, Vsn, R, DepsListAcc, Dep, CmpFun); - false -> - cmp_(BestMatch, MinVsn, R, DepsListAcc, Dep, CmpFun) - end. - -%% We need to treat this differently since we want a version that is LOWER but -%% the higest possible one. -cmpl({_Pkg, _PkgVsn, Dep} = Dep1, Vsn, HexRegistry, State, DepsListAcc, CmpFun) -> - {ok, Vsns} = rebar_packages:find_all(Dep, HexRegistry, State), - cmpl_(undefined, Vsn, Vsns, DepsListAcc, Dep1, CmpFun). - -cmpl_(undefined, _MaxVsn, [], DepsListAcc, {Pkg, PkgVsn, Dep}, _CmpFun) -> - ?WARN("[~s:~s] Missing registry entry for package ~s. Try to fix with `rebar3 update`", - [Pkg, PkgVsn, Dep]), - DepsListAcc; - -cmpl_(HighestDepVsn, _MaxVsn, [], DepsListAcc, {_Pkg, _PkgVsn, Dep}, _CmpFun) -> - [{Dep, HighestDepVsn} | DepsListAcc]; - -cmpl_(undefined, MaxVsn, [Vsn | R], DepsListAcc, Dep, CmpFun) -> - case CmpFun(Vsn, MaxVsn) of - true -> - cmpl_(Vsn, MaxVsn, R, DepsListAcc, Dep, CmpFun); - false -> - cmpl_(undefined, MaxVsn, R, DepsListAcc, Dep, CmpFun) - end; -cmpl_(BestMatch, MaxVsn, [Vsn | R], DepsListAcc, Dep, CmpFun) -> - case CmpFun(Vsn, MaxVsn) of - true -> - case ec_semver:gte(Vsn, BestMatch) of - true -> - cmpl_(Vsn, MaxVsn, R, DepsListAcc, Dep, CmpFun); - false -> - cmpl_(BestMatch, MaxVsn, R, DepsListAcc, Dep, CmpFun) - end; - false -> - cmpl_(BestMatch, MaxVsn, R, DepsListAcc, Dep, CmpFun) +update_package(Name, RepoConfig, State) -> + case rebar_packages:update_package(Name, RepoConfig, State) of + fail -> + ?WARN("Failed to fetch updates for package ~ts from repo ~ts", [Name, maps:get(name, RepoConfig)]); + _ -> + ok end. diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl index 18c307b..565f342 100644 --- a/src/rebar_prv_upgrade.erl +++ b/src/rebar_prv_upgrade.erl @@ -32,7 +32,7 @@ init(State) -> {deps, ?DEPS}, {example, "rebar3 upgrade [cowboy[,ranch]]"}, {short_desc, "Upgrade dependencies."}, - {desc, "Upgrade project dependecies. Mentioning no application " + {desc, "Upgrade project dependencies. Mentioning no application " "will upgrade all dependencies. To upgrade specific dependencies, " "their names can be listed in the command."}, {opts, [ @@ -43,6 +43,19 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> + Cwd = rebar_state:dir(State), + Providers = rebar_state:providers(State), + rebar_hooks:run_project_and_app_hooks(Cwd, pre, ?PROVIDER, Providers, State), + case do_(State) of + {ok, NewState} -> + rebar_hooks:run_project_and_app_hooks(Cwd, post, ?PROVIDER, Providers, NewState), + {ok, NewState}; + Other -> + rebar_hooks:run_project_and_app_hooks(Cwd, post, ?PROVIDER, Providers, State), + Other + end. + +do_(State) -> {Args, _} = rebar_state:command_parsed_args(State), Locks = rebar_state:get(State, {locks, default}, []), %% We have 3 sources of dependencies to upgrade from: @@ -68,16 +81,23 @@ do(State) -> ProfileDeps = rebar_state:get(State, {deps, default}, []), Deps = [Dep || Dep <- TopDeps ++ ProfileDeps, % TopDeps > ProfileDeps is_atom(Dep) orelse is_atom(element(1, Dep))], - Names = parse_names(ec_cnv:to_binary(proplists:get_value(package, Args, <<"">>)), Locks), + Names = parse_names(rebar_utils:to_binary(proplists:get_value(package, Args, <<"">>)), Locks), + DepsDict = deps_dict(rebar_state:all_deps(State)), - case prepare_locks(Names, Deps, Locks, [], DepsDict) of + AltDeps = find_non_default_deps(Deps, State), + FilteredNames = cull_default_names_if_profiles(Names, Deps, State), + case prepare_locks(FilteredNames, Deps, Locks, [], DepsDict, AltDeps) of {error, Reason} -> {error, Reason}; - {Locks0, _Unlocks0} -> + {Locks0, Unlocks0} -> Deps0 = top_level_deps(Deps, Locks), State1 = rebar_state:set(State, {deps, default}, Deps0), DepsDir = rebar_prv_install_deps:profile_dep_dir(State, default), D = rebar_app_utils:parse_deps(root, DepsDir, Deps0, State1, Locks0, 0), + + %% first update the package index for the packages to be upgraded + update_pkg_deps(Unlocks0, D, State1), + State2 = rebar_state:set(State1, {parsed_deps, default}, D), State3 = rebar_state:set(State2, {locks, default}, Locks0), State4 = rebar_state:set(State3, upgrade, true), @@ -100,14 +120,44 @@ do(State) -> format_error({unknown_dependency, Name}) -> io_lib:format("Dependency ~ts not found", [Name]); format_error({transitive_dependency, Name}) -> - io_lib:format("Dependency ~ts is transient and cannot be safely upgraded. " + io_lib:format("Dependency ~ts is transitive and cannot be safely upgraded. " "Promote it to your top-level rebar.config file to upgrade it.", [Name]); format_error(Reason) -> io_lib:format("~p", [Reason]). +%% fetch updates for package deps that have been unlocked for upgrade +update_pkg_deps([], _, _) -> + ok; +update_pkg_deps([{Name, _, _} | Rest], AppInfos, State) -> + case rebar_app_utils:find(Name, AppInfos) of + {ok, AppInfo} -> + Source = rebar_app_info:source(AppInfo), + case element(1, Source) of + pkg -> + Resources = rebar_state:resources(State), + #{repos := RepoConfigs} = rebar_resource_v2:find_resource_state(pkg, Resources), + PkgName = element(2, Source), + [update_package(PkgName, RepoConfig, State) || RepoConfig <- RepoConfigs]; + _ -> + skip + end; + _ -> + %% this should be impossible... + skip + end, + update_pkg_deps(Rest, AppInfos, State). + +update_package(Name, RepoConfig, State) -> + case rebar_packages:update_package(Name, RepoConfig, State) of + fail -> + ?WARN("Failed to fetch updates for package ~ts from repo ~ts", [Name, maps:get(name, RepoConfig)]); + _ -> + ok + end. + parse_names(Bin, Locks) -> - case lists:usort(re:split(Bin, <<" *, *">>, [trim])) of + case lists:usort(re:split(Bin, <<" *, *">>, [trim, unicode])) of %% Nothing submitted, use *all* apps [<<"">>] -> [Name || {Name, _, 0} <- Locks]; [] -> [Name || {Name, _, 0} <- Locks]; @@ -115,20 +165,45 @@ parse_names(Bin, Locks) -> Other -> Other end. -prepare_locks([], _, Locks, Unlocks, _Dict) -> +%% Find alternative deps in non-default profiles since they may +%% need to be passed through (they are never locked) +find_non_default_deps(Deps, State) -> + AltProfiles = rebar_state:current_profiles(State) -- [default], + AltProfileDeps = lists:append([ + rebar_state:get(State, {deps, Profile}, []) || Profile <- AltProfiles] + ), + [Dep || Dep <- AltProfileDeps, + is_atom(Dep) orelse is_atom(element(1, Dep)) + andalso not lists:member(Dep, Deps)]. + +%% If any alt profiles are used, remove the default profiles from +%% the upgrade list and warn about it. +cull_default_names_if_profiles(Names, Deps, State) -> + case rebar_state:current_profiles(State) of + [default] -> + Names; + _ -> + ?INFO("Dependencies in the default profile will not be upgraded", []), + lists:filter(fun(Name) -> + AtomName = binary_to_atom(Name, utf8), + rebar_utils:tup_find(AtomName, Deps) == false + end, Names) + end. + +prepare_locks([], _, Locks, Unlocks, _Dict, _AltDeps) -> {Locks, Unlocks}; -prepare_locks([Name|Names], Deps, Locks, Unlocks, Dict) -> +prepare_locks([Name|Names], Deps, Locks, Unlocks, Dict, AltDeps) -> AtomName = binary_to_atom(Name, utf8), case lists:keyfind(Name, 1, Locks) of {_, _, 0} = Lock -> case rebar_utils:tup_find(AtomName, Deps) of false -> - ?WARN("Dependency ~s has been removed and will not be upgraded", [Name]), - prepare_locks(Names, Deps, Locks, Unlocks, Dict); + ?WARN("Dependency ~ts has been removed and will not be upgraded", [Name]), + prepare_locks(Names, Deps, Locks, Unlocks, Dict, AltDeps); Dep -> {Source, NewLocks, NewUnlocks} = prepare_lock(Dep, Lock, Locks, Dict), prepare_locks(Names, Deps, NewLocks, - [{Name, Source, 0} | NewUnlocks ++ Unlocks], Dict) + [{Name, Source, 0} | NewUnlocks ++ Unlocks], Dict, AltDeps) end; {_, _, Level} = Lock when Level > 0 -> case rebar_utils:tup_find(AtomName, Deps) of @@ -137,10 +212,15 @@ prepare_locks([Name|Names], Deps, Locks, Unlocks, Dict) -> Dep -> % Dep has been promoted {Source, NewLocks, NewUnlocks} = prepare_lock(Dep, Lock, Locks, Dict), prepare_locks(Names, Deps, NewLocks, - [{Name, Source, 0} | NewUnlocks ++ Unlocks], Dict) + [{Name, Source, 0} | NewUnlocks ++ Unlocks], Dict, AltDeps) end; false -> - ?PRV_ERROR({unknown_dependency, Name}) + case rebar_utils:tup_find(AtomName, AltDeps) of + false -> + ?PRV_ERROR({unknown_dependency, Name}); + _ -> % non-default profile dependency found, pass through + prepare_locks(Names, Deps, Locks, Unlocks, Dict, AltDeps) + end end. prepare_lock(Dep, Lock, Locks, Dict) -> @@ -149,7 +229,7 @@ prepare_lock(Dep, Lock, Locks, Dict) -> {Name, _, Src} -> {Name, Src}; _ when is_atom(Dep) -> %% version-free package. Must unlock whatever matches in locks - {_, Vsn, _} = lists:keyfind(ec_cnv:to_binary(Dep), 1, Locks), + {_, Vsn, _} = lists:keyfind(rebar_utils:to_binary(Dep), 1, Locks), {Dep, Vsn} end, Children = all_children(Name1, Dict), @@ -165,7 +245,7 @@ unlock_children(Children, Locks) -> unlock_children(_, [], Locks, Unlocks) -> {Locks, Unlocks}; unlock_children(Children, [App = {Name,_,_} | Apps], Locks, Unlocks) -> - case lists:member(ec_cnv:to_binary(Name), Children) of + case lists:member(rebar_utils:to_binary(Name), Children) of true -> unlock_children(Children, Apps, Locks, [App | Unlocks]); false -> @@ -183,7 +263,7 @@ all_children(Name, Dict) -> lists:flatten(all_children_(Name, Dict)). all_children_(Name, Dict) -> - case dict:find(ec_cnv:to_binary(Name), Dict) of + case dict:find(rebar_utils:to_binary(Name), Dict) of {ok, Children} -> [Children | [all_children_(Child, Dict) || Child <- Children]]; error -> diff --git a/src/rebar_prv_xref.erl b/src/rebar_prv_xref.erl index 45badd3..3c987b4 100644 --- a/src/rebar_prv_xref.erl +++ b/src/rebar_prv_xref.erl @@ -36,7 +36,7 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - code:add_pathsa(rebar_state:code_paths(State, all_deps)), + rebar_paths:set_paths([deps], State), XrefChecks = prepare(State), XrefIgnores = rebar_state:get(State, xref_ignores, []), %% Run xref checks @@ -47,7 +47,6 @@ do(State) -> QueryChecks = rebar_state:get(State, xref_queries, []), QueryResults = lists:foldl(fun check_query/2, [], QueryChecks), stopped = xref:stop(xref), - rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), case XrefResults =:= [] andalso QueryResults =:= [] of true -> {ok, State}; @@ -70,7 +69,7 @@ short_desc() -> desc() -> io_lib:format( - "~s~n" + "~ts~n" "~n" "Valid rebar.config options:~n" " ~p~n" @@ -97,8 +96,11 @@ prepare(State) -> rebar_state:get(State, xref_warnings, false)}, {verbose, rebar_log:is_verbose(State)}]), - [{ok, _} = xref:add_directory(xref, rebar_app_info:ebin_dir(App)) - || App <- rebar_state:project_apps(State)], + [{ok, _} = xref:add_directory(xref, Dir) + || App <- rebar_state:project_apps(State), + %% the directory may not exist in rare cases of a compile + %% hook of a dep running xref prior to the full job being done + Dir <- [rebar_app_info:ebin_dir(App)], filelib:is_dir(Dir)], %% Get list of xref checks we want to run ConfXrefChecks = rebar_state:get(State, xref_checks, @@ -158,14 +160,23 @@ get_xref_ignorelist(Mod, XrefCheck) -> %% And create a flat {M,F,A} list lists:foldl( fun({F, A}, Acc) -> [{Mod,F,A} | Acc]; - ({M, F, A}, Acc) -> [{M,F,A} | Acc] + ({M, F, A}, Acc) -> [{M,F,A} | Acc]; + (M, Acc) when is_atom(M) -> [M | Acc] end, [], lists:flatten([IgnoreXref, BehaviourCallbacks])). keyall(Key, List) -> lists:flatmap(fun({K, L}) when Key =:= K -> L; (_) -> [] end, List). get_behaviour_callbacks(exports_not_used, Attributes) -> - [B:behaviour_info(callbacks) || B <- keyall(behaviour, Attributes)]; + lists:map(fun(Mod) -> + try + Mod:behaviour_info(callbacks) + catch + error:undef -> + ?WARN("Behaviour ~p is used but cannot be found.", [Mod]), + [] + end + end, keyall(behaviour, Attributes) ++ keyall(behavior, Attributes)); get_behaviour_callbacks(_XrefCheck, _Attributes) -> []. @@ -185,14 +196,15 @@ filter_xref_results(XrefCheck, XrefIgnores, XrefResults) -> end, SearchModules), [Result || Result <- XrefResults, - not lists:member(parse_xref_result(Result), Ignores)]. + not lists:member(element(1, Result), Ignores) + andalso not lists:member(parse_xref_result(Result), Ignores)]. display_results(XrefResults, QueryResults) -> [lists:map(fun display_xref_results_for_type/1, XrefResults), lists:map(fun display_query_result/1, QueryResults)]. display_query_result({Query, Answer, Value}) -> - io_lib:format("Query ~s~n answer ~p~n did not match ~p~n", + io_lib:format("Query ~ts~n answer ~p~n did not match ~p~n", [Query, Answer, Value]). display_xref_results_for_type({Type, XrefResults}) -> @@ -213,37 +225,37 @@ display_xref_result_fun(Type) -> end, case Type of undefined_function_calls -> - io_lib:format("~sWarning: ~s calls undefined function ~s (Xref)\n", + io_lib:format("~tsWarning: ~ts calls undefined function ~ts (Xref)\n", [Source, SMFA, TMFA]); undefined_functions -> - io_lib:format("~sWarning: ~s is undefined function (Xref)\n", + io_lib:format("~tsWarning: ~ts is undefined function (Xref)\n", [Source, SMFA]); locals_not_used -> - io_lib:format("~sWarning: ~s is unused local function (Xref)\n", + io_lib:format("~tsWarning: ~ts is unused local function (Xref)\n", [Source, SMFA]); exports_not_used -> - io_lib:format("~sWarning: ~s is unused export (Xref)\n", + io_lib:format("~tsWarning: ~ts is unused export (Xref)\n", [Source, SMFA]); deprecated_function_calls -> - io_lib:format("~sWarning: ~s calls deprecated function ~s (Xref)\n", + io_lib:format("~tsWarning: ~ts calls deprecated function ~ts (Xref)\n", [Source, SMFA, TMFA]); deprecated_functions -> - io_lib:format("~sWarning: ~s is deprecated function (Xref)\n", + io_lib:format("~tsWarning: ~ts is deprecated function (Xref)\n", [Source, SMFA]); Other -> - io_lib:format("~sWarning: ~s - ~s xref check: ~s (Xref)\n", + io_lib:format("~tsWarning: ~ts - ~ts xref check: ~ts (Xref)\n", [Source, SMFA, TMFA, Other]) end end. format_mfa({M, F, A}) -> - ?FMT("~s:~s/~w", [M, F, A]). + ?FMT("~ts:~ts/~w", [M, F, A]). format_mfa_source(MFA) -> case find_mfa_source(MFA) of {module_not_found, function_not_found} -> ""; - {Source, function_not_found} -> ?FMT("~s: ", [Source]); - {Source, Line} -> ?FMT("~s:~w: ", [Source, Line]) + {Source, function_not_found} -> ?FMT("~ts: ", [Source]); + {Source, Line} -> ?FMT("~ts:~w: ", [Source, Line]) end. %% @@ -269,12 +281,21 @@ find_mfa_source({M, F, A}) -> end. find_function_source(M, F, A, Bin) -> - AbstractCode = beam_lib:chunks(Bin, [abstract_code]), - {ok, {M, [{abstract_code, {raw_abstract_v1, Code}}]}} = AbstractCode, + ChunksLookup = beam_lib:chunks(Bin, [abstract_code]), + {ok, {M, [{abstract_code, AbstractCodeLookup}]}} = ChunksLookup, + case AbstractCodeLookup of + no_abstract_code -> + % There isn't much else we can do at this point + {module_not_found, function_not_found}; + {raw_abstract_v1, AbstractCode} -> + find_function_source_in_abstract_code(F, A, AbstractCode) + end. + +find_function_source_in_abstract_code(F, A, AbstractCode) -> %% Extract the original source filename from the abstract code - [{attribute, 1, file, {Source, _}} | _] = Code, + [{attribute, _, file, {Source, _}} | _] = AbstractCode, %% Extract the line number for a given function def - Fn = [E || E <- Code, + Fn = [E || E <- AbstractCode, safe_element(1, E) == function, safe_element(3, E) == F, safe_element(4, E) == A], diff --git a/src/rebar_relx.erl b/src/rebar_relx.erl index 5c653a3..431e1bc 100644 --- a/src/rebar_relx.erl +++ b/src/rebar_relx.erl @@ -6,6 +6,10 @@ -export([do/4, format_error/1]). +-ifdef(TEST). +-export([merge_overlays/1]). +-endif. + -include("rebar.hrl"). %% =================================================================== @@ -23,22 +27,25 @@ do(Module, Command, Provider, State) -> LibDirs = rebar_utils:filtermap(fun ec_file:exists/1, [rebar_dir:checkouts_dir(State), DepsDir | ProjectAppDirs]), OutputDir = filename:join(rebar_dir:base_dir(State), ?DEFAULT_RELEASE_DIR), - AllOptions = string:join([Command | Options], " "), + AllOptions = rebar_string:join([Command | Options], " "), Cwd = rebar_state:dir(State), Providers = rebar_state:providers(State), + RebarOpts = rebar_state:opts(State), + ErlOpts = rebar_opts:erl_opts(RebarOpts), rebar_hooks:run_project_and_app_hooks(Cwd, pre, Provider, Providers, State), try case rebar_state:get(State, relx, []) of [] -> relx:main([{lib_dirs, LibDirs} ,{caller, api} - ,{log_level, LogLevel} | output_dir(OutputDir, Options)], AllOptions); + ,{log_level, LogLevel} | output_dir(OutputDir, Options)] ++ ErlOpts, AllOptions); Config -> - Config1 = merge_overlays(Config), + Config1 = [{overlay_vars, [{base_dir, rebar_dir:base_dir(State)}]} + | merge_overlays(Config)], relx:main([{lib_dirs, LibDirs} ,{config, Config1} ,{caller, api} - ,{log_level, LogLevel} | output_dir(OutputDir, Options)], AllOptions) + ,{log_level, LogLevel} | output_dir(OutputDir, Options)] ++ ErlOpts, AllOptions) end, rebar_hooks:run_project_and_app_hooks(Cwd, post, Provider, Providers, State), {ok, State} @@ -62,5 +69,5 @@ merge_overlays(Config) -> (_) -> false end, Config), %% Have profile overlay entries come before others to match how profiles work elsewhere - NewOverlay = lists:reverse(lists:flatmap(fun({overlay, Overlay}) -> Overlay end, Overlays)), + NewOverlay = lists:flatmap(fun({overlay, Overlay}) -> Overlay end, lists:reverse(Overlays)), [{overlay, NewOverlay} | Others]. diff --git a/src/rebar_resource.erl b/src/rebar_resource.erl index cdce7a8..a3a8edb 100644 --- a/src/rebar_resource.erl +++ b/src/rebar_resource.erl @@ -2,23 +2,53 @@ %% ex: ts=4 sw=4 et -module(rebar_resource). --export([]). +-export([new/3, + lock/2, + download/4, + needs_update/2, + make_vsn/2]). --export_type([resource/0 - ,type/0 - ,location/0 - ,ref/0]). +-export_type([source/0, + type/0, + location/0, + ref/0]). --type resource() :: {type(), location(), ref()}. +-include("rebar.hrl"). + +-type source() :: {type(), location(), ref()} | {type(), location(), ref(), binary()}. -type type() :: atom(). -type location() :: string(). -type ref() :: any(). -callback lock(file:filename_all(), tuple()) -> - rebar_resource:resource(). + source(). -callback download(file:filename_all(), tuple(), rebar_state:t()) -> {tarball, file:filename_all()} | {ok, any()} | {error, any()}. -callback needs_update(file:filename_all(), tuple()) -> boolean(). -callback make_vsn(file:filename_all()) -> {plain, string()} | {error, string()}. + +-spec new(type(), module(), term()) -> rebar_resource_v2:resource(). +new(Type, Module, State) -> + #resource{type=Type, + module=Module, + state=State, + implementation=?MODULE}. + +lock(Module, AppInfo) -> + Module:lock(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). + +download(Module, TmpDir, AppInfo, State) -> + case Module:download(TmpDir, rebar_app_info:source(AppInfo), State) of + {ok, _} -> + ok; + Error -> + Error + end. + +needs_update(Module, AppInfo) -> + Module:needs_update(rebar_app_info:dir(AppInfo), rebar_app_info:source(AppInfo)). + +make_vsn(Module, AppInfo) -> + Module:make_vsn(rebar_app_info:dir(AppInfo)). diff --git a/src/rebar_resource_v2.erl b/src/rebar_resource_v2.erl new file mode 100644 index 0000000..f032f6e --- /dev/null +++ b/src/rebar_resource_v2.erl @@ -0,0 +1,147 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +-module(rebar_resource_v2). + +-export([new/3, + find_resource_state/2, + format_source/1, + lock/2, + download/3, + needs_update/2, + make_vsn/3, + format_error/1]). + +-export_type([resource/0, + source/0, + type/0, + location/0, + ref/0, + resource_state/0]). + +-include("rebar.hrl"). +-include_lib("providers/include/providers.hrl"). + +-type resource() :: #resource{}. +-type source() :: {type(), location(), ref()} | {type(), location(), ref(), binary()}. +-type type() :: atom(). +-type location() :: string(). +-type ref() :: any(). +-type resource_state() :: term(). + +-callback init(type(), rebar_state:t()) -> {ok, resource()}. +-callback lock(rebar_app_info:t(), resource_state()) -> source(). +-callback download(file:filename_all(), rebar_app_info:t(), resource_state(), rebar_state:t()) -> + ok | {error, any()}. +-callback needs_update(rebar_app_info:t(), resource_state()) -> boolean(). +-callback make_vsn(rebar_app_info:t(), resource_state()) -> + {plain, string()} | {error, string()}. + +-spec new(type(), module(), term()) -> resource(). +new(Type, Module, State) -> + #resource{type=Type, + module=Module, + state=State, + implementation=?MODULE}. + +-spec find_resource(type(), [resource()]) -> {ok, resource()} | {error, not_found}. +find_resource(Type, Resources) -> + case ec_lists:find(fun(#resource{type=T}) -> T =:= Type end, Resources) of + error when is_atom(Type) -> + case code:which(Type) of + non_existing -> + {error, not_found}; + _ -> + {ok, rebar_resource:new(Type, Type, #{})} + end; + error -> + {error, not_found}; + {ok, Resource} -> + {ok, Resource} + end. + +find_resource_state(Type, Resources) -> + case lists:keyfind(Type, #resource.type, Resources) of + false -> + {error, not_found}; + #resource{state=State} -> + State + end. + +format_source({pkg, Name, Vsn, _Hash, _}) -> {pkg, Name, Vsn}; +format_source(Source) -> Source. + +lock(AppInfo, State) -> + resource_run(lock, rebar_app_info:source(AppInfo), [AppInfo], State). + +resource_run(Function, Source, Args, State) -> + Resources = rebar_state:resources(State), + case get_resource_type(Source, Resources) of + {ok, #resource{type=_, + module=Module, + state=ResourceState, + implementation=?MODULE}} -> + erlang:apply(Module, Function, Args++[ResourceState]); + {ok, #resource{type=_, + module=Module, + state=_, + implementation=rebar_resource}} -> + erlang:apply(rebar_resource, Function, [Module | Args]) + end. + +download(TmpDir, AppInfo, State) -> + resource_run(download, rebar_app_info:source(AppInfo), [TmpDir, AppInfo, State], State). + +needs_update(AppInfo, State) -> + resource_run(needs_update, rebar_app_info:source(AppInfo), [AppInfo], State). + +%% this is a special case since it is used for project apps as well, not just deps +make_vsn(AppInfo, VcsType, State) -> + Resources = rebar_state:resources(State), + case is_resource_type(VcsType, Resources) of + true -> + case find_resource(VcsType, Resources) of + {ok, #resource{type=_, + module=Module, + state=ResourceState, + implementation=?MODULE}} -> + Module:make_vsn(AppInfo, ResourceState); + {ok, #resource{type=_, + module=Module, + state=_, + implementation=rebar_resource}} -> + rebar_resource:make_vsn(Module, AppInfo) + end; + false -> + unknown + end. + +format_error({no_resource, Location, Type}) -> + io_lib:format("Cannot handle dependency ~ts.~n" + " No module found for resource type ~p.", [Location, Type]); +format_error({no_resource, Source}) -> + io_lib:format("Cannot handle dependency ~ts.~n" + " No module found for unknown resource type.", [Source]). + +is_resource_type(Type, Resources) -> + lists:any(fun(#resource{type=T}) -> T =:= Type end, Resources). + +-spec get_resource_type(term(), [resource()]) -> {ok, resource()}. +get_resource_type({Type, Location}, Resources) -> + get_resource(Type, Location, Resources); +get_resource_type({Type, Location, _}, Resources) -> + get_resource(Type, Location, Resources); +get_resource_type({Type, _, _, Location}, Resources) -> + get_resource(Type, Location, Resources); +get_resource_type(Location={Type, _, _, _, _}, Resources) -> + get_resource(Type, Location, Resources); +get_resource_type(Source, _) -> + throw(?PRV_ERROR({no_resource, Source})). + +-spec get_resource(type(), term(), [resource()]) -> {ok, resource()}. +get_resource(Type, Location, Resources) -> + case find_resource(Type, Resources) of + {error, not_found} -> + throw(?PRV_ERROR({no_resource, Location, Type})); + {ok, Resource} -> + {ok, Resource} + end. diff --git a/src/rebar_state.erl b/src/rebar_state.erl index bdd4aeb..31d3a08 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -38,6 +38,12 @@ to_list/1, + compilers/1, compilers/2, + prepend_compilers/2, append_compilers/2, + + project_builders/1, add_project_builder/3, + + create_resources/2, set_resources/2, resources/1, resources/2, add_resource/2, providers/1, providers/2, add_provider/2, allow_provider_overrides/1, allow_provider_overrides/2 @@ -59,12 +65,14 @@ command_args = [], command_parsed_args = {[], []}, - current_app :: rebar_app_info:t(), + current_app :: undefined | rebar_app_info:t(), project_apps = [] :: [rebar_app_info:t()], deps_to_build = [] :: [rebar_app_info:t()], all_plugin_deps = [] :: [rebar_app_info:t()], all_deps = [] :: [rebar_app_info:t()], + compilers = [] :: [{compiler_type(), extension(), extension(), compile_fun()}], + project_builders = [] :: [{rebar_app_info:project_type(), module()}], resources = [], providers = [], allow_provider_overrides = false :: boolean()}). @@ -73,28 +81,30 @@ -type t() :: #state_t{}. +-type compiler_type() :: atom(). +-type extension() :: string(). +-type compile_fun() :: fun(([file:filename()], rebar_app_info:t(), list()) -> ok). + -spec new() -> t(). new() -> - BaseState = base_state(), + BaseState = base_state(dict:new()), BaseState#state_t{dir = rebar_dir:get_cwd()}. -spec new(list()) -> t(). new(Config) when is_list(Config) -> - BaseState = base_state(), Opts = base_opts(Config), - BaseState#state_t { dir = rebar_dir:get_cwd(), - default = Opts, - opts = Opts }. + BaseState = base_state(Opts), + BaseState#state_t{dir=rebar_dir:get_cwd(), + default=Opts}. -spec new(t() | atom(), list()) -> t(). new(Profile, Config) when is_atom(Profile) , is_list(Config) -> - BaseState = base_state(), Opts = base_opts(Config), - BaseState#state_t { dir = rebar_dir:get_cwd(), - current_profiles = [Profile], - default = Opts, - opts = Opts }; + BaseState = base_state(Opts), + BaseState#state_t{dir = rebar_dir:get_cwd(), + current_profiles = [Profile], + default = Opts}; new(ParentState=#state_t{}, Config) -> %% Load terms from rebar.config, if it exists Dir = rebar_dir:get_cwd(), @@ -129,20 +139,15 @@ deps_from_config(Dir, Config) -> [{{locks, default}, D}, {{deps, default}, Deps}] end. -base_state() -> - case application:get_env(rebar, resources) of - undefined -> - Resources = []; - {ok, Resources} -> - Resources - end, - #state_t{resources=Resources}. +base_state(Opts) -> + #state_t{opts=Opts}. base_opts(Config) -> Deps = proplists:get_value(deps, Config, []), Plugins = proplists:get_value(plugins, Config, []), ProjectPlugins = proplists:get_value(project_plugins, Config, []), - Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins}, {{project_plugins, default}, ProjectPlugins} | Config], + Terms = [{{deps, default}, Deps}, {{plugins, default}, Plugins}, + {{project_plugins, default}, ProjectPlugins} | Config], true = rebar_config:verify_config_format(Terms), dict:from_list(Terms). @@ -182,7 +187,7 @@ all(_, []) -> all(Dir, [File|Artifacts]) -> case filelib:is_regular(filename:join(Dir, File)) of false -> - ?DEBUG("Missing artifact ~s", [filename:join(Dir, File)]), + ?DEBUG("Missing artifact ~ts", [filename:join(Dir, File)]), {false, File}; true -> all(Dir, Artifacts) @@ -257,12 +262,15 @@ apply_profiles(State, Profile) when not is_list(Profile) -> apply_profiles(State, [default]) -> State; apply_profiles(State=#state_t{default = Defaults, current_profiles=CurrentProfiles}, Profiles) -> + ProvidedProfiles = lists:prefix([default|Profiles], CurrentProfiles), AppliedProfiles = case Profiles of %% Head of list global profile is special, only for use by rebar3 %% It does not clash if a user does `rebar3 as global...` but when %% it is the head we must make sure not to prepend `default` [global | _] -> Profiles; + _ when ProvidedProfiles -> + deduplicate(CurrentProfiles); _ -> deduplicate(CurrentProfiles ++ Profiles) end, @@ -302,9 +310,9 @@ dir(State=#state_t{}, Dir) -> deps_names(Deps) when is_list(Deps) -> lists:map(fun(Dep) when is_tuple(Dep) -> - ec_cnv:to_binary(element(1, Dep)); + rebar_utils:to_binary(element(1, Dep)); (Dep) when is_atom(Dep) -> - ec_cnv:to_binary(Dep) + rebar_utils:to_binary(Dep) end, Deps); deps_names(State) -> Deps = rebar_state:get(State, deps, []), @@ -356,18 +364,80 @@ namespace(#state_t{namespace=Namespace}) -> namespace(State=#state_t{}, Namespace) -> State#state_t{namespace=Namespace}. --spec resources(t()) -> [{rebar_resource:type(), module()}]. +-spec resources(t()) -> [{rebar_resource_v2:type(), module()}]. resources(#state_t{resources=Resources}) -> Resources. --spec resources(t(), [{rebar_resource:type(), module()}]) -> t(). -resources(State, NewResources) -> - State#state_t{resources=NewResources}. +-spec set_resources(t(), [{rebar_resource_v2:type(), module()}]) -> t(). +set_resources(State, Resources) -> + State#state_t{resources=Resources}. --spec add_resource(t(), {rebar_resource:type(), module()}) -> t(). -add_resource(State=#state_t{resources=Resources}, Resource) -> +-spec resources(t(), [{rebar_resource_v2:type(), module()}]) -> t(). +resources(State, NewResources) -> + lists:foldl(fun(Resource, StateAcc) -> + add_resource(StateAcc, Resource) + end, State, NewResources). + +-spec add_resource(t(), {rebar_resource_v2:type(), module()}) -> t(). +add_resource(State=#state_t{resources=Resources}, {ResourceType, ResourceModule}) -> + _ = code:ensure_loaded(ResourceModule), + Resource = case erlang:function_exported(ResourceModule, init, 2) of + true -> + case ResourceModule:init(ResourceType, State) of + {ok, R=#resource{}} -> + R; + _ -> + %% init didn't return a resource + %% must be an old resource + warn_old_resource(ResourceModule), + rebar_resource:new(ResourceType, + ResourceModule, + #{}) + end; + false -> + %% no init, must be initial implementation + warn_old_resource(ResourceModule), + rebar_resource:new(ResourceType, + ResourceModule, + #{}) + end, State#state_t{resources=[Resource | Resources]}. +warn_old_resource(ResourceModule) -> + ?WARN("Using custom resource ~s that implements a deprecated api. " + "It should be upgraded to rebar_resource_v2.", [ResourceModule]). + +compilers(#state_t{compilers=Compilers}) -> + Compilers. + +prepend_compilers(State=#state_t{compilers=Compilers}, NewCompilers) -> + State#state_t{compilers=NewCompilers++Compilers}. + +append_compilers(State=#state_t{compilers=Compilers}, NewCompilers) -> + State#state_t{compilers=Compilers++NewCompilers}. + +compilers(State, Compilers) -> + State#state_t{compilers=Compilers}. + +project_builders(#state_t{project_builders=ProjectBuilders}) -> + ProjectBuilders. + +add_project_builder(State=#state_t{project_builders=ProjectBuilders}, Type, Module) -> + _ = code:ensure_loaded(Module), + case erlang:function_exported(Module, build, 1) of + true -> + State#state_t{project_builders=[{Type, Module} | ProjectBuilders]}; + false -> + ?WARN("Unable to add project builder for type ~s, required function ~s:build/1 not found.", + [Type, Module]), + State + end. + +create_resources(Resources, State) -> + lists:foldl(fun(R, StateAcc) -> + add_resource(StateAcc, R) + end, State, Resources). + providers(#state_t{providers=Providers}) -> Providers. @@ -391,7 +461,7 @@ add_provider(State=#state_t{providers=Providers, allow_provider_overrides=false} case {providers:impl(P), providers:namespace(P)} of {Name, Namespace} -> ?DEBUG("Not adding provider ~p ~p from module ~p because it already exists from module ~p", - [Namespace, Name, providers:module(P), Module]), + [Namespace, Name, Module, providers:module(P)]), true; _ -> false @@ -415,26 +485,33 @@ create_logic_providers(ProviderModules, State0) -> end end, State0, ProviderModules) catch - C:T -> - ?DEBUG("~p: ~p ~p", [C, T, erlang:get_stacktrace()]), - ?CRASHDUMP("~p: ~p~n~p~n~n~p", [C, T, erlang:get_stacktrace(), State0]), + ?WITH_STACKTRACE(C,T,S) + ?DEBUG("~p: ~p ~p", [C, T, S]), + ?CRASHDUMP("~p: ~p~n~p~n~n~p", [C, T, S, State0]), throw({error, "Failed creating providers. Run with DEBUG=1 for stacktrace or consult rebar3.crashdump."}) end. to_list(#state_t{} = State) -> Fields = record_info(fields, state_t), Values = tl(tuple_to_list(State)), - DictSz = tuple_size(dict:new()), - lists:zip(Fields, [reformat(I, DictSz) || I <- Values]). - -reformat({K,V}, DSz) when is_list(V) -> - {K, [reformat(I, DSz) || I <- V]}; -reformat(V, DSz) when is_tuple(V), element(1,V) =:= dict, tuple_size(V) =:= DSz -> - [reformat(I, DSz) || I <- dict:to_list(V)]; -reformat({K,V}, DSz) when is_tuple(V), element(1,V) =:= dict, tuple_size(V) =:= DSz -> - {K, [reformat(I, DSz) || I <- dict:to_list(V)]}; -reformat(Other, _DSz) -> - Other. + lists:zip(Fields, [reformat(I) || I <- Values]). + +reformat({K,V}) when is_list(V) -> + {K, [reformat(I) || I <- V]}; +reformat({K,V}) -> + try + {K, [reformat(I) || I <- dict:to_list(V)]} + catch + error:{badrecord,dict} -> + {K,V} + end; +reformat(V) -> + try + [reformat(I) || I <- dict:to_list(V)] + catch + error:{badrecord,dict} -> + V + end. %% =================================================================== %% Internal functions diff --git a/src/rebar_string.erl b/src/rebar_string.erl new file mode 100644 index 0000000..d03b14e --- /dev/null +++ b/src/rebar_string.erl @@ -0,0 +1,44 @@ +%%% @doc Compatibility module for string functionality +%%% for pre- and post-unicode support. +-module(rebar_string). +-export([join/2, split/2, lexemes/2, trim/3, uppercase/1, lowercase/1, chr/2]). + +-ifdef(unicode_str). + +%% string:join/2 copy; string:join/2 is getting obsoleted +%% and replaced by lists:join/2, but lists:join/2 is too new +%% for version support (only appeared in 19.0) so it cannot be +%% used. Instead we just adopt join/2 locally and hope it works +%% for most unicode use cases anyway. +join([], Sep) when is_list(Sep) -> + []; +join([H|T], Sep) -> + H ++ lists:append([Sep ++ X || X <- T]). + +split(Str, SearchPattern) -> string:split(Str, SearchPattern). +lexemes(Str, SepList) -> string:lexemes(Str, SepList). +trim(Str, Direction, Cluster=[_]) -> string:trim(Str, Direction, Cluster). +uppercase(Str) -> string:uppercase(Str). +lowercase(Str) -> string:lowercase(Str). + +chr(S, C) when is_integer(C) -> chr(S, C, 1). +chr([C|_Cs], C, I) -> I; +chr([_|Cs], C, I) -> chr(Cs, C, I+1); +chr([], _C, _I) -> 0. +-else. + +join(Strings, Separator) -> string:join(Strings, Separator). +split(Str, SearchPattern) when is_list(Str) -> string:split(Str, SearchPattern); +split(Str, SearchPattern) when is_binary(Str) -> binary:split(Str, SearchPattern). +lexemes(Str, SepList) -> string:tokens(Str, SepList). +trim(Str, Direction, [Char]) -> + Dir = case Direction of + both -> both; + leading -> left; + trailing -> right + end, + string:strip(Str, Dir, Char). +uppercase(Str) -> string:to_upper(Str). +lowercase(Str) -> string:to_lower(Str). +chr(Str, Char) -> string:chr(Str, Char). +-endif. diff --git a/src/rebar_templater.erl b/src/rebar_templater.erl index 2f33bfc..929ca47 100644 --- a/src/rebar_templater.erl +++ b/src/rebar_templater.erl @@ -33,7 +33,7 @@ -include("rebar.hrl"). --define(TEMPLATE_RE, "^[^._].*\\.template\$"). +-define(TEMPLATE_RE, "^(?!\\._).*\\.template\$"). %% =================================================================== %% Public API @@ -120,7 +120,8 @@ default_author_and_email() -> {ok, Name} -> case rebar_utils:sh("git config --global user.email", [return_on_error]) of {ok, Email} -> - {string:strip(Name, both, $\n), string:strip(Email, both, $\n)}; + {rebar_string:trim(Name, both, "\n"), + rebar_string:trim(Email, both, "\n")}; {error, _} -> %% Use neither if one doesn't exist {"Anonymous", "anonymous@example.org"} @@ -129,7 +130,7 @@ default_author_and_email() -> %% Ok, try mecurial case rebar_utils:sh("hg showconfig ui.username", [return_on_error]) of {ok, NameEmail} -> - case re:run(NameEmail, "^(.*) <(.*)>$", [{capture, [1,2], list}]) of + case re:run(NameEmail, "^(.*) <(.*)>$", [{capture, [1,2], list}, unicode]) of {match, [Name, Email]} -> {Name, Email}; _ -> @@ -169,7 +170,7 @@ maybe_warn_about_name(Vars) -> invalid -> ?WARN("The 'name' variable is often associated with Erlang " "module names and/or file names. The value submitted " - "(~s) isn't an unquoted Erlang atom. Templates " + "(~ts) isn't an unquoted Erlang atom. Templates " "generated may contain errors.", [Name]); valid -> @@ -189,7 +190,7 @@ validate_atom(Str) -> %% Run template instructions one at a time. execute_template([], _, {Template,_,_}, _, _) -> - ?DEBUG("Template ~s applied", [Template]), + ?DEBUG("Template ~ts applied", [Template]), ok; %% We can't execute the description execute_template([{description, _} | Terms], Files, Template, Vars, Force) -> @@ -242,7 +243,7 @@ execute_template([{template, From, To} | Terms], Files, {Template, Type, Cwd}, V execute_template(Terms, Files, {Template, Type, Cwd}, Vars, Force); %% Unknown execute_template([Instruction|Terms], Files, Tpl={Template,_,_}, Vars, Force) -> - ?WARN("Unknown template instruction ~p in template ~s", + ?WARN("Unknown template instruction ~p in template ~ts", [Instruction, Template]), execute_template(Terms, Files, Tpl, Vars, Force). @@ -267,8 +268,8 @@ find_templates(State) -> PluginTemplates = find_plugin_templates(State), {MainTemplates, Files} = case rebar_state:escript_path(State) of - undefined -> - {find_priv_templates(State), []}; + undefined -> % running in local install + {find_localinstall_templates(State), []}; _ -> %% Cache the files since we'll potentially need to walk it several times %% over the course of a run. @@ -305,13 +306,12 @@ cache_escript_files(State) -> find_escript_templates(Files) -> [{escript, Name} || {Name, _Bin} <- Files, - re:run(Name, ?TEMPLATE_RE, [{capture, none}]) == match]. + re:run(Name, ?TEMPLATE_RE, [{capture, none}, unicode]) == match]. -find_priv_templates(State) -> - OtherTemplates = rebar_utils:find_files(code:priv_dir(rebar), ?TEMPLATE_RE), - HomeFiles = rebar_utils:find_files(rebar_dir:template_dir(State), - ?TEMPLATE_RE, true), % recursive - [{file, F} || F <- OtherTemplates ++ HomeFiles]. +find_localinstall_templates(_State) -> + Templates = rebar_utils:find_files(code:priv_dir(rebar), ?TEMPLATE_RE), + %% Pretend we're still running escripts; should work transparently. + [{builtin, F} || F <- Templates]. %% Fetch template indexes that sit on disk in the user's HOME find_disk_templates(State) -> @@ -326,7 +326,7 @@ find_other_templates(State) -> undefined -> []; TemplateDir -> - rebar_utils:find_files(TemplateDir, ?TEMPLATE_RE) + rebar_utils:find_files(TemplateDir, ?TEMPLATE_RE, true) % recursive end. %% Fetch template indexes that sit on disk in plugins @@ -335,8 +335,19 @@ find_plugin_templates(State) -> || App <- rebar_state:all_plugin_deps(State), Priv <- [rebar_app_info:priv_dir(App)], Priv =/= undefined, + File <- rebar_utils:find_files(Priv, ?TEMPLATE_RE)] + ++ %% and add global plugins too + [{plugin, File} + || PSource <- rebar_state:get(State, {plugins, global}, []), + Plugin <- [plugin_provider(PSource)], + is_atom(Plugin), + Priv <- [code:priv_dir(Plugin)], + Priv =/= undefined, File <- rebar_utils:find_files(Priv, ?TEMPLATE_RE)]. +plugin_provider(P) when is_atom(P) -> P; +plugin_provider(T) when is_tuple(T) -> element(1, T). + %% Take an existing list of templates and tag them by name the way %% the user would enter it from the CLI tag_names(List) -> @@ -354,13 +365,17 @@ prioritize_templates([{Name, Type, File} | Rest], Valid) -> ?DEBUG("Skipping template ~p, due to presence of a built-in " "template with the same name", [Name]), prioritize_templates(Rest, Valid); + {_, builtin, _} -> + ?DEBUG("Skipping template ~p, due to presence of a built-in " + "template with the same name", [Name]), + prioritize_templates(Rest, Valid); {_, plugin, _} -> ?DEBUG("Skipping template ~p, due to presence of a plugin " "template with the same name", [Name]), prioritize_templates(Rest, Valid); {_, file, _} -> ?DEBUG("Skipping template ~p, due to presence of a custom " - "template at ~s", [Name, File]), + "template at ~ts", [Name, File]), prioritize_templates(Rest, Valid) end. @@ -369,6 +384,9 @@ prioritize_templates([{Name, Type, File} | Rest], Valid) -> load_file(Files, escript, Name) -> {Name, Bin} = lists:keyfind(Name, 1, Files), Bin; +load_file(_Files, builtin, Name) -> + {ok, Bin} = file:read_file(Name), + Bin; load_file(_Files, plugin, Name) -> {ok, Bin} = file:read_file(Name), Bin; @@ -412,10 +430,10 @@ write_file(Output, Data, Force) -> ok = filelib:ensure_dir(Output), case {Force, FileExists} of {true, true} -> - ?INFO("Writing ~s (forcibly overwriting)", + ?INFO("Writing ~ts (forcibly overwriting)", [Output]); _ -> - ?INFO("Writing ~s", [Output]) + ?INFO("Writing ~ts", [Output]) end, case file:write_file(Output, Data) of ok -> @@ -432,4 +450,4 @@ write_file(Output, Data, Force) -> %% Render a binary to a string, using mustache and the specified context %% render(Bin, Context) -> - bbmustache:render(ec_cnv:to_binary(Bin), Context, [{key_type, atom}]). + bbmustache:render(rebar_utils:to_binary(Bin), Context, [{key_type, atom}]). diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index aa9e268..11add61 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -37,8 +37,9 @@ escript_foldl/3, find_files/2, find_files/3, + find_files_in_dirs/3, + find_source/3, beam_to_mod/1, - beam_to_mod/2, erl_to_mod/1, beams/1, find_executable/1, @@ -55,6 +56,8 @@ get_arch/0, wordsize/0, deps_to_binary/1, + to_binary/1, + to_list/1, tup_dedup/1, tup_umerge/2, tup_sort/1, @@ -70,12 +73,17 @@ info_useless/2, list_dir/1, user_agent/0, - reread_config/1]). + reread_config/1, reread_config/2, + get_proxy_auth/0, + is_list_of_strings/1, + ssl_opts/1]). + %% for internal use only -export([otp_release/0]). -include("rebar.hrl"). +-include_lib("public_key/include/OTP-PUB-KEY.hrl"). -define(ONE_LEVEL_INDENT, " "). -define(APP_NAME_INDEX, 2). @@ -95,6 +103,12 @@ sort_deps(Deps) -> droplast(L) -> lists:reverse(tl(lists:reverse(L))). +%% @doc filtermap takes in a function that is either or both +%% a predicate and a map, and returns the matching and valid elements. +-spec filtermap(F, [In]) -> [Out] when + F :: fun((In) -> boolean() | {true, Out}), + In :: term(), + Out :: term(). filtermap(F, [Hd|Tail]) -> case F(Hd) of true -> @@ -107,18 +121,23 @@ filtermap(F, [Hd|Tail]) -> filtermap(F, []) when is_function(F, 1) -> []. is_arch(ArchRegex) -> - case re:run(get_arch(), ArchRegex, [{capture, none}]) of + case re:run(get_arch(), ArchRegex, [{capture, none}, unicode]) of match -> true; nomatch -> false end. +%% @doc returns the sytem architecture, in strings like +%% `"19.0.4-x86_64-unknown-linux-gnu-64"'. +-spec get_arch() -> string(). get_arch() -> Words = wordsize(), otp_release() ++ "-" ++ erlang:system_info(system_architecture) ++ "-" ++ Words. +%% @doc returns the size of a word on the system, as a string +-spec wordsize() -> string(). wordsize() -> try erlang:system_info({wordsize, external}) of Val -> @@ -129,7 +148,7 @@ wordsize() -> end. sh_send(Command0, String, Options0) -> - ?INFO("sh_send info:\n\tcwd: ~p\n\tcmd: ~s < ~s\n", + ?INFO("sh_send info:\n\tcwd: ~p\n\tcmd: ~ts < ~ts\n", [rebar_dir:get_cwd(), Command0, String]), ?DEBUG("\topts: ~p\n", [Options0]), @@ -137,7 +156,7 @@ sh_send(Command0, String, Options0) -> Options = [expand_sh_flag(V) || V <- proplists:compact(Options0 ++ DefaultOptions)], - Command = lists:flatten(patch_on_windows(Command0, proplists:get_value(env, Options, []))), + Command = lists:flatten(patch_on_windows(Command0, proplists:get_value(env, Options0, []))), PortSettings = proplists:get_all_values(port_settings, Options) ++ [exit_status, {line, 16384}, use_stdio, stderr_to_stdout, hide], Port = open_port({spawn, Command}, PortSettings), @@ -158,7 +177,7 @@ sh_send(Command0, String, Options0) -> %% Val = string() | false %% sh(Command0, Options0) -> - ?DEBUG("sh info:\n\tcwd: ~p\n\tcmd: ~s\n", [rebar_dir:get_cwd(), Command0]), + ?DEBUG("sh info:\n\tcwd: ~p\n\tcmd: ~ts\n", [rebar_dir:get_cwd(), Command0]), ?DEBUG("\topts: ~p\n", [Options0]), DefaultOptions = [{use_stdout, false}, debug_and_abort_on_error], @@ -168,10 +187,10 @@ sh(Command0, Options0) -> ErrorHandler = proplists:get_value(error_handler, Options), OutputHandler = proplists:get_value(output_handler, Options), - Command = lists:flatten(patch_on_windows(Command0, proplists:get_value(env, Options, []))), + Command = lists:flatten(patch_on_windows(Command0, proplists:get_value(env, Options0, []))), PortSettings = proplists:get_all_values(port_settings, Options) ++ [exit_status, {line, 16384}, use_stdio, stderr_to_stdout, hide, eof], - ?DEBUG("Port Cmd: ~s\nPort Opts: ~p\n", [Command, PortSettings]), + ?DEBUG("Port Cmd: ~ts\nPort Opts: ~p\n", [Command, PortSettings]), Port = open_port({spawn, Command}, PortSettings), try @@ -188,6 +207,12 @@ sh(Command0, Options0) -> find_files(Dir, Regex) -> find_files(Dir, Regex, true). +find_files_in_dirs([], _Regex, _Recursive) -> + []; +find_files_in_dirs([Dir | T], Regex, Recursive) -> + find_files(Dir, Regex, Recursive) ++ find_files_in_dirs(T, Regex, Recursive). + + find_files(Dir, Regex, Recursive) -> filelib:fold_files(Dir, Regex, Recursive, fun(F, Acc) -> [F | Acc] end, []). @@ -219,7 +244,7 @@ deprecated(Old, New, When) -> <<"WARNING: deprecated ~p option used~n" "Option '~p' has been deprecated~n" "in favor of '~p'.~n" - "'~p' will be removed ~s.~n">>, + "'~p' will be removed ~ts.~n">>, [Old, Old, New, Old, When]). %% for use by `do` task @@ -231,11 +256,17 @@ args_to_tasks(Args) -> new_task(Args, []). deps_to_binary([]) -> []; deps_to_binary([{Name, _, Source} | T]) -> - [{ec_cnv:to_binary(Name), Source} | deps_to_binary(T)]; + [{to_binary(Name), Source} | deps_to_binary(T)]; deps_to_binary([{Name, Source} | T]) -> - [{ec_cnv:to_binary(Name), Source} | deps_to_binary(T)]; + [{to_binary(Name), Source} | deps_to_binary(T)]; deps_to_binary([Name | T]) -> - [ec_cnv:to_binary(Name) | deps_to_binary(T)]. + [to_binary(Name) | deps_to_binary(T)]. + +to_binary(A) when is_atom(A) -> atom_to_binary(A, unicode); +to_binary(Str) -> unicode:characters_to_binary(Str). + +to_list(A) when is_atom(A) -> atom_to_list(A); +to_list(Str) -> unicode:characters_to_list(Str). tup_dedup(List) -> tup_dedup_(tup_sort(List)). @@ -370,7 +401,7 @@ compare({Priority, A}, {Secondary, B}) when not is_tuple(A), is_tuple(B) -> %% Implements wc -l functionality used to determine patchcount from git output line_count(PatchLines) -> - Tokenized = string:tokens(PatchLines, "\n"), + Tokenized = rebar_string:lexemes(PatchLines, "\n"), {ok, length(Tokenized)}. check_min_otp_version(undefined) -> @@ -383,10 +414,10 @@ check_min_otp_version(MinOtpVersion) -> case ParsedVsn >= ParsedMin of true -> - ?DEBUG("~s satisfies the requirement for minimum OTP version ~s", + ?DEBUG("~ts satisfies the requirement for minimum OTP version ~ts", [OtpRelease, MinOtpVersion]); false -> - ?ABORT("OTP release ~s or later is required. Version in use: ~s", + ?ABORT("OTP release ~ts or later is required. Version in use: ~ts", [MinOtpVersion, OtpRelease]) end. @@ -402,28 +433,103 @@ check_blacklisted_otp_versions(BlacklistedRegexes) -> abort_if_blacklisted(BlacklistedRegex, OtpRelease) -> case re:run(OtpRelease, BlacklistedRegex, [{capture, none}]) of match -> - ?ABORT("OTP release ~s matches blacklisted version ~s", + ?ABORT("OTP release ~ts matches blacklisted version ~ts", [OtpRelease, BlacklistedRegex]); nomatch -> - ?DEBUG("~s does not match blacklisted OTP version ~s", + ?DEBUG("~ts does not match blacklisted OTP version ~ts", [OtpRelease, BlacklistedRegex]) end. user_agent() -> {ok, Vsn} = application:get_key(rebar, vsn), - ?FMT("Rebar/~s (OTP/~s)", [Vsn, otp_release()]). + ?FMT("Rebar/~ts (OTP/~ts)", [Vsn, otp_release()]). reread_config(ConfigList) -> + %% Default to not re-configuring the logger for now; + %% this can leak logs in CT redirection when setting up hooks + %% for example. If we want to turn it on by default, we may + %% want to disable it in CT at the same time or figure out a + %% way to silence it. + %% The same pattern may apply to other tasks, so let's enable + %% case-by-case. + reread_config(ConfigList, []). + +reread_config(ConfigList, Opts) -> + UpdateLoggerConfig = erlang:function_exported(logger, module_info, 0) andalso + proplists:get_value(update_logger, Opts, false), + %% NB: we attempt to mimic -config here, which survives app reload, + %% hence {persistent, true}. + SetEnv = case version_tuple(?MODULE:otp_release()) of + {X, _, _} when X =< 17 -> + fun application:set_env/3; + _ -> + fun (App, Key, Val) -> application:set_env(App, Key, Val, [{persistent, true}]) end + end, try - [application:set_env(Application, Key, Val) + Res = + [SetEnv(Application, Key, Val) || Config <- ConfigList, {Application, Items} <- Config, - {Key, Val} <- Items] + {Key, Val} <- Items], + case UpdateLoggerConfig of + true -> reread_logger_config(); + false -> ok + end, + Res catch _:_ -> ?ERROR("The configuration file submitted could not be read " "and will be ignored.", []) end. +%% @private since the kernel app is already booted, re-reading its config +%% requires doing some magic to dynamically patch running handlers to +%% deal with the current value. +reread_logger_config() -> + KernelCfg = application:get_all_env(kernel), + LogCfg = proplists:get_value(logger, KernelCfg), + case LogCfg of + undefined -> + ok; + _ -> + %% Extract and apply settings related to primary configuration + %% -- primary config is used for settings shared across handlers + LogLvlPrimary = proplists:get_value(logger_info, KernelCfg, all), + {FilterDefault, Filters} = + case lists:keyfind(filters, 1, KernelCfg) of + false -> {log, []}; + {filters, FoundDef, FoundFilter} -> {FoundDef, FoundFilter} + end, + Primary = #{level => LogLvlPrimary, + filter_default => FilterDefault, + filters => Filters}, + %% Load the correct handlers based on their individual config. + [case Id of + default -> logger:update_handler_config(Id, Cfg); + _ -> logger:add_handler(Id, Mod, Cfg) + end || {handler, Id, Mod, Cfg} <- LogCfg], + logger:set_primary_config(Primary), + ok + end. + + +%% @doc Given env. variable `FOO' we want to expand all references to +%% it in `InStr'. References can have two forms: `$FOO' and `${FOO}' +%% The end of form `$FOO' is delimited with whitespace or EOL +-spec expand_env_variable(string(), string(), term()) -> string(). +expand_env_variable(InStr, VarName, RawVarValue) -> + case rebar_string:chr(InStr, $$) of + 0 -> + %% No variables to expand + InStr; + _ -> + ReOpts = [global, unicode, {return, list}], + VarValue = re:replace(RawVarValue, "\\\\", "\\\\\\\\", ReOpts), + %% Use a regex to match/replace: + %% Given variable "FOO": match $FOO\s | $FOOeol | ${FOO} + RegEx = io_lib:format("\\\$(~ts(\\W|$)|{~ts})", [VarName, VarName]), + re:replace(InStr, RegEx, [VarValue, "\\2"], ReOpts) + end. + %% ==================================================================== %% Internal functions %% ==================================================================== @@ -436,7 +542,7 @@ version_tuple(OtpRelease) -> {match, [_Full, Maj]} -> {list_to_integer(Maj), 0, 0}; nomatch -> - ?ABORT("Minimum OTP release unable to be parsed: ~s", [OtpRelease]) + ?ABORT("Minimum OTP release unable to be parsed: ~ts", [OtpRelease]) end. otp_release() -> @@ -459,11 +565,10 @@ otp_release1(Rel) -> %% It's fine to rely on the binary module here because we can %% be sure that it's available when the otp_release string does %% not begin with $R. - Size = byte_size(Vsn), %% The shortest vsn string consists of at least two digits %% followed by "\n". Therefore, it's safe to assume Size >= 3. - case binary:part(Vsn, {Size, -3}) of - <<"**\n">> -> + case binary:match(Vsn, <<"**">>) of + {Pos, _} -> %% The OTP documentation mentions that a system patched %% using the otp_patch_apply tool available to licensed %% customers will leave a '**' suffix in the version as a @@ -472,9 +577,9 @@ otp_release1(Rel) -> %% drop the suffix, given for all intents and purposes, we %% cannot obtain relevant information from it as far as %% tooling is concerned. - binary:bin_to_list(Vsn, {0, Size - 3}); - _ -> - binary:bin_to_list(Vsn, {0, Size - 1}) + binary:bin_to_list(Vsn, {0, Pos}); + nomatch -> + rebar_string:trim(binary:bin_to_list(Vsn), trailing, "\n") end end. @@ -489,30 +594,11 @@ patch_on_windows(Cmd, Env) -> end, Cmd, Env), %% Remove left-over vars re:replace(Cmd1, "\\\$\\w+|\\\${\\w+}", "", - [global, {return, list}]); + [global, {return, list}, unicode]); _ -> Cmd end. -%% -%% Given env. variable FOO we want to expand all references to -%% it in InStr. References can have two forms: $FOO and ${FOO} -%% The end of form $FOO is delimited with whitespace or eol -%% -expand_env_variable(InStr, VarName, RawVarValue) -> - case string:chr(InStr, $$) of - 0 -> - %% No variables to expand - InStr; - _ -> - ReOpts = [global, unicode, {return, list}], - VarValue = re:replace(RawVarValue, "\\\\", "\\\\\\\\", ReOpts), - %% Use a regex to match/replace: - %% Given variable "FOO": match $FOO\s | $FOOeol | ${FOO} - RegEx = io_lib:format("\\\$(~s(\\W|$)|{~s})", [VarName, VarName]), - re:replace(InStr, RegEx, [VarValue, "\\2"], ReOpts) - end. - expand_sh_flag(return_on_error) -> {error_handler, fun(_Command, Err) -> @@ -534,7 +620,7 @@ expand_sh_flag(use_stdout) -> {output_handler, fun(Line, Acc) -> %% Line already has a newline so don't use ?CONSOLE which adds one - io:format("~s", [Line]), + io:format("~ts", [Line]), [Line | Acc] end}; expand_sh_flag({use_stdout, false}) -> @@ -557,23 +643,23 @@ log_msg_and_abort(Message) -> -spec debug_log_msg_and_abort(string()) -> err_handler(). debug_log_msg_and_abort(Message) -> fun(Command, {Rc, Output}) -> - ?DEBUG("sh(~s)~n" + ?DEBUG("sh(~ts)~n" "failed with return code ~w and the following output:~n" - "~s", [Command, Rc, Output]), + "~ts", [Command, Rc, Output]), ?ABORT(Message, []) end. -spec log_and_abort(string(), {integer(), string()}) -> no_return(). log_and_abort(Command, {Rc, Output}) -> - ?ABORT("sh(~s)~n" + ?ABORT("sh(~ts)~n" "failed with return code ~w and the following output:~n" - "~s", [Command, Rc, Output]). + "~ts", [Command, Rc, Output]). -spec debug_and_abort(string(), {integer(), string()}) -> no_return(). debug_and_abort(Command, {Rc, Output}) -> - ?DEBUG("sh(~s)~n" + ?DEBUG("sh(~ts)~n" "failed with return code ~w and the following output:~n" - "~s", [Command, Rc, Output]), + "~ts", [Command, Rc, Output]), throw(rebar_abort). sh_loop(Port, Fun, Acc) -> @@ -592,10 +678,6 @@ sh_loop(Port, Fun, Acc) -> end end. -beam_to_mod(Dir, Filename) -> - [Dir | Rest] = filename:split(Filename), - list_to_atom(filename:basename(string:join(Rest, "."), ".beam")). - beam_to_mod(Filename) -> list_to_atom(filename:basename(Filename, ".beam")). @@ -633,36 +715,40 @@ escript_foldl(Fun, Acc, File) -> Error end. -vcs_vsn(Vcs, Dir, Resources) -> - case vcs_vsn_cmd(Vcs, Dir, Resources) of +%% TODO: this is just for rebar3_hex and maybe other plugins +%% but eventually it should be dropped +vcs_vsn(OriginalVsn, Dir, Resources) when is_list(Dir) , + is_list(Resources) -> + ?WARN("Using deprecated rebar_utils:vcs_vsn/3. Please upgrade your plugins.", []), + FakeState = rebar_state:new(), + {ok, AppInfo} = rebar_app_info:new(fake, OriginalVsn, Dir), + vcs_vsn(AppInfo, OriginalVsn, + rebar_state:set_resources(FakeState, Resources)); +vcs_vsn(AppInfo, Vcs, State) -> + case vcs_vsn_cmd(AppInfo, Vcs, State) of {plain, VsnString} -> VsnString; {cmd, CmdString} -> - vcs_vsn_invoke(CmdString, Dir); + vcs_vsn_invoke(CmdString, rebar_app_info:dir(AppInfo)); unknown -> ?ABORT("vcs_vsn: Unknown vsn format: ~p", [Vcs]); {error, Reason} -> - ?ABORT("vcs_vsn: ~s", [Reason]) + ?ABORT("vcs_vsn: ~ts", [Reason]) end. %% Temp work around for repos like relx that use "semver" -vcs_vsn_cmd(Vsn, _, _) when is_binary(Vsn) -> +vcs_vsn_cmd(_, Vsn, _) when is_binary(Vsn) -> {plain, Vsn}; -vcs_vsn_cmd(VCS, Dir, Resources) when VCS =:= semver ; VCS =:= "semver" -> - vcs_vsn_cmd(git, Dir, Resources); -vcs_vsn_cmd({cmd, _Cmd}=Custom, _, _) -> +vcs_vsn_cmd(AppInfo, VCS, State) when VCS =:= semver ; VCS =:= "semver" -> + vcs_vsn_cmd(AppInfo, git, State); +vcs_vsn_cmd(_AppInfo, {cmd, _Cmd}=Custom, _) -> Custom; -vcs_vsn_cmd(VCS, Dir, Resources) when is_atom(VCS) -> - case find_resource_module(VCS, Resources) of - {ok, Module} -> - Module:make_vsn(Dir); - {error, _} -> - unknown - end; -vcs_vsn_cmd(VCS, Dir, Resources) when is_list(VCS) -> +vcs_vsn_cmd(AppInfo, VCS, State) when is_atom(VCS) -> + rebar_resource_v2:make_vsn(AppInfo, VCS, State); +vcs_vsn_cmd(AppInfo, VCS, State) when is_list(VCS) -> try list_to_existing_atom(VCS) of AVCS -> - case vcs_vsn_cmd(AVCS, Dir, Resources) of + case vcs_vsn_cmd(AppInfo, AVCS, State) of unknown -> {plain, VCS}; Other -> Other end @@ -675,20 +761,7 @@ vcs_vsn_cmd(_, _, _) -> vcs_vsn_invoke(Cmd, Dir) -> {ok, VsnString} = rebar_utils:sh(Cmd, [{cd, Dir}, {use_stdout, false}]), - string:strip(VsnString, right, $\n). - -find_resource_module(Type, Resources) -> - case lists:keyfind(Type, 1, Resources) of - false -> - case code:which(Type) of - non_existing -> - {error, unknown}; - _ -> - {ok, Type} - end; - {Type, Module} -> - {ok, Module} - end. + rebar_string:trim(VsnString, trailing, "\n"). %% @doc ident to the level specified -spec indent(non_neg_integer()) -> iolist(). @@ -735,11 +808,20 @@ remove_from_code_path(Paths) -> ok; {ok, Modules} -> application:unload(App), - [begin code:purge(M), code:delete(M) end || M <- Modules] + [case erlang:check_process_code(self(), M) of + false -> + code:purge(M), code:delete(M); + _ -> + ?DEBUG("~p can't purge ~p safely, doing a soft purge", [self(), M]), + code:soft_purge(M) andalso code:delete(M) + end || M <- Modules] end, code:del_path(Path) - end, Paths). + end, lists:usort(Paths)). +%% @doc Revert to only having the beams necessary for running rebar3 and +%% plugins in the path +-spec cleanup_code_path([string()]) -> true | {error, term()}. cleanup_code_path(OrigPath) -> CurrentPath = code:get_path(), AddedPaths = CurrentPath -- OrigPath, @@ -756,7 +838,7 @@ cleanup_code_path(OrigPath) -> new_task([], Acc) -> lists:reverse(Acc); new_task([TaskList|Rest], Acc) -> - case re:split(TaskList, ",", [{return, list}, {parts, 2}]) of + case re:split(TaskList, ",", [{return, list}, {parts, 2}, unicode]) of %% `do` consumes all remaining args ["do" = Task] -> lists:reverse([{Task, Rest}|Acc]); @@ -783,7 +865,7 @@ arg_or_flag(["-" ++ _ = Flag|Rest], [{Task, Args}|Acc]) -> end; %% an argument or a sequence of arguments arg_or_flag([ArgList|Rest], [{Task, Args}|Acc]) -> - case re:split(ArgList, ",", [{return, list}, {parts, 2}]) of + case re:split(ArgList, ",", [{return, list}, {parts, 2}, unicode]) of %% single arg terminated by a comma [Arg, ""] -> new_task(Rest, [{Task, lists:reverse([Arg|Args])}|Acc]); @@ -817,8 +899,17 @@ set_httpc_options(_, []) -> ok; set_httpc_options(Scheme, Proxy) -> - {ok, {_, _, Host, Port, _, _}} = http_uri:parse(Proxy), - httpc:set_options([{Scheme, {{Host, Port}, []}}], rebar). + URI = normalise_proxy(Scheme, Proxy), + {ok, {_, UserInfo, Host, Port, _, _}} = http_uri:parse(URI), + httpc:set_options([{Scheme, {{Host, Port}, []}}], rebar), + set_proxy_auth(UserInfo). + +normalise_proxy(Scheme, URI) -> + case re:run(URI, "://", [unicode]) of + nomatch when Scheme =:= https_proxy -> "https://" ++ URI; + nomatch when Scheme =:= proxy -> "http://" ++ URI; + _ -> URI + end. url_append_path(Url, ExtraPath) -> case http_uri:parse(Url) of @@ -833,15 +924,18 @@ url_append_path(Url, ExtraPath) -> escape_chars(Str) when is_atom(Str) -> escape_chars(atom_to_list(Str)); escape_chars(Str) -> - re:replace(Str, "([ ()?`!$&;])", "\\\\&", [global, {return, list}]). + re:replace(Str, "([ ()?`!$&;\"\'])", "\\\\&", + [global, {return, list}, unicode]). %% "escape inside these" escape_double_quotes(Str) -> - re:replace(Str, "([\"\\\\`!$&*;])", "\\\\&", [global, {return, list}]). + re:replace(Str, "([\"\\\\`!$&*;])", "\\\\&", + [global, {return, list}, unicode]). %% "escape inside these" but allow * escape_double_quotes_weak(Str) -> - re:replace(Str, "([\"\\\\`!$&;])", "\\\\&", [global, {return, list}]). + re:replace(Str, "([\"\\\\`!$&;])", "\\\\&", + [global, {return, list}, unicode]). info_useless(Old, New) -> [?INFO("App ~ts is no longer needed and can be deleted.", [Name]) @@ -857,3 +951,212 @@ list_dir(Dir) -> true -> file:list_dir_all(Dir); false -> file:list_dir(Dir) end. + +set_proxy_auth([]) -> + ok; +set_proxy_auth(UserInfo) -> + [Username, Password] = re:split(UserInfo, ":", + [{return, list}, {parts,2}, unicode]), + %% password may contain url encoded characters, need to decode them first + application:set_env(rebar, proxy_auth, [{proxy_auth, {Username, http_uri:decode(Password)}}]). + +get_proxy_auth() -> + case application:get_env(rebar, proxy_auth) of + undefined -> []; + {ok, ProxyAuth} -> ProxyAuth + end. + +-spec rebar_utils:is_list_of_strings(term()) -> boolean(). +is_list_of_strings(List) when not is_list(hd(List)) -> + false; +is_list_of_strings(List) when is_list(hd(List)) -> + true; +is_list_of_strings(List) when is_list(List) -> + true. + +%%------------------------------------------------------------------------------ +%% @doc +%% Return the SSL options adequate for the project based on +%% its configuration, including for validation of certs. +%% @end +%%------------------------------------------------------------------------------ +-spec ssl_opts(Url) -> Res when + Url :: string(), + Res :: proplists:proplist(). +ssl_opts(Url) -> + case get_ssl_config() of + ssl_verify_enabled -> + ssl_opts(ssl_verify_enabled, Url); + ssl_verify_disabled -> + [{verify, verify_none}] + end. + +%%------------------------------------------------------------------------------ +%% @doc +%% Return the SSL options adequate for the project based on +%% its configuration, including for validation of certs. +%% @end +%%------------------------------------------------------------------------------ +-spec ssl_opts(Enabled, Url) -> Res when + Enabled :: atom(), + Url :: string(), + Res :: proplists:proplist(). +ssl_opts(ssl_verify_enabled, Url) -> + case check_ssl_version() of + true -> + {ok, {_, _, Hostname, _, _, _}} = + http_uri:parse(rebar_utils:to_list(Url)), + VerifyFun = {fun ssl_verify_hostname:verify_fun/3, + [{check_hostname, Hostname}]}, + CACerts = certifi:cacerts(), + [{verify, verify_peer}, {depth, 2}, {cacerts, CACerts}, + {partial_chain, fun partial_chain/1}, {verify_fun, VerifyFun}]; + false -> + ?WARN("Insecure HTTPS request (peer verification disabled), " + "please update to OTP 17.4 or later", []), + [{verify, verify_none}] + end. + +-spec partial_chain(Certs) -> Res when + Certs :: list(any()), + Res :: unknown_ca | {trusted_ca, any()}. +partial_chain(Certs) -> + Certs1 = [{Cert, public_key:pkix_decode_cert(Cert, otp)} || Cert <- Certs], + CACerts = certifi:cacerts(), + CACerts1 = [public_key:pkix_decode_cert(Cert, otp) || Cert <- CACerts], + case ec_lists:find(fun({_, Cert}) -> + check_cert(CACerts1, Cert) + end, Certs1) of + {ok, Trusted} -> + {trusted_ca, element(1, Trusted)}; + _ -> + unknown_ca + end. + +-spec extract_public_key_info(Cert) -> Res when + Cert :: #'OTPCertificate'{tbsCertificate::#'OTPTBSCertificate'{}}, + Res :: any(). +extract_public_key_info(Cert) -> + ((Cert#'OTPCertificate'.tbsCertificate)#'OTPTBSCertificate'.subjectPublicKeyInfo). + +-spec check_cert(CACerts, Cert) -> Res when + CACerts :: list(any()), + Cert :: any(), + Res :: boolean(). +check_cert(CACerts, Cert) -> + lists:any(fun(CACert) -> + extract_public_key_info(CACert) == extract_public_key_info(Cert) + end, CACerts). + +-spec check_ssl_version() -> + boolean(). +check_ssl_version() -> + case application:get_key(ssl, vsn) of + {ok, Vsn} -> + parse_vsn(Vsn) >= {5, 3, 6}; + _ -> + false + end. + +-spec get_ssl_config() -> + ssl_verify_disabled | ssl_verify_enabled. +get_ssl_config() -> + GlobalConfigFile = rebar_dir:global_config(), + Config = rebar_config:consult_file(GlobalConfigFile), + case proplists:get_value(ssl_verify, Config, []) of + false -> + ssl_verify_disabled; + _ -> + ssl_verify_enabled + end. + +-spec parse_vsn(Vsn) -> Res when + Vsn :: string(), + Res :: {integer(), integer(), integer()}. +parse_vsn(Vsn) -> + version_pad(rebar_string:lexemes(Vsn, ".-")). + +-spec version_pad(list(nonempty_string())) -> Res when + Res :: {integer(), integer(), integer()}. +version_pad([Major]) -> + {list_to_integer(Major), 0, 0}; +version_pad([Major, Minor]) -> + {list_to_integer(Major), list_to_integer(Minor), 0}; +version_pad([Major, Minor, Patch]) -> + {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}; +version_pad([Major, Minor, Patch | _]) -> + {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}. + + +-ifdef(filelib_find_source). +find_source(Filename, Dir, Rules) -> + filelib:find_source(Filename, Dir, Rules). +-else. +%% Looks for a file relative to a given directory + +-type find_file_rule() :: {ObjDirSuffix::string(), SrcDirSuffix::string()}. + +%% Looks for a source file relative to the object file name and directory + +-type find_source_rule() :: {ObjExtension::string(), SrcExtension::string(), + [find_file_rule()]}. + +keep_suffix_search_rules(Rules) -> + [T || {_,_,_}=T <- Rules]. + +-spec find_source(file:filename(), file:filename(), [find_source_rule()]) -> + {ok, file:filename()} | {error, not_found}. +find_source(Filename, Dir, Rules) -> + try_suffix_rules(keep_suffix_search_rules(Rules), Filename, Dir). + +try_suffix_rules(Rules, Filename, Dir) -> + Ext = filename:extension(Filename), + try_suffix_rules(Rules, filename:rootname(Filename, Ext), Dir, Ext). + +try_suffix_rules([{Ext,Src,Rules}|Rest], Root, Dir, Ext) + when is_list(Src), is_list(Rules) -> + case try_dir_rules(add_local_search(Rules), Root ++ Src, Dir) of + {ok, File} -> {ok, File}; + _Other -> + try_suffix_rules(Rest, Root, Dir, Ext) + end; +try_suffix_rules([_|Rest], Root, Dir, Ext) -> + try_suffix_rules(Rest, Root, Dir, Ext); +try_suffix_rules([], _Root, _Dir, _Ext) -> + {error, not_found}. + +%% ensuring we check the directory of the object file before any other directory +add_local_search(Rules) -> + Local = {"",""}, + [Local] ++ lists:filter(fun (X) -> X =/= Local end, Rules). + +try_dir_rules([{From, To}|Rest], Filename, Dir) + when is_list(From), is_list(To) -> + case try_dir_rule(Dir, Filename, From, To) of + {ok, File} -> {ok, File}; + error -> try_dir_rules(Rest, Filename, Dir) + end; +try_dir_rules([], _Filename, _Dir) -> + {error, not_found}. + +try_dir_rule(Dir, Filename, From, To) -> + case lists:suffix(From, Dir) of + true -> + NewDir = lists:sublist(Dir, 1, length(Dir)-length(From))++To, + Src = filename:join(NewDir, Filename), + case filelib:is_regular(Src) of + true -> {ok, Src}; + false -> find_regular_file(filelib:wildcard(Src)) + end; + false -> + error + end. + +find_regular_file([]) -> + error; +find_regular_file([File|Files]) -> + case filelib:is_regular(File) of + true -> {ok, File}; + false -> find_regular_file(Files) + end. +-endif. diff --git a/systest/all_SUITE.erl b/systest/all_SUITE.erl new file mode 100644 index 0000000..083fae9 --- /dev/null +++ b/systest/all_SUITE.erl @@ -0,0 +1,117 @@ +-module(all_SUITE). +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-compile([export_all, nowarn_export_all]). + +init_per_suite(Config) -> + %% TODO: figure out how to use a local rebar3 copy? + %% Ensure global rebar3 has the same version as current one! + {ok, Vsn} = application:get_key(rebar, vsn), + {ok, ExecVsn} = rebar3("version", [{path, "."} | Config]), + case rebar_string:lexemes(ExecVsn, " ") of + ["rebar", Vsn | _] -> + %% Copy all base cases to priv_dir + rebar_file_utils:cp_r([?config(data_dir, Config)], + ?config(priv_dir, Config)), + Config; + _ -> + {skip, "expected current version "++Vsn++" in path " + "and found '"++ ExecVsn ++ "'"} + end. + +end_per_suite(Config) -> + Config. + +init_per_testcase(Name, Config) -> + set_name_config(Name, Config). + +end_per_testcase(_Name, Config) -> + Config. + +all() -> + [noop, resource_plugins, alias_clash, grisp_explode, compile_deps]. + +%groups() -> +% [{plugins, [shuffle], []}, +% {deps, [shuffle], []}]. + +%%%%%%%%%%%%%%%%%% +%%% TEST CASES %%% +%%%%%%%%%%%%%%%%%% + +noop() -> + [{doc, "just a sanity check on the handling of the test suite init/end"}]. +noop(_Config) -> + true. + +resource_plugins() -> + [{doc, "Issue #1673: " + "Ensure that deps using resource plugins with semver compile."}]. +resource_plugins(Config) -> + %% When the environment handling is wrong, the run fails violently. + {ok, Output} = rebar3("compile", Config), + ct:pal("Rebar3 Output:~n~s",[Output]), + ok. + +alias_clash() -> + [{doc, "checking that the provider won't get plugin interference."}, + {timetrap, 10000}]. +alias_clash(Config) -> + {ok, Help} = rebar3("help", Config), % should be redefined, but by the plugin + ?assertNotEqual(nomatch, + re:run(Help, "Alias help is already the name of a command[a-z ]+and will be ignored") + ), + {ok, Output} = rebar3("test", Config, [{env, [{"DEBUG", "1"}]}]), + ?assertNotEqual(nomatch, re:run(Output, "cover summary written to:")), + ?assertNotEqual(nomatch, + re:run(Output, "Not adding provider default test from module rebar_prv_alias_test " + "because it already exists from module rebar_prv_alias_test")), + ok. + +grisp_explode() -> + [{doc, "Don't force purge a plugin that runs the compile job itself"}]. +grisp_explode(Config) -> + %% When the purge handling is wrong, the run fails violently. + {error, {_,Output}} = rebar3("grisp deploy -n robot -v 0.1.0", Config), + ct:pal("Rebar3 Output:~n~s",[Output]), + ?assertNotEqual(nomatch, + re:run(Output, "No releases exist in the system for robot:0.1.0!") + ), + ok. + +compile_deps() -> + [{doc, "When compiling a project multiple times, the deps should always be built event if refetched"}]. +compile_deps(Config) -> + rebar3("compile", Config), + rebar3("compile", Config), + + PrivDir = ?config(path, Config), + EbinDir = filename:join([PrivDir, "_build", "default", "lib", "fake_dep", "ebin"]), + + {ok, Beams} = file:list_dir(EbinDir), + ?assert(length(Beams) > 1). + + +%%%%%%%%%%%%%%% +%%% Helpers %%% +%%%%%%%%%%%%%%% +set_name_config(Atom, Config) -> + [{path, + filename:join([?config(priv_dir, Config), + atom_to_list(?MODULE)++"_data", atom_to_list(Atom)])} + | Config]. + +rebar3(Args, Config) -> rebar3(Args, Config, []). + +rebar3(Args, Config, UserOpts) -> + Exec = case os:type() of + {win32, _} -> + "rebar3.cmd"; + _ -> + "rebar3" + end, + Cmd = Exec ++ " " ++ Args, + Opts = [{cd, ?config(path, Config)}, return_on_error, use_stdout + | UserOpts], + ct:pal("Calling rebar3 ~s with options ~p", [Cmd, Opts]), + rebar_utils:sh(Cmd, Opts). diff --git a/systest/all_SUITE_data/alias_clash/rebar.config b/systest/all_SUITE_data/alias_clash/rebar.config new file mode 100644 index 0000000..baf20a9 --- /dev/null +++ b/systest/all_SUITE_data/alias_clash/rebar.config @@ -0,0 +1,4 @@ +{alias, [{help, [version]}, % should be skipped, but be overriden by plugin + {test, [compile, {eunit, "-c"}, cover]}]}. + +{plugins, [rebar_alias]}. % should be overridden diff --git a/systest/all_SUITE_data/alias_clash/src/alias_clash.app.src b/systest/all_SUITE_data/alias_clash/src/alias_clash.app.src new file mode 100644 index 0000000..b4cdda2 --- /dev/null +++ b/systest/all_SUITE_data/alias_clash/src/alias_clash.app.src @@ -0,0 +1,15 @@ +{application, alias_clash, + [{description, "An OTP library"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, + [kernel, + stdlib + ]}, + {env,[]}, + {modules, []}, + + {maintainers, []}, + {licenses, ["Apache 2.0"]}, + {links, []} + ]}. diff --git a/systest/all_SUITE_data/alias_clash/src/alias_clash.erl b/systest/all_SUITE_data/alias_clash/src/alias_clash.erl new file mode 100644 index 0000000..9249cdb --- /dev/null +++ b/systest/all_SUITE_data/alias_clash/src/alias_clash.erl @@ -0,0 +1,13 @@ +-module(alias_clash). + +%% API exports +-export([]). + +%%==================================================================== +%% API functions +%%==================================================================== + + +%%==================================================================== +%% Internal functions +%%==================================================================== diff --git a/systest/all_SUITE_data/compile_deps/rebar.config b/systest/all_SUITE_data/compile_deps/rebar.config new file mode 100644 index 0000000..08e2f62 --- /dev/null +++ b/systest/all_SUITE_data/compile_deps/rebar.config @@ -0,0 +1,8 @@ +{deps, [ + {fake_dep, {localdep, "fake_dep"}} + ]}. + +{plugins, [{rebar_localdep, + {git, "https://github.com/alinpopa/rebar3-localdep-plugin.git", + {branch, "master"}}}]}. + diff --git a/systest/all_SUITE_data/compile_deps/rebar.config.script b/systest/all_SUITE_data/compile_deps/rebar.config.script new file mode 100644 index 0000000..c0ebab1 --- /dev/null +++ b/systest/all_SUITE_data/compile_deps/rebar.config.script @@ -0,0 +1,2 @@ +os:putenv("LOCALDEP_DIR", "./vendored/"). +CONFIG. diff --git a/systest/all_SUITE_data/compile_deps/vendored/fake_dep/rebar.config b/systest/all_SUITE_data/compile_deps/vendored/fake_dep/rebar.config new file mode 100644 index 0000000..f618f3e --- /dev/null +++ b/systest/all_SUITE_data/compile_deps/vendored/fake_dep/rebar.config @@ -0,0 +1,2 @@ +{erl_opts, [debug_info]}. +{deps, []}.
\ No newline at end of file diff --git a/systest/all_SUITE_data/compile_deps/vendored/fake_dep/src/fake_dep.app.src b/systest/all_SUITE_data/compile_deps/vendored/fake_dep/src/fake_dep.app.src new file mode 100644 index 0000000..8547c35 --- /dev/null +++ b/systest/all_SUITE_data/compile_deps/vendored/fake_dep/src/fake_dep.app.src @@ -0,0 +1,15 @@ +{application, fake_dep, + [{description, "An OTP library"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, + [kernel, + stdlib + ]}, + {env,[]}, + {modules, []}, + + {maintainers, []}, + {licenses, ["Apache 2.0"]}, + {links, []} + ]}. diff --git a/systest/all_SUITE_data/compile_deps/vendored/fake_dep/src/fake_dep.erl b/systest/all_SUITE_data/compile_deps/vendored/fake_dep/src/fake_dep.erl new file mode 100644 index 0000000..db8d9f0 --- /dev/null +++ b/systest/all_SUITE_data/compile_deps/vendored/fake_dep/src/fake_dep.erl @@ -0,0 +1,13 @@ +-module(fake_dep). + +%% API exports +-export([]). + +%%==================================================================== +%% API functions +%%==================================================================== + + +%%==================================================================== +%% Internal functions +%%==================================================================== diff --git a/systest/all_SUITE_data/grisp_explode/rebar.config b/systest/all_SUITE_data/grisp_explode/rebar.config new file mode 100644 index 0000000..43c9f63 --- /dev/null +++ b/systest/all_SUITE_data/grisp_explode/rebar.config @@ -0,0 +1,16 @@ +{deps, [grisp]}. + +{plugins, [{rebar3_grisp, "0.1.0"}]}. + +{erl_opts, [debug_info]}. + +{grisp, [ + {otp_release, "19"}, + {deploy, [ + {destination, "/path/to/SD-card"} + ]} +]}. + +{relx, [ + {release, {mygrispproject, "0.1.0"}, [mygrispproject]} +]}. diff --git a/systest/all_SUITE_data/grisp_explode/rebar.lock b/systest/all_SUITE_data/grisp_explode/rebar.lock new file mode 100644 index 0000000..2523b13 --- /dev/null +++ b/systest/all_SUITE_data/grisp_explode/rebar.lock @@ -0,0 +1,8 @@ +{"1.1.0", +[{<<"grisp">>,{pkg,<<"grisp">>,<<"0.1.1">>},0}, + {<<"mapz">>,{pkg,<<"mapz">>,<<"0.3.0">>},1}]}. +[ +{pkg_hash,[ + {<<"grisp">>, <<"5A1318E7B1582D7C5B1E446D149A6F93428A380BCFE7D740E57E4F6B6CDB19DD">>}, + {<<"mapz">>, <<"438D24746CE5A252101E00B2032EFDF7FC69EB32689D3B805DE5E6DD7F52614F">>}]} +]. diff --git a/systest/all_SUITE_data/grisp_explode/src/mygrispproject.app.src b/systest/all_SUITE_data/grisp_explode/src/mygrispproject.app.src new file mode 100644 index 0000000..0f0a396 --- /dev/null +++ b/systest/all_SUITE_data/grisp_explode/src/mygrispproject.app.src @@ -0,0 +1,17 @@ +{application, mygrispproject, [ + {description, "A GRiSP application"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, {mygrispproject, []}}, + {applications, [ + kernel, + stdlib, + grisp + ]}, + {env,[]}, + {modules, []}, + + {maintainers, []}, + {licenses, ["Apache 2.0"]}, + {links, []} +]}. diff --git a/systest/all_SUITE_data/grisp_explode/src/mygrispproject.erl b/systest/all_SUITE_data/grisp_explode/src/mygrispproject.erl new file mode 100644 index 0000000..a9152fe --- /dev/null +++ b/systest/all_SUITE_data/grisp_explode/src/mygrispproject.erl @@ -0,0 +1,15 @@ +% @doc mygrispproject public API. +% @end +-module(mygrispproject). + +-behavior(application). + +% Callbacks +-export([start/2]). +-export([stop/1]). + +%--- Callbacks ----------------------------------------------------------------- + +start(_Type, _Args) -> mygrispproject_sup:start_link(). + +stop(_State) -> ok. diff --git a/systest/all_SUITE_data/grisp_explode/src/mygrispproject_sup.erl b/systest/all_SUITE_data/grisp_explode/src/mygrispproject_sup.erl new file mode 100644 index 0000000..aef0d4f --- /dev/null +++ b/systest/all_SUITE_data/grisp_explode/src/mygrispproject_sup.erl @@ -0,0 +1,19 @@ +% @doc mygrispproject top level supervisor. +% @end +-module(mygrispproject_sup). + +-behavior(supervisor). + +% API +-export([start_link/0]). + +% Callbacks +-export([init/1]). + +%--- API ----------------------------------------------------------------------- + +start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +%--- Callbacks ----------------------------------------------------------------- + +init([]) -> {ok, { {one_for_all, 0, 1}, []} }. diff --git a/systest/all_SUITE_data/resource_plugins/rebar.config b/systest/all_SUITE_data/resource_plugins/rebar.config new file mode 100644 index 0000000..7df4a4a --- /dev/null +++ b/systest/all_SUITE_data/resource_plugins/rebar.config @@ -0,0 +1,23 @@ +%% Sample provided by @tothlac +{plugins, [ + {rebar_tidy_deps, ".*", {git, "https://github.com/tothlac/rebar3-tidy-deps-plugin.git"}}, + {rebar_alias, {git, "https://github.com/tsloughter/rebar_alias.git"}}, + rebar3_appup_plugin +]}. + +{deps, [ + {hackney, {git, "https://github.com/benoitc/hackney.git", {tag, "1.10.1"}}} + ]}. + + +%% Make work despite compat issues with strings and warnings +{overrides, [ + {override, rebar3_appup_plugin, [ + {erl_opts, [ + {platform_define, "^19", brutal_purge_fixed}, + {platform_define, "^2", brutal_purge_fixed}, + %% warnings_as_errors, + debug_info + ]} + ]} +]}. diff --git a/test/mock_git_resource.erl b/test/mock_git_resource.erl index e922af3..5673349 100644 --- a/test/mock_git_resource.erl +++ b/test/mock_git_resource.erl @@ -27,7 +27,7 @@ mock(Opts) -> mock(Opts, create_app). mock(Opts, CreateType) -> - meck:new(?MOD, [no_link]), + meck:new(?MOD, [no_link, passthrough]), mock_lock(Opts), mock_update(Opts), mock_vsn(Opts), @@ -46,8 +46,8 @@ unmock() -> mock_lock(_) -> meck:expect( ?MOD, lock, - fun(_AppDir, Git) -> - case Git of + fun(AppInfo, _) -> + case rebar_app_info:source(AppInfo) of {git, Url, {tag, Ref}} -> {git, Url, {ref, Ref}}; {git, Url, {ref, Ref}} -> {git, Url, {ref, Ref}}; {git, Url} -> {git, Url, {ref, "0.0.0"}}; @@ -62,7 +62,8 @@ mock_update(Opts) -> % ct:pal("TOUp: ~p", [ToUpdate]), meck:expect( ?MOD, needs_update, - fun(_Dir, {git, Url, _Ref}) -> + fun(AppInfo, _) -> + {git, Url, _Ref} = rebar_app_info:source(AppInfo), App = app(Url), % ct:pal("Needed update? ~p (~p) -> ~p", [App, {Url,_Ref}, lists:member(App, ToUpdate)]), lists:member(App, ToUpdate) @@ -78,7 +79,8 @@ mock_vsn(Opts) -> Default = proplists:get_value(default_vsn, Opts, "0.0.0"), meck:expect( ?MOD, make_vsn, - fun(Dir) -> + fun(AppInfo, _) -> + Dir = rebar_app_info:dir(AppInfo), case filelib:wildcard("*.app.src", filename:join([Dir,"src"])) of [AppSrc] -> {ok, App} = file:consult(AppSrc), @@ -108,7 +110,8 @@ mock_download(Opts, CreateType) -> Overrides = proplists:get_value(override_vsn, Opts, []), meck:expect( ?MOD, download, - fun (Dir, Git, _) -> + fun (Dir, AppInfo, _, _) -> + Git = rebar_app_info:source(AppInfo), filelib:ensure_dir(Dir), {git, Url, {_, Vsn}} = normalize_git(Git, Overrides, Default), App = app(Url), @@ -118,7 +121,7 @@ mock_download(Opts, CreateType) -> [kernel, stdlib] ++ [element(1,D) || D <- AppDeps] ), rebar_test_utils:create_config(Dir, [{deps, AppDeps}]++Config), - {ok, 'WHATEVER'} + ok end). %%%%%%%%%%%%%%% diff --git a/test/mock_pkg_resource.erl b/test/mock_pkg_resource.erl index f837713..a169efd 100644 --- a/test/mock_pkg_resource.erl +++ b/test/mock_pkg_resource.erl @@ -3,6 +3,8 @@ -export([mock/0, mock/1, unmock/0]). -define(MOD, rebar_pkg_resource). +-include("rebar.hrl"). + %%%%%%%%%%%%%%%%% %%% Interface %%% %%%%%%%%%%%%%%%%% @@ -26,7 +28,7 @@ mock() -> mock([]). Vsn :: string(), Hash :: string() | undefined. mock(Opts) -> - meck:new(?MOD, [no_link]), + meck:new(?MOD, [no_link, passthrough]), mock_lock(Opts), mock_update(Opts), mock_vsn(Opts), @@ -44,7 +46,10 @@ unmock() -> %% @doc creates values for a lock file. mock_lock(_) -> - meck:expect(?MOD, lock, fun(_AppDir, Source) -> Source end). + meck:expect(?MOD, lock, fun(AppInfo, _) -> + {pkg, Name, Vsn, Hash, _RepoConfig} = rebar_app_info:source(AppInfo), + {pkg, Name, Vsn, Hash} + end). %% @doc The config passed to the `mock/2' function can specify which apps %% should be updated on a per-name basis: `{update, ["App1", "App3"]}'. @@ -52,7 +57,8 @@ mock_update(Opts) -> ToUpdate = proplists:get_value(upgrade, Opts, []), meck:expect( ?MOD, needs_update, - fun(_Dir, {pkg, App, _Vsn, _Hash}) -> + fun(AppInfo, _) -> + {pkg, App, _Vsn, _Hash, _} = rebar_app_info:source(AppInfo), lists:member(binary_to_list(App), ToUpdate) end). @@ -60,7 +66,7 @@ mock_update(Opts) -> mock_vsn(_Opts) -> meck:expect( ?MOD, make_vsn, - fun(_Dir) -> + fun(_AppInfo, _) -> {error, "Replacing version of type pkg not supported."} end). @@ -77,30 +83,32 @@ mock_download(Opts) -> Config = proplists:get_value(config, Opts, []), meck:expect( ?MOD, download, - fun (Dir, {pkg, AppBin, Vsn, _}, _) -> - App = binary_to_list(AppBin), + fun (Dir, AppInfo, _, _) -> + {pkg, AppBin, Vsn, _, _} = rebar_app_info:source(AppInfo), + App = rebar_utils:to_list(AppBin), filelib:ensure_dir(Dir), AppDeps = proplists:get_value({App,Vsn}, Deps, []), - {ok, AppInfo} = rebar_test_utils:create_app( - Dir, App, binary_to_list(Vsn), + {ok, AppInfo1} = rebar_test_utils:create_app( + Dir, App, rebar_utils:to_list(Vsn), [kernel, stdlib] ++ [element(1,D) || D <- AppDeps] ), rebar_test_utils:create_config(Dir, [{deps, AppDeps}]++Config), - TarApp = App++"-"++binary_to_list(Vsn)++".tar", - Tarball = filename:join([Dir, TarApp]), - Contents = filename:join([Dir, "contents.tar.gz"]), - Files = all_files(rebar_app_info:dir(AppInfo)), - ok = erl_tar:create(Contents, - archive_names(Dir, App, Vsn, Files), - [compressed]), - ok = erl_tar:create(Tarball, - [{"contents.tar.gz", Contents}], - []), + + TarApp = App++"-"++rebar_utils:to_list(Vsn)++".tar", + + Metadata = #{<<"app">> => AppBin, + <<"version">> => Vsn}, + + Files = all_files(rebar_app_info:dir(AppInfo1)), + {ok, {Tarball, _Checksum}} = hex_tarball:create(Metadata, archive_names(Dir, Files)), + Archive = filename:join([Dir, TarApp]), + file:write_file(Archive, Tarball), + Cache = proplists:get_value(cache_dir, Opts, filename:join(Dir,"cache")), Cached = filename:join([Cache, TarApp]), filelib:ensure_dir(Cached), - rebar_file_utils:mv(Tarball, Cached), - {ok, true} + rebar_file_utils:mv(Archive, Cached), + ok end). %% @doc On top of the pkg resource mocking, we need to mock the package @@ -110,16 +118,18 @@ mock_download(Opts) -> %% specific applications otherwise listed. mock_pkg_index(Opts) -> Deps = proplists:get_value(pkgdeps, Opts, []), + Repos = proplists:get_value(repos, Opts, [<<"hexpm">>]), Skip = proplists:get_value(not_in_index, Opts, []), %% Dict: {App, Vsn}: [{<<"link">>, <<>>}, {<<"deps">>, []}] %% Index: all apps and deps in the index Dict = find_parts(Deps, Skip), + to_index(Deps, Dict, Repos), meck:new(rebar_packages, [passthrough, no_link]), - meck:expect(rebar_packages, packages, - fun(_State) -> to_index(Deps, Dict) end), + meck:expect(rebar_packages, update_package, + fun(_, _, _State) -> ok end), meck:expect(rebar_packages, verify_table, - fun(_State) -> to_index(Deps, Dict), true end). + fun(_State) -> true end). %%%%%%%%%%%%%%% %%% Helpers %%% @@ -128,7 +138,7 @@ mock_pkg_index(Opts) -> all_files(Dir) -> filelib:wildcard(filename:join([Dir, "**"])). -archive_names(Dir, _App, _Vsn, Files) -> +archive_names(Dir, Files) -> [{(F -- Dir) -- "/", F} || F <- Files]. find_parts(Apps, Skip) -> find_parts(Apps, Skip, dict:new()). @@ -143,21 +153,42 @@ find_parts([{AppName, Deps}|Rest], Skip, Acc) -> Acc), find_parts(Rest, Skip, AccNew) end. +parse_deps(Deps) -> + [{maps:get(app, D, Name), {pkg, Name, Constraint, undefined}} || D=#{package := Name, + requirement := Constraint} <- Deps]. + +to_index(AllDeps, Dict, Repos) -> + catch ets:delete(?PACKAGE_TABLE), + rebar_packages:new_package_table(), -to_index(AllDeps, Dict) -> - catch ets:delete(package_index), - ets:new(package_index, [named_table, public]), dict:fold( - fun(K, Deps, _) -> - DepsList = [{ec_cnv:to_binary(DK), ec_cnv:to_binary(DV)} || {DK, DV} <- Deps], - ets:insert(package_index, {K, DepsList, <<"checksum">>}) + fun({N, V}, Deps, _) -> + DepsList = [#{package => DKB, + app => DKB, + requirement => DVB, + source => {pkg, DKB, DVB, undefined}} + || {DK, DV} <- Deps, + DKB <- [ec_cnv:to_binary(DK)], + DVB <- [ec_cnv:to_binary(DV)]], + Repo = rebar_test_utils:random_element(Repos), + ets:insert(?PACKAGE_TABLE, #package{key={N, ec_semver:parse(V), Repo}, + dependencies=parse_deps(DepsList), + retired=false, + checksum = <<"checksum">>}) end, ok, Dict), - ets:insert(package_index, {package_index_version, 3}), + lists:foreach(fun({{Name, Vsn}, _}) -> - case ets:lookup(package_index, ec_cnv:to_binary(Name)) of - [{_, Vsns}] -> - ets:insert(package_index, {ec_cnv:to_binary(Name), [ec_cnv:to_binary(Vsn) | Vsns]}); - _ -> - ets:insert(package_index, {ec_cnv:to_binary(Name), [ec_cnv:to_binary(Vsn)]}) + case lists:any(fun(R) -> + ets:member(?PACKAGE_TABLE, {ec_cnv:to_binary(Name), ec_semver:parse(Vsn), R}) + end, Repos) of + false -> + Repo = rebar_test_utils:random_element(Repos), + ets:insert(?PACKAGE_TABLE, #package{key={ec_cnv:to_binary(Name), ec_semver:parse(Vsn), Repo}, + dependencies=[], + retired=false, + checksum = <<"checksum">>}); + true -> + ok end end, AllDeps). + diff --git a/test/rebar_alias_SUITE.erl b/test/rebar_alias_SUITE.erl new file mode 100644 index 0000000..d6e27d9 --- /dev/null +++ b/test/rebar_alias_SUITE.erl @@ -0,0 +1,165 @@ +-module(rebar_alias_SUITE). +-compile([export_all]). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +init_per_suite(Config) -> Config. +end_per_suite(_Config) -> ok. + +init_per_testcase(_, Config) -> + rebar_test_utils:init_rebar_state(Config, "alias_"). + +end_per_testcase(_, _Config) -> + ok. + +all() -> [command, args, many, override_default, no_circular, release, + check_namespaces, create_lib]. + +command() -> + [{doc, "Runs multiple regular commands as one alias"}]. +command(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("alias_command_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + RebarConfig = [{alias, [{test, [compile, unlock]}]}], + + %% compile job ran + rebar_test_utils:run_and_check(Config, RebarConfig, + ["test"], {ok, [{app, Name}]}), + %% unlock job also ran + Lockfile = filename:join(?config(apps, Config), "rebar.lock"), + ?assertNot(filelib:is_file(Lockfile)), + ok. + +args() -> + [{doc, "Runs multiple regular commands as one alias, some of " + "which have default arguments"}]. +args(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("alias_args_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + RebarConfig = [{alias, [{test, [{eunit,"-c"}, cover]}]}], + + %% test job ran (compiled and succeeded) + rebar_test_utils:run_and_check(Config, RebarConfig, + ["test"], {ok, [{app, Name}]}), + %% cover job also ran, meaning eunit had coverage on, otherwise + %% the index file is not generated. + CoverFile = filename:join([?config(apps, Config), + "_build", "test", "cover", "index.html"]), + ?assert(filelib:is_file(CoverFile)), + ok. + +many() -> + [{doc, "Multiple aliases may be registered"}]. +many(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("alias_args_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + RebarConfig = [{alias, [{test, [{eunit,"-c"}, cover]}, + {nolock, [compile, unlock]}]}], + + %% test job ran (compiled and succeeded) + rebar_test_utils:run_and_check(Config, RebarConfig, + ["test"], {ok, [{app, Name}]}), + rebar_test_utils:run_and_check(Config, RebarConfig, + ["nolock"], {ok, [{app, Name}]}), + %% both jobs ran (see args/1 and command/1) + CoverFile = filename:join([?config(apps, Config), + "_build", "test", "cover", "index.html"]), + ?assert(filelib:is_file(CoverFile)), + Lockfile = filename:join(?config(apps, Config), "rebar.lock"), + ?assertNot(filelib:is_file(Lockfile)), + ok. + +override_default() -> + [{doc, "An alias cannot take over a default provider"}]. +override_default(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("alias_override_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + RebarConfig = [{alias, [{compile, [help]}]}], + + %% App compiles anyway + rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], + {ok, [{app, Name}]}), + ok. + +no_circular() -> + [{doc, "An alias cannot define itself as itself"}, + {timetrap, 2000}]. +no_circular(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("alias_circular_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + RebarConfig = [{alias, [{test, [help, {test,"-a"}, compile]}]}], + + %% Code does not deadlock forever and errors by not knowing + %% the command + rebar_test_utils:run_and_check(Config, RebarConfig, ["test"], + {error, [$C,$o,$m,$m,$a,$n,$d,$ ,"test",$ ,$n,$o,$t,$ , + $f,$o,$u,$n,$d]}), + ok. + +release() -> + [{doc, "An alias for a release command"}]. +release(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("alias_release_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + RebarConfig = [{relx, + [{release, {skipped_release, Vsn}, []}, + {release, {the_release, Vsn}, [list_to_atom(Name)]}, + {lib_dirs, [AppDir]}]}, + {alias, + [{the_rel1, [clean, {release, "-n the_release"}]}, + {the_rel2, [clean, {release, "--relname=the_release"}]}]}], + rebar_test_utils:run_and_check( + Config, RebarConfig, + ["the_rel1"], + {ok, [{release, the_release, Vsn, false}]}), + rebar_test_utils:run_and_check( + Config, RebarConfig, + ["the_rel2"], + {ok, [{release, the_release, Vsn, false}]}), + ok. + +check_namespaces() -> + [{doc, "Test calling commands with namespaces from rebar3"}]. +check_namespaces(Config) -> + AppDir = ?config(apps, Config), + Name = rebar_test_utils:create_random_name("alias_args_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + RebarConfig = [{alias, [{test, [{eunit,"-c"}, {plugins, list}]}]}], + rebar_test_utils:run_and_check(Config, RebarConfig, + ["test"], {ok, [{app, Name}]}), + ok. + +create_lib() -> + [{doc, "Test calling commands with namespaces from rebar3"}]. +create_lib(Config) -> + AppDir = ?config(apps, Config), + Name = rebar_test_utils:create_random_name("create_lib_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + RebarConfig = [{alias, [{test, [compile, {do, "new lib shouldexist"}]}]}], + rebar_test_utils:run_and_check(Config, RebarConfig, + ["test"], {ok, [{app, Name}]}), + AppFile = filename:join(?config(apps, Config), + "../../../../shouldexist/src/shouldexist.app.src"), + ?assert(filelib:is_file(AppFile)), + ok. diff --git a/test/rebar_as_SUITE.erl b/test/rebar_as_SUITE.erl index 0f37dc8..78ea8ae 100644 --- a/test/rebar_as_SUITE.erl +++ b/test/rebar_as_SUITE.erl @@ -14,6 +14,7 @@ as_dir_name/1, as_with_task_args/1, warn_on_empty_profile/1, + error_on_empty_tasks/1, clean_as_profile/1]). -include_lib("common_test/include/ct.hrl"). @@ -33,7 +34,7 @@ all() -> [as_basic, as_multiple_profiles, as_multiple_tasks, as_multiple_profiles_multiple_tasks, as_comma_placement, as_comma_then_space, as_dir_name, as_with_task_args, - warn_on_empty_profile, clean_as_profile]. + warn_on_empty_profile, error_on_empty_tasks, clean_as_profile]. as_basic(Config) -> AppDir = ?config(apps, Config), @@ -159,9 +160,23 @@ warn_on_empty_profile(Config) -> meck:unload(rebar_log), ok. +error_on_empty_tasks(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("as_error_empty_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + meck:new(rebar_log, [passthrough]), + rebar_test_utils:run_and_check(Config, + [], + ["as", "default"], + {error, "At least one task must be specified when using `as`"}), + ok. + warn_match(App, History) -> lists:any( - fun({_, {rebar_log,log, [warn, "No entry for profile ~s in config.", + fun({_, {rebar_log,log, [warn, "No entry for profile ~ts in config.", [ArgApp]]}, _}) -> ArgApp =:= App ; (_) -> false diff --git a/test/rebar_compile_SUITE.erl b/test/rebar_compile_SUITE.erl index d9b75e4..ddaad0c 100644 --- a/test/rebar_compile_SUITE.erl +++ b/test/rebar_compile_SUITE.erl @@ -1,48 +1,6 @@ -module(rebar_compile_SUITE). --export([suite/0, - init_per_suite/1, - end_per_suite/1, - init_per_testcase/2, - end_per_testcase/2, - init_per_group/2, - end_per_group/2, - all/0, - groups/0, - build_basic_app/1, paths_basic_app/1, clean_basic_app/1, - build_release_apps/1, paths_release_apps/1, clean_release_apps/1, - build_checkout_apps/1, paths_checkout_apps/1, - build_checkout_deps/1, paths_checkout_deps/1, - build_basic_srcdirs/1, paths_basic_srcdirs/1, - build_release_srcdirs/1, paths_release_srcdirs/1, - build_unbalanced_srcdirs/1, paths_unbalanced_srcdirs/1, - build_basic_extra_dirs/1, paths_basic_extra_dirs/1, clean_basic_extra_dirs/1, - build_release_extra_dirs/1, paths_release_extra_dirs/1, clean_release_extra_dirs/1, - build_unbalanced_extra_dirs/1, paths_unbalanced_extra_dirs/1, - build_extra_dirs_in_project_root/1, - paths_extra_dirs_in_project_root/1, - clean_extra_dirs_in_project_root/1, - recompile_when_hrl_changes/1, - recompile_when_included_hrl_changes/1, - recompile_when_opts_change/1, - dont_recompile_when_opts_dont_change/1, - dont_recompile_yrl_or_xrl/1, - deps_in_path/1, - delete_beam_if_source_deleted/1, - checkout_priority/1, - highest_version_of_pkg_dep/1, - parse_transform_test/1, - erl_first_files_test/1, - mib_test/1, - umbrella_mib_first_test/1, - only_default_transitive_deps/1, - clean_all/1, - override_deps/1, - profile_override_deps/1, - deps_build_in_prod/1, - include_file_relative_to_working_directory/1, - include_file_in_src/1, - always_recompile_when_erl_compiler_options_set/1]). +-compile(export_all). -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -58,18 +16,30 @@ all() -> {group, basic_extras}, {group, release_extras}, {group, unbalanced_extras}, {group, root_extras}, recompile_when_hrl_changes, recompile_when_included_hrl_changes, + recompile_when_opts_included_hrl_changes, recompile_when_opts_change, dont_recompile_when_opts_dont_change, dont_recompile_yrl_or_xrl, delete_beam_if_source_deleted, deps_in_path, checkout_priority, highest_version_of_pkg_dep, parse_transform_test, erl_first_files_test, mib_test, - umbrella_mib_first_test, only_default_transitive_deps, - clean_all, override_deps, profile_override_deps, deps_build_in_prod, - include_file_relative_to_working_directory, include_file_in_src] ++ - case erlang:function_exported(os, unsetenv, 1) of - true -> [always_recompile_when_erl_compiler_options_set]; - false -> [] - end. + umbrella_mib_first_test, only_default_transitive_deps, clean_all, + profile_deps, deps_build_in_prod, only_deps, + override_deps, override_add_deps, override_del_deps, + override_opts, override_add_opts, override_del_opts, + apply_overrides_exactly_once, + profile_override_deps, profile_override_add_deps, profile_override_del_deps, + profile_override_opts, profile_override_add_opts, profile_override_del_opts, + include_file_relative_to_working_directory, include_file_in_src, + include_file_relative_to_working_directory_test, include_file_in_src_test, + include_file_in_src_test_multiapp, + recompile_when_parse_transform_as_opt_changes, + recompile_when_parse_transform_inline_changes, + regex_filter_skip, regex_filter_regression, + recursive, no_recursive, + always_recompile_when_erl_compiler_options_set, + dont_recompile_when_erl_compiler_options_env_does_not_change, + recompile_when_erl_compiler_options_env_changes, + rebar_config_os_var]. groups() -> [{basic_app, [], [build_basic_app, paths_basic_app, clean_basic_app]}, @@ -243,7 +213,31 @@ init_per_suite(Config) -> end_per_suite(_Config) -> ok. -init_per_testcase(_, Config) -> +init_per_testcase(Test, Config) when + Test == dont_recompile_when_erl_compiler_options_env_does_not_change + orelse + Test == recompile_when_erl_compiler_options_env_changes -> + _ = code:ensure_loaded(os), + UnSetEnv = erlang:function_exported(os, unsetenv, 1), + _ = code:ensure_loaded(compile), + EnvOpts = erlang:function_exported(compile, env_compiler_options, 0), + case {UnSetEnv, EnvOpts} of + {true, true} -> maybe_init_config(Config); + _ -> {skip, "compile:env_compiler_options/0 unavailable"} + end; +init_per_testcase(always_recompile_when_erl_compiler_options_set, Config) -> + _ = code:ensure_loaded(os), + UnSetEnv = erlang:function_exported(os, unsetenv, 1), + _ = code:ensure_loaded(compile), + EnvOpts = erlang:function_exported(compile, env_compiler_options, 0), + case {UnSetEnv, EnvOpts} of + {true, true} -> {skip, "compile:env_compiler_options/0 available"}; + {true, false} -> maybe_init_config(Config); + _ -> {skip, "os:unsetenv/1 unavailable"} + end; +init_per_testcase(_, Config) -> maybe_init_config(Config). + +maybe_init_config(Config) -> case ?config(apps, Config) of undefined -> rebar_test_utils:init_rebar_state(Config); _ -> Config @@ -253,7 +247,6 @@ end_per_testcase(_, _Config) -> catch meck:unload(). - %% test cases build_basic_app(Config) -> @@ -717,6 +710,53 @@ recompile_when_included_hrl_changes(Config) -> ?assert(ModTime =/= NewModTime). +recompile_when_opts_included_hrl_changes(Config) -> + AppsDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("app1_"), + Vsn = rebar_test_utils:create_random_vsn(), + AppDir = filename:join([AppsDir, "apps", Name]), + + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + ExtraSrc = <<"-module(test_header_include).\n" + "-export([main/0]).\n" + "-include(\"test_header_include.hrl\").\n" + "main() -> ?SOME_DEFINE.\n">>, + + ExtraHeader = <<"-define(SOME_DEFINE, true).\n">>, + ok = filelib:ensure_dir(filename:join([AppsDir, "include", "dummy"])), + HeaderFile = filename:join([AppsDir, "include", "test_header_include.hrl"]), + ok = file:write_file(filename:join([AppDir, "src", "test_header_include.erl"]), ExtraSrc), + ok = file:write_file(HeaderFile, ExtraHeader), + + %% Using relative path from the project root + RebarConfig = [{erl_opts, [{i, "include/"}]}], + {ok,Cwd} = file:get_cwd(), + ok = file:set_cwd(AppsDir), + + rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], {ok, [{app, Name}]}), + + EbinDir = filename:join([AppsDir, "_build", "default", "lib", Name, "ebin"]), + {ok, Files} = rebar_utils:list_dir(EbinDir), + ModTime = [filelib:last_modified(filename:join([EbinDir, F])) + || F <- Files, filename:extension(F) == ".beam"], + + timer:sleep(1000), + + NewExtraHeader = <<"-define(SOME_DEFINE, false).\n">>, + ok = file:write_file(HeaderFile, NewExtraHeader), + + rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], {ok, [{app, Name}]}), + + {ok, NewFiles} = rebar_utils:list_dir(EbinDir), + NewModTime = [filelib:last_modified(filename:join([EbinDir, F])) + || F <- NewFiles, filename:extension(F) == ".beam"], + + ok = file:set_cwd(Cwd), + + ?assert(ModTime =/= NewModTime). + recompile_when_opts_change(Config) -> AppDir = ?config(apps, Config), @@ -735,7 +775,7 @@ recompile_when_opts_change(Config) -> rebar_test_utils:create_config(AppDir, [{erl_opts, [{d, some_define}]}]), - rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name}]}), + rebar_test_utils:run_and_check(Config, [{erl_opts, [{d, some_define}]}], ["compile"], {ok, [{app, Name}]}), {ok, NewFiles} = rebar_utils:list_dir(EbinDir), NewModTime = [filelib:last_modified(filename:join([EbinDir, F])) @@ -765,7 +805,7 @@ dont_recompile_when_opts_dont_change(Config) -> NewModTime = [filelib:last_modified(filename:join([EbinDir, F])) || F <- NewFiles, filename:extension(F) == ".beam"], - ?assert(ModTime == NewModTime). + ?assertEqual(ModTime, NewModTime). dont_recompile_yrl_or_xrl(Config) -> AppDir = ?config(apps, Config), @@ -792,19 +832,55 @@ dont_recompile_yrl_or_xrl(Config) -> "Erlang code.", ok = ec_file:write(Xrl, XrlBody), - XrlBeam = filename:join([AppDir, "ebin", filename:basename(Xrl, ".xrl") ++ ".beam"]), + Yrl = filename:join([AppDir, "src", "not_a_real_yrl_" ++ Name ++ ".yrl"]), + ok = filelib:ensure_dir(Yrl), + YrlBody = ["Nonterminals E T F.\n" + "Terminals '+' '*' '(' ')' number.\n" + "Rootsymbol E.\n" + "E -> E '+' T: {'$2', '$1', '$3'}.\n" + "E -> T : '$1'.\n" + "T -> T '*' F: {'$2', '$1', '$3'}.\n" + "T -> F : '$1'.\n" + "F -> '(' E ')' : '$2'.\n" + "F -> number : '$1'.\n"], + ok = ec_file:write(Yrl, YrlBody), + + XrlErl = filename:join([AppDir, "src", filename:basename(Xrl, ".xrl") ++ ".erl"]), + YrlErl = filename:join([AppDir, "src", filename:basename(Yrl, ".yrl") ++ ".erl"]), - rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name}]}), + EbinDir = filename:join([AppDir, "_build", "default", "lib", Name, "ebin"]), + XrlBeam = filename:join([EbinDir, filename:basename(Xrl, ".xrl") ++ ".beam"]), + YrlBeam = filename:join([EbinDir, filename:basename(Yrl, ".yrl") ++ ".beam"]), + + Hrl = filename:join([AppDir, "include", "some_header.hrl"]), + ok = filelib:ensure_dir(Hrl), + HrlBody = yeccpre_hrl(), + ok = ec_file:write(Hrl, HrlBody), + RebarConfig = [{yrl_opts, [{includefile, "include/some_header.hrl"}]}], + + rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], {ok, [{app, Name}]}), - ModTime = filelib:last_modified(XrlBeam), + XrlModTime = filelib:last_modified(XrlErl), + YrlModTime = filelib:last_modified(YrlErl), + + XrlBeamModTime = filelib:last_modified(XrlBeam), + YrlBeamModTime = filelib:last_modified(YrlBeam), timer:sleep(1000), - rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name}]}), + rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], {ok, [{app, Name}]}), + + NewXrlModTime = filelib:last_modified(XrlErl), + NewYrlModTime = filelib:last_modified(YrlErl), - NewModTime = filelib:last_modified(XrlBeam), + NewXrlBeamModTime = filelib:last_modified(XrlBeam), + NewYrlBeamModTime = filelib:last_modified(YrlBeam), - ?assert(ModTime == NewModTime). + ?assert(XrlBeamModTime == NewXrlBeamModTime), + ?assert(YrlBeamModTime == NewYrlBeamModTime), + + ?assert(XrlModTime == NewXrlModTime), + ?assert(YrlModTime == NewYrlModTime). delete_beam_if_source_deleted(Config) -> AppDir = ?config(apps, Config), @@ -1085,17 +1161,19 @@ umbrella_mib_first_test(Config) -> rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), - MibsSrc = <<"-- SIMPLE-MIB.\n" + BExporterSrc = <<"-- BEXPORTER-MIB.\n" "-- This is just a simple MIB used for testing!\n" "--\n" -"SIMPLE-MIB DEFINITIONS ::= BEGIN\n" +"BEXPORTER-MIB DEFINITIONS ::= BEGIN\n" "IMPORTS\n" +" TEXTUAL-CONVENTION\n" +" FROM SNMPv2-TC\n" " MODULE-IDENTITY, enterprises\n" " FROM SNMPv2-SMI;\n" "\n" "ericsson MODULE-IDENTITY\n" " LAST-UPDATED\n" -" \"201403060000Z\"\n" +" \"201812050000Z\"\n" " ORGANIZATION\n" " \"rebar\"\n" " CONTACT-INFO\n" @@ -1107,25 +1185,71 @@ umbrella_mib_first_test(Config) -> " \"This very small module is made available\n" " for mib-compilation testing.\"\n" " ::= { enterprises 999 }\n" +"\n" +"Something ::= TEXTUAL-CONVENTION\n" +" STATUS current\n" +" DESCRIPTION \"\"\n" +" SYNTAX OCTET STRING (SIZE (4))\n" +"END\n">>, + + AImporterSrc = <<"-- AIMPORTER-MIB.\n" +"-- This is just a simple MIB used for testing!\n" +"--\n" +"AIMPORTER-MIB DEFINITIONS ::= BEGIN\n" +"IMPORTS\n" +" Something\n" +" FROM BEXPORTER-MIB\n" +" MODULE-IDENTITY, enterprises\n" +" FROM SNMPv2-SMI;\n" +"\n" +"ericsson MODULE-IDENTITY\n" +" LAST-UPDATED\n" +" \"201812050000Z\"\n" +" ORGANIZATION\n" +" \"rebar\"\n" +" CONTACT-INFO\n" +" \"rebar <rebar@example.com>\n" +" or\n" +" whoever is currently responsible for the SIMPLE\n" +" enterprise MIB tree branch (enterprises.999).\"\n" +" DESCRIPTION\n" +" \"This very small module is made available\n" +" for mib-compilation testing.\"\n" +" ::= { enterprises 1000 }\n" "END\n">>, + + ok = filelib:ensure_dir(filename:join([AppDir, "mibs", "dummy"])), - ok = file:write_file(filename:join([AppDir, "mibs", "SIMPLE-MIB.mib"]), MibsSrc), + ok = file:write_file(filename:join([AppDir, "mibs", "AIMPORTER-MIB.mib"]), AImporterSrc), + ok = file:write_file(filename:join([AppDir, "mibs", "BEXPORTER-MIB.mib"]), BExporterSrc), - RebarConfig = [{mib_first_files, ["mibs/SIMPLE-MIB.mib"]}], + FailureRebarConfig = [{mib_first_files, ["mibs/AIMPORTER-MIB.mib"]}], + SuccessRebarConfig = [{mib_first_files, ["mibs/BEXPORTER-MIB.mib"]}], - rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], {ok, [{app, Name}]}), + PrivMibsDir = filename:join([AppsDir, "_build", "default", "lib", Name, "priv", "mibs"]), + + FailureRebarConfig = [{mib_first_files, ["mibs/AIMPORTER-MIB.mib"]}], + catch ( + rebar_test_utils:run_and_check(Config, FailureRebarConfig, ["compile"], {ok, [{app, Name}]}) ), + + %% check that the bin file was NOT cretated + false = filelib:is_file(filename:join([PrivMibsDir, "AIMPORTER-MIB.bin"])), + + + SuccessRebarConfig = [{mib_first_files, ["mibs/BEXPORTER-MIB.mib"]}], + rebar_test_utils:run_and_check(Config, SuccessRebarConfig, ["compile"], {ok, [{app, Name}]}), %% check a bin corresponding to the mib in the mibs dir exists in priv/mibs - PrivMibsDir = filename:join([AppsDir, "_build", "default", "lib", Name, "priv", "mibs"]), - true = filelib:is_file(filename:join([PrivMibsDir, "SIMPLE-MIB.bin"])), + true = filelib:is_file(filename:join([PrivMibsDir, "AIMPORTER-MIB.bin"])), %% check a hrl corresponding to the mib in the mibs dir exists in include - true = filelib:is_file(filename:join([AppDir, "include", "SIMPLE-MIB.hrl"])), + true = filelib:is_file(filename:join([AppDir, "include", "AIMPORTER-MIB.hrl"])), %% check the mibs dir was linked into the _build dir true = filelib:is_dir(filename:join([AppsDir, "_build", "default", "lib", Name, "mibs"])). + only_default_transitive_deps(Config) -> AppDir = ?config(apps, Config), @@ -1184,41 +1308,455 @@ clean_all(Config) -> {app, PkgName, invalid}]}). override_deps(Config) -> - mock_git_resource:mock([{deps, [{some_dep, "0.0.1"},{other_dep, "0.0.1"}]}]), Deps = rebar_test_utils:expand_deps(git, [{"some_dep", "0.0.1", [{"other_dep", "0.0.1", []}]}]), TopDeps = rebar_test_utils:top_level_deps(Deps), + {SrcDeps, _} = rebar_test_utils:flat_deps(Deps), + mock_git_resource:mock([{deps, SrcDeps}]), + RebarConfig = [ {deps, TopDeps}, {overrides, [ {override, some_dep, [ - {deps, []} - ]} - ]} - ], + {deps, []} + ]} + ]} + ], rebar_test_utils:run_and_check( Config, RebarConfig, ["compile"], - {ok, [{dep, "some_dep"},{dep_not_exist, "other_dep"}]} + {ok, [{dep, "some_dep"}, + {dep_not_exist, "other_dep"}]} ). +override_add_deps(Config) -> + Deps = rebar_test_utils:expand_deps(git, [{"some_dep", "0.0.1", [{"other_dep", "0.0.1", []}]}]), + TopDeps = rebar_test_utils:top_level_deps(Deps), + + DepA = {dep_a, "0.0.1", {git, "http://site.com/dep_a.git", {tag, "0.0.1"}}}, + DepB = {dep_b, "0.0.1", {git, "http://site.com/dep_b.git", {tag, "0.0.1"}}}, + DepC = {dep_c, "0.0.1", {git, "http://site.com/dep_c.git", {tag, "0.0.1"}}}, + + {SrcDeps, _} = rebar_test_utils:flat_deps(Deps), + mock_git_resource:mock([{deps, [DepA, DepB, DepC | SrcDeps]}]), + + RebarConfig = [ + {deps, TopDeps}, + {overrides, [ + {add, some_dep, [ + {deps, [DepA, DepB]} + ]}, + {add, [ + {deps, [DepC]} + ]} + ]} + ], + rebar_test_utils:run_and_check( + Config, RebarConfig, ["compile"], + {ok, [{dep, "some_dep"}, + {dep, "other_dep"}, + {dep, "dep_a"}, + {dep, "dep_b"}, + {dep, "dep_c"}]} + ). + +override_del_deps(Config) -> + Deps = rebar_test_utils:expand_deps(git, [{"some_dep", "0.0.1", [{"dep_a", "0.0.1", []}, + {"dep_b", "0.0.1", []}, + {"dep_c", "0.0.1", []}]}, + {"other_dep", "0.0.1", [{"dep_c", "0.0.1", []}, + {"dep_d", "0.0.1", []}]}]), + TopDeps = rebar_test_utils:top_level_deps(Deps), + + DepA = {dep_a, "0.0.1", {git, "https://example.org/user/dep_a.git", {tag, "0.0.1"}}}, + DepB = {dep_b, "0.0.1", {git, "https://example.org/user/dep_b.git", {tag, "0.0.1"}}}, + DepC = {dep_c, "0.0.1", {git, "https://example.org/user/dep_c.git", {tag, "0.0.1"}}}, + + {SrcDeps, _} = rebar_test_utils:flat_deps(Deps), + mock_git_resource:mock([{deps, SrcDeps}]), + + RebarConfig = [ + {deps, TopDeps}, + {overrides, [ + {del, some_dep, [ + {deps, [DepA, DepB]} + ]}, + {del, [ + {deps, [DepC]} + ]} + ]} + ], + + rebar_test_utils:run_and_check( + Config, RebarConfig, ["compile"], + {ok, [{dep, "some_dep"}, + {dep, "other_dep"}, + {dep_not_exist, "dep_a"}, + {dep_not_exist, "dep_b"}, + {dep_not_exist, "dep_c"}, + {dep, "dep_d"}]} + ). + +override_opts(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("app1_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + RebarConfig = [ + {erl_opts, [ + compressed, + warn_missing_spec + ]}, + {overrides, [ + {override, [ + {erl_opts, [compressed]} + ]} + ]} + ], + + rebar_test_utils:create_config(AppDir, RebarConfig), + + rebar_test_utils:run_and_check( + Config, RebarConfig, ["compile"], {ok, [{app, Name}]}), + + Path = filename:join([AppDir, "_build", "default", "lib", Name, "ebin"]), + code:add_patha(Path), + + Mod = list_to_atom("not_a_real_src_" ++ Name), + + true = lists:member(compressed, proplists:get_value(options, Mod:module_info(compile), [])), + false = lists:member(warn_missing_spec, proplists:get_value(options, Mod:module_info(compile), [])). + +%% test for fix of https://github.com/erlang/rebar3/issues/1801 +%% only apply overrides once +%% verify by having an override add the macro TEST to the dep some_dep +%% building under `ct` will fail if the `add` is applied more than once +apply_overrides_exactly_once(Config) -> + AppDir = ?config(apps, Config), + + Deps = rebar_test_utils:expand_deps(git, [{"some_dep", "0.0.1", [{"other_dep", "0.0.1", []}]}]), + TopDeps = rebar_test_utils:top_level_deps(Deps), + + {SrcDeps, _} = rebar_test_utils:flat_deps(Deps), + mock_git_resource:mock([{deps, SrcDeps}]), + + Name = rebar_test_utils:create_random_name("app1_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + RebarConfig = [{deps, TopDeps}, + {overrides, [ + {add, some_dep, [ + {erl_opts, [{d, 'TEST'}]} + ]} + ]}], + + rebar_test_utils:create_config(AppDir, RebarConfig), + + rebar_test_utils:run_and_check( + Config, RebarConfig, ["ct", "--compile_only"], {ok, [{app, Name}, {dep, "some_dep"}], "test"}). + +override_add_opts(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("app1_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + RebarConfig = [ + {erl_opts, [ + warn_missing_spec + ]}, + {overrides, [ + {add, [ + {erl_opts, [compressed]} + ]} + ]} + ], + + rebar_test_utils:create_config(AppDir, RebarConfig), + + rebar_test_utils:run_and_check( + Config, RebarConfig, ["compile"], {ok, [{app, Name}]}), + + Path = filename:join([AppDir, "_build", "default", "lib", Name, "ebin"]), + code:add_patha(Path), + + Mod = list_to_atom("not_a_real_src_" ++ Name), + + true = lists:member(compressed, proplists:get_value(options, Mod:module_info(compile), [])), + true = lists:member(warn_missing_spec, proplists:get_value(options, Mod:module_info(compile), [])). + +override_del_opts(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("app1_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + RebarConfig = [ + {erl_opts, [ + compressed, + warn_missing_spec + ]}, + {overrides, [ + {del, [ + {erl_opts, [warn_missing_spec]} + ]} + ]} + ], + + rebar_test_utils:create_config(AppDir, RebarConfig), + + rebar_test_utils:run_and_check( + Config, RebarConfig, ["compile"], {ok, [{app, Name}]}), + + Path = filename:join([AppDir, "_build", "default", "lib", Name, "ebin"]), + code:add_patha(Path), + + Mod = list_to_atom("not_a_real_src_" ++ Name), + + true = lists:member(compressed, proplists:get_value(options, Mod:module_info(compile), [])), + false = lists:member(warn_missing_spec, proplists:get_value(options, Mod:module_info(compile), [])). + profile_override_deps(Config) -> - mock_git_resource:mock([{deps, [{some_dep, "0.0.1"},{other_dep, "0.0.1"}]}]), Deps = rebar_test_utils:expand_deps(git, [{"some_dep", "0.0.1", [{"other_dep", "0.0.1", []}]}]), TopDeps = rebar_test_utils:top_level_deps(Deps), + {SrcDeps, _} = rebar_test_utils:flat_deps(Deps), + mock_git_resource:mock([{deps, SrcDeps}]), + RebarConfig = [ {deps, TopDeps}, - {profiles, [{a, - [{overrides, [ - {override, some_dep, [ - {deps, []} - ]} - ]} + {profiles, [ + {a, [ + {overrides, [ + {override, some_dep, [ + {deps, []} ]} + ]} + ]} ]}], rebar_test_utils:run_and_check( Config, RebarConfig, ["as", "a", "compile"], - {ok, [{dep, "some_dep"},{dep_not_exist, "other_dep"}]} + {ok, [{dep, "some_dep"}, + {dep_not_exist, "other_dep"}]} + ). + +profile_override_add_deps(Config) -> + Deps = rebar_test_utils:expand_deps(git, [{"some_dep", "0.0.1", [{"other_dep", "0.0.1", []}]}]), + TopDeps = rebar_test_utils:top_level_deps(Deps), + + DepA = {dep_a, "0.0.1", {git, "http://site.com/dep_a.git", {tag, "0.0.1"}}}, + DepB = {dep_b, "0.0.1", {git, "http://site.com/dep_b.git", {tag, "0.0.1"}}}, + DepC = {dep_c, "0.0.1", {git, "http://site.com/dep_c.git", {tag, "0.0.1"}}}, + + {SrcDeps, _} = rebar_test_utils:flat_deps(Deps), + mock_git_resource:mock([{deps, [DepA, DepB, DepC | SrcDeps]}]), + + RebarConfig = [ + {deps, TopDeps}, + {profiles, [ + {a, [ + {overrides, [ + {add, some_dep, [ + {deps, [DepA, DepB]} + ]}, + {add, [ + {deps, [DepC]} + ]} + ]} + ]} + ]} + ], + rebar_test_utils:run_and_check( + Config, RebarConfig, ["as", "a", "compile"], + {ok, [{dep, "some_dep"}, + {dep, "other_dep"}, + {dep, "dep_a"}, + {dep, "dep_b"}, + {dep, "dep_c"}]} + ). + +profile_override_del_deps(Config) -> + Deps = rebar_test_utils:expand_deps(git, [{"some_dep", "0.0.1", [{"dep_a", "0.0.1", []}, + {"dep_b", "0.0.1", []}, + {"dep_c", "0.0.1", []}]}, + {"other_dep", "0.0.1", [{"dep_c", "0.0.1", []}, + {"dep_d", "0.0.1", []}]}]), + TopDeps = rebar_test_utils:top_level_deps(Deps), + + DepA = {dep_a, "0.0.1", {git, "https://example.org/user/dep_a.git", {tag, "0.0.1"}}}, + DepB = {dep_b, "0.0.1", {git, "https://example.org/user/dep_b.git", {tag, "0.0.1"}}}, + DepC = {dep_c, "0.0.1", {git, "https://example.org/user/dep_c.git", {tag, "0.0.1"}}}, + + {SrcDeps, _} = rebar_test_utils:flat_deps(Deps), + mock_git_resource:mock([{deps, SrcDeps}]), + + RebarConfig = [ + {deps, TopDeps}, + {profiles, [ + {a, [ + {overrides, [ + {del, some_dep, [ + {deps, [DepA, DepB]} + ]}, + {del, [ + {deps, [DepC]} + ]} + ]} + ]} + ]} + ], + rebar_test_utils:run_and_check( + Config, RebarConfig, ["as", "a", "compile"], + {ok, [{dep, "some_dep"}, + {dep, "other_dep"}, + {dep_not_exist, "dep_a"}, + {dep_not_exist, "dep_b"}, + {dep_not_exist, "dep_c"}, + {dep, "dep_d"}]} + ). + +profile_override_opts(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("app1_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + RebarConfig = [ + {erl_opts, [ + compressed, + warn_missing_spec + ]}, + {profiles, [ + {a, [ + {overrides, [ + {override, [ + {erl_opts, [compressed]} + ]} + ]} + ]} + ]} + ], + + rebar_test_utils:create_config(AppDir, RebarConfig), + + rebar_test_utils:run_and_check( + Config, RebarConfig, ["as", "a", "compile"], {ok, [{app, Name}]}), + + Path = filename:join([AppDir, "_build", "a", "lib", Name, "ebin"]), + code:add_patha(Path), + + Mod = list_to_atom("not_a_real_src_" ++ Name), + + true = lists:member(compressed, proplists:get_value(options, Mod:module_info(compile), [])), + false = lists:member(warn_missing_spec, proplists:get_value(options, Mod:module_info(compile), [])). + +profile_override_add_opts(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("app1_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + RebarConfig = [ + {erl_opts, [ + warn_missing_spec + ]}, + {profiles, [ + {a, [ + {overrides, [ + {add, [ + {erl_opts, [compressed]} + ]} + ]} + ]} + ]} + ], + + rebar_test_utils:create_config(AppDir, RebarConfig), + + rebar_test_utils:run_and_check( + Config, RebarConfig, ["as", "a", "compile"], {ok, [{app, Name}]}), + + Path = filename:join([AppDir, "_build", "a", "lib", Name, "ebin"]), + code:add_patha(Path), + + Mod = list_to_atom("not_a_real_src_" ++ Name), + + true = lists:member(compressed, proplists:get_value(options, Mod:module_info(compile), [])), + true = lists:member(warn_missing_spec, proplists:get_value(options, Mod:module_info(compile), [])). + +profile_override_del_opts(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("app1_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + RebarConfig = [ + {erl_opts, [ + compressed, + warn_missing_spec + ]}, + {profiles, [ + {a, [ + {overrides, [ + {del, [ + {erl_opts, [warn_missing_spec]} + ]} + ]} + ]} + ]} + ], + + rebar_test_utils:create_config(AppDir, RebarConfig), + + rebar_test_utils:run_and_check( + Config, RebarConfig, ["as", "a", "compile"], {ok, [{app, Name}]}), + + Path = filename:join([AppDir, "_build", "a", "lib", Name, "ebin"]), + code:add_patha(Path), + + Mod = list_to_atom("not_a_real_src_" ++ Name), + + true = lists:member(compressed, proplists:get_value(options, Mod:module_info(compile), [])), + false = lists:member(warn_missing_spec, proplists:get_value(options, Mod:module_info(compile), [])). + +profile_deps(Config) -> + Deps = rebar_test_utils:expand_deps(git, [{"some_dep", "0.0.1", [{"other_dep", "0.0.1", []}]}]), + TopDeps = rebar_test_utils:top_level_deps(Deps), + {SrcDeps, _} = rebar_test_utils:flat_deps(Deps), + mock_git_resource:mock([{deps, SrcDeps}]), + + RebarConfig = [ + {deps, TopDeps}, + {profiles, [{a, []}]}], + rebar_test_utils:run_and_check( + Config, RebarConfig, ["as", "a", "compile"], + {ok, [{dep, "some_dep"},{dep, "other_dep"}]} + ). + +only_deps(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("app1_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + Deps = rebar_test_utils:expand_deps(git, [{"some_dep", "0.0.1", [{"other_dep", "0.0.1", []}]}]), + TopDeps = rebar_test_utils:top_level_deps(Deps), + {SrcDeps, _} = rebar_test_utils:flat_deps(Deps), + mock_git_resource:mock([{deps, SrcDeps}]), + + RConfFile = rebar_test_utils:create_config(AppDir, [{deps, TopDeps}]), + {ok, RConf} = file:consult(RConfFile), + rebar_test_utils:run_and_check( + Config, RConf, ["compile", "--deps_only"], + {ok, [{app_not_exist, Name}, {dep, "some_dep"},{dep, "other_dep"}]} ). %% verify a deps prod profile is used @@ -1310,7 +1848,151 @@ include_file_in_src(Config) -> ["compile"], {ok, [{app, Name}]}). -always_recompile_when_erl_compiler_options_set(Config) -> +%% verify that the proper include path is defined +%% according the erlang doc which states: +%% If the filename File is absolute (possibly after variable substitution), +%% the include file with that name is included. Otherwise, the specified file +%% is searched for in the following directories, and in this order: +%% * The current working directory +%% * The directory where the module is being compiled +%% * The directories given by the include option +%% +%% This test ensures that things keep working when additional directories +%% are used for apps, such as the test/ directory within the test profile. +include_file_relative_to_working_directory_test(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("app1_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + Src = <<"-module(test).\n" +"\n" +"-include(\"include/test.hrl\").\n" +"\n" +"test() -> ?TEST_MACRO.\n" +"\n">>, + Include = <<"-define(TEST_MACRO, test).\n">>, + + ok = filelib:ensure_dir(filename:join([AppDir, "src", "dummy"])), + ok = filelib:ensure_dir(filename:join([AppDir, "test", "dummy"])), + ok = file:write_file(filename:join([AppDir, "test", "test.erl"]), Src), + + ok = filelib:ensure_dir(filename:join([AppDir, "include", "dummy"])), + ok = file:write_file(filename:join([AppDir, "include", "test.hrl"]), Include), + + RebarConfig = [], + rebar_test_utils:run_and_check(Config, RebarConfig, + ["as", "test", "compile"], + {ok, [{app, Name}]}). + +%% Same as `include_file_in_src/1' but using the `test/' directory +%% within the test profile. +include_file_in_src_test(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("app1_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + Src = <<"-module(test).\n" +"\n" +"-include(\"test.hrl\").\n" +"\n" +"test() -> ?TEST_MACRO.\n" +"\n">>, + Include = <<"-define(TEST_MACRO, test).\n">>, + + ok = filelib:ensure_dir(filename:join([AppDir, "src", "dummy"])), + ok = filelib:ensure_dir(filename:join([AppDir, "test", "dummy"])), + ok = file:write_file(filename:join([AppDir, "test", "test.erl"]), Src), + + ok = file:write_file(filename:join([AppDir, "src", "test.hrl"]), Include), + + RebarConfig = [], + rebar_test_utils:run_and_check(Config, RebarConfig, + ["as", "test", "compile"], + {ok, [{app, Name}]}). + +%% Same as `include_file_in_src_test/1' but using multiple top-level +%% apps as dependencies. +include_file_in_src_test_multiapp(Config) -> + + Name1 = rebar_test_utils:create_random_name("app2_"), + Name2 = rebar_test_utils:create_random_name("app1_"), + AppDir1 = filename:join([?config(apps, Config), "lib", Name1]), + AppDir2 = filename:join([?config(apps, Config), "lib", Name2]), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir1, Name1, Vsn, [kernel, stdlib, list_to_atom(Name2)]), + rebar_test_utils:create_app(AppDir2, Name2, Vsn, [kernel, stdlib]), + + Src = "-module(test).\n" +"\n" +"-include_lib(\"" ++ Name2 ++ "/include/test.hrl\").\n" +"\n" +"test() -> ?TEST_MACRO.\n" +"\n", + Include = <<"-define(TEST_MACRO, test).\n">>, + + ok = filelib:ensure_dir(filename:join([AppDir1, "src", "dummy"])), + ok = filelib:ensure_dir(filename:join([AppDir1, "test", "dummy"])), + ok = filelib:ensure_dir(filename:join([AppDir2, "src", "dummy"])), + ok = filelib:ensure_dir(filename:join([AppDir2, "include", "dummy"])), + ok = file:write_file(filename:join([AppDir1, "test", "test.erl"]), Src), + + ok = file:write_file(filename:join([AppDir2, "include", "test.hrl"]), Include), + + RebarConfig = [], + rebar_test_utils:run_and_check(Config, RebarConfig, + ["as", "test", "compile"], + {ok, [{app, Name1}]}), + ok. + +%% this test sets the env var, compiles, records the file last modified timestamp, +%% recompiles and compares the file last modified timestamp to ensure it hasn't +%% changed. this test should run on 19.x+ +dont_recompile_when_erl_compiler_options_env_does_not_change(Config) -> + %% save existing env to restore after test + ExistingEnv = os:getenv("ERL_COMPILER_OPTIONS"), + + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("erl_compiler_options_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + true = os:unsetenv("ERL_COMPILER_OPTIONS"), + + true = os:putenv("ERL_COMPILER_OPTIONS", "[{d, some_macro}]"), + + rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name}]}), + + EbinDir = filename:join([AppDir, "_build", "default", "lib", Name, "ebin"]), + + {ok, Files} = rebar_utils:list_dir(EbinDir), + ModTime = [filelib:last_modified(filename:join([EbinDir, F])) + || F <- Files, filename:extension(F) == ".beam"], + + timer:sleep(1000), + + rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name}]}), + + {ok, NewFiles} = rebar_utils:list_dir(EbinDir), + NewModTime = [filelib:last_modified(filename:join([EbinDir, F])) + || F <- NewFiles, filename:extension(F) == ".beam"], + + ?assert(ModTime == NewModTime), + + %% restore existing env + case ExistingEnv of + false -> ok; + _ -> os:putenv("ERL_COMPILER_OPTIONS", ExistingEnv) + end. + +%% this test compiles, records the file last modified timestamp, sets the env +%% var, recompiles and compares the file last modified timestamp to ensure it +%% has changed. this test should run on 19.x+ +recompile_when_erl_compiler_options_env_changes(Config) -> %% save existing env to restore after test ExistingEnv = os:getenv("ERL_COMPILER_OPTIONS"), @@ -1348,5 +2030,381 @@ always_recompile_when_erl_compiler_options_set(Config) -> _ -> os:putenv("ERL_COMPILER_OPTIONS", ExistingEnv) end. +rebar_config_os_var(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("rebar_config_os_var_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + rebar_test_utils:create_config(AppDir, [{erl_opts, []}]), + + AltConfig = filename:join(AppDir, "test.rebar.config"), + file:write_file(AltConfig, "{erl_opts, [compressed]}."), + true = os:putenv("REBAR_CONFIG", AltConfig), + + rebar_test_utils:run_and_check(Config, ["compile"], {ok, [{app, Name}]}), + + Path = filename:join([AppDir, "_build", "default", "lib", Name, "ebin"]), + code:add_patha(Path), + + Mod = list_to_atom("not_a_real_src_" ++ Name), + + true = lists:member(compressed, proplists:get_value(options, Mod:module_info(compile), [])), + ok. + +%% this test sets the env var, compiles, records the file last modified +%% timestamp, recompiles and compares the file last modified timestamp to +%% ensure it has changed. this test should run on 18.x +always_recompile_when_erl_compiler_options_set(Config) -> + %% save existing env to restore after test + ExistingEnv = os:getenv("ERL_COMPILER_OPTIONS"), + + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("erl_compiler_options_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + true = os:unsetenv("ERL_COMPILER_OPTIONS"), + + true = os:putenv("ERL_COMPILER_OPTIONS", "[{d, some_macro}]"), + + rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name}]}), + + EbinDir = filename:join([AppDir, "_build", "default", "lib", Name, "ebin"]), + + {ok, Files} = rebar_utils:list_dir(EbinDir), + ModTime = [filelib:last_modified(filename:join([EbinDir, F])) + || F <- Files, filename:extension(F) == ".beam"], + + timer:sleep(1000), + + rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name}]}), + + {ok, NewFiles} = rebar_utils:list_dir(EbinDir), + NewModTime = [filelib:last_modified(filename:join([EbinDir, F])) + || F <- NewFiles, filename:extension(F) == ".beam"], + + ?assert(ModTime =/= NewModTime), + + %% restore existing env + case ExistingEnv of + false -> ok; + _ -> os:putenv("ERL_COMPILER_OPTIONS", ExistingEnv) + end. + +recompile_when_parse_transform_inline_changes(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("parse_transform_inline_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + ok = filelib:ensure_dir(filename:join([AppDir, "src", "dummy"])), + + ModSrc = <<"-module(example).\n" + "-export([foo/2]).\n" + "-compile([{parse_transform, example_parse_transform}]).\n" + "foo(_, _) -> ok.">>, + + ok = file:write_file(filename:join([AppDir, "src", "example.erl"]), + ModSrc), + + ParseTransform = <<"-module(example_parse_transform).\n" + "-export([parse_transform/2]).\n" + "parse_transform(AST, _) -> AST.\n">>, + + ok = file:write_file(filename:join([AppDir, "src", "example_parse_transform.erl"]), + ParseTransform), + + rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name}]}), + + EbinDir = filename:join([AppDir, "_build", "default", "lib", Name, "ebin"]), + {ok, Files} = rebar_utils:list_dir(EbinDir), + ModTime = [filelib:last_modified(filename:join([EbinDir, F])) + || F <- Files, filename:basename(F, ".beam") == "example"], + + timer:sleep(1000), + + NewParseTransform = <<"-module(example_parse_transform).\n" + "-export([parse_transform/2]).\n" + "parse_transform(AST, _) -> identity(AST).\n" + "identity(AST) -> AST.\n">>, + + ok = file:write_file(filename:join([AppDir, "src", "example_parse_transform.erl"]), + NewParseTransform), + + rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name}]}), + + {ok, NewFiles} = rebar_utils:list_dir(EbinDir), + NewModTime = [filelib:last_modified(filename:join([EbinDir, F])) + || F <- NewFiles, filename:basename(F, ".beam") == "example"], + + ?assert(ModTime =/= NewModTime). + +recompile_when_parse_transform_as_opt_changes(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("parse_transform_opt_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + ok = filelib:ensure_dir(filename:join([AppDir, "src", "dummy"])), + + ModSrc = <<"-module(example).\n" + "-export([foo/2]).\n" + "foo(_, _) -> ok.">>, + ok = file:write_file(filename:join([AppDir, "src", "example.erl"]), + ModSrc), + + ParseTransform = <<"-module(example_parse_transform).\n" + "-export([parse_transform/2]).\n" + "parse_transform(AST, _) -> AST.">>, + + ok = file:write_file(filename:join([AppDir, "src", "example_parse_transform.erl"]), + ParseTransform), + + RebarConfig = [{erl_opts, [{parse_transform, example_parse_transform}]}], + + rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], {ok, [{app, Name}]}), + + EbinDir = filename:join([AppDir, "_build", "default", "lib", Name, "ebin"]), + {ok, Files} = rebar_utils:list_dir(EbinDir), + ModTime = [filelib:last_modified(filename:join([EbinDir, F])) + || F <- Files, filename:basename(F, ".beam") == "example"], + + timer:sleep(1000), + + NewParseTransform = <<"-module(example_parse_transform).\n" + "-export([parse_transform/2]).\n" + "parse_transform(AST, _) -> identity(AST).\n" + "identity(AST) -> AST.">>, + + ok = file:write_file(filename:join([AppDir, "src", "example_parse_transform.erl"]), + NewParseTransform), + + rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], {ok, [{app, Name}]}), + + {ok, NewFiles} = rebar_utils:list_dir(EbinDir), + NewModTime = [filelib:last_modified(filename:join([EbinDir, F])) + || F <- NewFiles, filename:basename(F, ".beam") == "example"], + + ?assert(ModTime =/= NewModTime). + +recursive(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("app1_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + rebar_test_utils:write_src_file(filename:join(AppDir,src),"rec.erl"), + + rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name}]}), + + EbinDir = filename:join([AppDir, "_build", "default", "lib", Name, "ebin"]), + {ok, Files} = rebar_utils:list_dir(EbinDir), + ?assert(lists:member("rec.beam",Files)), + + %% check that rec is in modules list of .app file + AppFile = filename:join(EbinDir, Name++".app"), + {ok, [{application, _, List}]} = file:consult(AppFile), + {modules, Modules} = lists:keyfind(modules, 1, List), + ?assert(lists:member(rec, Modules)). + +no_recursive(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("app1_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + rebar_test_utils:write_src_file(filename:join(AppDir,src),"rec.erl"), + + RebarConfig1 = [{erlc_compiler,[{recursive,false}]}], + rebar_test_utils:run_and_check(Config, RebarConfig1, ["compile"], + {ok, [{app, Name}]}), + EbinDir = filename:join([AppDir, "_build", "default", "lib", Name, "ebin"]), + {ok, Files1} = rebar_utils:list_dir(EbinDir), + ?assert(false==lists:member("rec.beam",Files1)), + + RebarConfig2 = [{src_dirs,[{"src",[{recursive,false}]}]}], + rebar_test_utils:run_and_check(Config, RebarConfig2, ["compile"], + {ok, [{app, Name}]}), + {ok, Files2} = rebar_utils:list_dir(EbinDir), + ?assert(false==lists:member("rec.beam",Files2)), + ok. + +regex_filter_skip(Config) -> + AppDir = ?config(apps, Config), + Name = rebar_test_utils:create_random_name("regex_skip"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + rebar_test_utils:write_src_file(filename:join(AppDir,src),"._rec.erl"), + Expected = filename:join([AppDir, "_build", "default", "lib", Name, "ebin","._rec.beam"]), + + RebarConfig = [], + try + rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], + {ok, [{file, Expected}]}), + throw(should_not_be_found) + catch + %% the file was not found, as desired! + error:{assertion_failed,_} -> %% OTP =< 17 + ok; + error:{assert,_} -> %% OTP >= 18 + ok + end. + +regex_filter_regression(Config) -> + AppDir = ?config(apps, Config), + Name = rebar_test_utils:create_random_name("regex_regression"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + rebar_test_utils:write_src_file(filename:join(AppDir,src),"r_f.erl"), + Expected = filename:join([AppDir, "_build", "default", "lib", Name, "ebin","r_f.beam"]), + RebarConfig = [], + rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], + {ok, [{file, Expected}]}), + ok. + +%% + +%% a copy of lib/parsetools/include/yeccpre.hrl so we can test yrl includefile +yeccpre_hrl() -> + <<"-type yecc_ret() :: {'error', _} | {'ok', _}. + +-spec parse(Tokens :: list()) -> yecc_ret(). +parse(Tokens) -> + yeccpars0(Tokens, {no_func, no_line}, 0, [], []). + +-spec parse_and_scan({function() | {atom(), atom()}, [_]} + | {atom(), atom(), [_]}) -> yecc_ret(). +parse_and_scan({F, A}) -> + yeccpars0([], {{F, A}, no_line}, 0, [], []); +parse_and_scan({M, F, A}) -> + Arity = length(A), + yeccpars0([], {{fun M:F/Arity, A}, no_line}, 0, [], []). + +-spec format_error(any()) -> [char() | list()]. +format_error(Message) -> + case io_lib:deep_char_list(Message) of + true -> + Message; + _ -> + io_lib:write(Message) + end. + +%% To be used in grammar files to throw an error message to the parser +%% toplevel. Doesn't have to be exported! +-compile({nowarn_unused_function, return_error/2}). +-spec return_error(integer(), any()) -> no_return(). +return_error(Line, Message) -> + throw({error, {Line, ?MODULE, Message}}). + +-define(CODE_VERSION, \"1.4\"). + +yeccpars0(Tokens, Tzr, State, States, Vstack) -> + try yeccpars1(Tokens, Tzr, State, States, Vstack) + catch + error:Error -> + try yecc_error_type(Error, []) of + Desc -> + erlang:raise(error, {yecc_bug, ?CODE_VERSION, Desc}, + []) + catch _:_ -> erlang:raise(error, Error, []) + end; + %% Probably thrown from return_error/2: + throw: {error, {_Line, ?MODULE, _M}} = Error -> + Error + end. + +yecc_error_type(function_clause, _) -> + not_implemented. + +yeccpars1([Token | Tokens], Tzr, State, States, Vstack) -> + yeccpars2(State, element(1, Token), States, Vstack, Token, Tokens, Tzr); +yeccpars1([], {{F, A},_Line}, State, States, Vstack) -> + case apply(F, A) of + {ok, Tokens, Endline} -> + yeccpars1(Tokens, {{F, A}, Endline}, State, States, Vstack); + {eof, Endline} -> + yeccpars1([], {no_func, Endline}, State, States, Vstack); + {error, Descriptor, _Endline} -> + {error, Descriptor} + end; +yeccpars1([], {no_func, no_line}, State, States, Vstack) -> + Line = 999999, + yeccpars2(State, '$end', States, Vstack, yecc_end(Line), [], + {no_func, Line}); +yeccpars1([], {no_func, Endline}, State, States, Vstack) -> + yeccpars2(State, '$end', States, Vstack, yecc_end(Endline), [], + {no_func, Endline}). + +%% yeccpars1/7 is called from generated code. +%% +%% When using the {includefile, Includefile} option, make sure that +%% yeccpars1/7 can be found by parsing the file without following +%% include directives. yecc will otherwise assume that an old +%% yeccpre.hrl is included (one which defines yeccpars1/5). +yeccpars1(State1, State, States, Vstack, Token0, [Token | Tokens], Tzr) -> + yeccpars2(State, element(1, Token), [State1 | States], + [Token0 | Vstack], Token, Tokens, Tzr); +yeccpars1(State1, State, States, Vstack, Token0, [], {{_F,_A}, _Line}=Tzr) -> + yeccpars1([], Tzr, State, [State1 | States], [Token0 | Vstack]); +yeccpars1(State1, State, States, Vstack, Token0, [], {no_func, no_line}) -> + Line = yecctoken_end_location(Token0), + yeccpars2(State, '$end', [State1 | States], [Token0 | Vstack], + yecc_end(Line), [], {no_func, Line}); +yeccpars1(State1, State, States, Vstack, Token0, [], {no_func, Line}) -> + yeccpars2(State, '$end', [State1 | States], [Token0 | Vstack], + yecc_end(Line), [], {no_func, Line}). + +%% For internal use only. +yecc_end({Line,_Column}) -> + {'$end', Line}; +yecc_end(Line) -> + {'$end', Line}. + +yecctoken_end_location(Token) -> + try erl_anno:end_location(element(2, Token)) of + undefined -> yecctoken_location(Token); + Loc -> Loc + catch _:_ -> yecctoken_location(Token) + end. + +-compile({nowarn_unused_function, yeccerror/1}). +yeccerror(Token) -> + Text = yecctoken_to_string(Token), + Location = yecctoken_location(Token), + {error, {Location, ?MODULE, [\"syntax error before: \", Text]}}. + +-compile({nowarn_unused_function, yecctoken_to_string/1}). +yecctoken_to_string(Token) -> + try erl_scan:text(Token) of + undefined -> yecctoken2string(Token); + Txt -> Txt + catch _:_ -> yecctoken2string(Token) + end. + +yecctoken_location(Token) -> + try erl_scan:location(Token) + catch _:_ -> element(2, Token) + end. +-compile({nowarn_unused_function, yecctoken2string/1}). +yecctoken2string({atom, _, A}) -> io_lib:write_atom(A); +yecctoken2string({integer,_,N}) -> io_lib:write(N); +yecctoken2string({float,_,F}) -> io_lib:write(F); +yecctoken2string({char,_,C}) -> io_lib:write_char(C); +yecctoken2string({var,_,V}) -> io_lib:format(\"~s\", [V]); +yecctoken2string({string,_,S}) -> io_lib:write_string(S); +yecctoken2string({reserved_symbol, _, A}) -> io_lib:write(A); +yecctoken2string({_Cat, _, Val}) -> io_lib:format(\"~p\", [Val]); +yecctoken2string({dot, _}) -> \"'.'\"; +yecctoken2string({'$end', _}) -> []; +yecctoken2string({Other, _}) when is_atom(Other) -> + io_lib:write_atom(Other); +yecctoken2string(Other) -> + io_lib:format(\"~p\", [Other]). +">>. diff --git a/test/rebar_cover_SUITE.erl b/test/rebar_cover_SUITE.erl index 841e29f..8d6429d 100644 --- a/test/rebar_cover_SUITE.erl +++ b/test/rebar_cover_SUITE.erl @@ -7,12 +7,17 @@ all/0, flag_coverdata_written/1, config_coverdata_written/1, + config_coverdata_overridden_name_written/1, basic_extra_src_dirs/1, release_extra_src_dirs/1, root_extra_src_dirs/1, index_written/1, flag_verbose/1, - config_verbose/1]). + config_verbose/1, + excl_mods_and_apps/1, + coverdata_is_reset_on_write/1, + flag_min_coverage/1, + config_min_coverage/1]). -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -32,10 +37,13 @@ init_per_testcase(_, Config) -> all() -> [flag_coverdata_written, config_coverdata_written, + config_coverdata_overridden_name_written, basic_extra_src_dirs, release_extra_src_dirs, root_extra_src_dirs, index_written, - flag_verbose, config_verbose]. + flag_verbose, config_verbose, + excl_mods_and_apps, coverdata_is_reset_on_write, + flag_min_coverage, config_min_coverage]. flag_coverdata_written(Config) -> AppDir = ?config(apps, Config), @@ -67,6 +75,21 @@ config_coverdata_written(Config) -> true = filelib:is_file(filename:join([AppDir, "_build", "test", "cover", "eunit.coverdata"])). +config_coverdata_overridden_name_written(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("cover_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_eunit_app(AppDir, Name, Vsn, [kernel, stdlib]), + + RebarConfig = [{erl_opts, [{d, some_define}]}, {cover_enabled, true}], + rebar_test_utils:run_and_check(Config, + RebarConfig, + ["eunit", "--cover_export_name=test_name"], + {ok, [{app, Name}]}), + + true = filelib:is_file(filename:join([AppDir, "_build", "test", "cover", "test_name.coverdata"])). + basic_extra_src_dirs(Config) -> AppDir = ?config(apps, Config), @@ -206,3 +229,94 @@ config_verbose(Config) -> {ok, [{app, Name}]}), true = filelib:is_file(filename:join([AppDir, "_build", "test", "cover", "index.html"])). + +excl_mods_and_apps(Config) -> + AppDir = ?config(apps, Config), + + Name1 = rebar_test_utils:create_random_name("relapp1_"), + Vsn1 = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(filename:join([AppDir, "apps", Name1]), Name1, Vsn1, [kernel, stdlib]), + + Name2 = rebar_test_utils:create_random_name("relapp2_"), + Vsn2 = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(filename:join([AppDir, "apps", Name2]), Name2, Vsn2, [kernel, stdlib]), + + Name3 = rebar_test_utils:create_random_name("excludeme_"), + Vsn3 = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(filename:join([AppDir, "apps", Name3]), Name3, Vsn3, [kernel, stdlib]), + + Mod1 = list_to_atom(Name1), + Mod2 = list_to_atom(Name2), + Mod3 = list_to_atom(Name3), + RebarConfig = [{erl_opts, [{d, some_define}]}, + {cover_excl_mods, [Mod2]}, + {cover_excl_apps, [Name3]}], + + rebar_test_utils:run_and_check(Config, + RebarConfig, + ["eunit", "--cover"], + {ok, [{app, Name1}, {app, Name2}, {app, Name3}]}), + + {file, _} = cover:is_compiled(Mod1), + false = cover:is_compiled(Mod2), + false = cover:is_compiled(Mod3). + +coverdata_is_reset_on_write(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("coverdata_is_reset_on_write_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_eunit_app(AppDir, Name, Vsn, [kernel, stdlib]), + + RebarConfig = [{erl_opts, [{d, some_define}]}, {cover_enabled, true}], + rebar_test_utils:run_and_check(Config, + RebarConfig, + ["eunit"], + {ok, [{app, Name}]}), + + Res = lists:map(fun(M) -> cover:analyse(M) end, cover:modules()), + Ok = lists:foldl(fun({ok, R}, Acc) -> R ++ Acc end, [], Res), + [] = lists:filter(fun({_, {0,_}}) -> false; (_) -> true end, Ok). + +flag_min_coverage(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("min_cover_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_eunit_app(AppDir, Name, Vsn, [kernel, stdlib]), + + RebarConfig = [{erl_opts, [{d, some_define}]}], + ?assertMatch({ok, _}, + rebar_test_utils:run_and_check( + Config, RebarConfig, + ["do", "eunit", "--cover", ",", "cover", "--min_coverage=5"], + return)), + + ?assertMatch({error,{rebar_prv_cover,{min_coverage_failed,{65,_}}}}, + rebar_test_utils:run_and_check( + Config, RebarConfig, + ["do", "eunit", "--cover", ",", "cover", "--min_coverage=65"], + return)), + ok. + +config_min_coverage(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("cover_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_eunit_app(AppDir, Name, Vsn, [kernel, stdlib]), + + RebarConfig1 = [{erl_opts, [{d, some_define}]}, {cover_opts, [{min_coverage,5}]}], + ?assertMatch({ok, _}, + rebar_test_utils:run_and_check( + Config, RebarConfig1, + ["do", "eunit", "--cover", ",", "cover"], + return)), + + RebarConfig2 = [{erl_opts, [{d, some_define}]}, {cover_opts, [{min_coverage,65}]}], + ?assertMatch({error,{rebar_prv_cover,{min_coverage_failed,{65,_}}}}, + rebar_test_utils:run_and_check( + Config, RebarConfig2, + ["do", "eunit", "--cover", ",", "cover"], + return)), + ok. diff --git a/test/rebar_ct_SUITE.erl b/test/rebar_ct_SUITE.erl index c10875b..15dc63e 100644 --- a/test/rebar_ct_SUITE.erl +++ b/test/rebar_ct_SUITE.erl @@ -24,6 +24,8 @@ data_dir_correct/1, cmd_label/1, cmd_config/1, + cmd_spec/1, + cmd_join_specs/1, cmd_allow_user_terms/1, cmd_logdir/1, cmd_logopts/1, @@ -44,13 +46,19 @@ cmd_sys_config/1, cfg_opts/1, cfg_arbitrary_opts/1, - cfg_test_spec/1, cfg_cover_spec/1, cfg_atom_suites/1, cover_compiled/1, + cover_export_name/1, misspecified_ct_opts/1, misspecified_ct_compile_opts/1, - misspecified_ct_first_files/1]). + misspecified_ct_first_files/1, + testspec/1, + testspec_at_root/1, + testspec_parse_error/1, + cmd_vs_cfg_opts/1, + single_testspec_in_ct_opts/1, + compile_only/1]). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -62,11 +70,17 @@ all() -> [{group, basic_app}, {group, ct_opts}, {group, cover}, cfg_opts, cfg_arbitrary_opts, - cfg_test_spec, cfg_cover_spec, + cfg_cover_spec, cfg_atom_suites, misspecified_ct_opts, misspecified_ct_compile_opts, - misspecified_ct_first_files]. + misspecified_ct_first_files, + testspec, + testspec_at_root, + testspec_parse_error, + cmd_vs_cfg_opts, + single_testspec_in_ct_opts, + compile_only]. groups() -> [{basic_app, [], [basic_app_default_dirs, basic_app_default_beams, @@ -88,6 +102,8 @@ groups() -> [{basic_app, [], [basic_app_default_dirs, {data_dirs, [], [data_dir_correct]}, {ct_opts, [], [cmd_label, cmd_config, + cmd_spec, + cmd_join_specs, cmd_allow_user_terms, cmd_logdir, cmd_logopts, @@ -106,7 +122,8 @@ groups() -> [{basic_app, [], [basic_app_default_dirs, cmd_create_priv_dir, cmd_include_dir, cmd_sys_config]}, - {cover, [], [cover_compiled]}]. + {cover, [], [cover_compiled, + cover_export_name]}]. init_per_group(basic_app, Config) -> C = rebar_test_utils:init_rebar_state(Config, "ct_"), @@ -686,7 +703,33 @@ suite_at_root(Config) -> true = filelib:is_dir(DataDir), DataFile = filename:join([AppDir, "_build", "test", "extras", "root_SUITE_data", "some_data.txt"]), - true = filelib:is_file(DataFile). + true = filelib:is_file(DataFile), + + %% Same test again, but using relative path to the suite from the + %% project root + {ok,Cwd} = file:get_cwd(), + ok = file:set_cwd(AppDir), + rebar_file_utils:rm_rf("_build"), + + {ok, GetOptResult2} = getopt:parse(GetOptSpec, ["--suite=" ++ "root_SUITE"]), + + State3 = rebar_state:command_parsed_args(State1, GetOptResult2), + + Tests2 = rebar_prv_common_test:prepare_tests(State3), + {ok, NewState2} = rebar_prv_common_test:compile(State3, Tests2), + {ok, T2} = Tests2, + Opts2 = rebar_prv_common_test:translate_paths(NewState2, T2), + + ok = file:set_cwd(Cwd), + + Suite2 = proplists:get_value(suite, Opts2), + [Expected] = Suite2, + true = filelib:is_file(TestHrl), + true = filelib:is_file(TestBeam), + true = filelib:is_dir(DataDir), + true = filelib:is_file(DataFile), + + ok. suite_at_app_root(Config) -> AppDir = ?config(apps, Config), @@ -723,7 +766,32 @@ suite_at_app_root(Config) -> true = filelib:is_dir(DataDir), DataFile = filename:join([AppDir, "_build", "test", "lib", Name2, "app_root_SUITE_data", "some_data.txt"]), - true = filelib:is_file(DataFile). + true = filelib:is_file(DataFile), + + %% Same test again using relative path to the suite from the project root + {ok,Cwd} = file:get_cwd(), + ok = file:set_cwd(AppDir), + rebar_file_utils:rm_rf("_build"), + + {ok, GetOptResult2} = getopt:parse(GetOptSpec, ["--suite=" ++ filename:join(["apps", Name2, "app_root_SUITE"])]), + + State3 = rebar_state:command_parsed_args(State1, GetOptResult2), + + Tests2 = rebar_prv_common_test:prepare_tests(State3), + {ok, NewState2} = rebar_prv_common_test:compile(State3, Tests2), + {ok, T2} = Tests2, + Opts2 = rebar_prv_common_test:translate_paths(NewState2, T2), + + ok = file:set_cwd(Cwd), + + Suite2 = proplists:get_value(suite, Opts2), + [Expected] = Suite2, + true = filelib:is_file(TestHrl), + true = filelib:is_file(TestBeam), + true = filelib:is_dir(DataDir), + true = filelib:is_file(DataFile), + + ok. %% this test probably only fails when this suite is run via rebar3 with the --cover flag data_dir_correct(Config) -> @@ -761,6 +829,36 @@ cmd_config(Config) -> true = lists:member({config, ["config/foo", "config/bar", "config/baz"]}, TestOpts). +cmd_spec(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--spec=foo.spec,bar.spec,baz.spec"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({spec, ["foo.spec", "bar.spec", "baz.spec"]}, TestOpts). + +cmd_join_specs(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--join_specs=true"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState), + + true = lists:member({join_specs, true}, TestOpts). + cmd_allow_user_terms(Config) -> State = ?config(result, Config), @@ -1036,12 +1134,19 @@ cmd_sys_config(Config) -> CfgFile = filename:join([AppDir, "config", "cfg_sys.config"]), ok = filelib:ensure_dir(CfgFile), ok = file:write_file(CfgFile, cfg_sys_config_file(AppName)), + + OtherCfgFile = filename:join([AppDir, "config", "other.config"]), + ok = filelib:ensure_dir(OtherCfgFile), + ok = file:write_file(OtherCfgFile, other_sys_config_file(AppName)), + RebarConfig = [{ct_opts, [{sys_config, CfgFile}]}], {ok, State1} = rebar_test_utils:run_and_check(Config, RebarConfig, ["as", "test", "lock"], return), {ok, _} = rebar_prv_common_test:prepare_tests(State1), ?assertEqual({ok, cfg_value}, application:get_env(AppName, key)), + ?assertEqual({ok, other_cfg_value}, application:get_env(AppName, other_key)), + Providers = rebar_state:providers(State1), Namespace = rebar_state:namespace(State1), CommandProvider = providers:get_provider(ct, Providers, Namespace), @@ -1096,23 +1201,6 @@ cfg_arbitrary_opts(Config) -> true = lists:member({bar, 2}, TestOpts), true = lists:member({baz, 3}, TestOpts). -cfg_test_spec(Config) -> - C = rebar_test_utils:init_rebar_state(Config, "ct_cfg_test_spec_opts_"), - - AppDir = ?config(apps, C), - - Name = rebar_test_utils:create_random_name("ct_cfg_test_spec_opts_"), - Vsn = rebar_test_utils:create_random_vsn(), - rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), - - RebarConfig = [{ct_opts, [Opt = {test_spec, "spec/foo.spec"}]}], - - {ok, State} = rebar_test_utils:run_and_check(C, RebarConfig, ["as", "test", "lock"], return), - - {ok, TestOpts} = rebar_prv_common_test:prepare_tests(State), - - false = lists:member(Opt, TestOpts). - cfg_cover_spec(Config) -> C = rebar_test_utils:init_rebar_state(Config, "ct_cfg_cover_spec_opts_"), @@ -1165,6 +1253,29 @@ cover_compiled(Config) -> Mod = list_to_atom(Name), {file, _} = cover:is_compiled(Mod). +cover_export_name(Config) -> + State = ?config(result, Config), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--cover", "--cover_export_name=export_name"]), + + NewState = rebar_state:command_parsed_args(State, GetOptResult), + + Tests = rebar_prv_common_test:prepare_tests(NewState), + {ok, _} = rebar_prv_common_test:compile(NewState, Tests), + rebar_prv_common_test:maybe_write_coverdata(NewState), + + Name = ?config(name, Config), + Mod = list_to_atom(Name), + {file, _} = cover:is_compiled(Mod), + + Dir = rebar_dir:profile_dir(rebar_state:opts(NewState), [default, test]), + ct:pal("DIR ~s", [Dir]), + true = filelib:is_file(filename:join([Dir, "cover", "export_name.coverdata"])). + misspecified_ct_opts(Config) -> C = rebar_test_utils:init_rebar_state(Config, "ct_cfg_atom_suites_"), @@ -1218,6 +1329,312 @@ misspecified_ct_first_files(Config) -> {badconfig, {"Value `~p' of option `~p' must be a list", {some_file, ct_first_files}}} = Error. +testspec(Config) -> + C = rebar_test_utils:init_rebar_state(Config, "ct_testspec_"), + + AppDir = ?config(apps, C), + + Name = rebar_test_utils:create_random_name("ct_testspec_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + Spec1 = filename:join([AppDir, "test", "some.spec"]), + ok = filelib:ensure_dir(Spec1), + ok = file:write_file(Spec1, "{suites,\".\",all}.\n"), + Spec2 = filename:join([AppDir, "specs", "another.spec"]), + ok = filelib:ensure_dir(Spec2), + Suites2 = filename:join([AppDir,"suites","*"]), + ok = filelib:ensure_dir(Suites2), + ok = file:write_file(Spec2, "{suites,\"../suites/\",all}.\n"), + + {ok,Wd} = file:get_cwd(), + ok = file:set_cwd(AppDir), + + {ok, State} = rebar_test_utils:run_and_check(C, + [], + ["as", "test", "lock"], + return), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + + %% Testspec in "test" directory + {ok, GetOptResult1} = getopt:parse(GetOptSpec, ["--spec","test/some.spec"]), + State1 = rebar_state:command_parsed_args(State, GetOptResult1), + Tests1 = rebar_prv_common_test:prepare_tests(State1), + {ok, NewState1} = rebar_prv_common_test:compile(State1, Tests1), + {ok, T1} = Tests1, + Opts1= rebar_prv_common_test:translate_paths(NewState1, T1), + + %% check that extra src dir is added + [App1] = rebar_state:project_apps(NewState1), + ["test"] = rebar_dir:extra_src_dirs(rebar_app_info:opts(App1)), + + %% check that path is translated + ExpectedSpec1 = filename:join([AppDir, "_build", "test", "lib", Name, + "test", "some.spec"]), + [ExpectedSpec1] = proplists:get_value(spec, Opts1), + + + %% Testspec in directory other than "test" + {ok, GetOptResult2} = getopt:parse(GetOptSpec, + ["--spec","specs/another.spec"]), + State2 = rebar_state:command_parsed_args(State, GetOptResult2), + Tests2 = {ok, T2} =rebar_prv_common_test:prepare_tests(State2), + {ok, NewState2} = rebar_prv_common_test:compile(State2, Tests2), + Opts2= rebar_prv_common_test:translate_paths(NewState2, T2), + + %% check that extra src dirs are added + [App2] = rebar_state:project_apps(NewState2), + ["specs","suites","test"] = + lists:sort(rebar_dir:extra_src_dirs(rebar_app_info:opts(App2))), + + %% check that paths are translated + ExpectedSpec2 = filename:join([AppDir, "_build", "test", "lib", Name, + "specs", "another.spec"]), + [ExpectedSpec2] = proplists:get_value(spec, Opts2), + + ok = file:set_cwd(Wd), + + ok. + +testspec_at_root(Config) -> + C = rebar_test_utils:init_rebar_state(Config, "ct_testspec_at_root_"), + + AppDir = ?config(apps, C), + + Name = rebar_test_utils:create_random_name("ct_testspec_at_root_"), + Vsn = rebar_test_utils:create_random_vsn(), + AppDir1 = filename:join([AppDir, "apps", Name]), + rebar_test_utils:create_app(AppDir1, Name, Vsn, [kernel, stdlib]), + + Spec1 = filename:join([AppDir, "root.spec"]), + ok = filelib:ensure_dir(Spec1), + ok = file:write_file(Spec1, "{suites,\"test\",all}."), + Spec2 = filename:join([AppDir, "root1.spec"]), + ok = file:write_file(Spec2, "{suites,\".\",all}."), + Spec3 = filename:join([AppDir, "root2.spec"]), + ok = file:write_file(Spec3, "{suites,\"suites\",all}."), + Suite1 = filename:join(AppDir,"root_SUITE.erl"), + ok = file:write_file(Suite1, test_suite("root")), + Suite2 = filename:join([AppDir,"suites","test_SUITE.erl"]), + ok = filelib:ensure_dir(Suite2), + ok = file:write_file(Suite2, test_suite("test")), + + {ok, State} = rebar_test_utils:run_and_check(C, + [], + ["as", "test", "lock"], + return), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + + SpecArg1 = rebar_string:join([Spec1,Spec2,Spec3],","), + {ok, GetOptResult1} = getopt:parse(GetOptSpec, ["--spec",SpecArg1]), + State1 = rebar_state:command_parsed_args(State, GetOptResult1), + Tests1 = rebar_prv_common_test:prepare_tests(State1), + {ok, NewState1} = rebar_prv_common_test:compile(State1, Tests1), + {ok, T1} = Tests1, + Opts1= rebar_prv_common_test:translate_paths(NewState1, T1), + + %% check that extra src dir is added + ExtraDir = filename:join([AppDir, "_build", "test", "extras"]), + [ExtraDir,"suites","test"] = + rebar_dir:extra_src_dirs(rebar_state:opts(NewState1)), + + %% check that path is translated + ExpectedSpec1 = filename:join([AppDir, "_build", "test", + "extras", "root.spec"]), + ExpectedSpec2 = filename:join([AppDir, "_build", "test", + "extras", "root1.spec"]), + ExpectedSpec3 = filename:join([AppDir, "_build", "test", + "extras", "root2.spec"]), + [ExpectedSpec1,ExpectedSpec2,ExpectedSpec3] = + lists:sort(proplists:get_value(spec, Opts1)), + + %% check that test specs are copied + [ExpectedSpec1,ExpectedSpec2,ExpectedSpec3] = + lists:sort(filelib:wildcard(filename:join([AppDir, "_build", "test", + "extras", "*.spec"]))), + + %% Same test again, using relative path + {ok,Cwd} = file:get_cwd(), + ok = file:set_cwd(AppDir), + ok = rebar_file_utils:rm_rf("_build"), + + SpecArg2 = "root.spec,root1.spec,root2.spec", + {ok, GetOptResult2} = getopt:parse(GetOptSpec, ["--spec",SpecArg2]), + State2 = rebar_state:command_parsed_args(State, GetOptResult2), + Tests2 = rebar_prv_common_test:prepare_tests(State2), + {ok, NewState2} = rebar_prv_common_test:compile(State2, Tests2), + {ok, T2} = Tests2, + Opts2= rebar_prv_common_test:translate_paths(NewState2, T2), + + %% check that extra src dir is added + [ExtraDir,"suites","test"] = + rebar_dir:extra_src_dirs(rebar_state:opts(NewState2)), + + %% check that path is translated + [ExpectedSpec1,ExpectedSpec2,ExpectedSpec3] = + lists:sort(proplists:get_value(spec, Opts2)), + + %% check that test specs are copied + [ExpectedSpec1,ExpectedSpec2,ExpectedSpec3] = + lists:sort(filelib:wildcard(filename:join([AppDir, "_build", "test", + "extras", "root*.spec"]))), + + ok = file:set_cwd(Cwd), + + ok. + +testspec_parse_error(Config) -> + C = rebar_test_utils:init_rebar_state(Config, "ct_testspec_error"), + + AppDir = ?config(apps, C), + + Name = rebar_test_utils:create_random_name("ct_testspec_error"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + Spec1 = filename:join([AppDir, "test", "nonexisting.spec"]), + Spec2 = filename:join([AppDir, "test", "some.spec"]), + ok = filelib:ensure_dir(Spec2), + ok = file:write_file(Spec2, ".\n"), + + {ok, State} = rebar_test_utils:run_and_check(C, + [], + ["as", "test", "lock"], + return), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + + %% Non existing testspec + {ok, GetOptResult1} = getopt:parse(GetOptSpec, ["--spec",Spec1]), + State1 = rebar_state:command_parsed_args(State, GetOptResult1), + Tests1 = rebar_prv_common_test:prepare_tests(State1), + {error, + {rebar_prv_common_test, + {error_reading_testspec, + {Spec1,"no such file or directory"}}}} = + rebar_prv_common_test:compile(State1, Tests1), + + %% Syntax error + {ok, GetOptResult2} = getopt:parse(GetOptSpec, ["--spec",Spec2]), + State2 = rebar_state:command_parsed_args(State, GetOptResult2), + Tests2 = rebar_prv_common_test:prepare_tests(State2), + {error, + {rebar_prv_common_test, + {error_reading_testspec, + {Spec2,"1: syntax error before: '.'"}}}} = + rebar_prv_common_test:compile(State2, Tests2), + + ok. + +cmd_vs_cfg_opts(Config) -> + C = rebar_test_utils:init_rebar_state(Config, "ct_cmd_vs_cfg_opts_"), + + AppDir = ?config(apps, C), + + Name = rebar_test_utils:create_random_name("ct_cmd_vs_cfg_opts_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + RebarConfig = [{ct_opts, [{spec,"mytest.spec"}, + {dir,"test"}, + {suite,"some_SUITE"}, + {group,"some_group"}, + {testcase,"some_test"}]}], + + {ok, State} = rebar_test_utils:run_and_check(C, RebarConfig, ["as", "test", "lock"], return), + + {ok, TestOpts} = rebar_prv_common_test:prepare_tests(State), + true = lists:member({spec, "mytest.spec"}, TestOpts), + true = lists:member({dir, "test"}, TestOpts), + true = lists:member({suite, "some_SUITE"}, TestOpts), + true = lists:member({group, "some_group"}, TestOpts), + true = lists:member({testcase, "some_test"}, TestOpts), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + + {ok, GetOptResult1} = getopt:parse(GetOptSpec, ["--spec","test/some.spec"]), + State1 = rebar_state:command_parsed_args(State, GetOptResult1), + {ok, TestOpts1} = rebar_prv_common_test:prepare_tests(State1), + true = lists:member({spec, ["test/some.spec"]}, TestOpts1), + false = lists:keymember(dir, 1, TestOpts1), + false = lists:keymember(suite, 1, TestOpts1), + false = lists:keymember(group, 1, TestOpts1), + false = lists:keymember(testcase, 1, TestOpts1), + + {ok, GetOptResult2} = getopt:parse(GetOptSpec, ["--suite","test/some_SUITE"]), + State2 = rebar_state:command_parsed_args(State, GetOptResult2), + {ok, TestOpts2} = rebar_prv_common_test:prepare_tests(State2), + true = lists:member({suite, ["test/some_SUITE"]}, TestOpts2), + false = lists:keymember(spec, 1, TestOpts2), + false = lists:keymember(dir, 1, TestOpts2), + false = lists:keymember(group, 1, TestOpts2), + false = lists:keymember(testcase, 1, TestOpts2), + + ok. + +single_testspec_in_ct_opts(Config) -> + C = rebar_test_utils:init_rebar_state(Config, "ct_testspec_"), + + AppDir = ?config(apps, C), + + Name = rebar_test_utils:create_random_name("ct_testspec_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + Spec = filename:join([AppDir, "test", "some.spec"]), + ok = filelib:ensure_dir(Spec), + ok = file:write_file(Spec, "{suites,\".\",all}.\n"), + + {ok,Wd} = file:get_cwd(), + ok = file:set_cwd(AppDir), + + RebarConfig = [{ct_opts, [{spec,"test/some.spec"}]}], + + {ok, State} = rebar_test_utils:run_and_check(C, RebarConfig, ["as", "test", "lock"], return), + + Providers = rebar_state:providers(State), + Namespace = rebar_state:namespace(State), + CommandProvider = providers:get_provider(ct, Providers, Namespace), + GetOptSpec = providers:opts(CommandProvider), + + %% Testspec in "test" directory + {ok, GetOptResult1} = getopt:parse(GetOptSpec, []), + State1 = rebar_state:command_parsed_args(State, GetOptResult1), + Tests1 = rebar_prv_common_test:prepare_tests(State1), + {ok, T1} = Tests1, + "test/some.spec" = proplists:get_value(spec,T1), + {ok, _NewState} = rebar_prv_common_test:compile(State1, Tests1), + + ok = file:set_cwd(Wd), + ok. + +compile_only(Config) -> + C = rebar_test_utils:init_rebar_state(Config, "compile_only_"), + + AppDir = ?config(apps, C), + + Name = rebar_test_utils:create_random_name(atom_to_list(basic_app) ++ "_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + Suite = filename:join([AppDir, "test", Name ++ "_SUITE.erl"]), + ok = filelib:ensure_dir(Suite), + ok = file:write_file(Suite, test_suite(Name)), + + {ok, _State} = rebar_test_utils:run_and_check(C, [], ["ct", "--compile_only"], {ok, [{app,Name}], "test"}). + + %% helper for generating test data test_suite(Name) -> io_lib:format("-module(~ts_SUITE).\n" @@ -1226,7 +1643,10 @@ test_suite(Name) -> "some_test(_) -> ok.\n", [Name]). cmd_sys_config_file(AppName) -> - io_lib:format("[{~s, [{key, cmd_value}]}].", [AppName]). + io_lib:format("[{~ts, [{key, cmd_value}]}].", [AppName]). cfg_sys_config_file(AppName) -> - io_lib:format("[{~s, [{key, cfg_value}]}].", [AppName]). + io_lib:format("[{~ts, [{key, cfg_value}]}, \"config/other\"].", [AppName]). + +other_sys_config_file(AppName) -> + io_lib:format("[{~ts, [{other_key, other_cfg_value}]}].", [AppName]). diff --git a/test/rebar_deps_SUITE.erl b/test/rebar_deps_SUITE.erl index 24bf2a0..8a3362d 100644 --- a/test/rebar_deps_SUITE.erl +++ b/test/rebar_deps_SUITE.erl @@ -7,7 +7,7 @@ all() -> [sub_app_deps, newly_added_dep, newly_added_after_empty_lock, http_proxy_settings, https_proxy_settings, http_os_proxy_settings, https_os_proxy_settings, semver_matching_lt, semver_matching_lte, semver_matching_gt, - valid_version, {group, git}, {group, pkg}]. + valid_version, top_override, {group, git}, {group, pkg}]. groups() -> [{all, [], [flat, pick_highest_left, pick_highest_right, @@ -47,6 +47,8 @@ init_per_testcase(newly_added_dep, Config) -> rebar_test_utils:init_rebar_state(Config); init_per_testcase(sub_app_deps, Config) -> rebar_test_utils:init_rebar_state(Config); +init_per_testcase(top_override, Config) -> + rebar_test_utils:init_rebar_state(Config); init_per_testcase(http_proxy_settings, Config) -> %% Create private rebar.config Priv = ?config(priv_dir, Config), @@ -186,27 +188,27 @@ deps(flat) -> [], {ok, ["B", "C"]}}; deps(pick_highest_left) -> - {[{"B", [{"C", "2", []}]}, - {"C", "1", []}], - [{"C","2"}], - {ok, ["B", {"C", "1"}]}}; + {[{"B", [{"C", "2.0.0", []}]}, + {"C", "1.0.0", []}], + [{"C","2.0.0"}], + {ok, ["B", {"C", "1.0.0"}]}}; deps(pick_highest_right) -> - {[{"B", "1", []}, - {"C", [{"B", "2", []}]}], - [{"B","2"}], - {ok, [{"B","1"}, "C"]}}; + {[{"B", "1.0.0", []}, + {"C", [{"B", "2.0.0", []}]}], + [{"B","2.0.0"}], + {ok, [{"B","1.0.0"}, "C"]}}; deps(pick_smallest1) -> - {[{"B", [{"D", "1", []}]}, - {"C", [{"D", "2", []}]}], - [{"D","2"}], + {[{"B", [{"D", "1.0.0", []}]}, + {"C", [{"D", "2.0.0", []}]}], + [{"D","2.0.0"}], %% we pick D1 because B < C - {ok, ["B","C",{"D","1"}]}}; + {ok, ["B","C",{"D","1.0.0"}]}}; deps(pick_smallest2) -> - {[{"C", [{"D", "2", []}]}, - {"B", [{"D", "1", []}]}], - [{"D","2"}], + {[{"C", [{"D", "2.0.0", []}]}, + {"B", [{"D", "1.0.0", []}]}], + [{"D","2.0.0"}], %% we pick D1 because B < C - {ok, ["B","C",{"D","1"}]}}; + {ok, ["B","C",{"D","1.0.0"}]}}; deps(circular1) -> {[{"B", [{"A", []}]}, % A is the top-level app {"C", []}], @@ -220,15 +222,17 @@ deps(circular2) -> deps(circular_skip) -> %% Never spot the circular dep due to being to low in the deps tree %% in source deps - {[{"B", [{"C", "2", [{"B", []}]}]}, - {"C", "1", [{"D",[]}]}], - [{"C","2"}], - {ok, ["B", {"C","1"}, "D"]}}. + {[{"B", [{"C", "2.0.0", [{"B", []}]}]}, + {"C", "1.0.0", [{"D",[]}]}], + [{"C","2.0.0"}], + {ok, ["B", {"C","1.0.0"}, "D"]}}. setup_project(Case, Config0, Deps) -> DepsType = ?config(deps_type, Config0), + %% spread packages across 3 repos randomly + Repos = [<<"test-repo-1">>, <<"test-repo-2">>, <<"hexpm">>], Config = rebar_test_utils:init_rebar_state( - Config0, + [{repos, Repos} | Config0], atom_to_list(Case)++"_"++atom_to_list(DepsType)++"_" ), AppDir = ?config(apps, Config), @@ -237,7 +241,7 @@ setup_project(Case, Config0, Deps) -> RebarConf = rebar_test_utils:create_config(AppDir, [{deps, TopDeps}]), {SrcDeps, PkgDeps} = rebar_test_utils:flat_deps(Deps), mock_git_resource:mock([{deps, SrcDeps}]), - mock_pkg_resource:mock([{pkgdeps, PkgDeps}]), + mock_pkg_resource:mock([{pkgdeps, PkgDeps}, {repos, Repos}]), [{rebarconfig, RebarConf} | Config]. mock_warnings() -> @@ -255,6 +259,32 @@ circular1(Config) -> run(Config). circular2(Config) -> run(Config). circular_skip(Config) -> run(Config). +%% Test that a top-level application overtakes dependencies, and +%% works even if said deps do not exist. +top_override(Config) -> + AppDir = ?config(apps, Config), + ct:pal("dir: ~p", [AppDir]), + Name1 = rebar_test_utils:create_random_name("sub_app1_"), + Name2 = rebar_test_utils:create_random_name("sub_app2_"), + SubAppsDir1 = filename:join([AppDir, "apps", Name1]), + SubAppsDir2 = filename:join([AppDir, "apps", Name2]), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(SubAppsDir1, Name1, Vsn, [kernel, stdlib]), + rebar_test_utils:create_app(SubAppsDir2, Name2, Vsn, [kernel, stdlib]), + rebar_test_utils:create_config( + SubAppsDir1, + [{deps, [list_to_atom(Name2)]}] + ), + rebar_test_utils:create_config( + SubAppsDir2, + [{deps, [{list_to_atom(Name1), + {git, "https://example.org", {branch, "master"}}}]}] + ), + rebar_test_utils:run_and_check( + Config, [], ["compile"], + {ok, [{app, Name1}, {app,Name2}]} + ). + %% Test that the deps of project apps that have their own rebar.config %% are included, but that top level rebar.config deps take precedence sub_app_deps(Config) -> @@ -384,70 +414,62 @@ https_os_proxy_settings(_Config) -> httpc:get_option(https_proxy, rebar)). semver_matching_lt(_Config) -> - Dep = <<"test">>, - Dep1 = {Dep, <<"1.0.0">>, Dep}, MaxVsn = <<"0.2.0">>, Vsns = [<<"0.1.7">>, <<"0.1.9">>, <<"0.1.8">>, <<"0.2.0">>, <<"0.2.1">>], - ?assertEqual([{Dep, <<"0.1.9">>}], - rebar_prv_update:cmpl_(undefined, MaxVsn, Vsns, [], Dep1, + ?assertEqual({ok, <<"0.1.9">>}, + rebar_packages:cmpl_(undefined, MaxVsn, Vsns, fun ec_semver:lt/2)). semver_matching_lte(_Config) -> - Dep = <<"test">>, - Dep1 = {Dep, <<"1.0.0">>, Dep}, MaxVsn = <<"0.2.0">>, Vsns = [<<"0.1.7">>, <<"0.1.9">>, <<"0.1.8">>, <<"0.2.0">>, <<"0.2.1">>], - ?assertEqual([{Dep, <<"0.2.0">>}], - rebar_prv_update:cmpl_(undefined, MaxVsn, Vsns, [], Dep1, + ?assertEqual({ok, <<"0.2.0">>}, + rebar_packages:cmpl_(undefined, MaxVsn, Vsns, fun ec_semver:lte/2)). semver_matching_gt(_Config) -> - Dep = <<"test">>, - Dep1 = {Dep, <<"1.0.0">>, Dep}, MaxVsn = <<"0.2.0">>, Vsns = [<<"0.1.7">>, <<"0.1.9">>, <<"0.1.8">>, <<"0.2.0">>, <<"0.2.1">>], - ?assertEqual([{Dep, <<"0.2.1">>}], - rebar_prv_update:cmp_(undefined, MaxVsn, Vsns, [], Dep1, + ?assertEqual({ok, <<"0.2.1">>}, + rebar_packages:cmp_(undefined, MaxVsn, Vsns, fun ec_semver:gt/2)). semver_matching_gte(_Config) -> - Dep = <<"test">>, - Dep1 = {Dep, <<"1.0.0">>, Dep}, MaxVsn = <<"0.2.0">>, Vsns = [<<"0.1.7">>, <<"0.1.9">>, <<"0.1.8">>, <<"0.2.0">>], - ?assertEqual([{Dep, <<"0.2.0">>}], - rebar_prv_update:cmp_(undefined, MaxVsn, Vsns, [], Dep1, + ?assertEqual({ok, <<"0.2.0">>}, + rebar_packages:cmp_(undefined, MaxVsn, Vsns, fun ec_semver:gt/2)). valid_version(_Config) -> - ?assert(rebar_prv_update:valid_vsn(<<"0.1">>)), - ?assert(rebar_prv_update:valid_vsn(<<"0.1.0">>)), - ?assert(rebar_prv_update:valid_vsn(<<" 0.1.0">>)), - ?assert(rebar_prv_update:valid_vsn(<<" 0.1.0">>)), - ?assert(rebar_prv_update:valid_vsn(<<"<0.1">>)), - ?assert(rebar_prv_update:valid_vsn(<<"<0.1.0">>)), - ?assert(rebar_prv_update:valid_vsn(<<"< 0.1.0">>)), - ?assert(rebar_prv_update:valid_vsn(<<"< 0.1.0">>)), - ?assert(rebar_prv_update:valid_vsn(<<">0.1">>)), - ?assert(rebar_prv_update:valid_vsn(<<">0.1.0">>)), - ?assert(rebar_prv_update:valid_vsn(<<"> 0.1.0">>)), - ?assert(rebar_prv_update:valid_vsn(<<"> 0.1.0">>)), - ?assert(rebar_prv_update:valid_vsn(<<"<=0.1">>)), - ?assert(rebar_prv_update:valid_vsn(<<"<=0.1.0">>)), - ?assert(rebar_prv_update:valid_vsn(<<"<= 0.1.0">>)), - ?assert(rebar_prv_update:valid_vsn(<<"<= 0.1.0">>)), - ?assert(rebar_prv_update:valid_vsn(<<">=0.1">>)), - ?assert(rebar_prv_update:valid_vsn(<<">=0.1.0">>)), - ?assert(rebar_prv_update:valid_vsn(<<">= 0.1.0">>)), - ?assert(rebar_prv_update:valid_vsn(<<">= 0.1.0">>)), - ?assert(rebar_prv_update:valid_vsn(<<"==0.1">>)), - ?assert(rebar_prv_update:valid_vsn(<<"==0.1.0">>)), - ?assert(rebar_prv_update:valid_vsn(<<"== 0.1.0">>)), - ?assert(rebar_prv_update:valid_vsn(<<"== 0.1.0">>)), - ?assert(rebar_prv_update:valid_vsn(<<"~>0.1">>)), - ?assert(rebar_prv_update:valid_vsn(<<"~>0.1.0">>)), - ?assert(rebar_prv_update:valid_vsn(<<"~> 0.1.0">>)), - ?assert(rebar_prv_update:valid_vsn(<<"~> 0.1.0">>)), - ?assertNot(rebar_prv_update:valid_vsn(<<"> 0.1.0 and < 0.2.0">>)), + ?assert(rebar_packages:valid_vsn(<<"0.1">>)), + ?assert(rebar_packages:valid_vsn(<<"0.1.0">>)), + ?assert(rebar_packages:valid_vsn(<<" 0.1.0">>)), + ?assert(rebar_packages:valid_vsn(<<" 0.1.0">>)), + ?assert(rebar_packages:valid_vsn(<<"<0.1">>)), + ?assert(rebar_packages:valid_vsn(<<"<0.1.0">>)), + ?assert(rebar_packages:valid_vsn(<<"< 0.1.0">>)), + ?assert(rebar_packages:valid_vsn(<<"< 0.1.0">>)), + ?assert(rebar_packages:valid_vsn(<<">0.1">>)), + ?assert(rebar_packages:valid_vsn(<<">0.1.0">>)), + ?assert(rebar_packages:valid_vsn(<<"> 0.1.0">>)), + ?assert(rebar_packages:valid_vsn(<<"> 0.1.0">>)), + ?assert(rebar_packages:valid_vsn(<<"<=0.1">>)), + ?assert(rebar_packages:valid_vsn(<<"<=0.1.0">>)), + ?assert(rebar_packages:valid_vsn(<<"<= 0.1.0">>)), + ?assert(rebar_packages:valid_vsn(<<"<= 0.1.0">>)), + ?assert(rebar_packages:valid_vsn(<<">=0.1">>)), + ?assert(rebar_packages:valid_vsn(<<">=0.1.0">>)), + ?assert(rebar_packages:valid_vsn(<<">= 0.1.0">>)), + ?assert(rebar_packages:valid_vsn(<<">= 0.1.0">>)), + ?assert(rebar_packages:valid_vsn(<<"==0.1">>)), + ?assert(rebar_packages:valid_vsn(<<"==0.1.0">>)), + ?assert(rebar_packages:valid_vsn(<<"== 0.1.0">>)), + ?assert(rebar_packages:valid_vsn(<<"== 0.1.0">>)), + ?assert(rebar_packages:valid_vsn(<<"~>0.1">>)), + ?assert(rebar_packages:valid_vsn(<<"~>0.1.0">>)), + ?assert(rebar_packages:valid_vsn(<<"~> 0.1.0">>)), + ?assert(rebar_packages:valid_vsn(<<"~> 0.1.0">>)), + ?assertNot(rebar_packages:valid_vsn(<<"> 0.1.0 and < 0.2.0">>)), ok. @@ -476,5 +498,5 @@ in_warnings(git, Warns, NameRaw, VsnRaw) -> in_warnings(pkg, Warns, NameRaw, VsnRaw) -> Name = iolist_to_binary(NameRaw), Vsn = iolist_to_binary(VsnRaw), - 1 =< length([1 || {_, [AppName, {pkg, _, AppVsn, _}]} <- Warns, + 1 =< length([1 || {_, [AppName, {pkg, _, AppVsn}]} <- Warns, AppName =:= Name, AppVsn =:= Vsn]). diff --git a/test/rebar_dialyzer_SUITE.erl b/test/rebar_dialyzer_SUITE.erl index e5d8c52..6579afb 100644 --- a/test/rebar_dialyzer_SUITE.erl +++ b/test/rebar_dialyzer_SUITE.erl @@ -14,7 +14,8 @@ update_base_plt/1, update_app_plt/1, build_release_plt/1, - plt_apps_option/1]). + plt_apps_option/1, + exclude_and_extra/1]). -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -57,7 +58,7 @@ all() -> groups() -> [{empty, [empty_base_plt, empty_app_plt, empty_app_succ_typings]}, - {build_and_check, [build_release_plt, plt_apps_option]}, + {build_and_check, [build_release_plt, plt_apps_option, exclude_and_extra]}, {update, [update_base_plt, update_app_plt]}]. empty_base_plt(Config) -> @@ -275,6 +276,39 @@ plt_apps_option(Config) -> {ok, PltFiles2} = plt_files(Plt), ?assertEqual([App1, App2, erts], get_apps_from_beam_files(PltFiles2)). +exclude_and_extra(Config) -> + AppDir = ?config(apps, Config), + RebarConfig = ?config(rebar_config, Config), + BasePlt = ?config(base_plt, Config), + Plt = ?config(plt, Config), + + {value, {dialyzer, Opts}, Rest} = lists:keytake(dialyzer, 1, RebarConfig), + % Remove erts => [] + % Add erlang+zlib => [erlang, zlib], + % Add erl_prim_loader+init => [erl_prim_loader, init, erlang, zlib] + % Remove zlib+init => [erl_prim_loader, erlang] + Opts2 = [{exclude_apps, [erts]}, + {base_plt_mods, [erlang, zlib]}, + {plt_extra_mods, [erl_prim_loader, init]}, + {exclude_mods, [zlib, init]} | + Opts], + RebarConfig2 = [{dialyzer, Opts2} | Rest], + + Name = rebar_test_utils:create_random_name("app1_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [erts]), + + rebar_test_utils:run_and_check(Config, RebarConfig2, ["dialyzer"], + {ok, [{app, Name}]}), + + Erlang = code:where_is_file("erlang.beam"), + {ok, BasePltFiles} = plt_files(BasePlt), + ?assertEqual([Erlang], BasePltFiles), + + Pair = lists:sort([Erlang, code:where_is_file("erl_prim_loader.beam")]), + {ok, PltFiles} = plt_files(Plt), + ?assertEqual(Pair, PltFiles). + %% Helpers erts_files() -> @@ -328,6 +362,6 @@ get_apps_from_beam_files(BeamFiles) -> lists:usort( [begin AppNameVsn = filename:basename(filename:dirname(filename:dirname(File))), - [AppName | _] = string:tokens(AppNameVsn ++ "-", "-"), + [AppName | _] = rebar_string:lexemes(AppNameVsn ++ "-", "-"), ec_cnv:to_atom(AppName) end || File <- BeamFiles]). diff --git a/test/rebar_dir_SUITE.erl b/test/rebar_dir_SUITE.erl index 9734830..81051e6 100644 --- a/test/rebar_dir_SUITE.erl +++ b/test/rebar_dir_SUITE.erl @@ -3,8 +3,11 @@ -export([all/0, init_per_testcase/2, end_per_testcase/2]). -export([default_src_dirs/1, default_extra_src_dirs/1, default_all_src_dirs/1]). --export([src_dirs/1, extra_src_dirs/1, all_src_dirs/1]). +-export([src_dirs/1, alt_src_dir_nested/1, src_dirs_with_opts/1, extra_src_dirs/1, all_src_dirs/1]). +-export([src_dir_opts/1, recursive/1]). +-export([top_src_dirs/1]). -export([profile_src_dirs/1, profile_extra_src_dirs/1, profile_all_src_dirs/1]). +-export([profile_src_dir_opts/1]). -export([retarget_path/1, alt_base_dir_abs/1, alt_base_dir_rel/1]). -export([global_cache_dir/1, default_global_cache_dir/1, overwrite_default_global_cache_dir/1]). @@ -14,8 +17,9 @@ all() -> [default_src_dirs, default_extra_src_dirs, default_all_src_dirs, - src_dirs, extra_src_dirs, all_src_dirs, + src_dirs, alt_src_dir_nested, extra_src_dirs, all_src_dirs, src_dir_opts, recursive, profile_src_dirs, profile_extra_src_dirs, profile_all_src_dirs, + profile_src_dir_opts, top_src_dirs, retarget_path, alt_base_dir_abs, alt_base_dir_rel, global_cache_dir, default_global_cache_dir, overwrite_default_global_cache_dir]. @@ -65,10 +69,35 @@ default_all_src_dirs(Config) -> ["src", "test"] = rebar_dir:all_src_dirs(rebar_state:opts(State), ["src"], ["test"]). src_dirs(Config) -> - RebarConfig = [{erl_opts, [{src_dirs, ["foo", "bar", "baz"]}]}], + RebarConfig = [{erl_opts, [{src_dirs, ["foo", "./bar", "bar", "bar/", "./bar/", "baz", + "./", ".", "../", "..", "./../", "../.", ".././../"]}]}], {ok, State} = rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], return), - ["bar", "baz", "foo"] = rebar_dir:src_dirs(rebar_state:opts(State)). + [".", "..", "../..", "bar", "baz", "foo"] = rebar_dir:src_dirs(rebar_state:opts(State)). + +alt_src_dir_nested(Config) -> + RebarConfig = [{src_dirs, ["src", "alt/nested"]}], + AppsDir = ?config(apps, Config), + Name1 = ?config(app_one, Config), + ModDir = filename:join([AppsDir, "apps", Name1, "alt", "nested"]), + Mod = "-module(altmod). -export([main/0]). main() -> ok.", + + ec_file:mkdir_path(ModDir), + ok = file:write_file(filename:join([ModDir, "altmod.erl"]), Mod), + + Ebin = filename:join([AppsDir, "_build", "default", "lib", Name1, "ebin", "altmod.beam"]), + {ok, State} = rebar_test_utils:run_and_check( + Config, RebarConfig, ["compile"], + {ok, [{file, Ebin}]} + ), + ["alt/nested", "src"] = rebar_dir:src_dirs(rebar_state:opts(State)). + +src_dirs_with_opts(Config) -> + RebarConfig = [{erl_opts, [{src_dirs, ["foo", "bar", "baz"]}, + {src_dirs, [{"foo",[{recursive,false}]}, "qux"]}]}], + {ok, State} = rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], return), + + ["bar", "baz", "foo", "qux"] = rebar_dir:src_dirs(rebar_state:opts(State)). extra_src_dirs(Config) -> RebarConfig = [{erl_opts, [{extra_src_dirs, ["foo", "bar", "baz"]}]}], @@ -77,11 +106,50 @@ extra_src_dirs(Config) -> ["bar", "baz", "foo"] = rebar_dir:extra_src_dirs(rebar_state:opts(State)). all_src_dirs(Config) -> - RebarConfig = [{erl_opts, [{src_dirs, ["foo", "bar"]}, {extra_src_dirs, ["baz", "qux"]}]}], + RebarConfig = [{erl_opts, [{src_dirs, ["foo", "bar"]}, {extra_src_dirs, ["baz", "qux"]}, {src_dirs, [{"foo", [{recursive,false}]}]}]}], {ok, State} = rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], return), ["bar", "baz", "foo", "qux"] = rebar_dir:all_src_dirs(rebar_state:opts(State)). +src_dir_opts(Config) -> + RebarConfig = + [{erl_opts, [{src_dirs, [{"foo",[{recursive,true}]}, "bar"]}, + {extra_src_dirs, ["baz", {"foo", [{recursive,false}]}]}, + {src_dirs, [{"foo", [{recursive,false}]}]}]}], + {ok, State} = rebar_test_utils:run_and_check(Config, RebarConfig, + ["compile"], return), + [{recursive,true}] = rebar_dir:src_dir_opts(rebar_state:opts(State), "foo"), + [] = rebar_dir:src_dir_opts(rebar_state:opts(State), "bar"), + [] = rebar_dir:src_dir_opts(rebar_state:opts(State), "nonexisting"). + +recursive(Config) -> + RebarConfig1 = + [{erl_opts, [{src_dirs, ["foo", "bar"]}, + {extra_src_dirs, ["baz", {"foo", [{recursive,true}]}]}, + {src_dirs, [{"foo", [{recursive,false}]}]}]}], + {ok, State1} = rebar_test_utils:run_and_check(Config, RebarConfig1, + ["compile"], return), + false = rebar_dir:recursive(rebar_state:opts(State1), "foo"), + true = rebar_dir:recursive(rebar_state:opts(State1), "bar"), + + RebarConfig2 = [{erlc_compiler,[{recursive,false}]}, + {erl_opts,[{src_dirs,["foo",{"bar",[{recursive,true}]}]}]}], + {ok, State2} = rebar_test_utils:run_and_check(Config, RebarConfig2, + ["compile"], return), + false = rebar_dir:recursive(rebar_state:opts(State2), "foo"), + true = rebar_dir:recursive(rebar_state:opts(State2), "bar"), + + ok. + +top_src_dirs(Config) -> + %% We can get the same result out of specifying src_dirs from the config root, + %% not just the erl_opts + RebarConfig = [{src_dirs, ["foo", "./bar", "bar", "bar/", "./bar/", "baz", + "./", ".", "../", "..", "./../", "../.", ".././../"]}], + {ok, State} = rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], return), + + [".", "..", "../..", "bar", "baz", "foo"] = rebar_dir:src_dirs(rebar_state:opts(State)). + profile_src_dirs(Config) -> RebarConfig = [ {erl_opts, [{src_dirs, ["foo", "bar"]}]}, @@ -118,6 +186,26 @@ profile_all_src_dirs(Config) -> R = lists:sort(["foo", "bar", "baz", "qux"]), R = rebar_dir:all_src_dirs(rebar_state:opts(State)). +profile_src_dir_opts(Config) -> + RebarConfig = [ + {erl_opts, [{src_dirs, ["foo"]}, + {extra_src_dirs, [{"bar",[recursive]}]}]}, + {profiles, [ + {more, [{erl_opts, [{src_dirs, [{"bar",[{recursive,false}]}]}, + {extra_src_dirs, ["qux"]}]}]} + ]} + ], + {ok, State} = rebar_test_utils:run_and_check(Config, RebarConfig, + ["as", "more", "compile"], + return), + + [{recursive,false}] = rebar_dir:src_dir_opts(rebar_state:opts(State),"bar"), + + {ok, State1} = rebar_test_utils:run_and_check(Config, RebarConfig, + ["compile"], return), + + [{recursive,true}] = rebar_dir:src_dir_opts(rebar_state:opts(State1),"bar"). + retarget_path(Config) -> {ok, State} = rebar_test_utils:run_and_check(Config, [], ["compile"], return), diff --git a/test/rebar_edoc_SUITE.erl b/test/rebar_edoc_SUITE.erl new file mode 100644 index 0000000..2c4aba5 --- /dev/null +++ b/test/rebar_edoc_SUITE.erl @@ -0,0 +1,72 @@ +-module(rebar_edoc_SUITE). +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-compile(export_all). + +all() -> [multiapp, error_survival]. + +init_per_testcase(multiapp, Config) -> + application:load(rebar), + DataDir = ?config(data_dir, Config), + PrivDir = ?config(priv_dir, Config), + Name = rebar_test_utils:create_random_name("multiapp"), + AppsDir = filename:join([PrivDir, rebar_test_utils:create_random_name(Name)]), + ec_file:copy(filename:join([DataDir, "foo"]), AppsDir, [recursive]), + Verbosity = rebar3:log_level(), + rebar_log:init(command_line, Verbosity), + State = rebar_state:new([{base_dir, filename:join([AppsDir, "_build"])} + ,{root_dir, AppsDir}]), + [{apps, AppsDir}, {state, State}, {name, Name} | Config]; +init_per_testcase(error_survival, Config) -> + application:load(rebar), + DataDir = ?config(data_dir, Config), + PrivDir = ?config(priv_dir, Config), + Name = rebar_test_utils:create_random_name("error_survival"), + AppsDir = filename:join([PrivDir, rebar_test_utils:create_random_name(Name)]), + ec_file:copy(filename:join([DataDir, "bad"]), AppsDir, [recursive]), + Verbosity = rebar3:log_level(), + rebar_log:init(command_line, Verbosity), + State = rebar_state:new([{base_dir, filename:join([AppsDir, "_build"])} + ,{root_dir, AppsDir}]), + [{apps, AppsDir}, {state, State}, {name, Name} | Config]. + +end_per_testcase(_, Config) -> + Config. + +multiapp(Config) -> + %% With an empty config (no `dir'), links are being processed + RebarConfig = [], + rebar_test_utils:run_and_check(Config, RebarConfig, ["edoc"], {ok, []}), + %% validate that all doc entries are generated and links work + AppsDir = ?config(apps, Config), + ct:pal("AppsDir: ~s", [AppsDir]), + ?assert(file_content_matches( + filename:join([AppsDir, "apps", "bar1", "doc", "bar1.html"]), + "barer1")), + ?assert(file_content_matches( + filename:join([AppsDir, "apps", "bar2", "doc", "bar2.html"]), + "barer2")), + %% Links are in place for types + ?assert(file_content_matches( + filename:join([AppsDir, "apps", "foo", "doc", "foo.html"]), + "barer1")), + ?assert(file_content_matches( + filename:join([AppsDir, "apps", "foo", "doc", "foo.html"]), + "apps/bar1/doc/bar1.html")). + +error_survival(Config) -> + RebarConfig = [], + rebar_test_utils:run_and_check( + Config, RebarConfig, ["edoc"], + {error,{rebar_prv_edoc,{app_failed,"bar2"}}} + ), + ok. + + +file_content_matches(Path, Regex) -> + case file:read_file(Path) of + {ok, Bin} -> + nomatch =/= re:run(Bin, Regex); + {error, Reason} -> + Reason + end. diff --git a/test/rebar_edoc_SUITE_data/bad/apps/bar1/src/bar1.app.src b/test/rebar_edoc_SUITE_data/bad/apps/bar1/src/bar1.app.src new file mode 100644 index 0000000..6e7ec24 --- /dev/null +++ b/test/rebar_edoc_SUITE_data/bad/apps/bar1/src/bar1.app.src @@ -0,0 +1,16 @@ +{application, bar1, + [{description, "An OTP application"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, { bar1_app, []}}, + {applications, + [kernel, + stdlib + ]}, + {env,[]}, + {modules, []}, + + {maintainers, []}, + {licenses, []}, + {links, []} + ]}. diff --git a/test/rebar_edoc_SUITE_data/bad/apps/bar1/src/bar1.erl b/test/rebar_edoc_SUITE_data/bad/apps/bar1/src/bar1.erl new file mode 100644 index 0000000..2700aef --- /dev/null +++ b/test/rebar_edoc_SUITE_data/bad/apps/bar1/src/bar1.erl @@ -0,0 +1,9 @@ +-module(bar1).
+-export([bar1/0]).
+-export_type([barer1/0]).
+
+-type barer1() :: string().
+
+% @doc Bar1 bars the bar.
+-spec bar1() -> barer1().
+bar1() -> "Barer1".
\ No newline at end of file diff --git a/test/rebar_edoc_SUITE_data/bad/apps/bar1/src/bar1_app.erl b/test/rebar_edoc_SUITE_data/bad/apps/bar1/src/bar1_app.erl new file mode 100644 index 0000000..414ac30 --- /dev/null +++ b/test/rebar_edoc_SUITE_data/bad/apps/bar1/src/bar1_app.erl @@ -0,0 +1,26 @@ +%%%------------------------------------------------------------------- +%% @doc bar1 public API +%% @end +%%%------------------------------------------------------------------- + +-module(bar1_app). + +-behaviour(application). + +%% Application callbacks +-export([start/2, stop/1]). + +%%==================================================================== +%% API +%%==================================================================== + +start(_StartType, _StartArgs) -> + bar1_sup:start_link(). + +%%-------------------------------------------------------------------- +stop(_State) -> + ok. + +%%==================================================================== +%% Internal functions +%%==================================================================== diff --git a/test/rebar_edoc_SUITE_data/bad/apps/bar1/src/bar1_sup.erl b/test/rebar_edoc_SUITE_data/bad/apps/bar1/src/bar1_sup.erl new file mode 100644 index 0000000..f9d6670 --- /dev/null +++ b/test/rebar_edoc_SUITE_data/bad/apps/bar1/src/bar1_sup.erl @@ -0,0 +1,35 @@ +%%%------------------------------------------------------------------- +%% @doc bar1 top level supervisor. +%% @end +%%%------------------------------------------------------------------- + +-module(bar1_sup). + +-behaviour(supervisor). + +%% API +-export([start_link/0]). + +%% Supervisor callbacks +-export([init/1]). + +-define(SERVER, ?MODULE). + +%%==================================================================== +%% API functions +%%==================================================================== + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +%%==================================================================== +%% Supervisor callbacks +%%==================================================================== + +%% Child :: {Id,StartFunc,Restart,Shutdown,Type,Modules} +init([]) -> + {ok, { {one_for_all, 0, 1}, []} }. + +%%==================================================================== +%% Internal functions +%%==================================================================== diff --git a/test/rebar_edoc_SUITE_data/bad/apps/bar2/src/bar2.app.src b/test/rebar_edoc_SUITE_data/bad/apps/bar2/src/bar2.app.src new file mode 100644 index 0000000..58de8bc --- /dev/null +++ b/test/rebar_edoc_SUITE_data/bad/apps/bar2/src/bar2.app.src @@ -0,0 +1,16 @@ +{application, bar2, + [{description, "An OTP application"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, { bar2_app, []}}, + {applications, + [kernel, + stdlib + ]}, + {env,[]}, + {modules, []}, + + {maintainers, []}, + {licenses, []}, + {links, []} + ]}. diff --git a/test/rebar_edoc_SUITE_data/bad/apps/bar2/src/bar2.erl b/test/rebar_edoc_SUITE_data/bad/apps/bar2/src/bar2.erl new file mode 100644 index 0000000..2afb745 --- /dev/null +++ b/test/rebar_edoc_SUITE_data/bad/apps/bar2/src/bar2.erl @@ -0,0 +1,12 @@ +%% @doc one docline is fine
+%% @doc a second docline causes a failure
+%% @doc if not, then a & causes a bad ref error.
+-module(bar2).
+-export([bar2/0]).
+-export_type([barer2/0]).
+
+-type barer2() :: string().
+
+% @doc Bar2 bars the bar2.
+-spec bar2() -> barer2().
+bar2() -> "Barer2".
diff --git a/test/rebar_edoc_SUITE_data/bad/apps/bar2/src/bar2_app.erl b/test/rebar_edoc_SUITE_data/bad/apps/bar2/src/bar2_app.erl new file mode 100644 index 0000000..d0058a0 --- /dev/null +++ b/test/rebar_edoc_SUITE_data/bad/apps/bar2/src/bar2_app.erl @@ -0,0 +1,26 @@ +%%%------------------------------------------------------------------- +%% @doc bar2 public API +%% @end +%%%------------------------------------------------------------------- + +-module(bar2_app). + +-behaviour(application). + +%% Application callbacks +-export([start/2, stop/1]). + +%%==================================================================== +%% API +%%==================================================================== + +start(_StartType, _StartArgs) -> + bar2_sup:start_link(). + +%%-------------------------------------------------------------------- +stop(_State) -> + ok. + +%%==================================================================== +%% Internal functions +%%==================================================================== diff --git a/test/rebar_edoc_SUITE_data/bad/apps/bar2/src/bar2_sup.erl b/test/rebar_edoc_SUITE_data/bad/apps/bar2/src/bar2_sup.erl new file mode 100644 index 0000000..0bdaf4a --- /dev/null +++ b/test/rebar_edoc_SUITE_data/bad/apps/bar2/src/bar2_sup.erl @@ -0,0 +1,35 @@ +%%%------------------------------------------------------------------- +%% @doc bar2 top level supervisor. +%% @end +%%%------------------------------------------------------------------- + +-module(bar2_sup). + +-behaviour(supervisor). + +%% API +-export([start_link/0]). + +%% Supervisor callbacks +-export([init/1]). + +-define(SERVER, ?MODULE). + +%%==================================================================== +%% API functions +%%==================================================================== + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +%%==================================================================== +%% Supervisor callbacks +%%==================================================================== + +%% Child :: {Id,StartFunc,Restart,Shutdown,Type,Modules} +init([]) -> + {ok, { {one_for_all, 0, 1}, []} }. + +%%==================================================================== +%% Internal functions +%%==================================================================== diff --git a/test/rebar_edoc_SUITE_data/bad/apps/foo/src/foo.app.src b/test/rebar_edoc_SUITE_data/bad/apps/foo/src/foo.app.src new file mode 100644 index 0000000..9987fd5 --- /dev/null +++ b/test/rebar_edoc_SUITE_data/bad/apps/foo/src/foo.app.src @@ -0,0 +1,17 @@ +{application, foo, + [{description, "An OTP application"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, { foo_app, []}}, + {applications, + [kernel, + stdlib, + bar1, bar2 + ]}, + {env,[]}, + {modules, []}, + + {maintainers, []}, + {licenses, []}, + {links, []} + ]}. diff --git a/test/rebar_edoc_SUITE_data/bad/apps/foo/src/foo.erl b/test/rebar_edoc_SUITE_data/bad/apps/foo/src/foo.erl new file mode 100644 index 0000000..52e3d0a --- /dev/null +++ b/test/rebar_edoc_SUITE_data/bad/apps/foo/src/foo.erl @@ -0,0 +1,19 @@ +-module(foo).
+
+-export([foo/0, bar1/0, bar2/0]).
+
+-export_type([fooer/0]).
+
+-type fooer() :: string().
+
+% @doc Foo function returns fooer.
+-spec foo() -> fooer().
+foo() -> "fooer".
+
+% @doc Bar1 function returns barer1.
+-spec bar1() -> bar1:barer1().
+bar1() -> bar1:bar1().
+
+% @doc Bar2 functions returns barer2.
+-spec bar2() -> bar2:barer2().
+bar2() -> bar2:bar2().
\ No newline at end of file diff --git a/test/rebar_edoc_SUITE_data/bad/apps/foo/src/foo_app.erl b/test/rebar_edoc_SUITE_data/bad/apps/foo/src/foo_app.erl new file mode 100644 index 0000000..d0158d7 --- /dev/null +++ b/test/rebar_edoc_SUITE_data/bad/apps/foo/src/foo_app.erl @@ -0,0 +1,26 @@ +%%%------------------------------------------------------------------- +%% @doc foo public API +%% @end +%%%------------------------------------------------------------------- + +-module(foo_app). + +-behaviour(application). + +%% Application callbacks +-export([start/2, stop/1]). + +%%==================================================================== +%% API +%%==================================================================== + +start(_StartType, _StartArgs) -> + foo_sup:start_link(). + +%%-------------------------------------------------------------------- +stop(_State) -> + ok. + +%%==================================================================== +%% Internal functions +%%==================================================================== diff --git a/test/rebar_edoc_SUITE_data/bad/apps/foo/src/foo_sup.erl b/test/rebar_edoc_SUITE_data/bad/apps/foo/src/foo_sup.erl new file mode 100644 index 0000000..67e88b4 --- /dev/null +++ b/test/rebar_edoc_SUITE_data/bad/apps/foo/src/foo_sup.erl @@ -0,0 +1,35 @@ +%%%------------------------------------------------------------------- +%% @doc foo top level supervisor. +%% @end +%%%------------------------------------------------------------------- + +-module(foo_sup). + +-behaviour(supervisor). + +%% API +-export([start_link/0]). + +%% Supervisor callbacks +-export([init/1]). + +-define(SERVER, ?MODULE). + +%%==================================================================== +%% API functions +%%==================================================================== + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +%%==================================================================== +%% Supervisor callbacks +%%==================================================================== + +%% Child :: {Id,StartFunc,Restart,Shutdown,Type,Modules} +init([]) -> + {ok, { {one_for_all, 0, 1}, []} }. + +%%==================================================================== +%% Internal functions +%%==================================================================== diff --git a/test/rebar_edoc_SUITE_data/foo/apps/bar1/src/bar1.app.src b/test/rebar_edoc_SUITE_data/foo/apps/bar1/src/bar1.app.src new file mode 100644 index 0000000..6e7ec24 --- /dev/null +++ b/test/rebar_edoc_SUITE_data/foo/apps/bar1/src/bar1.app.src @@ -0,0 +1,16 @@ +{application, bar1, + [{description, "An OTP application"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, { bar1_app, []}}, + {applications, + [kernel, + stdlib + ]}, + {env,[]}, + {modules, []}, + + {maintainers, []}, + {licenses, []}, + {links, []} + ]}. diff --git a/test/rebar_edoc_SUITE_data/foo/apps/bar1/src/bar1.erl b/test/rebar_edoc_SUITE_data/foo/apps/bar1/src/bar1.erl new file mode 100644 index 0000000..2700aef --- /dev/null +++ b/test/rebar_edoc_SUITE_data/foo/apps/bar1/src/bar1.erl @@ -0,0 +1,9 @@ +-module(bar1).
+-export([bar1/0]).
+-export_type([barer1/0]).
+
+-type barer1() :: string().
+
+% @doc Bar1 bars the bar.
+-spec bar1() -> barer1().
+bar1() -> "Barer1".
\ No newline at end of file diff --git a/test/rebar_edoc_SUITE_data/foo/apps/bar1/src/bar1_app.erl b/test/rebar_edoc_SUITE_data/foo/apps/bar1/src/bar1_app.erl new file mode 100644 index 0000000..414ac30 --- /dev/null +++ b/test/rebar_edoc_SUITE_data/foo/apps/bar1/src/bar1_app.erl @@ -0,0 +1,26 @@ +%%%------------------------------------------------------------------- +%% @doc bar1 public API +%% @end +%%%------------------------------------------------------------------- + +-module(bar1_app). + +-behaviour(application). + +%% Application callbacks +-export([start/2, stop/1]). + +%%==================================================================== +%% API +%%==================================================================== + +start(_StartType, _StartArgs) -> + bar1_sup:start_link(). + +%%-------------------------------------------------------------------- +stop(_State) -> + ok. + +%%==================================================================== +%% Internal functions +%%==================================================================== diff --git a/test/rebar_edoc_SUITE_data/foo/apps/bar1/src/bar1_sup.erl b/test/rebar_edoc_SUITE_data/foo/apps/bar1/src/bar1_sup.erl new file mode 100644 index 0000000..f9d6670 --- /dev/null +++ b/test/rebar_edoc_SUITE_data/foo/apps/bar1/src/bar1_sup.erl @@ -0,0 +1,35 @@ +%%%------------------------------------------------------------------- +%% @doc bar1 top level supervisor. +%% @end +%%%------------------------------------------------------------------- + +-module(bar1_sup). + +-behaviour(supervisor). + +%% API +-export([start_link/0]). + +%% Supervisor callbacks +-export([init/1]). + +-define(SERVER, ?MODULE). + +%%==================================================================== +%% API functions +%%==================================================================== + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +%%==================================================================== +%% Supervisor callbacks +%%==================================================================== + +%% Child :: {Id,StartFunc,Restart,Shutdown,Type,Modules} +init([]) -> + {ok, { {one_for_all, 0, 1}, []} }. + +%%==================================================================== +%% Internal functions +%%==================================================================== diff --git a/test/rebar_edoc_SUITE_data/foo/apps/bar2/src/bar2.app.src b/test/rebar_edoc_SUITE_data/foo/apps/bar2/src/bar2.app.src new file mode 100644 index 0000000..58de8bc --- /dev/null +++ b/test/rebar_edoc_SUITE_data/foo/apps/bar2/src/bar2.app.src @@ -0,0 +1,16 @@ +{application, bar2, + [{description, "An OTP application"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, { bar2_app, []}}, + {applications, + [kernel, + stdlib + ]}, + {env,[]}, + {modules, []}, + + {maintainers, []}, + {licenses, []}, + {links, []} + ]}. diff --git a/test/rebar_edoc_SUITE_data/foo/apps/bar2/src/bar2.erl b/test/rebar_edoc_SUITE_data/foo/apps/bar2/src/bar2.erl new file mode 100644 index 0000000..c639db0 --- /dev/null +++ b/test/rebar_edoc_SUITE_data/foo/apps/bar2/src/bar2.erl @@ -0,0 +1,9 @@ +-module(bar2).
+-export([bar2/0]).
+-export_type([barer2/0]).
+
+-type barer2() :: string().
+
+% @doc Bar2 bars the bar2.
+-spec bar2() -> barer2().
+bar2() -> "Barer2".
\ No newline at end of file diff --git a/test/rebar_edoc_SUITE_data/foo/apps/bar2/src/bar2_app.erl b/test/rebar_edoc_SUITE_data/foo/apps/bar2/src/bar2_app.erl new file mode 100644 index 0000000..d0058a0 --- /dev/null +++ b/test/rebar_edoc_SUITE_data/foo/apps/bar2/src/bar2_app.erl @@ -0,0 +1,26 @@ +%%%------------------------------------------------------------------- +%% @doc bar2 public API +%% @end +%%%------------------------------------------------------------------- + +-module(bar2_app). + +-behaviour(application). + +%% Application callbacks +-export([start/2, stop/1]). + +%%==================================================================== +%% API +%%==================================================================== + +start(_StartType, _StartArgs) -> + bar2_sup:start_link(). + +%%-------------------------------------------------------------------- +stop(_State) -> + ok. + +%%==================================================================== +%% Internal functions +%%==================================================================== diff --git a/test/rebar_edoc_SUITE_data/foo/apps/bar2/src/bar2_sup.erl b/test/rebar_edoc_SUITE_data/foo/apps/bar2/src/bar2_sup.erl new file mode 100644 index 0000000..0bdaf4a --- /dev/null +++ b/test/rebar_edoc_SUITE_data/foo/apps/bar2/src/bar2_sup.erl @@ -0,0 +1,35 @@ +%%%------------------------------------------------------------------- +%% @doc bar2 top level supervisor. +%% @end +%%%------------------------------------------------------------------- + +-module(bar2_sup). + +-behaviour(supervisor). + +%% API +-export([start_link/0]). + +%% Supervisor callbacks +-export([init/1]). + +-define(SERVER, ?MODULE). + +%%==================================================================== +%% API functions +%%==================================================================== + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +%%==================================================================== +%% Supervisor callbacks +%%==================================================================== + +%% Child :: {Id,StartFunc,Restart,Shutdown,Type,Modules} +init([]) -> + {ok, { {one_for_all, 0, 1}, []} }. + +%%==================================================================== +%% Internal functions +%%==================================================================== diff --git a/test/rebar_edoc_SUITE_data/foo/apps/foo/src/foo.app.src b/test/rebar_edoc_SUITE_data/foo/apps/foo/src/foo.app.src new file mode 100644 index 0000000..9987fd5 --- /dev/null +++ b/test/rebar_edoc_SUITE_data/foo/apps/foo/src/foo.app.src @@ -0,0 +1,17 @@ +{application, foo, + [{description, "An OTP application"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, { foo_app, []}}, + {applications, + [kernel, + stdlib, + bar1, bar2 + ]}, + {env,[]}, + {modules, []}, + + {maintainers, []}, + {licenses, []}, + {links, []} + ]}. diff --git a/test/rebar_edoc_SUITE_data/foo/apps/foo/src/foo.erl b/test/rebar_edoc_SUITE_data/foo/apps/foo/src/foo.erl new file mode 100644 index 0000000..52e3d0a --- /dev/null +++ b/test/rebar_edoc_SUITE_data/foo/apps/foo/src/foo.erl @@ -0,0 +1,19 @@ +-module(foo).
+
+-export([foo/0, bar1/0, bar2/0]).
+
+-export_type([fooer/0]).
+
+-type fooer() :: string().
+
+% @doc Foo function returns fooer.
+-spec foo() -> fooer().
+foo() -> "fooer".
+
+% @doc Bar1 function returns barer1.
+-spec bar1() -> bar1:barer1().
+bar1() -> bar1:bar1().
+
+% @doc Bar2 functions returns barer2.
+-spec bar2() -> bar2:barer2().
+bar2() -> bar2:bar2().
\ No newline at end of file diff --git a/test/rebar_edoc_SUITE_data/foo/apps/foo/src/foo_app.erl b/test/rebar_edoc_SUITE_data/foo/apps/foo/src/foo_app.erl new file mode 100644 index 0000000..d0158d7 --- /dev/null +++ b/test/rebar_edoc_SUITE_data/foo/apps/foo/src/foo_app.erl @@ -0,0 +1,26 @@ +%%%------------------------------------------------------------------- +%% @doc foo public API +%% @end +%%%------------------------------------------------------------------- + +-module(foo_app). + +-behaviour(application). + +%% Application callbacks +-export([start/2, stop/1]). + +%%==================================================================== +%% API +%%==================================================================== + +start(_StartType, _StartArgs) -> + foo_sup:start_link(). + +%%-------------------------------------------------------------------- +stop(_State) -> + ok. + +%%==================================================================== +%% Internal functions +%%==================================================================== diff --git a/test/rebar_edoc_SUITE_data/foo/apps/foo/src/foo_sup.erl b/test/rebar_edoc_SUITE_data/foo/apps/foo/src/foo_sup.erl new file mode 100644 index 0000000..67e88b4 --- /dev/null +++ b/test/rebar_edoc_SUITE_data/foo/apps/foo/src/foo_sup.erl @@ -0,0 +1,35 @@ +%%%------------------------------------------------------------------- +%% @doc foo top level supervisor. +%% @end +%%%------------------------------------------------------------------- + +-module(foo_sup). + +-behaviour(supervisor). + +%% API +-export([start_link/0]). + +%% Supervisor callbacks +-export([init/1]). + +-define(SERVER, ?MODULE). + +%%==================================================================== +%% API functions +%%==================================================================== + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +%%==================================================================== +%% Supervisor callbacks +%%==================================================================== + +%% Child :: {Id,StartFunc,Restart,Shutdown,Type,Modules} +init([]) -> + {ok, { {one_for_all, 0, 1}, []} }. + +%%==================================================================== +%% Internal functions +%%==================================================================== diff --git a/test/rebar_escriptize_SUITE.erl b/test/rebar_escriptize_SUITE.erl index 1817d6b..a6901f9 100644 --- a/test/rebar_escriptize_SUITE.erl +++ b/test/rebar_escriptize_SUITE.erl @@ -5,7 +5,10 @@ end_per_suite/1, init_per_testcase/2, all/0, - build_and_clean_app/1]). + escriptize_with_name/1, + escriptize_with_bad_name/1, + build_and_clean_app/1, + escriptize_with_ebin_subdir/1]). -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -24,7 +27,12 @@ init_per_testcase(_, Config) -> rebar_test_utils:init_rebar_state(Config). all() -> - [build_and_clean_app]. + [ + build_and_clean_app, + escriptize_with_name, + escriptize_with_bad_name, + escriptize_with_ebin_subdir + ]. %% Test escriptize builds and runs the app's escript build_and_clean_app(Config) -> @@ -35,3 +43,42 @@ build_and_clean_app(Config) -> rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), rebar_test_utils:run_and_check(Config, [], ["escriptize"], {ok, [{app, Name, valid}]}). + +escriptize_with_name(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("app1_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + rebar_test_utils:run_and_check(Config, [{escript_main_app, Name}], ["escriptize"], + {ok, [{app, Name, valid}]}). + +escriptize_with_bad_name(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("app1_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + rebar_test_utils:run_and_check(Config, [{escript_main_app, boogers}], ["escriptize"], + {error,{rebar_prv_escriptize, {bad_name, boogers}}}). + +escriptize_with_ebin_subdir(Config) -> + AppDir = ?config(apps, Config), + Name = rebar_test_utils:create_random_name("app1_"), + Vsn = rebar_test_utils:create_random_vsn(), + + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + filelib:ensure_dir(filename:join([AppDir, "ebin", "subdir", "subdirfile"])), + + %% To work, this test must run from the AppDir itself. To avoid breaking + %% other tests, be careful with cwd + Cwd = file:get_cwd(), + try + file:set_cwd(AppDir), + {ok, _} = rebar3:run(rebar_state:new(?config(state,Config), [], AppDir), + ["escriptize"]) + after + file:set_cwd(Cwd) % reset always + end, + ok. diff --git a/test/rebar_eunit_SUITE.erl b/test/rebar_eunit_SUITE.erl index 41ab6ff..1a8bade 100644 --- a/test/rebar_eunit_SUITE.erl +++ b/test/rebar_eunit_SUITE.erl @@ -1,7 +1,8 @@ -module(rebar_eunit_SUITE). -export([all/0, groups/0]). --export([init_per_suite/1, init_per_group/2, end_per_group/2]). +-export([init_per_suite/1, end_per_suite/1]). +-export([init_per_group/2, end_per_group/2]). -export([basic_app_compiles/1, basic_app_files/1]). -export([basic_app_exports/1, basic_app_testset/1]). -export([basic_app_eunit_macro/1]). @@ -18,6 +19,7 @@ -export([misspecified_eunit_tests/1]). -export([misspecified_eunit_compile_opts/1]). -export([misspecified_eunit_first_files/1]). +-export([alternate_test_regex/1]). -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -27,7 +29,8 @@ all() -> [{group, basic_app}, {group, multi_app}, {group, cmd_line_args}, misspecified_eunit_tests, misspecified_eunit_compile_opts, - misspecified_eunit_first_files]. + misspecified_eunit_first_files, + alternate_test_regex]. groups() -> [{basic_app, [sequence], [basic_app_compiles, {group, basic_app_results}]}, @@ -58,6 +61,8 @@ init_per_suite(Config) -> {ok, _} = zip:extract(filename:join([PrivDir, "multi_app.zip"]), [{cwd, PrivDir}]), Config. +end_per_suite(Config) -> Config. + init_per_group(basic_app, Config) -> GroupState = rebar_test_utils:init_rebar_state(Config, "basic_app_"), @@ -579,3 +584,25 @@ misspecified_eunit_first_files(Config) -> {error, {rebar_prv_eunit, Error}} = rebar_test_utils:run_and_check(State, RebarConfig, ["eunit"], return), {badconfig, {"Value `~p' of option `~p' must be a list", {some_file, eunit_first_files}}} = Error. + +alternate_test_regex(Config) -> + State = rebar_test_utils:init_rebar_state(Config, "alternate_test_regex_"), + + AppDir = ?config(apps, State), + PrivDir = ?config(priv_dir, State), + + AppDirs = ["src", "include", "test"], + + lists:foreach(fun(F) -> ec_file:copy(filename:join([PrivDir, "basic_app", F]), + filename:join([AppDir, F]), + [recursive]) end, AppDirs), + + BaseConfig = [{erl_opts, [{d, config_define}]}, {eunit_compile_opts, [{d, eunit_compile_define}]}], + + RebarConfig = [{eunit_test_regex, "basic_app_tests.erl"}|BaseConfig], + + {ok, S} = rebar_test_utils:run_and_check(State, RebarConfig, ["as", "test", "lock"], return), + + Set = {ok, [{application, basic_app}, + {module, basic_app_tests}]}, + Set = rebar_prv_eunit:prepare_tests(S). diff --git a/test/rebar_file_utils_SUITE.erl b/test/rebar_file_utils_SUITE.erl index a44a06d..4cc6a93 100644 --- a/test/rebar_file_utils_SUITE.erl +++ b/test/rebar_file_utils_SUITE.erl @@ -4,6 +4,8 @@ groups/0, init_per_group/2, end_per_group/2, + init_per_testcase/2, + end_per_testcase/2, raw_tmpdir/1, empty_tmpdir/1, simple_tmpdir/1, @@ -14,7 +16,14 @@ path_from_ancestor/1, canonical_path/1, resolve_link/1, - split_dirname/1]). + split_dirname/1, + mv_warning_is_ignored/1, + mv_dir/1, + mv_file_same/1, + mv_file_diff/1, + mv_file_dir_same/1, + mv_file_dir_diff/1, + mv_no_clobber/1]). -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -24,14 +33,18 @@ all() -> [{group, tmpdir}, {group, reset_dir}, + {group, mv}, path_from_ancestor, canonical_path, resolve_link, - split_dirname]. + split_dirname, + mv_warning_is_ignored]. groups() -> [{tmpdir, [], [raw_tmpdir, empty_tmpdir, simple_tmpdir, multi_tmpdir]}, - {reset_dir, [], [reset_nonexistent_dir, reset_empty_dir, reset_dir]}]. + {reset_dir, [], [reset_nonexistent_dir, reset_empty_dir, reset_dir]}, + {mv, [], [mv_dir, mv_file_same, mv_file_diff, + mv_file_dir_same, mv_file_dir_diff, mv_no_clobber]}]. init_per_group(reset_dir, Config) -> TmpDir = rebar_file_utils:system_tmpdir(["rebar_file_utils_SUITE", "resetable"]), @@ -39,6 +52,20 @@ init_per_group(reset_dir, Config) -> init_per_group(_, Config) -> Config. end_per_group(_, Config) -> Config. +init_per_testcase(Test, Config) -> + case os:type() of + {win32, _} -> + case lists:member(Test, [resolve_link, mv_warning_is_ignored]) of + true -> {skip, "broken in windows"}; + false -> Config + end; + _ -> + Config + end. + +end_per_testcase(_Test, Config) -> + Config. + raw_tmpdir(_Config) -> case rebar_file_utils:system_tmpdir() of "/tmp" -> ok; @@ -135,3 +162,170 @@ split_dirname(_Config) -> ?assertEqual({".", "foo"}, rebar_file_utils:split_dirname("foo")), ?assertEqual({"/foo", "bar"}, rebar_file_utils:split_dirname("/foo/bar")), ?assertEqual({"foo", "bar"}, rebar_file_utils:split_dirname("foo/bar")). + +mv_warning_is_ignored(_Config) -> + meck:new(rebar_utils, [passthrough]), + meck:expect(rebar_utils, sh, fun("mv ding dong", _) -> {ok, "Warning"} end), + ok = rebar_file_utils:mv("ding", "dong"), + meck:unload(rebar_utils). + +%%% Ensure Windows & Unix operations to move files + +mv_dir(Config) -> + %% Move a directory to another one location + PrivDir = ?config(priv_dir, Config), + BaseDir = mk_base_dir(PrivDir, mv_dir), + SrcDir = filename:join(BaseDir, "src/"), + ec_file:mkdir_p(SrcDir), + ?assert(filelib:is_dir(SrcDir)), + %% empty dir movement + DstDir1 = filename:join(BaseDir, "dst1/"), + ?assertNot(filelib:is_dir(DstDir1)), + ?assertEqual(ok, rebar_file_utils:mv(SrcDir, DstDir1)), + ?assert(filelib:is_dir(DstDir1)), + + %% move files from dir to empty dir + F1 = filename:join(SrcDir, "file1"), + F2 = filename:join(SrcDir, "subdir/file2"), + filelib:ensure_dir(F2), + file:write_file(F1, "hello"), + file:write_file(F2, "world"), + DstDir2 = filename:join(BaseDir, "dst2/"), + D2F1 = filename:join(DstDir2, "file1"), + D2F2 = filename:join(DstDir2, "subdir/file2"), + ?assertNot(filelib:is_dir(DstDir2)), + ?assertEqual(ok, rebar_file_utils:mv(SrcDir, DstDir2)), + ?assert(filelib:is_file(D2F1)), + ?assert(filelib:is_file(D2F2)), + + %% move files from dir to existing dir moves it to + %% a subdir + filelib:ensure_dir(F2), + file:write_file(F1, "hello"), + file:write_file(F2, "world"), + DstDir3 = filename:join(BaseDir, "dst3/"), + D3F1 = filename:join(DstDir3, "src/file1"), + D3F2 = filename:join(DstDir3, "src/subdir/file2"), + ec_file:mkdir_p(DstDir3), + ?assert(filelib:is_dir(DstDir3)), + ?assertEqual(ok, rebar_file_utils:mv(SrcDir, DstDir3)), + ?assertNot(filelib:is_file(F1)), + ?assertNot(filelib:is_file(F2)), + ?assert(filelib:is_file(D3F1)), + ?assert(filelib:is_file(D3F2)), + ?assertNot(filelib:is_dir(SrcDir)), + ok. + +mv_file_same(Config) -> + %% Move a file from a directory to the other without renaming + PrivDir = ?config(priv_dir, Config), + BaseDir = mk_base_dir(PrivDir, mv_file_same), + SrcDir = filename:join(BaseDir, "src/"), + ec_file:mkdir_p(SrcDir), + ?assert(filelib:is_dir(SrcDir)), + F = filename:join(SrcDir, "file"), + file:write_file(F, "hello"), + DstDir = filename:join(BaseDir, "dst/"), + ec_file:mkdir_p(DstDir), + Dst = filename:join(DstDir, "file"), + ?assertEqual(ok, rebar_file_utils:mv(F, Dst)), + ?assert(filelib:is_file(Dst)), + ?assertNot(filelib:is_file(F)), + ok. + +mv_file_diff(Config) -> + %% Move a file from a directory to another one while renaming + %% into a pre-existing file + PrivDir = ?config(priv_dir, Config), + BaseDir = mk_base_dir(PrivDir, mv_file_diff), + SrcDir = filename:join(BaseDir, "src/"), + ec_file:mkdir_p(SrcDir), + ?assert(filelib:is_dir(SrcDir)), + F = filename:join(SrcDir, "file"), + file:write_file(F, "hello"), + DstDir = filename:join(BaseDir, "dst/"), + ec_file:mkdir_p(DstDir), + Dst = filename:join(DstDir, "file-rename"), + file:write_file(Dst, "not-the-right-content"), + ?assert(filelib:is_file(Dst)), + ?assertEqual(ok, rebar_file_utils:mv(F, Dst)), + ?assert(filelib:is_file(Dst)), + ?assertEqual({ok, <<"hello">>}, file:read_file(Dst)), + ?assertNot(filelib:is_file(F)), + ok. + +mv_file_dir_same(Config) -> + %% Move a file to a directory without renaming + PrivDir = ?config(priv_dir, Config), + BaseDir = mk_base_dir(PrivDir, mv_file_dir_same), + SrcDir = filename:join(BaseDir, "src/"), + ec_file:mkdir_p(SrcDir), + ?assert(filelib:is_dir(SrcDir)), + F = filename:join(SrcDir, "file"), + file:write_file(F, "hello"), + DstDir = filename:join(BaseDir, "dst/"), + ec_file:mkdir_p(DstDir), + Dst = filename:join(DstDir, "file"), + ?assert(filelib:is_dir(DstDir)), + ?assertEqual(ok, rebar_file_utils:mv(F, DstDir)), + ?assert(filelib:is_file(Dst)), + ?assertNot(filelib:is_file(F)), + ok. + +mv_file_dir_diff(Config) -> + %% Move a file to a directory while renaming + PrivDir = ?config(priv_dir, Config), + BaseDir = mk_base_dir(PrivDir, mv_file_dir_diff), + SrcDir = filename:join(BaseDir, "src/"), + ec_file:mkdir_p(SrcDir), + ?assert(filelib:is_dir(SrcDir)), + F = filename:join(SrcDir, "file"), + file:write_file(F, "hello"), + DstDir = filename:join(BaseDir, "dst/"), + ec_file:mkdir_p(DstDir), + Dst = filename:join(DstDir, "file-rename"), + ?assert(filelib:is_dir(DstDir)), + ?assertNot(filelib:is_file(Dst)), + ?assertEqual(ok, rebar_file_utils:mv(F, Dst)), + ?assert(filelib:is_file(Dst)), + ?assertNot(filelib:is_file(F)), + ok. + +mv_no_clobber(Config) -> + %% Moving a file while renaming does not clobber other files + PrivDir = ?config(priv_dir, Config), + BaseDir = mk_base_dir(PrivDir, mv_no_clobber), + SrcDir = filename:join(BaseDir, "src/"), + ec_file:mkdir_p(SrcDir), + ?assert(filelib:is_dir(SrcDir)), + F = filename:join(SrcDir, "file"), + file:write_file(F, "hello"), + FBad = filename:join(SrcDir, "file-alt"), + file:write_file(FBad, "wrong-data"), + DstDir = filename:join(BaseDir, "dst/"), + ec_file:mkdir_p(DstDir), + Dst = filename:join(DstDir, "file-alt"), + DstBad = filename:join(DstDir, "file"), + file:write_file(DstBad, "wrong-data"), + ?assert(filelib:is_file(F)), + ?assert(filelib:is_file(FBad)), + ?assert(filelib:is_dir(DstDir)), + ?assertNot(filelib:is_file(Dst)), + ?assert(filelib:is_file(DstBad)), + ?assertEqual(ok, rebar_file_utils:mv(F, Dst)), + ?assert(filelib:is_file(Dst)), + ?assertNot(filelib:is_file(F)), + ?assert(filelib:is_file(DstBad)), + ?assert(filelib:is_file(FBad)), + ?assertEqual({ok, <<"hello">>}, file:read_file(Dst)), + ?assertEqual({ok, <<"wrong-data">>}, file:read_file(FBad)), + ?assertEqual({ok, <<"wrong-data">>}, file:read_file(DstBad)), + ok. + + +mk_base_dir(BasePath, Name) -> + {_,_,Micro} = os:timestamp(), + Index = integer_to_list(Micro), + Path = filename:join(BasePath, atom_to_list(Name) ++ Index), + ec_file:mkdir_p(Path), + Path. diff --git a/test/rebar_hooks_SUITE.erl b/test/rebar_hooks_SUITE.erl index b121dd5..29e343f 100644 --- a/test/rebar_hooks_SUITE.erl +++ b/test/rebar_hooks_SUITE.erl @@ -1,17 +1,6 @@ -module(rebar_hooks_SUITE). --export([suite/0, - init_per_suite/1, - end_per_suite/1, - init_per_testcase/2, - end_per_testcase/2, - all/0, - build_and_clean_app/1, - escriptize_artifacts/1, - run_hooks_once/1, - run_hooks_for_plugins/1, - eunit_app_hooks/1, - deps_hook_namespace/1]). +-compile(export_all). -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -33,8 +22,10 @@ end_per_testcase(_, _Config) -> catch meck:unload(). all() -> - [build_and_clean_app, run_hooks_once, escriptize_artifacts, - run_hooks_for_plugins, deps_hook_namespace, eunit_app_hooks]. + [build_and_clean_app, run_hooks_once, run_hooks_once_profiles, + escriptize_artifacts, run_hooks_for_plugins, deps_hook_namespace, + bare_compile_hooks_default_ns, deps_clean_hook_namespace, eunit_app_hooks, + sub_app_hooks, root_hooks]. %% Test post provider hook cleans compiled project app, leaving it invalid build_and_clean_app(Config) -> @@ -93,11 +84,23 @@ run_hooks_once(Config) -> Name = rebar_test_utils:create_random_name("app1_"), Vsn = rebar_test_utils:create_random_vsn(), - RebarConfig = [{pre_hooks, [{compile, "mkdir blah"}]}], + RebarConfig = [{pre_hooks, [{compile, "mkdir \"$REBAR_ROOT_DIR/blah\""}]}], rebar_test_utils:create_config(AppDir, RebarConfig), rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], {ok, [{app, Name, valid}]}). +%% test that even if a hook is defined at the project level in a used profile +%% the hook is not run for each application in the project umbrella +run_hooks_once_profiles(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("app1_"), + Vsn = rebar_test_utils:create_random_vsn(), + RebarConfig = [{profiles, [{hooks, [{pre_hooks, [{compile, "mkdir \"$REBAR_ROOT_DIR/blah\""}]}]}]}], + rebar_test_utils:create_config(AppDir, RebarConfig), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + rebar_test_utils:run_and_check(Config, RebarConfig, ["as", "hooks", "compile"], {ok, [{app, Name, valid}]}). + deps_hook_namespace(Config) -> mock_git_resource:mock([{deps, [{some_dep, "0.0.1"}]}]), Deps = rebar_test_utils:expand_deps(git, [{"some_dep", "0.0.1", []}]), @@ -120,6 +123,44 @@ deps_hook_namespace(Config) -> {ok, [{dep, "some_dep"}]} ). +%% tests that hooks to compile when running bare compile run in the default namespace +bare_compile_hooks_default_ns(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("app1_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + RConfFile = rebar_test_utils:create_config(AppDir, + [{provider_hooks, [{post, [{compile, clean}]}]}]), + {ok, RConf} = file:consult(RConfFile), + rebar_test_utils:run_and_check( + Config, RConf, ["bare", "compile", "--paths", "."], + {ok, []} + ). + +deps_clean_hook_namespace(Config) -> + mock_git_resource:mock([{deps, [{some_dep, "0.0.1"}]}]), + Deps = rebar_test_utils:expand_deps(git, [{"some_dep", "0.0.1", []}]), + TopDeps = rebar_test_utils:top_level_deps(Deps), + + RebarConfig = [ + {deps, TopDeps}, + {overrides, [ + {override, some_dep, [ + {provider_hooks, [ + {pre, [ + {compile, clean} + ]} + ]} + ]} + ]} + ], + rebar_test_utils:run_and_check( + Config, RebarConfig, ["clean"], + {ok, [{dep, "some_dep"}]} + ). + %% Checks that a hook that is defined on an app (not a top level hook of a project with subapps) is run eunit_app_hooks(Config) -> AppDir = ?config(apps, Config), @@ -161,3 +202,45 @@ run_hooks_for_plugins(Config) -> rebar_test_utils:run_and_check(Config, RConf, ["compile"], {ok, [{app, Name, valid}, {plugin, PluginName}, {file, filename:join([AppDir, "_build", "default", "plugins", PluginName, "randomfile"])}]}). + +%% test that a subapp of a project keeps its hooks +sub_app_hooks(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("sub_app1_"), + Vsn = rebar_test_utils:create_random_vsn(), + + SubAppsDir = filename:join([AppDir, "apps", Name]), + + rebar_test_utils:create_app(SubAppsDir, Name, Vsn, [kernel, stdlib]), + rebar_test_utils:create_config(SubAppsDir, [{provider_hooks, [{post, [{compile, clean}]}]}]), + + RConfFile = rebar_test_utils:create_config(AppDir, []), + {ok, RConf} = file:consult(RConfFile), + + %% Build with deps. + rebar_test_utils:run_and_check( + Config, RConf, ["compile"], + {ok, [{app, Name, invalid}]} + ). + +%% test that hooks at the top level don't run in the subapps +root_hooks(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("sub_app1_"), + Vsn = rebar_test_utils:create_random_vsn(), + + SubAppsDir = filename:join([AppDir, "apps", Name]), + + rebar_test_utils:create_app(SubAppsDir, Name, Vsn, [kernel, stdlib]), + rebar_test_utils:create_config(SubAppsDir, [{provider_hooks, [{post, [{compile, clean}]}]}]), + + RConfFile = rebar_test_utils:create_config(AppDir, [{pre_hooks, [{compile, "mkdir \"$REBAR_ROOT_DIR/blah\""}]}]), + {ok, RConf} = file:consult(RConfFile), + + %% Build with deps. + rebar_test_utils:run_and_check( + Config, RConf, ["compile"], + {ok, [{app, Name, invalid}]} + ). diff --git a/test/rebar_install_deps_SUITE.erl b/test/rebar_install_deps_SUITE.erl index 9ff28c7..96b9d38 100644 --- a/test/rebar_install_deps_SUITE.erl +++ b/test/rebar_install_deps_SUITE.erl @@ -17,7 +17,7 @@ groups() -> {mixed, [], [ m_flat1, m_flat2, m_circular1, m_circular2, m_pick_source1, m_pick_source2, m_pick_source3, - m_pick_source4, m_pick_source5, m_source_to_pkg, + m_pick_source4, m_pick_source5, m_pick_source6, m_source_to_pkg, m_pkg_level1, m_pkg_level2, m_pkg_level3, m_pkg_level3_alpha_order ]} ]. @@ -82,22 +82,24 @@ format_expected_mdeps(Deps) -> lists:append([ case Dep of {N,V} when hd(N) >= $a, hd(N) =< $z -> - UN = string:to_upper(N), + UN = rebar_string:uppercase(N), [{dep, UN, V}, {lock, pkg, UN, V}]; {N,V} when hd(N) >= $A, hd(N) =< $Z -> [{dep, N, V}, {lock, src, N, V}]; N when hd(N) >= $a, hd(N) =< $z -> - UN = string:to_upper(N), + UN = rebar_string:uppercase(N), [{dep, UN}, {lock, pkg, UN, "0.0.0"}]; N when hd(N) >= $A, hd(N) =< $Z -> [{dep, N}, {lock, src, N, "0.0.0"}] end || Dep <- Deps]). +format_expected_mixed_warnings(none) -> + none; format_expected_mixed_warnings(Warnings) -> [case W of - {N, Vsn} when hd(N) >= $a, hd(N) =< $z -> {pkg, string:to_upper(N), Vsn}; + {N, Vsn} when hd(N) >= $a, hd(N) =< $z -> {pkg, rebar_string:uppercase(N), Vsn}; {N, Vsn} when hd(N) >= $A, hd(N) =< $Z -> {git, N, Vsn}; - N when hd(N) >= $a, hd(N) =< $z -> {pkg, string:to_upper(N), "0.0.0"}; + N when hd(N) >= $a, hd(N) =< $z -> {pkg, rebar_string:uppercase(N), "0.0.0"}; N when hd(N) >= $A, hd(N) =< $Z -> {git, N, "0.0.0"} end || W <- Warnings]. @@ -121,27 +123,27 @@ deps(flat) -> [], {ok, ["B", "C"]}}; deps(pick_highest_left) -> - {[{"B", [{"C", "2", []}]}, - {"C", "1", []}], - [{"C","2"}], - {ok, ["B", {"C", "1"}]}}; + {[{"B", [{"C", "2.0.0", []}]}, + {"C", "1.0.0", []}], + [{"C","2.0.0"}], + {ok, ["B", {"C", "1.0.0"}]}}; deps(pick_highest_right) -> - {[{"B", "1", []}, - {"C", [{"B", "2", []}]}], - [{"B","2"}], - {ok, [{"B","1"}, "C"]}}; + {[{"B", "1.0.0", []}, + {"C", [{"B", "2.0.0", []}]}], + [{"B","2.0.0"}], + {ok, [{"B","1.0.0"}, "C"]}}; deps(pick_smallest1) -> - {[{"B", [{"D", "1", []}]}, - {"C", [{"D", "2", []}]}], - [{"D","2"}], + {[{"B", [{"D", "1.0.0", []}]}, + {"C", [{"D", "2.0.0", []}]}], + [{"D","2.0.0"}], %% we pick D1 because B < C - {ok, ["B","C",{"D","1"}]}}; + {ok, ["B","C",{"D","1.0.0"}]}}; deps(pick_smallest2) -> - {[{"C", [{"D", "2", []}]}, - {"B", [{"D", "1", []}]}], - [{"D","2"}], + {[{"C", [{"D", "2.0.0", []}]}, + {"B", [{"D", "1.0.0", []}]}], + [{"D","2.0.0"}], %% we pick D1 because B < C - {ok, ["B","C",{"D","1"}]}}; + {ok, ["B","C",{"D","1.0.0"}]}}; deps(circular1) -> {[{"B", [{"A", []}]}, % A is the top-level app {"C", []}], @@ -155,14 +157,14 @@ deps(circular2) -> deps(circular_skip) -> %% Never spot the circular dep due to being to low in the deps tree %% in source deps - {[{"B", [{"C", "2", [{"B", []}]}]}, - {"C", "1", [{"D",[]}]}], - [{"C","2"}], - {ok, ["B", {"C","1"}, "D"]}}; + {[{"B", [{"C", "2.0.0", [{"B", []}]}]}, + {"C", "1.0.0", [{"D",[]}]}], + [{"C","2.0.0"}], + {ok, ["B", {"C","1.0.0"}, "D"]}}; deps(fail_conflict) -> - {[{"B", [{"C", "2", []}]}, - {"C", "1", []}], - [{"C","2"}], + {[{"B", [{"C", "2.0.0", []}]}, + {"C", "1.0.0", []}], + [{"C","2.0.0"}], rebar_abort}; deps(default_profile) -> {[{"B", []}, @@ -216,39 +218,44 @@ mdeps(m_pick_source3) -> [], {ok, ["B"]}}; mdeps(m_pick_source4) -> - {[{"b", [{"d", "1", []}]}, - {"C", [{"D", "1", []}]}], - [{"D", "1"}], - {ok, ["b", "C", {"d", "1"}]}}; + {[{"b", [{"d", "1.0.0", []}]}, + {"C", [{"D", "1.0.0", []}]}], + [{"D", "1.0.0"}], + {ok, ["b", "C", {"d", "1.0.0"}]}}; mdeps(m_pick_source5) -> - {[{"B", [{"d", "1", []}]}, - {"C", [{"D", "1", []}]}], - [{"D", "1"}], - {ok, ["B", "C", {"d", "1"}]}}; + {[{"B", [{"d", "1.0.0", []}]}, + {"C", [{"D", "1.0.0", []}]}], + [{"D", "1.0.0"}], + {ok, ["B", "C", {"d", "1.0.0"}]}}; +mdeps(m_pick_source6) -> + {[{"B", [{"D", "1.0.0", []}]}, + {"C", [{"D", "1.0.0", []}]}], + none, + {ok, ["B", "C", {"D", "1.0.0"}]}}; mdeps(m_source_to_pkg) -> {[{"B", [{"c",[{"d", []}]}]}], [], {ok, ["B", "c", "d"]}}; mdeps(m_pkg_level1) -> - {[{"B", [{"D", [{"e", "2", []}]}]}, - {"C", [{"e", "1", []}]}], - [{"e","2"}], - {ok, ["B","C","D",{"e","1"}]}}; + {[{"B", [{"D", [{"e", "2.0.0", []}]}]}, + {"C", [{"e", "1.0.0", []}]}], + [{"e","2.0.0"}], + {ok, ["B","C","D",{"e","1.0.0"}]}}; mdeps(m_pkg_level2) -> - {[{"B", [{"e", "1", []}]}, - {"C", [{"D", [{"e", "2", []}]}]}], - [{"e","2"}], - {ok, ["B","C","D",{"e","1"}]}}; + {[{"B", [{"e", "1.0.0", []}]}, + {"C", [{"D", [{"e", "2.0.0", []}]}]}], + [{"e","2.0.0"}], + {ok, ["B","C","D",{"e","1.0.0"}]}}; mdeps(m_pkg_level3_alpha_order) -> - {[{"B", [{"d", [{"f", "1", []}]}]}, - {"C", [{"E", [{"f", "2", []}]}]}], - [{"f","2"}], - {ok, ["B","C","d","E",{"f","1"}]}}; + {[{"B", [{"d", [{"f", "1.0.0", []}]}]}, + {"C", [{"E", [{"f", "2.0.0", []}]}]}], + [{"f","2.0.0"}], + {ok, ["B","C","d","E",{"f","1.0.0"}]}}; mdeps(m_pkg_level3) -> - {[{"B", [{"d", [{"f", "1", []}]}]}, - {"C", [{"E", [{"G", [{"f", "2", []}]}]}]}], - [{"f","2"}], - {ok, ["B","C","d","E","G",{"f","1"}]}}. + {[{"B", [{"d", [{"f", "1.0.0", []}]}]}, + {"C", [{"E", [{"G", [{"f", "2.0.0", []}]}]}]}], + [{"f","2.0.0"}], + {ok, ["B","C","d","E","G",{"f","1.0.0"}]}}. setup_project(fail_conflict, Config0, Deps) -> DepsType = ?config(deps_type, Config0), @@ -289,8 +296,8 @@ setup_project(nondefault_pick_highest, Config0, _) -> ), AppDir = ?config(apps, Config), rebar_test_utils:create_app(AppDir, "A", "0.0.0", [kernel, stdlib]), - DefaultDeps = rebar_test_utils:expand_deps(DepsType, [{"B", [{"C", "1", []}]}]), - ProfileDeps = rebar_test_utils:expand_deps(DepsType, [{"C", "2", []}]), + DefaultDeps = rebar_test_utils:expand_deps(DepsType, [{"B", [{"C", "1.0.0", []}]}]), + ProfileDeps = rebar_test_utils:expand_deps(DepsType, [{"C", "2.0.0", []}]), DefaultTop = rebar_test_utils:top_level_deps(DefaultDeps), ProfileTop = rebar_test_utils:top_level_deps(ProfileDeps), RebarConf = rebar_test_utils:create_config( @@ -412,19 +419,19 @@ nondefault_pick_highest(Config) -> {ok, RebarConfig} = file:consult(?config(rebarconfig, Config)), rebar_test_utils:run_and_check( Config, RebarConfig, ["lock"], - {ok, [{dep, "B"}, {lock, "B"}, {lock, "C", "1"}, {dep, "C", "1"}], "default"} + {ok, [{dep, "B"}, {lock, "B"}, {lock, "C", "1.0.0"}, {dep, "C", "1.0.0"}], "default"} ), rebar_test_utils:run_and_check( Config, RebarConfig, ["as", "nondef", "lock"], - {ok, [{dep, "B"}, {lock, "B"}, {lock, "C", "1"}, {dep, "C", "2"}], "nondef"} + {ok, [{dep, "B"}, {lock, "B"}, {lock, "C", "1.0.0"}, {dep, "C", "2.0.0"}], "nondef"} ), rebar_test_utils:run_and_check( Config, RebarConfig, ["lock"], - {ok, [{dep, "B"}, {lock, "B"}, {dep, "C", "1"}, {lock, "C", "1"}], "default"} + {ok, [{dep, "B"}, {lock, "B"}, {dep, "C", "1.0.0"}, {lock, "C", "1.0.0"}], "default"} ), rebar_test_utils:run_and_check( Config, RebarConfig, ["as", "nondef", "lock"], - {ok, [{dep, "B"}, {lock, "B"}, {lock, "C", "1"}, {dep, "C", "2"}], "nondef"} + {ok, [{dep, "B"}, {lock, "B"}, {lock, "C", "1.0.0"}, {dep, "C", "2.0.0"}], "nondef"} ). m_flat1(Config) -> run(Config). @@ -436,6 +443,7 @@ m_pick_source2(Config) -> run(Config). m_pick_source3(Config) -> run(Config). m_pick_source4(Config) -> run(Config). m_pick_source5(Config) -> run(Config). +m_pick_source6(Config) -> run(Config). m_source_to_pkg(Config) -> run(Config). m_pkg_level1(Config) -> run(Config). m_pkg_level2(Config) -> run(Config). @@ -466,7 +474,10 @@ check_warnings(Warns, [{Type, Name, Vsn} | Rest], mixed) -> check_warnings(Warns, [{Name, Vsn} | Rest], Type) -> ct:pal("Checking for warning ~p in ~p", [{Name,Vsn},Warns]), ?assert(in_warnings(Type, Warns, Name, Vsn)), - check_warnings(Warns, Rest, Type). + check_warnings(Warns, Rest, Type); +check_warnings(Warns, none, _Type) -> + ct:pal("Checking that there were no warnings", []), + ?assert(Warns == []). in_warnings(git, Warns, NameRaw, VsnRaw) -> Name = iolist_to_binary(NameRaw), @@ -475,5 +486,5 @@ in_warnings(git, Warns, NameRaw, VsnRaw) -> in_warnings(pkg, Warns, NameRaw, VsnRaw) -> Name = iolist_to_binary(NameRaw), Vsn = iolist_to_binary(VsnRaw), - 1 =< length([1 || {_, [AppName, {pkg, _, AppVsn, _}]} <- Warns, + 1 =< length([1 || {_, [AppName, {pkg, _, AppVsn}]} <- Warns, AppName =:= Name, AppVsn =:= Vsn]). diff --git a/test/rebar_localfs_resource.erl b/test/rebar_localfs_resource.erl index d60421e..3d1296a 100644 --- a/test/rebar_localfs_resource.erl +++ b/test/rebar_localfs_resource.erl @@ -2,6 +2,7 @@ %% ex: ts=4 sw=4 et %% %% @doc A localfs custom resource (for testing purposes only) +%% implementing the deprecated rebar_resource instead of v2 %% %% ``` %% {deps, [ @@ -13,13 +14,18 @@ -behaviour(rebar_resource). --export([lock/2 +-export([init/1 + ,lock/2 ,download/3 ,needs_update/2 ,make_vsn/1]). -include_lib("eunit/include/eunit.hrl"). +-spec init(rebar_state:t()) -> {ok, term()}. +init(_State) -> + {ok, #{}}. + lock(AppDir, {localfs, Path, _Ref}) -> lock(AppDir, {localfs, Path}); lock(_AppDir, {localfs, Path}) -> diff --git a/test/rebar_localfs_resource_v2.erl b/test/rebar_localfs_resource_v2.erl new file mode 100644 index 0000000..52af4d4 --- /dev/null +++ b/test/rebar_localfs_resource_v2.erl @@ -0,0 +1,50 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +%% +%% @doc A localfs custom resource (for testing purposes only) +%% +%% ``` +%% {deps, [ +%% %% Application files are copied from "/path/to/app_name" +%% {app_name, {localfs, "/path/to/app_name", undefined}} +%% ]}. +%% ''' +-module(rebar_localfs_resource_v2). + +-behaviour(rebar_resource_v2). + +-export([init/2 + ,lock/2 + ,download/4 + ,needs_update/2 + ,make_vsn/2]). + +-include_lib("eunit/include/eunit.hrl"). + +-spec init(atom(), rebar_state:t()) -> {ok, term()}. +init(Type, _State) -> + Resource = rebar_resource_v2:new(Type, ?MODULE, #{}), + {ok, Resource}. + +lock(AppInfo, _) -> + case rebar_app_info:source(AppInfo) of + {localfs, Path, _Ref} -> + {localfs, Path, undefined}; + {localfs, Path} -> + {localfs, Path, undefined} + end. + +needs_update(_AppInfo, _) -> + false. + +download(TmpDir, AppInfo, State, _) -> + download_(TmpDir, rebar_app_info:source(AppInfo), State). + +download_(TmpDir, {localfs, Path, _Ref}, State) -> + download_(TmpDir, {localfs, Path}, State); +download_(TmpDir, {localfs, Path}, _State) -> + ok = rebar_file_utils:cp_r(filelib:wildcard(Path ++ "/*"), TmpDir), + {ok, undefined}. + +make_vsn(_AppInfo, _) -> + {plain, "undefined"}. diff --git a/test/rebar_paths_SUITE.erl b/test/rebar_paths_SUITE.erl new file mode 100644 index 0000000..96cda45 --- /dev/null +++ b/test/rebar_paths_SUITE.erl @@ -0,0 +1,240 @@ +-module(rebar_paths_SUITE). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-compile(export_all). + +all() -> + [clashing_apps, + check_modules, + set_paths, + misloaded_mods + ]. + +%%%%%%%%%%%%%%%%%% +%%% TEST SETUP %%% +%%%%%%%%%%%%%%%%%% + +init_per_testcase(Case, Config) -> + BasePaths = code:get_path(), + %% This test checks that the right module sets get loaded; however, we must + %% ensure that we do not have clashes with other test suites' loaded modules, + %% which we cannot track. As such, we have to ensure all module names here are + %% unique. + %% + %% This is done by hand; if you see this test suite failing on its own, you + %% probably wrote a test suite that clashes! + Dir = filename:join([?config(priv_dir, Config), atom_to_list(?MODULE), + atom_to_list(Case)]), + InDir = fun(Path) -> filename:join([Dir, Path]) end, + ADep = fake_app(<<"rp_a">>, <<"1.0.0">>, InDir("_build/default/lib/rp_a/")), + BDep = fake_app(<<"rp_b">>, <<"1.0.0">>, InDir("_build/default/lib/rp_b/")), + CDep = fake_app(<<"rp_c">>, <<"1.0.0">>, InDir("_build/default/lib/rp_c/")), + DDep = fake_app(<<"rp_d">>, <<"1.0.0">>, InDir("_build/default/lib/rp_d/")), + RelxDep = fake_app(<<"relx">>, <<"1.0.0">>, InDir("_build/default/lib/relx/")), + + APlug = fake_app(<<"rp_a">>, <<"1.0.0">>, + InDir("_build/default/plugins/lib/rp_a/")), + RelxPlug = fake_app(<<"relx">>, <<"1.1.1">>, + InDir("_build/default/plugins/lib/relx")), + EPlug = fake_app(<<"rp_e">>, <<"1.0.0">>, + InDir("_build/default/plugins/lib/rp_e/")), + + S0 = rebar_state:new(), + S1 = rebar_state:all_deps(S0, [ADep, BDep, CDep, DDep, RelxDep]), + S2 = rebar_state:all_plugin_deps(S1, [APlug, RelxPlug]), + S3 = rebar_state:code_paths(S2, default, code:get_path()), + S4 = rebar_state:code_paths( + S3, + all_deps, + [rebar_app_info:ebin_dir(A) || A <- [ADep, BDep, CDep, DDep, RelxDep]] + ), + S5 = rebar_state:code_paths( + S4, + all_plugin_deps, + [rebar_app_info:ebin_dir(A) || A <- [APlug, RelxPlug, EPlug]] + ), + [{base_paths, BasePaths}, {root_dir, Dir}, {state, S5} | Config]. + +end_per_testcase(_, Config) -> + %% this is deeply annoying because we interfere with rebar3's own + %% path handling! + rebar_paths:unset_paths([plugins, deps], ?config(state, Config)), + Config. + +fake_app(Name, Vsn, OutDir) -> + {ok, App} = rebar_app_info:new(Name, Vsn, OutDir), + compile_fake_appmod(App), + App. + +compile_fake_appmod(App) -> + OutDir = rebar_app_info:ebin_dir(App), + Vsn = rebar_app_info:original_vsn(App), + Name = rebar_app_info:name(App), + + ok = filelib:ensure_dir(filename:join([OutDir, ".touch"])), + + AppFile = [ + "{application,", Name, ", " + " [{description, \"some app\"}, " + " {vsn, \"", Vsn, "\"}, " + " {modules, [",Name,"]}, " + " {registered, []}, " + " {applications, [stdlib, kernel]} " + " ]}. "], + + ok = file:write_file(filename:join([OutDir, <<Name/binary, ".app">>]), AppFile), + + Mod = [{attribute, 1, module, binary_to_atom(Name, utf8)}, + {attribute, 2, export, [{f,0}]}, + {function,3,f,0, + [{clause,3, [], [], + [{string,3,OutDir}] + }]} + ], + + {ok, _, Bin} = compile:forms(Mod), + ok = file:write_file(filename:join([OutDir, <<Name/binary, ".beam">>]), Bin). + +%%%%%%%%%%%%% +%%% TESTS %%% +%%%%%%%%%%%%% + +clashing_apps(Config) -> + Clashes = rebar_paths:clashing_apps([deps, plugins], + ?config(state, Config)), + ct:pal("Clashes: ~p", [Clashes]), + + ?assertEqual([<<"relx">>, <<"rp_a">>], lists:sort(proplists:get_value(deps, Clashes))), + ?assertEqual([], proplists:get_value(plugins, Clashes)), + ok. + +set_paths(Config) -> + State = ?config(state, Config), + RootDir = filename:split(?config(root_dir, Config)), + rebar_paths:set_paths([plugins, deps], State), + PluginPaths = code:get_path(), + rebar_paths:set_paths([deps, plugins], State), + DepPaths = code:get_path(), + + ?assertEqual( + RootDir ++ ["_build", "default", "plugins", "lib", "rp_a", "ebin"], + find_first_instance("rp_a", PluginPaths) + ), + ?assertEqual( + RootDir ++ ["_build", "default", "lib", "rp_b", "ebin"], + find_first_instance("rp_b", PluginPaths) + ), + ?assertEqual( + RootDir ++ ["_build", "default", "lib", "rp_c", "ebin"], + find_first_instance("rp_c", PluginPaths) + ), + ?assertEqual( + RootDir ++ ["_build", "default", "lib", "rp_d", "ebin"], + find_first_instance("rp_d", PluginPaths) + ), + ?assertEqual( + RootDir ++ ["_build", "default", "plugins", "lib", "rp_e", "ebin"], + find_first_instance("rp_e", PluginPaths) + ), + ?assertEqual( + RootDir ++ ["_build", "default", "plugins", "lib", "relx", "ebin"], + find_first_instance("relx", PluginPaths) + ), + + + ?assertEqual( + RootDir ++ ["_build", "default", "lib", "rp_a", "ebin"], + find_first_instance("rp_a", DepPaths) + ), + ?assertEqual( + RootDir ++ ["_build", "default", "lib", "rp_b", "ebin"], + find_first_instance("rp_b", DepPaths) + ), + ?assertEqual( + RootDir ++ ["_build", "default", "lib", "rp_c", "ebin"], + find_first_instance("rp_c", DepPaths) + ), + ?assertEqual( + RootDir ++ ["_build", "default", "lib", "rp_d", "ebin"], + find_first_instance("rp_d", DepPaths) + ), + ?assertEqual( + RootDir ++ ["_build", "default", "plugins", "lib", "rp_e", "ebin"], + find_first_instance("rp_e", DepPaths) + ), + ?assertEqual( + RootDir ++ ["_build", "default", "lib", "relx", "ebin"], + find_first_instance("relx", DepPaths) + ), + ok. + +check_modules(Config) -> + State = ?config(state, Config), + RootDir = ?config(root_dir, Config)++"/", + rebar_paths:set_paths([plugins, deps], State), + ct:pal("code:get_path() -> ~p", [code:get_path()]), + + ?assertEqual(RootDir ++ "_build/default/plugins/lib/rp_a/ebin", rp_a:f()), + ct:pal("~p", [catch file:list_dir(RootDir ++ "_build/default/lib/")]), + ct:pal("~p", [catch file:list_dir(RootDir ++ "_build/default/lib/rp_b/")]), + ct:pal("~p", [catch file:list_dir(RootDir ++ "_build/default/lib/rp_b/ebin")]), + ct:pal("~p", [catch b:module_info()]), + ?assertEqual(RootDir ++ "_build/default/lib/rp_b/ebin", rp_b:f()), + ?assertEqual(RootDir ++ "_build/default/lib/rp_c/ebin", rp_c:f()), + ?assertEqual(RootDir ++ "_build/default/lib/rp_d/ebin", rp_d:f()), + ?assertEqual(RootDir ++ "_build/default/plugins/lib/rp_e/ebin", rp_e:f()), + ?assertEqual(RootDir ++ "_build/default/plugins/lib/relx/ebin", relx:f()), + ?assertEqual(3, length(relx:module_info(exports))), % can't replace bundled + + rebar_paths:set_paths([deps, plugins], State), + ct:pal("code:get_path() -> ~p", [code:get_path()]), + + ?assertEqual(RootDir ++ "_build/default/lib/rp_a/ebin", rp_a:f()), + ?assertEqual(RootDir ++ "_build/default/lib/rp_b/ebin", rp_b:f()), + ?assertEqual(RootDir ++ "_build/default/lib/rp_c/ebin", rp_c:f()), + ?assertEqual(RootDir ++ "_build/default/lib/rp_d/ebin", rp_d:f()), + ?assertEqual(RootDir ++ "_build/default/plugins/lib/rp_e/ebin", rp_e:f()), + ?assertEqual(RootDir ++ "_build/default/lib/relx/ebin", relx:f()), + ?assertEqual(3, length(relx:module_info(exports))), % can't replace bundled + + %% once again + rebar_paths:set_paths([plugins, deps], State), + ct:pal("code:get_path() -> ~p", [code:get_path()]), + + ?assertEqual(RootDir ++ "_build/default/plugins/lib/rp_a/ebin", rp_a:f()), + ?assertEqual(RootDir ++ "_build/default/lib/rp_b/ebin", rp_b:f()), + ?assertEqual(RootDir ++ "_build/default/lib/rp_c/ebin", rp_c:f()), + ?assertEqual(RootDir ++ "_build/default/lib/rp_d/ebin", rp_d:f()), + ?assertEqual(RootDir ++ "_build/default/plugins/lib/rp_e/ebin", rp_e:f()), + ?assertEqual(RootDir ++ "_build/default/plugins/lib/relx/ebin", relx:f()), + ?assertEqual(3, length(relx:module_info(exports))), % can't replace bundled + ok. + +misloaded_mods(_Config) -> + Res = rebar_paths:misloaded_modules( + ["/1/2/3/4", + "/1/2/4", + "/2/1/1", + "/3/4/5"], + [{a, "/0/1/2/file.beam"}, + {b, "/1/2/3/4/file.beam"}, + {c, "/2/1/file.beam"}, + {f, preloaded}, + {d, "/3/5/7/file.beam"}, + {e, "/3/4/5/file.beam"}] + ), + ?assertEqual([a,c,d], Res), + ok. + +%%%%%%%%%%%%%%% +%%% HELPERS %%% +%%%%%%%%%%%%%%% + +find_first_instance(Frag, []) -> + {not_found, Frag}; +find_first_instance(Frag, [Path|Rest]) -> + Frags = filename:split(Path), + case lists:member(Frag, Frags) of + true -> Frags; + false -> find_first_instance(Frag, Rest) + end. diff --git a/test/rebar_pkg_SUITE.erl b/test/rebar_pkg_SUITE.erl index 30cc0a8..ee74af5 100644 --- a/test/rebar_pkg_SUITE.erl +++ b/test/rebar_pkg_SUITE.erl @@ -4,16 +4,19 @@ -compile(export_all). -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). +-include("rebar.hrl"). --define(bad_etag, "abcdef"). --define(good_etag, "22e1d7387c9085a462340088a2a8ba67"). +-define(bad_etag, <<"abcdef">>). +-define(good_etag, <<"22e1d7387c9085a462340088a2a8ba67">>). +-define(badpkg_checksum, <<"A14E3718B33F8124E98004433193509EC6660F6CA03302657CAB8785751D77A0">>). +-define(badindex_checksum, <<"7B2CBED315C89F3126B5BF553DD7FF0FB5FE94B064888DD1B095CE8BF4B6A16A">>). -define(bad_checksum, <<"D576B442A68C7B92BACDE1EFE9C6E54D8D6C74BDB71D8175B9D3C6EC8C7B62A7">>). --define(good_checksum, <<"1C6CE379D191FBAB41B7905075E0BF87CBBE23C77CECE775C5A0B786B2244C35">>). +-define(good_checksum, <<"12726BDE1F65583A0817A7E8AADCA73F03FD8CB06F01E6CD29117C4A0DA0AFCF">>). +-define(BADPKG_ETAG, <<"BADETAG">>). -all() -> [good_uncached, good_cached, badindexchk, badpkg, - badhash_nocache, badhash_cache, - bad_to_good, good_disconnect, bad_disconnect, pkgs_provider, - find_highest_matching]. +all() -> [good_uncached, good_cached, badpkg, badhash_nocache, + badindexchk, badhash_cache, bad_to_good, good_disconnect, + bad_disconnect, pkgs_provider, find_highest_matching]. init_per_suite(Config) -> application:start(meck), @@ -31,10 +34,6 @@ init_per_testcase(pkgs_provider=Name, Config) -> CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]), filelib:ensure_dir(filename:join([CacheDir, "registry"])), ok = ets:tab2file(Tid, filename:join([CacheDir, "registry"])), - meck:new(rebar_packages, [passthrough]), - meck:expect(rebar_packages, registry_dir, fun(_) -> {ok, CacheDir} end), - meck:expect(rebar_packages, package_dir, fun(_) -> {ok, CacheDir} end), - rebar_prv_update:hex_to_index(rebar_state:new()), Config; init_per_testcase(good_uncached=Name, Config0) -> Config = [{good_cache, false}, @@ -83,14 +82,14 @@ init_per_testcase(bad_to_good=Name, Config0) -> Config; init_per_testcase(good_disconnect=Name, Config0) -> Pkg = {<<"goodpkg">>, <<"1.0.0">>}, - Config1 = [{good_cache, true}, + Config1 = [{good_cache, false}, {pkg, Pkg} | Config0], Config = mock_config(Name, Config1), copy_to_cache(Pkg, Config), - meck:unload(httpc), + %% meck:unload(httpc), meck:new(httpc, [passthrough, unsticky]), - meck:expect(httpc, request, fun(_, _, _, _, _) -> {error, econnrefused} end), + meck:expect(httpc, request, fun(_, _, _, _) -> {error, econnrefused} end), Config; init_per_testcase(bad_disconnect=Name, Config0) -> Pkg = {<<"goodpkg">>, <<"1.0.0">>}, @@ -98,9 +97,9 @@ init_per_testcase(bad_disconnect=Name, Config0) -> {pkg, Pkg} | Config0], Config = mock_config(Name, Config1), - meck:unload(httpc), - meck:new(httpc, [passthrough, unsticky]), - meck:expect(httpc, request, fun(_, _, _, _, _) -> {error, econnrefused} end), + meck:expect(hex_repo, get_tarball, fun(_, _, _) -> + {error, econnrefused} + end), Config; init_per_testcase(Name, Config0) -> Config = [{good_cache, false}, @@ -116,8 +115,8 @@ good_uncached(Config) -> Tmp = ?config(tmp_dir, Config), {Pkg,Vsn} = ?config(pkg, Config), State = ?config(state, Config), - ?assertEqual({ok, true}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum}, State)), + ?assertEqual(ok, + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, #{}, true)), Cache = ?config(cache_dir, Config), ?assert(filelib:is_regular(filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>))). @@ -129,16 +128,17 @@ good_cached(Config) -> CachedFile = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>), ?assert(filelib:is_regular(CachedFile)), {ok, Content} = file:read_file(CachedFile), - ?assertEqual({ok, true}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum}, State)), + ?assertEqual(ok, + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, #{}, true)), {ok, Content} = file:read_file(CachedFile). + badindexchk(Config) -> Tmp = ?config(tmp_dir, Config), {Pkg,Vsn} = ?config(pkg, Config), State = ?config(state, Config), - ?assertMatch({bad_registry_checksum, _Path}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum}, State)), + ?assertMatch({error, {rebar_pkg_resource, {bad_registry_checksum, _, _, _, _}}}, + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum, #{}}, State, #{}, true)), %% The cached file is there for forensic purposes Cache = ?config(cache_dir, Config), ?assert(filelib:is_regular(filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>))). @@ -147,18 +147,22 @@ badpkg(Config) -> Tmp = ?config(tmp_dir, Config), {Pkg,Vsn} = ?config(pkg, Config), State = ?config(state, Config), - ?assertMatch({bad_download, _Path}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum}, State)), - %% The cached file is there for forensic purposes Cache = ?config(cache_dir, Config), - ?assert(filelib:is_regular(filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>))). + CachePath = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>), + ETagPath = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".etag">>), + rebar_pkg_resource:store_etag_in_cache(ETagPath, ?BADPKG_ETAG), + ?assertMatch({error, {hex_tarball, {tarball, {checksum_mismatch, _, _}}}}, + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?badpkg_checksum, #{}}, State, #{}, false)), + %% The cached/etag files are there for forensic purposes + ?assert(filelib:is_regular(ETagPath)), + ?assert(filelib:is_regular(CachePath)). badhash_nocache(Config) -> Tmp = ?config(tmp_dir, Config), {Pkg,Vsn} = ?config(pkg, Config), State = ?config(state, Config), - ?assertMatch({unexpected_hash, _Path, ?bad_checksum, ?good_checksum}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum}, State)), + ?assertMatch({error, {rebar_pkg_resource, {bad_registry_checksum, _, _, _, _}}}, + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum, #{}}, State, #{}, true)), %% The cached file is there for forensic purposes Cache = ?config(cache_dir, Config), ?assert(filelib:is_regular(filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>))). @@ -171,8 +175,8 @@ badhash_cache(Config) -> CachedFile = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>), ?assert(filelib:is_regular(CachedFile)), {ok, Content} = file:read_file(CachedFile), - ?assertMatch({unexpected_hash, _Path, ?bad_checksum, ?good_checksum}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum}, State)), + ?assertMatch({error, {rebar_pkg_resource, {bad_registry_checksum, _, _, _, _}}}, + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum, #{}}, State, #{}, true)), %% The cached file is there still, unchanged. ?assert(filelib:is_regular(CachedFile)), ?assertEqual({ok, Content}, file:read_file(CachedFile)). @@ -185,8 +189,8 @@ bad_to_good(Config) -> CachedFile = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>), ?assert(filelib:is_regular(CachedFile)), {ok, Contents} = file:read_file(CachedFile), - ?assertEqual({ok, true}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum}, State)), + ?assertEqual(ok, + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, #{}, true)), %% Cache has refreshed ?assert({ok, Contents} =/= file:read_file(CachedFile)). @@ -196,10 +200,12 @@ good_disconnect(Config) -> State = ?config(state, Config), Cache = ?config(cache_dir, Config), CachedFile = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>), + ETagFile = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".etag">>), ?assert(filelib:is_regular(CachedFile)), {ok, Content} = file:read_file(CachedFile), - ?assertEqual({ok, true}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum}, State)), + rebar_pkg_resource:store_etag_in_cache(ETagFile, ?BADPKG_ETAG), + ?assertEqual(ok, + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, #{}, true)), {ok, Content} = file:read_file(CachedFile). bad_disconnect(Config) -> @@ -207,27 +213,36 @@ bad_disconnect(Config) -> {Pkg,Vsn} = ?config(pkg, Config), State = ?config(state, Config), ?assertEqual({fetch_fail, Pkg, Vsn}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum}, State)). + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, #{}, true)). pkgs_provider(Config) -> Config1 = rebar_test_utils:init_rebar_state(Config), rebar_test_utils:run_and_check( - Config1, [], ["pkgs"], + Config1, [], ["pkgs", "relx"], {ok, []} ). find_highest_matching(_Config) -> State = rebar_state:new(), - {ok, Vsn} = rebar_packages:find_highest_matching( - <<"test">>, <<"1.0.0">>, <<"goodpkg">>, <<"1.0.0">>, package_index, State), - ?assertEqual(<<"1.0.1">>, Vsn), + {ok, Vsn} = rebar_packages:find_highest_matching_( + <<"goodpkg">>, ec_semver:parse(<<"1.0.0">>), #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State), + ?assertEqual({{1,0,1},{[],[]}}, Vsn), {ok, Vsn1} = rebar_packages:find_highest_matching( - <<"test">>, <<"1.0.0">>, <<"goodpkg">>, <<"1.0">>, package_index, State), - ?assertEqual(<<"1.1.1">>, Vsn1), + <<"goodpkg">>, ec_semver:parse(<<"1.0">>), #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State), + ?assertEqual({{1,1,1},{[],[]}}, Vsn1), {ok, Vsn2} = rebar_packages:find_highest_matching( - <<"test">>, <<"1.0.0">>, <<"goodpkg">>, <<"2.0">>, package_index, State), - ?assertEqual(<<"2.0.0">>, Vsn2). + <<"goodpkg">>, ec_semver:parse(<<"2.0">>), #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State), + ?assertEqual({{2,0,0},{[],[]}}, Vsn2), + + %% regression test. ~> constraints higher than the available packages would result + %% in returning the first package version instead of 'none'. + ?assertEqual(none, rebar_packages:find_highest_matching_(<<"goodpkg">>, ec_semver:parse(<<"5.0">>), + #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State)), + + {ok, Vsn3} = rebar_packages:find_highest_matching_(<<"goodpkg">>, ec_semver:parse(<<"3.0.0-rc.0">>), + #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State), + ?assertEqual({{3,0,0},{[<<"rc">>,0],[]}}, Vsn3). %%%%%%%%%%%%%%% %%% Helpers %%% @@ -237,49 +252,85 @@ mock_config(Name, Config) -> CacheRoot = filename:join([Priv, "cache", atom_to_list(Name)]), TmpDir = filename:join([Priv, "tmp", atom_to_list(Name)]), Tid = ets:new(registry_table, [public]), - ets:insert_new(Tid, [ - {<<"badindexchk">>,[[<<"1.0.0">>]]}, - {<<"goodpkg">>,[[<<"1.0.0">>, <<"1.0.1">>, <<"1.1.1">>, <<"2.0.0">>]]}, - {<<"badpkg">>,[[<<"1.0.0">>]]}, + AllDeps = [ {{<<"badindexchk">>,<<"1.0.0">>}, [[], ?bad_checksum, [<<"rebar3">>]]}, {{<<"goodpkg">>,<<"1.0.0">>}, [[], ?good_checksum, [<<"rebar3">>]]}, {{<<"goodpkg">>,<<"1.0.1">>}, [[], ?good_checksum, [<<"rebar3">>]]}, {{<<"goodpkg">>,<<"1.1.1">>}, [[], ?good_checksum, [<<"rebar3">>]]}, {{<<"goodpkg">>,<<"2.0.0">>}, [[], ?good_checksum, [<<"rebar3">>]]}, - {{<<"badpkg">>,<<"1.0.0">>}, [[], ?good_checksum, [<<"rebar3">>]]} - ]), + {{<<"goodpkg">>,<<"3.0.0-rc.0">>}, [[], ?good_checksum, [<<"rebar3">>]]}, + {{<<"badpkg">>,<<"1.0.0">>}, [[], ?badpkg_checksum, [<<"rebar3">>]]} + ], + ets:insert_new(Tid, AllDeps), CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]), filelib:ensure_dir(filename:join([CacheDir, "registry"])), ok = ets:tab2file(Tid, filename:join([CacheDir, "registry"])), + catch ets:delete(?PACKAGE_TABLE), + rebar_packages:new_package_table(), + lists:foreach(fun({{N, Vsn}, [Deps, Checksum, _]}) -> + case ets:member(?PACKAGE_TABLE, {ec_cnv:to_binary(N), Vsn, <<"hexpm">>}) of + false -> + ets:insert(?PACKAGE_TABLE, #package{key={ec_cnv:to_binary(N), ec_semver:parse(Vsn), <<"hexpm">>}, + dependencies=Deps, + retired=false, + checksum=Checksum}); + true -> + ok + end + end, AllDeps), + + + meck:new(hex_repo, [passthrough]), + meck:expect(hex_repo, get_package, + fun(_Config, PkgName) -> + Matches = ets:match_object(Tid, {{PkgName,'_'}, '_'}), + Releases = + [#{checksum => Checksum, + version => Vsn, + dependencies => Deps} || + {{_, Vsn}, [Deps, Checksum, _]} <- Matches], + {ok, {200, #{}, #{releases => Releases}}} + end), + %% The state returns us a fake registry meck:new(rebar_state, [passthrough]), meck:expect(rebar_state, get, fun(_State, rebar_packages_cdn, _Default) -> - "http://test.com/" + "http://test.com/"; + (_, _, Default) -> + Default + end), + meck:expect(rebar_state, resources, + fun(_State) -> + DefaultConfig = hex_core:default_config(), + [rebar_resource_v2:new(pkg, rebar_pkg_resource, + #{repos => [DefaultConfig#{name => <<"hexpm">>}], + base_config => #{}})] end), meck:new(rebar_dir, [passthrough]), meck:expect(rebar_dir, global_cache_dir, fun(_) -> CacheRoot end), - meck:new(rebar_packages, [passthrough]), meck:expect(rebar_packages, registry_dir, fun(_) -> {ok, CacheDir} end), - meck:expect(rebar_packages, package_dir, fun(_) -> {ok, CacheDir} end), - rebar_prv_update:hex_to_index(rebar_state:new()), + meck:expect(rebar_packages, package_dir, fun(_, _) -> {ok, CacheDir} end), + + meck:new(rebar_prv_update, [passthrough]), + meck:expect(rebar_prv_update, do, fun(State) -> {ok, State} end), %% Cache fetches are mocked -- we assume the server and clients are %% correctly used. GoodCache = ?config(good_cache, Config), - {Pkg,Vsn} = ?config(pkg, Config), + {Pkg,Vsn} = ?config(pkg, Config), PkgFile = <<Pkg/binary, "-", Vsn/binary, ".tar">>, {ok, PkgContents} = file:read_file(filename:join(?config(data_dir, Config), PkgFile)), - meck:new(httpc, [passthrough, unsticky]), - meck:expect(httpc, request, - fun(get, {_Url, _Opts}, _, _, _) when GoodCache -> - {ok, {{Vsn, 304, <<"Not Modified">>}, [{"etag", ?good_etag}], <<>>}}; - (get, {_Url, _Opts}, _, _, _) -> - {ok, {{Vsn, 200, <<"OK">>}, [{"etag", ?good_etag}], PkgContents}} - end), + + meck:expect(hex_repo, get_tarball, fun(_, _, _) when GoodCache -> + {ok, {304, #{<<"etag">> => ?good_etag}, <<>>}}; + (_, _, _) -> + {ok, {200, #{<<"etag">> => ?good_etag}, PkgContents}} + end), + [{cache_root, CacheRoot}, {cache_dir, CacheDir}, {tmp_dir, TmpDir}, @@ -287,7 +338,7 @@ mock_config(Name, Config) -> unmock_config(Config) -> meck:unload(), - ets:delete(?config(mock_table, Config)). + catch ets:delete(?config(mock_table, Config)). copy_to_cache({Pkg,Vsn}, Config) -> Name = <<Pkg/binary, "-", Vsn/binary, ".tar">>, diff --git a/test/rebar_pkg_SUITE_data/badindexchk-1.0.0.tar b/test/rebar_pkg_SUITE_data/badindexchk-1.0.0.tar Binary files differindex e5b963f..1765bb3 100644 --- a/test/rebar_pkg_SUITE_data/badindexchk-1.0.0.tar +++ b/test/rebar_pkg_SUITE_data/badindexchk-1.0.0.tar diff --git a/test/rebar_pkg_SUITE_data/badpkg-1.0.0.tar b/test/rebar_pkg_SUITE_data/badpkg-1.0.0.tar Binary files differindex 4930cd2..37bb57d 100644 --- a/test/rebar_pkg_SUITE_data/badpkg-1.0.0.tar +++ b/test/rebar_pkg_SUITE_data/badpkg-1.0.0.tar diff --git a/test/rebar_pkg_SUITE_data/goodpkg-1.0.0.tar b/test/rebar_pkg_SUITE_data/goodpkg-1.0.0.tar Binary files differindex e5b963f..d0fa4cb 100644 --- a/test/rebar_pkg_SUITE_data/goodpkg-1.0.0.tar +++ b/test/rebar_pkg_SUITE_data/goodpkg-1.0.0.tar diff --git a/test/rebar_pkg_alias_SUITE.erl b/test/rebar_pkg_alias_SUITE.erl index 8915357..079a3fd 100644 --- a/test/rebar_pkg_alias_SUITE.erl +++ b/test/rebar_pkg_alias_SUITE.erl @@ -3,38 +3,61 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("kernel/include/file.hrl"). +-include("rebar.hrl"). -all() -> [same_alias, diff_alias, diff_alias_vsn]. +all() -> [same_alias, diff_alias, diff_alias_vsn, transitive_alias%% , + %% transitive_hash_mismatch + ]. %% {uuid, {pkg, uuid}} = uuid %% {uuid, {pkg, alias}} = uuid on disk %% another run should yield the same lock file without error init_per_suite(Config) -> - mock_config(?MODULE, Config). + Config. + %% mock_config(?MODULE, Config). end_per_suite(Config) -> - unmock_config(Config). + Config. + %% unmock_config(Config). init_per_testcase(same_alias, Config0) -> + mock_config(?MODULE, Config0), Config = rebar_test_utils:init_rebar_state(Config0,"same_alias_"), AppDir = ?config(apps, Config), rebar_test_utils:create_app(AppDir, "A", "0.0.0", [kernel, stdlib]), RebarConf = rebar_test_utils:create_config(AppDir, [{deps, [{fakelib, {pkg, fakelib}}]}]), [{rebarconfig, RebarConf} | Config]; init_per_testcase(diff_alias, Config0) -> + mock_config(?MODULE, Config0), Config = rebar_test_utils:init_rebar_state(Config0,"diff_alias_"), AppDir = ?config(apps, Config), rebar_test_utils:create_app(AppDir, "A", "0.0.0", [kernel, stdlib]), RebarConf = rebar_test_utils:create_config(AppDir, [{deps, [{fakelib, {pkg, goodpkg}}]}]), [{rebarconfig, RebarConf} | Config]; init_per_testcase(diff_alias_vsn, Config0) -> + mock_config(?MODULE, Config0), Config = rebar_test_utils:init_rebar_state(Config0,"diff_alias_vsn_"), AppDir = ?config(apps, Config), rebar_test_utils:create_app(AppDir, "A", "0.0.0", [kernel, stdlib]), RebarConf = rebar_test_utils:create_config(AppDir, [{deps, [{fakelib, "1.0.0", {pkg, goodpkg}}]}]), + [{rebarconfig, RebarConf} | Config]; +init_per_testcase(transitive_alias, Config0) -> + mock_config(?MODULE, Config0), + Config = rebar_test_utils:init_rebar_state(Config0,"transitive_alias_vsn_"), + AppDir = ?config(apps, Config), + rebar_test_utils:create_app(AppDir, "A", "0.0.0", [kernel, stdlib]), + RebarConf = rebar_test_utils:create_config(AppDir, [{deps, [{topdep, "1.0.0", {pkg, topdep}}]}]), + [{rebarconfig, RebarConf} | Config]; +init_per_testcase(transitive_hash_mismatch, Config0) -> + mock_config(?MODULE, Config0), + Config = rebar_test_utils:init_rebar_state(Config0,"transitive_alias_vsn_"), + AppDir = ?config(apps, Config), + rebar_test_utils:create_app(AppDir, "A", "0.0.0", [kernel, stdlib]), + RebarConf = rebar_test_utils:create_config(AppDir, [{deps, [{topdep, "1.0.0", {pkg, topdep}}]}]), [{rebarconfig, RebarConf} | Config]. end_per_testcase(_, Config) -> + unmock_config(Config), Config. same_alias(Config) -> @@ -73,42 +96,168 @@ diff_alias(Config) -> diff_alias_vsn(Config) -> diff_alias(Config). +transitive_alias(Config) -> + %% ensure that the apps fetched under transitive aliases are + %% locked properly, but also that they are stored in the right + %% directory in the build dir to avoid breaking includes and + %% static analysis tools that rely on the location to work + AppDir = ?config(apps, Config), + Lockfile = filename:join([AppDir, "rebar.lock"]), + {ok, RebarConfig} = file:consult(?config(rebarconfig, Config)), + rebar_test_utils:run_and_check( + Config, RebarConfig, ["lock"], + {ok, [{lock, "topdep"},{dep, "topdep"}, + {lock,"transitive_app"},{dep,"transitive_app"}]} + ), + {ok, [{_Vsn, LockData}|_]} = file:consult(Lockfile), + ?assert(lists:any(fun({<<"transitive_app">>,{pkg,<<"transitive">>,_},_}) -> true + ; (_) -> false end, LockData)), + AppDir = ?config(apps, Config), + AliasedName = filename:join([AppDir, "_build", "default", "lib", "transitive_app"]), + PkgName = filename:join([AppDir, "_build", "default", "lib", "transitive"]), + ?assert(filelib:is_dir(AliasedName)), + ?assertNot(filelib:is_dir(PkgName)), + %% An second run yields the same + rebar_test_utils:run_and_check( + Config, RebarConfig, ["lock"], + {ok, [{lock, "topdep"},{dep, "topdep"}, + {lock,"transitive_app"},{dep,"transitive_app"}]} + ), + {ok, [{_Vsn, LockData}|_]} = file:consult(Lockfile), + ?assert(filelib:is_dir(AliasedName)), + ?assertNot(filelib:is_dir(PkgName)), + %% So does an upgrade + rebar_test_utils:run_and_check( + Config, RebarConfig, ["upgrade"], + {ok, [{lock, "topdep"},{dep, "topdep"}, + {lock,"transitive_app"},{dep,"transitive_app"}]} + ), + {ok, [{_Vsn, LockData}|_]} = file:consult(Lockfile), + ?assert(filelib:is_dir(AliasedName)), + ?assertNot(filelib:is_dir(PkgName)), + ok. + +transitive_hash_mismatch(Config) -> + %% ensure that the apps fetched under transitive aliases are + %% locked properly, but also that they are stored in the right + %% directory in the build dir to avoid breaking includes and + %% static analysis tools that rely on the location to work + AppDir = ?config(apps, Config), + Lockfile = filename:join([AppDir, "rebar.lock"]), + {ok, RebarConfig} = file:consult(?config(rebarconfig, Config)), + rebar_test_utils:run_and_check( + Config, RebarConfig, ["lock"], + {ok, [{lock, "topdep"},{dep, "topdep"}, + {lock,"transitive_app"},{dep,"transitive_app"}]} + ), + {ok, [LockData|Attrs]} = file:consult(Lockfile), + %% Change Lock hash data to cause a failure next time, but on transitive + %% deps only + NewLock = [LockData|lists:map( + fun([{pkg_hash, Hashes}|Rest]) -> + [{pkg_hash, [{<<"transitive_app">>, <<"fakehash">>} + | lists:keydelete(<<"transitive_app">>, 1, Hashes)]} + | Rest] + ; (Attr) -> + Attr + end, Attrs)], + {ok, Io} = file:open(Lockfile, [write]), + [io:format(Io, "~p.~n", [Attr]) || Attr <- NewLock], + file:close(Io), + ct:pal("lock: ~p", [file:consult(Lockfile)]), + ec_file:remove(filename:join([AppDir, "_build"]), [recursive]), + ?assertMatch( + {error, {rebar_fetch, {unexpected_hash, _, _, _}}}, + rebar_test_utils:run_and_check(Config, RebarConfig, ["lock"], return) + ), + ok. + +parse_deps(Deps) -> + [{maps:get(app, D, Name), {pkg, Name, Constraint, undefined}} || D=#{package := Name, + requirement := Constraint} <- Deps]. + mock_config(Name, Config) -> + {ChkFake, Etag} = create_lib(Name, Config, "fakelib"), + {ChkTop, _} = create_lib(Name, Config, "topdep"), + {ChkTrans, _} = create_lib(Name, Config, "transitive_app", "transitive"), + ct:pal("{~p, _}",[ChkTop]), + ct:pal("{~p, _}",[ChkTrans]), Priv = ?config(priv_dir, Config), + TmpDir = filename:join([Priv, "tmp", atom_to_list(Name)]), + %% Add an alias for goodpkg -> fakelib by hand AppDir = filename:join([Priv, "fakelib"]), CacheRoot = filename:join([Priv, "cache", atom_to_list(Name)]), - TmpDir = filename:join([Priv, "tmp", atom_to_list(Name)]), CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]), - filelib:ensure_dir(filename:join([CacheDir, "registry"])), rebar_test_utils:create_app(AppDir, "fakelib", "1.0.0", [kernel, stdlib]), - {Chk,Etag} = rebar_test_utils:package_app(AppDir, CacheDir, "fakelib-1.0.0"), - {Chk,Etag} = rebar_test_utils:package_app(AppDir, CacheDir, "goodpkg-1.0.0"), + ct:pal("{~p, ~p}",[ChkFake, Etag]), + {ChkGood, EtagGood} = rebar_test_utils:package_app(AppDir, CacheDir, "goodpkg", "1.0.0"), + AllDeps = [ + {<<"fakelib">>,[[<<"1.0.0">>]]}, + {<<"goodpkg">>,[[<<"1.0.0">>]]}, + {<<"topdep">>,[[<<"1.0.0">>]]}, + {<<"transitive">>, [[<<"1.0.0">>]]}, + {{<<"fakelib">>,<<"1.0.0">>}, [[], ChkFake, [<<"rebar3">>]]}, + {{<<"goodpkg">>,<<"1.0.0">>}, [[], ChkGood, [<<"rebar3">>]]}, + {{<<"topdep">>,<<"1.0.0">>}, + [[ + {<<"transitive">>, <<"1.0.0">>, false, <<"transitive_app">>} + ], ChkTop, [<<"rebar3">>]]}, + {{<<"transitive">>,<<"1.0.0">>}, [[], ChkTrans, [<<"rebar3">>]]} + ], Tid = ets:new(registry_table, [public]), - ets:insert_new(Tid, [ - {<<"fakelib">>,[[<<"1.0.0">>]]}, - {<<"goodpkg">>,[[<<"1.0.0">>]]}, - {{<<"fakelib">>,<<"1.0.0">>}, [[], Chk, [<<"rebar3">>]]}, - {{<<"goodpkg">>,<<"1.0.0">>}, [[], Chk, [<<"rebar3">>]]} - ]), + ets:insert_new(Tid, AllDeps), ok = ets:tab2file(Tid, filename:join([CacheDir, "registry"])), - ets:delete(Tid), + %% ets:delete(Tid), %% The state returns us a fake registry meck:new(rebar_dir, [passthrough, no_link]), meck:expect(rebar_dir, global_cache_dir, fun(_) -> CacheRoot end), meck:new(rebar_packages, [passthrough, no_link]), meck:expect(rebar_packages, registry_dir, fun(_) -> {ok, CacheDir} end), - meck:expect(rebar_packages, package_dir, fun(_) -> {ok, CacheDir} end), - rebar_prv_update:hex_to_index(rebar_state:new()), - - %% Cache fetches are mocked -- we assume the server and clients are - %% correctly used. - meck:new(httpc, [passthrough, unsticky, no_link]), - meck:expect(httpc, request, - fun(get, {_Url, _Opts}, _, _, _) -> - {ok, {{<<"1.0.0">>, 304, <<"Not Modified">>}, [{"etag", Etag}], <<>>}} - end), + meck:expect(rebar_packages, package_dir, fun(_, _) -> {ok, CacheDir} end), + + %% TODO: is something else wrong that we need this for transitive_alias to pass + meck:expect(rebar_packages, update_package, fun(_, _, _) -> ok end), + + meck:new(rebar_prv_update, [passthrough]), + meck:expect(rebar_prv_update, do, fun(State) -> {ok, State} end), + + catch ets:delete(?PACKAGE_TABLE), + rebar_packages:new_package_table(), + + lists:foreach(fun({{N, Vsn}, [Deps, Checksum, _]}) -> + case ets:member(?PACKAGE_TABLE, {ec_cnv:to_binary(N), Vsn, <<"hexpm">>}) of + false -> + ets:insert(?PACKAGE_TABLE, #package{key={ec_cnv:to_binary(N), ec_semver:parse(Vsn), <<"hexpm">>}, + dependencies=[{DAppName, {pkg, DN, DV, undefined}} || {DN, DV, _, DAppName} <- Deps], + retired=false, + checksum=Checksum}); + true -> + ok + end; + ({_N, _Vsns}) -> + ok + + end, AllDeps), + + meck:new(hex_repo, [passthrough]), + meck:expect(hex_repo, get_package, + fun(_Config, PkgName) -> + Matches = ets:match_object(Tid, {{PkgName,'_'}, '_'}), + Releases = + [#{checksum => Checksum, + version => Vsn, + dependencies => [{DAppName, {pkg, DN, DV, undefined}} || + {DN, DV, _, DAppName} <- Deps]} || + {{_, Vsn}, [Deps, Checksum, _]} <- Matches], + {ok, {200, #{}, #{releases => Releases}}} + end), + + meck:expect(hex_repo, get_tarball, fun(_, _, _) -> + {ok, {304, #{<<"etag">> => EtagGood}, <<>>}} + end), + %% Move all packages to cache NewConf = [{cache_root, CacheRoot}, {cache_dir, CacheDir}, @@ -119,3 +268,15 @@ mock_config(Name, Config) -> unmock_config(Config) -> meck:unload(), Config. + +create_lib(Name, Config, PkgName) -> + create_lib(Name, Config, PkgName, PkgName). + +create_lib(Name, Config, AppName, PkgName) -> + Priv = ?config(priv_dir, Config), + AppDir = filename:join([Priv, PkgName]), + CacheRoot = filename:join([Priv, "cache", atom_to_list(Name)]), + CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]), + filelib:ensure_dir(filename:join([CacheDir, "registry"])), + rebar_test_utils:create_app(AppDir, AppName, "1.0.0", [kernel, stdlib]), + rebar_test_utils:package_app(AppDir, CacheDir, PkgName, "1.0.0"). diff --git a/test/rebar_pkg_repos_SUITE.erl b/test/rebar_pkg_repos_SUITE.erl new file mode 100644 index 0000000..c808475 --- /dev/null +++ b/test/rebar_pkg_repos_SUITE.erl @@ -0,0 +1,376 @@ +%% Test suite for the handling hexpm repo configurations +-module(rebar_pkg_repos_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include("rebar.hrl"). + +all() -> + [default_repo, repo_merging, repo_replacing, + auth_merging, organization_merging, {group, resolve_version}]. + +groups() -> + [{resolve_version, [use_first_repo_match, use_exact_with_hash, fail_repo_update, + ignore_match_in_excluded_repo, optional_prereleases]}]. + +init_per_group(resolve_version, Config) -> + Repo1 = <<"test-repo-1">>, + Repo2 = <<"test-repo-2">>, + Repo3 = <<"test-repo-3">>, + Hexpm = <<"hexpm">>, + Repos = [Repo1, Repo2, Repo3, Hexpm], + + Deps = [{"A", "0.1.1", <<"good checksum">>, Repo1, false}, + {"A", "0.1.1", <<"good checksum">>, Repo2, false}, + {"B", "1.0.0", Repo1, false}, + {"B", "2.0.0", Repo2, false}, + {"B", "1.4.0", Repo3, false}, + {"B", "1.4.3", Hexpm, false}, + {"B", "1.4.6", Hexpm, #{reason => 'RETIRED_INVALID'}}, + {"B", "1.5.0", Hexpm, false}, + {"B", "1.5.6-rc.0", Hexpm, true}, + {"C", "1.3.1", <<"bad checksum">>, Repo1, false}, + {"C", "1.3.1", <<"good checksum">>, Repo2, false}], + [{deps, Deps}, {repos, Repos} | Config]; +init_per_group(_, Config) -> + Config. + +end_per_group(_, _) -> + ok. + +init_per_testcase(use_first_repo_match, Config) -> + Deps = ?config(deps, Config), + Repos = ?config(repos, Config), + State = setup_deps_and_repos(Deps, Repos), + + meck:new(rebar_packages, [passthrough, no_link]), + + %% fail when the first repo is updated since it doesn't have a matching package + %% should continue anyway + meck:expect(rebar_packages, update_package, + fun(_, _, _State) -> ok end), + meck:expect(rebar_packages, verify_table, + fun(_State) -> true end), + + [{state, State} | Config]; +init_per_testcase(use_exact_with_hash, Config) -> + Deps = ?config(deps, Config), + Repos = ?config(repos, Config), + State = setup_deps_and_repos(Deps, Repos), + + meck:new(rebar_packages, [passthrough, no_link]), + + %% fail when the first repo is updated since it doesn't have a matching package + %% should continue anyway + meck:expect(rebar_packages, update_package, + fun(_, _, _State) -> ok end), + meck:expect(rebar_packages, verify_table, + fun(_State) -> true end), + + [{state, State} | Config]; +init_per_testcase(fail_repo_update, Config) -> + Deps = ?config(deps, Config), + Repos = ?config(repos, Config), + State = setup_deps_and_repos(Deps, Repos), + + meck:new(rebar_packages, [passthrough, no_link]), + + %% fail when the first repo is updated since it doesn't have a matching package + %% should continue anyway + [Repo1 | _] = Repos, + meck:expect(rebar_packages, update_package, + fun(_, #{name := Repo}, _State) when Repo =:= Repo1 -> fail; + (_, _, _State) -> ok end), + meck:expect(rebar_packages, verify_table, + fun(_State) -> true end), + + [{state, State} | Config]; +init_per_testcase(ignore_match_in_excluded_repo, Config) -> + Deps = ?config(deps, Config), + Repos = [Repo1, _, Repo3 | _] = ?config(repos, Config), + + %% drop repo1 and repo2 from the repos to be used by the pkg resource + State = setup_deps_and_repos(Deps, [R || R <- Repos, R =/= Repo3, R =/= Repo1]), + + meck:new(rebar_packages, [passthrough, no_link]), + + %% fail when the first repo is updated since it doesn't have a matching package + %% should continue anyway + [_, _, Repo3 | _] = Repos, + meck:expect(rebar_packages, update_package, + fun(_, _, _State) -> ok end), + meck:expect(rebar_packages, verify_table, + fun(_State) -> true end), + + [{state, State} | Config]; +init_per_testcase(optional_prereleases, Config) -> + Deps = ?config(deps, Config), + Repos = ?config(repos, Config), + + State = setup_deps_and_repos(Deps, Repos), + + meck:new(rebar_packages, [passthrough, no_link]), + + meck:expect(rebar_packages, update_package, + fun(_, _, _State) -> ok end), + meck:expect(rebar_packages, verify_table, + fun(_State) -> true end), + + [{state, State} | Config]; +init_per_testcase(auth_merging, Config) -> + meck:new(file, [passthrough, no_link, unstick]), + meck:new(rebar_packages, [passthrough, no_link]), + Config; +init_per_testcase(organization_merging, Config) -> + meck:new(file, [passthrough, no_link, unstick]), + meck:new(rebar_packages, [passthrough, no_link]), + Config; +init_per_testcase(_, Config) -> + Config. + +end_per_testcase(Case, _Config) when Case =:= auth_merging ; + Case =:= organization_merging -> + meck:unload(file), + meck:unload(rebar_packages); +end_per_testcase(Case, _Config) when Case =:= use_first_repo_match ; + Case =:= use_exact_with_hash ; + Case =:= fail_repo_update ; + Case =:= ignore_match_in_excluded_repo ; + Case =:= optional_prereleases -> + meck:unload(rebar_packages); +end_per_testcase(_, _) -> + ok. + + +default_repo(_Config) -> + Repo1 = #{name => <<"hexpm">>, + api_key => <<"asdf">>}, + + MergedRepos = rebar_hex_repos:repos([{repos, [Repo1]}]), + + ?assertMatch([#{name := <<"hexpm">>, + api_key := <<"asdf">>, + api_url := <<"https://hex.pm/api">>}], MergedRepos). + + +repo_merging(_Config) -> + Repo1 = #{name => <<"repo-1">>, + api_url => <<"repo-1/api">>}, + Repo2 = #{name => <<"repo-2">>, + repo_url => <<"repo-2/repo">>, + repo_verify => false}, + Result = rebar_hex_repos:merge_repos([Repo1, Repo2, + #{name => <<"repo-2">>, + api_url => <<"repo-2/api">>, + repo_url => <<"bad url">>, + repo_verify => true}, + #{name => <<"repo-1">>, + api_url => <<"bad url">>, + repo_verify => true}, + #{name => <<"repo-2">>, + api_url => <<"repo-2/api-2">>, + repo_url => <<"other/repo">>}]), + ?assertMatch([#{name := <<"repo-1">>, + api_url := <<"repo-1/api">>, + repo_verify := true}, + #{name := <<"repo-2">>, + api_url := <<"repo-2/api">>, + repo_url := <<"repo-2/repo">>, + repo_verify := false}], Result). + +repo_replacing(_Config) -> + Repo1 = #{name => <<"repo-1">>, + api_url => <<"repo-1/api">>}, + Repo2 = #{name => <<"repo-2">>, + repo_url => <<"repo-2/repo">>, + repo_verify => false}, + + ?assertMatch([Repo1, Repo2, #{name := <<"hexpm">>}], + rebar_hex_repos:repos([{repos, [Repo1]}, + {repos, [Repo2]}])), + + %% use of replace is ignored if found in later entries than the first + ?assertMatch([Repo1, Repo2, #{name := <<"hexpm">>}], + rebar_hex_repos:repos([{repos, [Repo1]}, + {repos, replace, [Repo2]}])), + + ?assertMatch([Repo1], + rebar_hex_repos:repos([{repos, replace, [Repo1]}, + {repos, [Repo2]}])). + +auth_merging(_Config) -> + Repo1 = #{name => <<"repo-1">>, + api_url => <<"repo-1/api">>}, + Repo2 = #{name => <<"repo-2">>, + repo_url => <<"repo-2/repo">>, + repo_verify => false}, + + State = rebar_state:new([{hex, [{repos, [Repo1, Repo2]}]}]), + meck:expect(file, consult, + fun(_) -> + {ok, [#{<<"repo-1">> => #{read_key => <<"read key">>, + write_key => <<"write key">>}, + <<"repo-2">> => #{read_key => <<"read key 2">>, + repos_key => <<"repos key 2">>, + write_key => <<"write key 2">>}, + <<"hexpm">> => #{write_key => <<"write key hexpm">>}}]} + end), + + ?assertMatch({ok, + #resource{state=#{repos := [#{name := <<"repo-1">>, + read_key := <<"read key">>, + write_key := <<"write key">>}, + #{name := <<"repo-2">>, + read_key := <<"read key 2">>, + repos_key := <<"repos key 2">>, + write_key := <<"write key 2">>}, + #{name := <<"hexpm">>, + write_key := <<"write key hexpm">>}]}}}, + rebar_pkg_resource:init(pkg, State)), + + ok. + +organization_merging(_Config) -> + Repo1 = #{name => <<"hexpm:repo-1">>, + api_url => <<"repo-1/api">>}, + Repo2 = #{name => <<"hexpm:repo-2">>, + repo_url => <<"repo-2/repo">>, + repo_verify => false}, + + State = rebar_state:new([{hex, [{repos, [Repo1, Repo2]}]}]), + meck:expect(file, consult, + fun(_) -> + {ok, [#{<<"hexpm:repo-1">> => #{read_key => <<"read key">>}, + <<"hexpm:repo-2">> => #{read_key => <<"read key 2">>, + repos_key => <<"repos key 2">>, + write_key => <<"write key 2">>}, + <<"hexpm">> => #{write_key => <<"write key hexpm">>}}]} + end), + + ?assertMatch({ok, + #resource{state=#{repos := [#{name := <<"hexpm:repo-1">>, + parent := <<"hexpm">>, + read_key := <<"read key">>, + write_key := <<"write key hexpm">>}, + #{name := <<"hexpm:repo-2">>, + parent := <<"hexpm">>, + read_key := <<"read key 2">>, + repos_key := <<"repos key 2">>, + write_key := <<"write key 2">>}, + #{name := <<"hexpm">>, + write_key := <<"write key hexpm">>}]}}}, + rebar_pkg_resource:init(pkg, State)), + + ok. + +use_first_repo_match(Config) -> + State = ?config(state, Config), + + ?assertMatch({ok,{package,{<<"B">>, {{2,0,0}, {[],[]}}, Repo2}, + <<"some checksum">>, false, []}, + #{name := Repo2, + http_adapter_config := #{profile := rebar}}}, + rebar_packages:resolve_version(<<"B">>, <<"> 1.4.0">>, undefined, + ?PACKAGE_TABLE, State)), + + ?assertMatch({ok,{package,{<<"B">>, {{1,4,0}, {[],[]}}, Repo3}, + <<"some checksum">>, false, []}, + #{name := Repo3, + http_adapter_config := #{profile := rebar}}}, + rebar_packages:resolve_version(<<"B">>, <<"~> 1.4.0">>, undefined, + ?PACKAGE_TABLE, State)). + +%% tests that even though an easier repo has C-1.3.1 it doesn't use it since its hash is different +use_exact_with_hash(Config) -> + State = ?config(state, Config), + + ?assertMatch({ok,{package,{<<"C">>, {{1,3,1}, {[],[]}}, Repo2}, + <<"good checksum">>, false, []}, + #{name := Repo2, + http_adapter_config := #{profile := rebar}}}, + rebar_packages:resolve_version(<<"C">>, <<"1.3.1">>, <<"good checksum">>, + ?PACKAGE_TABLE, State)). + +fail_repo_update(Config) -> + State = ?config(state, Config), + + ?assertMatch({ok,{package,{<<"B">>, {{1,4,0}, {[],[]}}, Repo3}, + <<"some checksum">>, false, []}, + #{name := Repo3, + http_adapter_config := #{profile := rebar}}}, + rebar_packages:resolve_version(<<"B">>, <<"~> 1.4.0">>, undefined, + ?PACKAGE_TABLE, State)). + +ignore_match_in_excluded_repo(Config) -> + State = ?config(state, Config), + Repos = ?config(repos, Config), + + ?assertMatch({ok,{package,{<<"B">>, {{1,4,6}, {[],[]}}, Hexpm}, + <<"some checksum">>, #{reason := 'RETIRED_INVALID'}, []}, + #{name := Hexpm, + http_adapter_config := #{profile := rebar}}}, + rebar_packages:resolve_version(<<"B">>, <<"~> 1.4.0">>, undefined, + ?PACKAGE_TABLE, State)), + + [_, Repo2 | _] = Repos, + ?assertMatch({ok,{package,{<<"A">>, {{0,1,1}, {[],[]}}, Repo2}, + <<"good checksum">>, false, []}, + #{name := Repo2, + http_adapter_config := #{profile := rebar}}}, + rebar_packages:resolve_version(<<"A">>, <<"0.1.1">>, <<"good checksum">>, + ?PACKAGE_TABLE, State)). + +optional_prereleases(Config) -> + State = ?config(state, Config), + + ?assertMatch({ok,{package,{<<"B">>, {{1,5,0}, {[],[]}}, Hexpm}, + <<"some checksum">>, false, []}, + #{name := Hexpm, + http_adapter_config := #{profile := rebar}}}, + rebar_packages:resolve_version(<<"B">>, <<"~> 1.5.0">>, undefined, + ?PACKAGE_TABLE, State)), + + ?assertMatch({ok,{package,{<<"B">>, {{1,5,6}, {[<<"rc">>,0],[]}}, Hexpm}, + <<"some checksum">>, true, []}, + #{name := Hexpm, + http_adapter_config := #{profile := rebar}}}, + rebar_packages:resolve_version(<<"B">>, <<"1.5.6-rc.0">>, <<"some checksum">>, + ?PACKAGE_TABLE, State)), + + %% allow prerelease through configuration + State1 = rebar_state:set(State, deps_allow_prerelease, true), + ?assertMatch({ok,{package,{<<"B">>, {{1,5,6}, {[<<"rc">>,0],[]}}, Hexpm}, + <<"some checksum">>, true, []}, + #{name := Hexpm, + http_adapter_config := #{profile := rebar}}}, + rebar_packages:resolve_version(<<"B">>, <<"~> 1.5.0">>, <<"some checksum">>, + ?PACKAGE_TABLE, State1)). + +%% + +setup_deps_and_repos(Deps, Repos) -> + catch ets:delete(?PACKAGE_TABLE), + true = rebar_packages:new_package_table(), + insert_deps(Deps), + State = rebar_state:new([{hex, [{repos, [#{name => R} || R <- Repos]}]}]), + rebar_state:create_resources([{pkg, rebar_pkg_resource}], State). + + +insert_deps(Deps) -> + lists:foreach(fun({Name, Version, Repo, Retired}) -> + ets:insert(?PACKAGE_TABLE, #package{key={rebar_utils:to_binary(Name), + ec_semver:parse(Version), + rebar_utils:to_binary(Repo)}, + dependencies=[], + retired=Retired, + checksum = <<"some checksum">>}); + ({Name, Version, Checksum, Repo, Retired}) -> + ets:insert(?PACKAGE_TABLE, #package{key={rebar_utils:to_binary(Name), + ec_semver:parse(Version), + rebar_utils:to_binary(Repo)}, + dependencies=[], + retired=Retired, + checksum = Checksum}) + end, Deps). diff --git a/test/rebar_plugins_SUITE.erl b/test/rebar_plugins_SUITE.erl index a313683..c7a5d51 100644 --- a/test/rebar_plugins_SUITE.erl +++ b/test/rebar_plugins_SUITE.erl @@ -14,7 +14,8 @@ upgrade_project_plugin/1, sub_app_plugins/1, sub_app_plugin_overrides/1, - project_plugins/1]). + project_plugins/1, + use_checkout_plugins/1]). -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -37,7 +38,7 @@ end_per_testcase(_, _Config) -> all() -> [compile_plugins, compile_global_plugins, complex_plugins, list, upgrade, upgrade_project_plugin, - sub_app_plugins, sub_app_plugin_overrides, project_plugins]. + sub_app_plugins, sub_app_plugin_overrides, project_plugins, use_checkout_plugins]. %% Tests that compiling a project installs and compiles the plugins of deps compile_plugins(Config) -> @@ -334,7 +335,7 @@ project_plugins(Config) -> rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), DepName = rebar_test_utils:create_random_name("dep1_"), - PluginName = "compile", + PluginName = "compile_plugin", PluginName2 = "release", Plugins = rebar_test_utils:expand_deps(git, [{PluginName, Vsn, []}, {PluginName2, Vsn, []}]), @@ -370,3 +371,26 @@ project_plugins(Config) -> ?assertEqual(length(Release), 2), ?assertEqual(length(Compile), 1). + +use_checkout_plugins(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("app1_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + PluginName = "checkedout", + CheckoutsDir = filename:join(AppDir, "_checkouts/checkedout"), + rebar_test_utils:create_plugin(CheckoutsDir, PluginName, "1.0.0", []), + + RConfFile = + rebar_test_utils:create_config(AppDir, + [{deps, []}, + {plugins, [list_to_atom(PluginName)]}]), + {ok, RConf} = file:consult(RConfFile), + + %% Verify we can run the plugin + ?assertMatch({ok, _}, rebar_test_utils:run_and_check( + Config, RConf, ["checkedout"], + {ok, []} + )). diff --git a/test/rebar_profiles_SUITE.erl b/test/rebar_profiles_SUITE.erl index a31a4c9..512832a 100644 --- a/test/rebar_profiles_SUITE.erl +++ b/test/rebar_profiles_SUITE.erl @@ -7,6 +7,7 @@ all/0, profile_new_key/1, profile_merge_keys/1, + profile_merge_umbrella_keys/1, explicit_profile_deduplicate_deps/1, implicit_profile_deduplicate_deps/1, all_deps_code_paths/1, @@ -20,14 +21,23 @@ test_profile_applied_at_completion/1, test_profile_applied_before_compile/1, test_profile_applied_before_eunit/1, - test_profile_applied_to_apps/1]). + test_profile_applied_to_apps/1, + test_profile_erl_opts_order_1/1, + test_profile_erl_opts_order_2/1, + test_profile_erl_opts_order_3/1, + test_profile_erl_opts_order_4/1, + test_profile_erl_opts_order_5/1, + test_erl_opts_debug_info/1, + test_profile_erl_opts_precedence/1, + first_files_exception/1]). -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("kernel/include/file.hrl"). all() -> - [profile_new_key, profile_merge_keys, all_deps_code_paths, profile_merges, + [profile_new_key, profile_merge_keys, profile_merge_umbrella_keys, + all_deps_code_paths, profile_merges, explicit_profile_deduplicate_deps, implicit_profile_deduplicate_deps, same_profile_deduplication, stack_deduplication, add_to_profile, add_to_existing_profile, @@ -36,7 +46,15 @@ all() -> test_profile_applied_at_completion, test_profile_applied_before_compile, test_profile_applied_before_eunit, - test_profile_applied_to_apps]. + test_profile_applied_to_apps, + test_profile_erl_opts_order_1, + test_profile_erl_opts_order_2, + test_profile_erl_opts_order_3, + test_profile_erl_opts_order_4, + test_profile_erl_opts_order_5, + test_erl_opts_debug_info, + test_profile_erl_opts_precedence, + first_files_exception]. init_per_suite(Config) -> application:start(meck), @@ -106,6 +124,35 @@ profile_merge_keys(Config) -> ,{dep, "a", "1.0.0"} ,{dep, "b", "2.0.0"}]}). +profile_merge_umbrella_keys(Config) -> + AppDir = ?config(apps, Config), + ct:pal("Path: ~s", [AppDir]), + Name = rebar_test_utils:create_random_name("profile_merge_umbrella_keys"), + Vsn = rebar_test_utils:create_random_vsn(), + SubAppDir = filename:join([AppDir, "apps", Name]), + + RebarConfig = [{vals, [{a,1},{b,1}]}, + {profiles, + [{ct, + [{vals, [{a,1},{b,2}]}]}]}], + + SubRebarConfig = [{vals, []}, + {profiles, [{ct, [{vals, [{c,1}]}]}]}], + + rebar_test_utils:create_app(SubAppDir, Name, Vsn, [kernel, stdlib]), + rebar_test_utils:create_config(SubAppDir, SubRebarConfig), + {ok, RebarConfigRead} = file:consult(rebar_test_utils:create_config(AppDir, RebarConfig)), + + {ok, State} = rebar_test_utils:run_and_check( + Config, RebarConfigRead, ["as", "ct", "compile"], return + ), + + [ProjectApp] = rebar_state:project_apps(State), + ?assertEqual(Name, binary_to_list(rebar_app_info:name(ProjectApp))), + Opts = rebar_app_info:opts(ProjectApp), + ?assertEqual([{a,1},{b,2},{b,1},{c,1}], dict:fetch(vals, Opts)), + ok. + explicit_profile_deduplicate_deps(Config) -> AppDir = ?config(apps, Config), @@ -166,7 +213,7 @@ implicit_profile_deduplicate_deps(Config) -> rebar_test_utils:run_and_check(Config, RebarConfig, ["as", "test,bar", "eunit"], {ok, [{app, Name} ,{dep, "a", "1.0.0"} - ,{dep, "b", "2.0.0"}]}). + ,{dep, "b", "1.0.0"}]}). all_deps_code_paths(Config) -> AppDir = ?config(apps, Config), @@ -432,3 +479,166 @@ test_profile_applied_to_apps(Config) -> ErlOpts = dict:fetch(erl_opts, Opts), true = lists:member({d, 'TEST'}, ErlOpts) end, Apps). + +test_profile_erl_opts_order_1(Config) -> + Opts = get_compiled_profile_erl_opts([default], Config), + Opt = last_erl_opt(Opts, [warn_export_all, nowarn_export_all], undefined), + undefined = Opt. + +test_profile_erl_opts_order_2(Config) -> + Opts = get_compiled_profile_erl_opts([strict], Config), + Opt = last_erl_opt(Opts, [warn_export_all, nowarn_export_all], undefined), + warn_export_all = Opt. + +test_profile_erl_opts_order_3(Config) -> + Opts = get_compiled_profile_erl_opts([loose], Config), + Opt = last_erl_opt(Opts, [warn_export_all, nowarn_export_all], undefined), + nowarn_export_all = Opt. + +test_profile_erl_opts_order_4(Config) -> + Opts = get_compiled_profile_erl_opts([strict, loose], Config), + Opt = last_erl_opt(Opts, [warn_export_all, nowarn_export_all], undefined), + nowarn_export_all = Opt. + +test_profile_erl_opts_order_5(Config) -> + Opts = get_compiled_profile_erl_opts([loose, strict], Config), + Opt = last_erl_opt(Opts, [warn_export_all, nowarn_export_all], undefined), + warn_export_all = Opt. + +test_erl_opts_debug_info(_Config) -> + ToOpts = fun(List) -> rebar_opts:erl_opts(dict:from_list([{erl_opts, List}])) end, + ?assertEqual([debug_info,a,b,c], + ToOpts([a,b,c])), + ?assertEqual([{debug_info,{mod,123}},a,b,c,debug_info], + ToOpts([{debug_info,{mod,123}},a,b,c,debug_info])), + ?assertEqual([a,b,debug_info,c], + ToOpts([no_debug_info,a,b,debug_info,c])), + ?assertEqual([a,b,c], + ToOpts([debug_info,a,b,no_debug_info,c])), + ?assertEqual([a,b,c,debug_info], + ToOpts([{debug_info_key, "12345"},a,b, + no_debug_info,c,debug_info])), + ?assertEqual([a,b,c], + ToOpts([{debug_info,{mod,123}},{debug_info_key, "12345"}, + a,no_debug_info,b,c,debug_info,no_debug_info])), + ?assertEqual([a,b,c,{debug_info_key,"123"}], + ToOpts([{debug_info_key, "12345"},a,b,no_debug_info,debug_info, + c,{debug_info_key, "123"}])), + ?assertEqual([{debug_info_key,"12345"},a,b,c,{debug_info,{mod,"123"}}], + ToOpts([debug_info,{debug_info_key,"12345"},a, + no_debug_info,b,c,{debug_info,{mod,"123"}}])), + ok. + +test_profile_erl_opts_precedence(Config) -> + AppDir = ?config(apps, Config), + Name = rebar_test_utils:create_random_name("profile_new_key_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + RebarConfig = [{erl_opts, [no_debug_info]}, + {profiles, [ + {test, [{erl_opts, [debug_info, {d,'HI'}]}]}, + {other, [{erl_opts, [debug_info, {d,'HI'}]}]} + ]}], + {ok, State1} = rebar_test_utils:run_and_check( + Config, RebarConfig, ["as", "test", "compile"], return + ), + {ok, State2} = rebar_test_utils:run_and_check( + Config, RebarConfig, ["as", "other", "compile"], return + ), + {ok, State3} = rebar_test_utils:run_and_check( + Config, RebarConfig, ["compile"], return + ), + Opts1 = rebar_state:opts(State1), + Opts2 = rebar_state:opts(State2), + Opts3 = rebar_state:opts(State3), + ErlOpts1 = rebar_opts:erl_opts(Opts1), + ErlOpts2 = rebar_opts:erl_opts(Opts2), + ErlOpts3 = rebar_opts:erl_opts(Opts3), + ?assertEqual([{d,'TEST'}, debug_info, {d,'HI'}], ErlOpts1), + ?assertEqual([debug_info, {d,'HI'}], ErlOpts2), + ?assertEqual([], ErlOpts3), + ok. + +first_files_exception(_Config) -> + RebarConfig = [{erl_first_files, ["c","a","b"]}, + {mib_first_files, ["c","a","b"]}, + {other, ["c","a","b"]}, + {profiles, + [{profile2, [{erl_first_files, ["a","e"]}, + {mib_first_files, ["a","e"]}, + {other, ["a","e"]} + ]}]}], + State = rebar_state:new(RebarConfig), + State1 = rebar_state:apply_profiles(State, [profile2]), + + %% Combine lists + ?assertEqual(["a","b","c","e"], rebar_state:get(State1, other)), + %% there is no specific reason not to dedupe "a" here aside from "this is how it is" + ?assertEqual(["c","a","b","a","e"], rebar_state:get(State1, erl_first_files)), + ?assertEqual(["c","a","b","a","e"], rebar_state:get(State1, mib_first_files)), + ok. + +get_compiled_profile_erl_opts(Profiles, Config) -> + AppDir = ?config(apps, Config), + PStrs = [atom_to_list(P) || P <- Profiles], + + Name = rebar_test_utils:create_random_name( + lists:flatten(["erl_opts_order_" | [[S, $_] || S <- PStrs]])), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + RebarConfig = [ + {erl_opts, [warnings_as_errors, {d, profile_default}]}, + {profiles, [ + {strict, [{erl_opts, [warn_export_all, {d, profile_strict}]}]}, + {loose, [{erl_opts, [nowarn_export_all, {d, profile_loose}]}]} ]}], + rebar_test_utils:create_config(AppDir, RebarConfig), + + Command = case Profiles of + [] -> + ["compile"]; + [default] -> + ["compile"]; + _ -> + ["as", rebar_string:join(PStrs, ","), "compile"] + end, + {ok, State} = rebar_test_utils:run_and_check( + Config, RebarConfig, Command, {ok, [{app, Name}]}), + code:add_paths(rebar_state:code_paths(State, all_deps)), + Mod = list_to_atom(Name), + proplists:get_value(options, Mod:module_info(compile), []). + +% macro definitions get special handling +last_erl_opt([{d, Macro} = Opt | Opts], Targets, Last) -> + case lists:any(erl_opt_macro_match_fun(Macro), Targets) of + true -> + last_erl_opt(Opts, Targets, Opt); + _ -> + last_erl_opt(Opts, Targets, Last) + end; +last_erl_opt([{d, Macro, _} = Opt | Opts], Targets, Last) -> + case lists:any(erl_opt_macro_match_fun(Macro), Targets) of + true -> + last_erl_opt(Opts, Targets, Opt); + _ -> + last_erl_opt(Opts, Targets, Last) + end; +last_erl_opt([Opt | Opts], Targets, Last) -> + case lists:member(Opt, Targets) of + true -> + last_erl_opt(Opts, Targets, Opt); + _ -> + last_erl_opt(Opts, Targets, Last) + end; +last_erl_opt([], _, Last) -> + Last. + +erl_opt_macro_match_fun(Macro) -> + fun({d, M}) -> + M == Macro; + ({d, M, _}) -> + M == Macro; + (_) -> + false + end. + diff --git a/test/rebar_release_SUITE.erl b/test/rebar_release_SUITE.erl index 1125a7e..1bcc61e 100644 --- a/test/rebar_release_SUITE.erl +++ b/test/rebar_release_SUITE.erl @@ -11,6 +11,7 @@ all() -> [release, profile_ordering_sys_config_extend_3_tuple_merge, extend_release, user_output_dir, profile_overlays, + profile_overlay_merge, overlay_vars]. init_per_testcase(Case, Config0) -> @@ -199,13 +200,34 @@ profile_overlays(Config) -> AppDir = ?config(apps, Config), Name = ?config(name, Config), Vsn = "1.0.0", + file:write_file(filename:join(AppDir, "dev.file"), "dev.\n"), + file:write_file(filename:join(AppDir, "prod.file"), "prod.\n"), + file:write_file(filename:join(AppDir, "dev.vars"), "{env, \"dev\"}.\n"), + file:write_file(filename:join(AppDir, "prod.vars"), "{env, \"prod\"}.\n"), {ok, RebarConfig} = - file:consult(rebar_test_utils:create_config(AppDir, - [{relx, [{release, {list_to_atom(Name), Vsn}, - [list_to_atom(Name)]}, - {overlay, [{mkdir, "randomdir"}]}, - {lib_dirs, [AppDir]}]}, - {profiles, [{prod, [{relx, [{overlay, [{mkdir, "otherrandomdir"}]}]}]}]}])), + file:consult(rebar_test_utils:create_config(AppDir, + %% Paths are relative, but to cwd in relx, not the project root as + %% seen by rebar3 (in non-test cases, they're the same). + %% Work around by being explicit. + [{relx, [{release, {list_to_atom(Name), Vsn}, + [list_to_atom(Name)]}, + {overlay_vars, filename:join(AppDir, "dev.vars")}, + {overlay, [{mkdir, "randomdir"}, + {copy, filename:join(AppDir,"./dev.file"), "profile.file"}, + {copy, filename:join(AppDir,"./dev.file"), "{{env}}.file"}, + {chmod, 8#00770, "profile.file"}]}, + {lib_dirs, [AppDir]}]}, + {profiles, [{prod, + [{relx, [ + {overlay_vars, filename:join(AppDir, "prod.vars")}, + {overlay, [{mkdir, "otherrandomdir"}, + {copy, filename:join(AppDir, "./prod.file"), "{{env}}.file"}, + {copy, filename:join(AppDir, "./prod.file"), "profile.file"}, + {chmod, 8#00770, "profile.file"}]} + + ]}] + }]} + ])), ReleaseDir = filename:join([AppDir, "./_build/prod/rel/", Name]), @@ -215,7 +237,36 @@ profile_overlays(Config) -> {ok, [{release, list_to_atom(Name), Vsn, false}, {dir, filename:join(ReleaseDir, "otherrandomdir")}, {dir, filename:join(ReleaseDir, "randomdir")}]} - ). + ), + ?assertMatch({ok,[prod]}, + file:consult(filename:join(ReleaseDir, "profile.file"))), + ?assertMatch({ok,[prod]}, + file:consult(filename:join(ReleaseDir, "prod.file"))), + ok. + +profile_overlay_merge (_Config) -> + % when profile and relx overlays both exist, the profile overlays should be + % first, then the relx overlays, all the rest of the config should come + % after, rebar_relx:merge_overlays/1 should do this. + RelxOverlay = [{mkdir, "1_from_relx"}, {mkdir, "2_from_relx"}], + ProfileOverlay = [{mkdir, "0_from_other_profile"}], + OtherConfig = [{other1, config}, {other2, config}], + + % test with no overlays + ?assertEqual([{overlay,[]}] ++ OtherConfig, + rebar_relx:merge_overlays(OtherConfig)), + + % test with relx only, just move overlays to the top + RelxOnly = OtherConfig ++ [{overlay, RelxOverlay}], + ?assertEqual([{overlay, RelxOverlay}]++OtherConfig, + rebar_relx:merge_overlays(RelxOnly)), + + % now test with a profile (profiles end up after relx overlays + ProfilesToMerge = OtherConfig ++ + [{overlay, RelxOverlay}, + {overlay, ProfileOverlay}], + ?assertEqual([{overlay, ProfileOverlay ++ RelxOverlay}] ++ OtherConfig, + rebar_relx:merge_overlays(ProfilesToMerge)). overlay_vars(Config) -> AppDir = ?config(apps, Config), diff --git a/test/rebar_resource_SUITE.erl b/test/rebar_resource_SUITE.erl index 15f14db..ddacb91 100644 --- a/test/rebar_resource_SUITE.erl +++ b/test/rebar_resource_SUITE.erl @@ -29,12 +29,15 @@ init_per_testcase(change_type_upgrade, Config) -> TypeStr = atom_to_list(Type), DirName = filename:join([?config(priv_dir, Config), "resource_"++TypeStr]), ec_file:mkdir_path(DirName), - [{path, DirName} | Config]. + + {ok, AppInfo} = rebar_app_info:new(test_app, "0.0.1", DirName), + AppInfo1 = rebar_app_info:source(AppInfo, ?config(resource, Config)), + + [{app, AppInfo1} | Config]. end_per_testcase(_, Config) -> Config. change_type_upgrade(Config) -> - ?assert(rebar_fetch:needs_update(?config(path, Config), - ?config(resource, Config), + ?assert(rebar_fetch:needs_update(?config(app, Config), ?config(state, Config))). diff --git a/test/rebar_src_dirs_SUITE.erl b/test/rebar_src_dirs_SUITE.erl index f854a94..bc22160 100644 --- a/test/rebar_src_dirs_SUITE.erl +++ b/test/rebar_src_dirs_SUITE.erl @@ -11,12 +11,16 @@ src_dirs_in_erl_opts/1, extra_src_dirs_in_erl_opts/1, src_dirs_at_root_and_in_erl_opts/1, + dupe_src_dirs_at_root_and_in_erl_opts/1, extra_src_dirs_at_root_and_in_erl_opts/1, build_basic_app/1, build_multi_apps/1, - src_dir_takes_precedence_over_extra/1]). + src_dir_takes_precedence_over_extra/1, + src_dir_checkout_dep/1, + app_src_info/1]). -include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). suite() -> []. @@ -35,8 +39,11 @@ end_per_testcase(_, _Config) -> ok. all() -> [src_dirs_at_root, extra_src_dirs_at_root, src_dirs_in_erl_opts, extra_src_dirs_in_erl_opts, - src_dirs_at_root_and_in_erl_opts, extra_src_dirs_at_root_and_in_erl_opts, - build_basic_app, build_multi_apps, src_dir_takes_precedence_over_extra]. + src_dirs_at_root_and_in_erl_opts, + dupe_src_dirs_at_root_and_in_erl_opts, + extra_src_dirs_at_root_and_in_erl_opts, + build_basic_app, build_multi_apps, src_dir_takes_precedence_over_extra, + src_dir_checkout_dep, app_src_info]. src_dirs_at_root(Config) -> AppDir = ?config(apps, Config), @@ -93,15 +100,47 @@ extra_src_dirs_in_erl_opts(Config) -> src_dirs_at_root_and_in_erl_opts(Config) -> AppDir = ?config(apps, Config), - Name = rebar_test_utils:create_random_name("app1_"), + Name = rebar_test_utils:create_random_name("src_dirs_root_erlopts_"), Vsn = rebar_test_utils:create_random_vsn(), rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), RebarConfig = [{erl_opts, [{src_dirs, ["foo", "bar"]}]}, {src_dirs, ["baz", "qux"]}], + %% move the .app.src file to one of the subdirs, out of src/ + filelib:ensure_dir(filename:join([AppDir, "qux", "fake"])), + rebar_file_utils:mv(filename:join([AppDir, "src", Name ++ ".app.src"]), + filename:join([AppDir, "qux", Name ++ ".app.src"])), + {ok, State} = rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], return), - ["bar", "baz", "foo", "qux"] = rebar_dir:src_dirs(rebar_state:opts(State), []). + ["bar", "baz", "foo", "qux"] = rebar_dir:src_dirs(rebar_state:opts(State), []), + rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], + {ok, [{app, Name}]}), + ok. + +dupe_src_dirs_at_root_and_in_erl_opts(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("dupe_src_dirs_root_erlopts_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + RebarConfig = [{erl_opts, [{src_dirs, ["foo", "bar"]}]}, {src_dirs, ["baz", "qux"]}], + + %% move the .app.src file to one of the subdirs, out of src/ + filelib:ensure_dir(filename:join([AppDir, "qux", "fake"])), + filelib:ensure_dir(filename:join([AppDir, "foo", "fake"])), + Src1 = filename:join([AppDir, "qux", Name ++ ".app.src"]), + Src2 = filename:join([AppDir, "foo", Name ++ ".app.src"]), + rebar_file_utils:mv(filename:join([AppDir, "src", Name ++ ".app.src"]), + Src1), + %% Then copy it over to create a conflict with dupes + file:copy(Src1, Src2), + + {error, {rebar_prv_app_discovery, {multiple_app_files, [Src2, Src1]}}} = + rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], return), + + ok. extra_src_dirs_at_root_and_in_erl_opts(Config) -> AppDir = ?config(apps, Config), @@ -236,3 +275,52 @@ src_dir_takes_precedence_over_extra(Config) -> [{application, _, KVs}] = App, Mods = proplists:get_value(modules, KVs), true = lists:member(extra, Mods). + +src_dir_checkout_dep(Config) -> + AppDir = ?config(apps, Config), + AppName = rebar_test_utils:create_random_name("src_dir_checkout_app"), + DepName = rebar_test_utils:create_random_name("src_dir_checkout_dep"), + AtomDep = list_to_atom(DepName), + + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, AppName, Vsn, [kernel, stdlib]), + RebarConfig = [{deps, [AtomDep]}], + + DepDir = filename:join([?config(checkouts, Config), DepName]), + ct:pal("checkouts dir: ~p", [DepDir]), + rebar_test_utils:create_app(DepDir, DepName, Vsn, [kernel, stdlib]), + + + %% move the .app.src file to one of the subdirs, out of src/ + rebar_file_utils:mv(filename:join([DepDir, "src"]), + filename:join([DepDir, "qux"])), + DepRebarConfig = [{erl_opts, [{src_dirs, ["foo", "bar"]}]}, + {src_dirs, ["baz", "qux"]}], + file:write_file(filename:join([DepDir, "rebar.config"]), + io_lib:format("~p.~n~p.~n", DepRebarConfig)), + + rebar_test_utils:run_and_check( + Config, RebarConfig, ["compile"], + {ok, [{checkout, DepName}, {app, AppName}]} + ), + ok. + +app_src_info(Config) -> + PrivDir = ?config(priv_dir, Config), + AppName1 = rebar_test_utils:create_random_name("app_src_info"), + AppDir1 = filename:join(PrivDir, AppName1), + {ok, Info1} = rebar_app_info:new(AppName1, "1.0.0", AppDir1), + AppSrc1 = filename:join([AppDir1, "src", AppName1 ++ ".app.src"]), + ok = filelib:ensure_dir(AppSrc1), + ok = file:write_file(AppSrc1, "[]."), + ?assertEqual(AppSrc1, rebar_app_info:app_file_src(Info1)), + + AppName2 = rebar_test_utils:create_random_name("app_src_info"), + AppDir2 = filename:join(PrivDir, AppName2), + {ok, Info2Tmp} = rebar_app_info:new(AppName2, "1.0.0", AppDir2), + Info2 = rebar_app_info:set(Info2Tmp, src_dirs, ["foo", "bar", "baz"]), + AppSrc2 = filename:join([AppDir2, "bar", AppName2 ++ ".app.src"]), + ok = filelib:ensure_dir(AppSrc2), + ok = file:write_file(AppSrc2, "[]."), + ?assertEqual(AppSrc2, rebar_app_info:app_file_src(Info2)), + ok. diff --git a/test/rebar_test_utils.erl b/test/rebar_test_utils.erl index 8c177c9..944efa0 100644 --- a/test/rebar_test_utils.erl +++ b/test/rebar_test_utils.erl @@ -1,11 +1,12 @@ -module(rebar_test_utils). -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). --export([init_rebar_state/1, init_rebar_state/2, run_and_check/4, check_results/3]). +-export([init_rebar_state/1, init_rebar_state/2, run_and_check/3, run_and_check/4, check_results/3]). -export([expand_deps/2, flat_deps/1, top_level_deps/1]). -export([create_app/4, create_plugin/4, create_eunit_app/4, create_empty_app/4, - create_config/2, create_config/3, package_app/3]). --export([create_random_name/1, create_random_vsn/0, write_src_file/2]). + create_config/2, create_config/3, package_app/4]). +-export([create_random_name/1, create_random_vsn/0, write_src_file/2, + random_element/1]). %% Pick the right random module -ifdef(rand_only). @@ -34,8 +35,10 @@ init_rebar_state(Config, Name) -> Verbosity = rebar3:log_level(), rebar_log:init(command_line, Verbosity), GlobalDir = filename:join([DataDir, "cache"]), + Repos = proplists:get_value(repos, Config, []), State = rebar_state:new([{base_dir, filename:join([AppsDir, "_build"])} ,{global_rebar_dir, GlobalDir} + ,{hex, [{repos, [#{name => R} || R <- Repos]}]} ,{root_dir, AppsDir}]), [{apps, AppsDir}, {checkouts, CheckoutsDir}, {state, State} | Config]. @@ -79,6 +82,33 @@ run_and_check(Config, RebarConfig, Command, Expect) -> rebar_abort when Expect =:= rebar_abort -> rebar_abort end. +run_and_check(Config, Command, Expect) -> + %% Assumes init_rebar_state has run first + AppDir = ?config(apps, Config), + {ok, Cwd} = file:get_cwd(), + try + ok = file:set_cwd(AppDir), + Res = rebar3:run(Command), + case Expect of + {error, Reason} -> + ?assertEqual({error, Reason}, Res); + {ok, Expected} -> + {ok, _} = Res, + check_results(AppDir, Expected, "*"), + Res; + {ok, Expected, ProfileRun} -> + {ok, _} = Res, + check_results(AppDir, Expected, ProfileRun), + Res; + return -> + Res + end + catch + rebar_abort when Expect =:= rebar_abort -> rebar_abort + after + ok = file:set_cwd(Cwd) + end. + %% @doc Creates a dummy application including: %% - src/<file>.erl %% - src/<file>.app.src @@ -167,14 +197,14 @@ expand_deps(pkg, [{Name, Vsn, Deps} | Rest]) -> [{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)]; expand_deps(mixed, [{Name, Deps} | Rest]) -> Dep = if hd(Name) >= $a, hd(Name) =< $z -> - {pkg, string:to_upper(Name), "0.0.0", undefined} + {pkg, rebar_string:uppercase(Name), "0.0.0", undefined} ; hd(Name) >= $A, hd(Name) =< $Z -> {Name, ".*", {git, "https://example.org/user/"++Name++".git", "master"}} end, [{Dep, expand_deps(mixed, Deps)} | expand_deps(mixed, Rest)]; expand_deps(mixed, [{Name, Vsn, Deps} | Rest]) -> Dep = if hd(Name) >= $a, hd(Name) =< $z -> - {pkg, string:to_upper(Name), Vsn, undefined} + {pkg, rebar_string:uppercase(Name), Vsn, undefined} ; hd(Name) >= $A, hd(Name) =< $Z -> {Name, Vsn, {git, "https://example.org/user/"++Name++".git", {tag, Vsn}}} end, @@ -218,7 +248,7 @@ check_results(AppDir, Expected, ProfileRun) -> BuildDirs = filelib:wildcard(filename:join([AppDir, "_build", ProfileRun, "lib", "*"])), PluginDirs = filelib:wildcard(filename:join([AppDir, "_build", ProfileRun, "plugins", "*"])), GlobalPluginDirs = filelib:wildcard(filename:join([AppDir, "global", "plugins", "*"])), - CheckoutsDir = filename:join([AppDir, "_checkouts", "*"]), + CheckoutsDirs = filelib:wildcard(filename:join([AppDir, "_checkouts", "*"])), LockFile = filename:join([AppDir, "rebar.lock"]), Locks = lists:flatten(rebar_config:consult_lock_file(LockFile)), @@ -230,7 +260,7 @@ check_results(AppDir, Expected, ProfileRun) -> Deps = rebar_app_discover:find_apps(BuildDirs, all), DepsNames = [{ec_cnv:to_list(rebar_app_info:name(App)), App} || App <- Deps], - Checkouts = rebar_app_discover:find_apps([CheckoutsDir], all), + Checkouts = rebar_app_discover:find_apps(CheckoutsDirs, all), CheckoutsNames = [{ec_cnv:to_list(rebar_app_info:name(App)), App} || App <- Checkouts], Plugins = rebar_app_discover:find_apps(PluginDirs, all), PluginsNames = [{ec_cnv:to_list(rebar_app_info:name(App)), App} || App <- Plugins], @@ -263,6 +293,14 @@ check_results(AppDir, Expected, ProfileRun) -> ok end ; ({dep_not_exist, Name}) -> + ct:pal("Dep Not Exist Name: ~p", [Name]), + case lists:keyfind(Name, 1, DepsNames) of + false -> + ok; + {Name, _App} -> + error({app_found, Name}) + end + ; ({app_not_exist, Name}) -> ct:pal("App Not Exist Name: ~p", [Name]), case lists:keyfind(Name, 1, DepsNames) of false -> @@ -349,7 +387,7 @@ check_results(AppDir, Expected, ProfileRun) -> iolist_to_binary(LockVsn)) end ; ({release, Name, Vsn, ExpectedDevMode}) -> - ct:pal("Release: ~p-~s", [Name, Vsn]), + ct:pal("Release: ~p-~ts", [Name, Vsn]), {ok, Cwd} = file:get_cwd(), try file:set_cwd(AppDir), @@ -377,14 +415,14 @@ check_results(AppDir, Expected, ProfileRun) -> file:set_cwd(Cwd) end ; ({tar, Name, Vsn}) -> - ct:pal("Tarball: ~s-~s", [Name, Vsn]), + ct:pal("Tarball: ~ts-~ts", [Name, Vsn]), Tarball = filename:join([AppDir, "_build", "rel", Name, Name++"-"++Vsn++".tar.gz"]), ?assertNotEqual([], filelib:is_file(Tarball)) ; ({file, Filename}) -> - ct:pal("Filename: ~s", [Filename]), + ct:pal("Filename: ~ts", [Filename]), ?assert(filelib:is_file(Filename)) ; ({dir, Dirname}) -> - ct:pal("Directory: ~s", [Dirname]), + ct:pal("Directory: ~ts", [Dirname]), ?assert(filelib:is_dir(Dirname)) end, Expected). @@ -425,15 +463,16 @@ erl_src_file(Name) -> plugin_src_file(Name) -> io_lib:format("-module('~s').\n" - "-export([init/1]).\n" + "-export([init/1, do/1]).\n" "init(State) -> \n" "Provider = providers:create([\n" "{name, '~s'},\n" "{module, '~s'}\n" "]),\n" - "{ok, rebar_state:add_provider(State, Provider)}.\n", [filename:basename(Name, ".erl"), - filename:basename(Name, ".erl"), - filename:basename(Name, ".erl")]). + "{ok, rebar_state:add_provider(State, Provider)}.\n" + "do(State) -> {ok, State}.\n", [filename:basename(Name, ".erl"), + filename:basename(Name, ".erl"), + filename:basename(Name, ".erl")]). erl_eunitized_src_file(Name) -> io_lib:format("-module('~s').\n" @@ -466,24 +505,25 @@ get_app_metadata(Name, Vsn, Deps) -> {registered, []}, {applications, Deps}]}. -package_app(AppDir, DestDir, PkgName) -> - Name = PkgName++".tar", - {ok, Fs} = rebar_utils:list_dir(AppDir), - ok = erl_tar:create(filename:join(DestDir, "contents.tar.gz"), - lists:zip(Fs, [filename:join(AppDir,F) || F <- Fs]), - [compressed]), - ok = file:write_file(filename:join(DestDir, "metadata.config"), "who cares"), - ok = file:write_file(filename:join(DestDir, "VERSION"), "3"), - {ok, Contents} = file:read_file(filename:join(DestDir, "contents.tar.gz")), - Blob = <<"3who cares", Contents/binary>>, - <<X:256/big-unsigned>> = crypto:hash(sha256, Blob), - BinChecksum = list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))), - ok = file:write_file(filename:join(DestDir, "CHECKSUM"), BinChecksum), - PkgFiles = ["contents.tar.gz", "VERSION", "metadata.config", "CHECKSUM"], +package_app(AppDir, DestDir, PkgName, PkgVsn) -> + AppSrc = filename:join(AppDir, "src"), + {ok, Fs} = rebar_utils:list_dir(AppSrc), + Files = lists:zip([filename:join("src", F) || F <- Fs], [filename:join(AppSrc,F) || F <- Fs]), + Metadata = #{<<"app">> => list_to_binary(PkgName), + <<"version">> => list_to_binary(PkgVsn)}, + {ok, {Tarball, <<Checksum:256/big-unsigned-integer>>}} = hex_tarball:create(Metadata, Files), + + Name = PkgName++"-"++PkgVsn++".tar", Archive = filename:join(DestDir, Name), - ok = erl_tar:create(Archive, - lists:zip(PkgFiles, [filename:join(DestDir,F) || F <- PkgFiles])), - {ok, BinFull} = file:read_file(Archive), - <<E:128/big-unsigned-integer>> = crypto:hash(md5, BinFull), - Etag = string:to_lower(lists:flatten(io_lib:format("~32.16.0b", [E]))), - {BinChecksum, Etag}. + file:write_file(Archive, Tarball), + + <<E:128/big-unsigned-integer>> = crypto:hash(md5, Tarball), + + Checksum1 = list_to_binary( + rebar_string:uppercase( + lists:flatten(io_lib:format("~64.16.0b", [Checksum])))), + {Checksum1, E}. + +random_element(Repos) -> + Index = ?random:uniform(length(Repos)), + lists:nth(Index, Repos). diff --git a/test/rebar_unlock_SUITE.erl b/test/rebar_unlock_SUITE.erl index 8dbdb3a..a8d1400 100644 --- a/test/rebar_unlock_SUITE.erl +++ b/test/rebar_unlock_SUITE.erl @@ -33,7 +33,7 @@ pkgunlock(Config) -> rebar_test_utils:run_and_check(Config, [], ["unlock", "cf,certifi"], {ok, []}), ?assertEqual(Locks -- ["bbmustache","cf","certifi"], read_locks(Config)), ?assertEqual(Hashes -- ["bbmustache","cf","certifi"], read_hashes(Config)), - rebar_test_utils:run_and_check(Config, [], ["unlock", string:join(Locks,",")], {ok, []}), + rebar_test_utils:run_and_check(Config, [], ["unlock", rebar_string:join(Locks,",")], {ok, []}), ?assertEqual({error, enoent}, read_locks(Config)), ?assertEqual({error, enoent}, read_hashes(Config)), ok. @@ -42,18 +42,21 @@ unlock(Config) -> Locks = read_locks(Config), rebar_test_utils:run_and_check(Config, [], ["unlock", "fakeapp"], {ok, []}), Locks = read_locks(Config), - rebar_test_utils:run_and_check(Config, [], ["unlock", "uuid"], {ok, []}), + {ok, State} = rebar_test_utils:run_and_check(Config, [], ["unlock", "uuid"], return), ?assertEqual(Locks -- ["uuid"], read_locks(Config)), + ?assert(false =:= lists:keyfind(<<"uuid">>, 1, rebar_state:get(State, {locks, default}))), + ?assert(false =/= lists:keyfind(<<"itc">>, 1, rebar_state:get(State, {locks, default}))), rebar_test_utils:run_and_check(Config, [], ["unlock", "gproc,itc"], {ok, []}), ?assertEqual(Locks -- ["uuid","gproc","itc"], read_locks(Config)), - rebar_test_utils:run_and_check(Config, [], ["unlock", string:join(Locks,",")], {ok, []}), + rebar_test_utils:run_and_check(Config, [], ["unlock", rebar_string:join(Locks,",")], {ok, []}), ?assertEqual({error, enoent}, read_locks(Config)), ok. unlock_all(Config) -> [_|_] = read_locks(Config), - rebar_test_utils:run_and_check(Config, [], ["unlock"], {ok, []}), + {ok, State} = rebar_test_utils:run_and_check(Config, [], ["unlock"], return), ?assertEqual({error, enoent}, read_locks(Config)), + ?assertEqual([], rebar_state:get(State, {locks, default})), ok. read_locks(Config) -> diff --git a/test/rebar_unlock_SUITE_data/pkg.rebar.lock b/test/rebar_unlock_SUITE_data/pkg.rebar.lock index 38e22e5..231e266 100644 --- a/test/rebar_unlock_SUITE_data/pkg.rebar.lock +++ b/test/rebar_unlock_SUITE_data/pkg.rebar.lock @@ -1,32 +1,24 @@ -{"1.1.0",[{<<"bbmustache">>,{pkg,<<"bbmustache">>,<<"1.0.4">>},0}, - {<<"certifi">>,{pkg,<<"certifi">>,<<"0.4.0">>},0}, - {<<"cf">>,{pkg,<<"cf">>,<<"0.2.1">>},0}, - {<<"cth_readable">>,{pkg,<<"cth_readable">>,<<"1.2.2">>},0}, - {<<"erlware_commons">>,{pkg,<<"erlware_commons">>,<<"0.21.0">>},0}, - {<<"eunit_formatters">>,{pkg,<<"eunit_formatters">>,<<"0.3.1">>},0}, - {<<"getopt">>,{pkg,<<"getopt">>,<<"0.8.2">>},0}, - {<<"providers">>,{pkg,<<"providers">>,<<"1.6.0">>},0}, - {<<"relx">>,{pkg,<<"relx">>,<<"3.19.0">>},0}, - {<<"ssl_verify_hostname">>, - {pkg,<<"ssl_verify_hostname">>,<<"1.0.5">>}, - 0}]}. -[{pkg_hash,[{<<"bbmustache">>, - <<"7BA94F971C5AFD7B6617918A4BB74705E36CAB36EB84B19B6A1B7EE06427AA38">>}, - {<<"certifi">>, - <<"A7966EFB868B179023618D29A407548F70C52466BF1849B9E8EBD0E34B7EA11F">>}, - {<<"cf">>, - <<"69D0B1349FD4D7D4DC55B7F407D29D7A840BF9A1EF5AF529F1EBE0CE153FC2AB">>}, - {<<"cth_readable">>, - <<"983913A8E8572310B7EAF5F2631148B7D70B3C090D2120DCFE777A93AA4165FB">>}, - {<<"erlware_commons">>, - <<"A04433071AD7D112EDEFC75AC77719DD3E6753E697AC09428FC83D7564B80B15">>}, - {<<"eunit_formatters">>, - <<"7A6FC351EB5B873E2356B8852EB751E20C13A72FBCA03393CF682B8483509573">>}, - {<<"getopt">>, - <<"B17556DB683000BA50370B16C0619DF1337E7AF7ECBF7D64FBF8D1D6BCE3109B">>}, - {<<"providers">>, - <<"DB0E2F9043AE60C0155205FCD238D68516331D0E5146155E33D1E79DC452964A">>}, - {<<"relx">>, - <<"286DD5244B4786F56AAC75D5C8E2D1FB4CFD306810D4EC8548F3AE1B3AADB8F7">>}, - {<<"ssl_verify_hostname">>, - <<"2E73E068CD6393526F9FA6D399353D7C9477D6886BA005F323B592D389FB47BE">>}]}]. +{"1.1.0", +[{<<"bbmustache">>,{pkg,<<"bbmustache">>,<<"1.0.4">>},0}, + {<<"certifi">>,{pkg,<<"certifi">>,<<"0.4.0">>},0}, + {<<"cf">>,{pkg,<<"cf">>,<<"0.2.1">>},0}, + {<<"cth_readable">>,{pkg,<<"cth_readable">>,<<"1.2.3">>},0}, + {<<"erlware_commons">>,{pkg,<<"erlware_commons">>,<<"0.21.0">>},0}, + {<<"eunit_formatters">>,{pkg,<<"eunit_formatters">>,<<"0.3.1">>},0}, + {<<"getopt">>,{pkg,<<"getopt">>,<<"0.8.2">>},0}, + {<<"providers">>,{pkg,<<"providers">>,<<"1.6.0">>},0}, + {<<"relx">>,{pkg,<<"relx">>,<<"3.20.0">>},0}, + {<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.1">>},0}]}. +[ +{pkg_hash,[ + {<<"bbmustache">>, <<"7BA94F971C5AFD7B6617918A4BB74705E36CAB36EB84B19B6A1B7EE06427AA38">>}, + {<<"certifi">>, <<"A7966EFB868B179023618D29A407548F70C52466BF1849B9E8EBD0E34B7EA11F">>}, + {<<"cf">>, <<"69D0B1349FD4D7D4DC55B7F407D29D7A840BF9A1EF5AF529F1EBE0CE153FC2AB">>}, + {<<"cth_readable">>, <<"293120673DFF82F0768612C5282E35C40CACC1B6F94FE99077438FD3749D0E27">>}, + {<<"erlware_commons">>, <<"A04433071AD7D112EDEFC75AC77719DD3E6753E697AC09428FC83D7564B80B15">>}, + {<<"eunit_formatters">>, <<"7A6FC351EB5B873E2356B8852EB751E20C13A72FBCA03393CF682B8483509573">>}, + {<<"getopt">>, <<"B17556DB683000BA50370B16C0619DF1337E7AF7ECBF7D64FBF8D1D6BCE3109B">>}, + {<<"providers">>, <<"DB0E2F9043AE60C0155205FCD238D68516331D0E5146155E33D1E79DC452964A">>}, + {<<"relx">>, <<"B515B8317D25B3A1508699294C3D1FA6DC0527851DFFC87446661BCE21A36710">>}, + {<<"ssl_verify_fun">>, <<"28A4D65B7F59893BC2C7DE786DEC1E1555BD742D336043FE644AE956C3497FBE">>}]} +]. diff --git a/test/rebar_upgrade_SUITE.erl b/test/rebar_upgrade_SUITE.erl index 66e1fdf..c55456c 100644 --- a/test/rebar_upgrade_SUITE.erl +++ b/test/rebar_upgrade_SUITE.erl @@ -11,7 +11,8 @@ groups() -> triplet_a, triplet_b, triplet_c, tree_a, tree_b, tree_c, tree_c2, tree_cj, tree_ac, tree_all, delete_d, promote, stable_lock, fwd_lock, - compile_upgrade_parity, umbrella_config]}, + compile_upgrade_parity, umbrella_config, + profiles, profiles_exclusion]}, {git, [], [{group, all}]}, {pkg, [], [{group, all}]}]. @@ -78,6 +79,23 @@ setup_project(Case=umbrella_config, Config0, Deps, UpDeps) -> [{rebarconfig, TopConf}, {rebarumbrella, RebarConf}, {next_top_deps, rebar_test_utils:top_level_deps(UpDeps)} | Config]; +setup_project(Case, Config0, Deps, UpDeps) when Case == profiles; + Case == profiles_exclusion -> + DepsType = ?config(deps_type, Config0), + NameRoot = atom_to_list(Case)++"_"++atom_to_list(DepsType), + Config = rebar_test_utils:init_rebar_state(Config0, NameRoot++"_"), + AppDir = filename:join([?config(apps, Config), "apps", NameRoot]), + rebar_test_utils:create_app(AppDir, "Root", "0.0.0", [kernel, stdlib]), + [Top|ProfileDeps] = rebar_test_utils:top_level_deps(Deps), + RebarConf = rebar_test_utils:create_config(AppDir, [ + {deps, [Top]}, + {profiles, [{fake, [{deps, ProfileDeps}]}]} + ]), + [NextTop|NextPDeps] = rebar_test_utils:top_level_deps(UpDeps), + NextConfig = [{deps, [NextTop]}, + {profiles, [{fake, [{deps, NextPDeps}]}]}], + [{rebarconfig, RebarConf}, + {next_config, NextConfig} | Config]; setup_project(Case, Config0, Deps, UpDeps) -> DepsType = ?config(deps_type, Config0), Config = rebar_test_utils:init_rebar_state( @@ -94,25 +112,25 @@ setup_project(Case, Config0, Deps, UpDeps) -> upgrades(top_a) -> %% Original tree - {[{"A", "1", [{"B", [{"D", "1", []}]}, - {"C", [{"D", "2", []}]}]} + {[{"A", "1.0.0", [{"B", [{"D", "1.0.0", []}]}, + {"C", [{"D", "2.0.0", []}]}]} ], %% Updated tree - [{"A", "1", [{"B", [{"D", "3", []}]}, - {"C", [{"D", "2", []}]}]} + [{"A", "1.0.0", [{"B", [{"D", "3.0.0", []}]}, + {"C", [{"D", "2.0.0", []}]}]} ], %% Modified apps, gobally ["A","B","D"], %% upgrade vs. new tree - {"A", [{"A","1"}, "B", "C", {"D","3"}]}}; + {"A", [{"A","1.0.0"}, "B", "C", {"D","3.0.0"}]}}; upgrades(top_b) -> %% Original tree - {[{"A", "1", [{"B", [{"D", "1", []}]}, - {"C", [{"D", "2", []}]}]} + {[{"A", "1.0.0", [{"B", [{"D", "1.0.0", []}]}, + {"C", [{"D", "2.0.0", []}]}]} ], %% Updated tree - [{"A", "1", [{"B", [{"D", "3", []}]}, - {"C", [{"D", "2", []}]}]} + [{"A", "1.0.0", [{"B", [{"D", "3.0.0", []}]}, + {"C", [{"D", "2.0.0", []}]}]} ], %% Modified apps, gobally ["A","B","D"], @@ -120,12 +138,12 @@ upgrades(top_b) -> {"B", {error, {rebar_prv_upgrade, {transitive_dependency, <<"B">>}}}}}; upgrades(top_c) -> %% Original tree - {[{"A", "1", [{"B", [{"D", "1", []}]}, - {"C", [{"D", "2", []}]}]} + {[{"A", "1.0.0", [{"B", [{"D", "1.0.0", []}]}, + {"C", [{"D", "2.0.0", []}]}]} ], %% Updated tree - [{"A", "1", [{"B", [{"D", "3", []}]}, - {"C", [{"D", "2", []}]}]} + [{"A", "1.0.0", [{"B", [{"D", "3.0.0", []}]}, + {"C", [{"D", "2.0.0", []}]}]} ], %% Modified apps, gobally ["A","B","D"], @@ -133,12 +151,12 @@ upgrades(top_c) -> {"C", {error, {rebar_prv_upgrade, {transitive_dependency, <<"C">>}}}}}; upgrades(top_d1) -> %% Original tree - {[{"A", "1", [{"B", [{"D", "1", []}]}, - {"C", [{"D", "2", []}]}]} + {[{"A", "1.0.0", [{"B", [{"D", "1.0.0", []}]}, + {"C", [{"D", "2.0.0", []}]}]} ], %% Updated tree - [{"A", "1", [{"B", [{"D", "3", []}]}, - {"C", [{"D", "2", []}]}]} + [{"A", "1.0.0", [{"B", [{"D", "3.0.0", []}]}, + {"C", [{"D", "2.0.0", []}]}]} ], %% Modified apps, gobally ["A","B","D"], @@ -146,12 +164,12 @@ upgrades(top_d1) -> {"D", {error, {rebar_prv_upgrade, {transitive_dependency, <<"D">>}}}}}; upgrades(top_d2) -> %% Original tree - {[{"A", "1", [{"B", [{"D", "1", []}]}, - {"C", [{"D", "2", []}]}]} + {[{"A", "1.0.0", [{"B", [{"D", "1.0.0", []}]}, + {"C", [{"D", "2.0.0", []}]}]} ], %% Updated tree - [{"A", "1", [{"B", [{"D", "3", []}]}, - {"C", [{"D", "2", []}]}]} + [{"A", "1.0.0", [{"B", [{"D", "3.0.0", []}]}, + {"C", [{"D", "2.0.0", []}]}]} ], %% Modified apps, gobally ["A","B","D"], @@ -159,302 +177,342 @@ upgrades(top_d2) -> {"D", {error, {rebar_prv_upgrade, {transitive_dependency, <<"D">>}}}}}; upgrades(top_e) -> %% Original tree - {[{"A", "1", [{"B", [{"D", "1", []}]}, - {"C", [{"D", "2", []}]}]} + {[{"A", "1.0.0", [{"B", [{"D", "1.0.0", []}]}, + {"C", [{"D", "2.0.0", []}]}]} ], %% Updated tree - [{"A", "1", [{"B", [{"D", "3", []}]}, - {"C", [{"D", "2", []}]}]} + [{"A", "1.0.0", [{"B", [{"D", "3.0.0", []}]}, + {"C", [{"D", "2.0.0", []}]}]} ], %% Modified apps, gobally ["A","B","D"], %% upgrade vs. new tree {"E", {error, {rebar_prv_upgrade, {unknown_dependency, <<"E">>}}}}}; upgrades(pair_a) -> - {[{"A", "1", [{"C", "1", []}]}, - {"B", "1", [{"D", "1", []}]} + {[{"A", "1.0.0", [{"C", "1.0.0", []}]}, + {"B", "1.0.0", [{"D", "1.0.0", []}]} ], - [{"A", "2", [{"C", "2", []}]}, - {"B", "2", [{"D", "2", []}]} + [{"A", "2.0.0", [{"C", "2.0.0", []}]}, + {"B", "2.0.0", [{"D", "2.0.0", []}]} ], ["A","B","C","D"], - {"A", [{"A","2"},{"C","2"},{"B","1"},{"D","1"}]}}; + {"A", [{"A","2.0.0"},{"C","2.0.0"},{"B","1.0.0"},{"D","1.0.0"}]}}; upgrades(pair_b) -> - {[{"A", "1", [{"C", "1", []}]}, - {"B", "1", [{"D", "1", []}]} + {[{"A", "1.0.0", [{"C", "1.0.0", []}]}, + {"B", "1.0.0", [{"D", "1.0.0", []}]} ], - [{"A", "2", [{"C", "2", []}]}, - {"B", "2", [{"D", "2", []}]} + [{"A", "2.0.0", [{"C", "2.0.0", []}]}, + {"B", "2.0.0", [{"D", "2.0.0", []}]} ], ["A","B","C","D"], - {"B", [{"A","1"},{"C","1"},{"B","2"},{"D","2"}]}}; + {"B", [{"A","1.0.0"},{"C","1.0.0"},{"B","2.0.0"},{"D","2.0.0"}]}}; upgrades(pair_ab) -> - {[{"A", "1", [{"C", "1", []}]}, - {"B", "1", [{"D", "1", []}]} + {[{"A", "1.0.0", [{"C", "1.0.0", []}]}, + {"B", "1.0.0", [{"D", "1.0.0", []}]} ], - [{"A", "2", [{"C", "2", []}]}, - {"B", "2", [{"D", "2", []}]} + [{"A", "2.0.0", [{"C", "2.0.0", []}]}, + {"B", "2.0.0", [{"D", "2.0.0", []}]} ], ["A","B","C","D"], - {"A,B", [{"A","2"},{"C","2"},{"B","2"},{"D","2"}]}}; + {"A,B", [{"A","2.0.0"},{"C","2.0.0"},{"B","2.0.0"},{"D","2.0.0"}]}}; upgrades(pair_c) -> - {[{"A", "1", [{"C", "1", []}]}, - {"B", "1", [{"D", "1", []}]} + {[{"A", "1.0.0", [{"C", "1.0.0", []}]}, + {"B", "1.0.0", [{"D", "1.0.0", []}]} ], - [{"A", "2", [{"C", "2", []}]}, - {"B", "2", [{"D", "2", []}]} + [{"A", "2.0.0", [{"C", "2.0.0", []}]}, + {"B", "2.0.0", [{"D", "2.0.0", []}]} ], ["A","B","C","D"], {"C", {error, {rebar_prv_upgrade, {transitive_dependency, <<"C">>}}}}}; upgrades(pair_all) -> - {[{"A", "1", [{"C", "1", []}]}, - {"B", "1", [{"D", "1", []}]} + {[{"A", "1.0.0", [{"C", "1.0.0", []}]}, + {"B", "1.0.0", [{"D", "1.0.0", []}]} ], - [{"A", "2", [{"C", "2", []}]}, - {"B", "2", [{"D", "2", []}]} + [{"A", "2.0.0", [{"C", "2.0.0", []}]}, + {"B", "2.0.0", [{"D", "2.0.0", []}]} ], ["A","B","C","D"], - {"", [{"A","2"},{"C","2"},{"B","2"},{"D","2"}]}}; + {"", [{"A","2.0.0"},{"C","2.0.0"},{"B","2.0.0"},{"D","2.0.0"}]}}; upgrades(triplet_a) -> - {[{"A", "1", [{"D",[]}, - {"E","3",[]}]}, - {"B", "1", [{"F","1",[]}, + {[{"A", "1.0.0", [{"D",[]}, + {"E","3.0.0",[]}]}, + {"B", "1.0.0", [{"F","1.0.0",[]}, {"G",[]}]}, - {"C", "0", [{"H","3",[]}, + {"C", "0.0.0", [{"H","3.0.0",[]}, {"I",[]}]}], - [{"A", "1", [{"D",[]}, - {"E","2",[]}]}, - {"B", "1", [{"F","1",[]}, + [{"A", "1.0.0", [{"D",[]}, + {"E","2.0.0",[]}]}, + {"B", "1.0.0", [{"F","1.0.0",[]}, {"G",[]}]}, - {"C", "1", [{"H","4",[]}, + {"C", "1.0.0", [{"H","4.0.0",[]}, {"I",[]}]}], ["A","C","E","H"], - {"A", [{"A","1"}, "D", {"E","2"}, - {"B","1"}, {"F","1"}, "G", - {"C","0"}, {"H","3"}, "I"]}}; + {"A", [{"A","1.0.0"}, "D", {"E","2.0.0"}, + {"B","1.0.0"}, {"F","1.0.0"}, "G", + {"C","0.0.0"}, {"H","3.0.0"}, "I"]}}; upgrades(triplet_b) -> - {[{"A", "1", [{"D",[]}, - {"E","3",[]}]}, - {"B", "1", [{"F","1",[]}, + {[{"A", "1.0.0", [{"D",[]}, + {"E","3.0.0",[]}]}, + {"B", "1.0.0", [{"F","1.0.0",[]}, {"G",[]}]}, - {"C", "0", [{"H","3",[]}, + {"C", "0.0.0", [{"H","3.0.0",[]}, {"I",[]}]}], - [{"A", "2", [{"D",[]}, - {"E","2",[]}]}, - {"B", "1", [{"F","1",[]}, + [{"A", "2.0.0", [{"D",[]}, + {"E","2.0.0",[]}]}, + {"B", "1.0.0", [{"F","1.0.0",[]}, {"G",[]}]}, - {"C", "1", [{"H","4",[]}, + {"C", "1.0.0", [{"H","4.0.0",[]}, {"I",[]}]}], ["A","C","E","H"], - {"B", [{"A","1"}, "D", {"E","3"}, - {"B","1"}, {"F","1"}, "G", - {"C","0"}, {"H","3"}, "I"]}}; + {"B", [{"A","1.0.0"}, "D", {"E","3.0.0"}, + {"B","1.0.0"}, {"F","1.0.0"}, "G", + {"C","0.0.0"}, {"H","3.0.0"}, "I"]}}; upgrades(triplet_c) -> - {[{"A", "1", [{"D",[]}, - {"E","3",[]}]}, - {"B", "1", [{"F","1",[]}, + {[{"A", "1.0.0", [{"D",[]}, + {"E","3.0.0",[]}]}, + {"B", "1.0.0", [{"F","1.0.0",[]}, {"G",[]}]}, - {"C", "0", [{"H","3",[]}, + {"C", "0.0.0", [{"H","3.0.0",[]}, {"I",[]}]}], - [{"A", "2", [{"D",[]}, - {"E","2",[]}]}, - {"B", "1", [{"F","1",[]}, + [{"A", "2.0.0", [{"D",[]}, + {"E","2.0.0",[]}]}, + {"B", "1.0.0", [{"F","1.0.0",[]}, {"G",[]}]}, - {"C", "1", [{"H","4",[]}, + {"C", "1.0.0", [{"H","4.0.0",[]}, {"I",[]}]}], ["A","C","E","H"], - {"C", [{"A","1"}, "D", {"E","3"}, - {"B","1"}, {"F","1"}, "G", - {"C","1"}, {"H","4"}, "I"]}}; + {"C", [{"A","1.0.0"}, "D", {"E","3.0.0"}, + {"B","1.0.0"}, {"F","1.0.0"}, "G", + {"C","1.0.0"}, {"H","4.0.0"}, "I"]}}; upgrades(tree_a) -> - {[{"A", "1", [{"D",[{"J",[]}]}, - {"E",[{"I","1",[]}]}]}, - {"B", "1", [{"F",[]}, + {[{"A", "1.0.0", [{"D",[{"J",[]}]}, + {"E",[{"I","1.0.0",[]}]}]}, + {"B", "1.0.0", [{"F",[]}, {"G",[]}]}, - {"C", "1", [{"H",[]}, - {"I","2",[]}]} + {"C", "1.0.0", [{"H",[]}, + {"I","2.0.0",[]}]} ], - [{"A", "1", [{"D",[{"J",[]}]}, - {"E",[{"I","1",[]}]}]}, - {"B", "1", [{"F",[]}, + [{"A", "1.0.0", [{"D",[{"J",[]}]}, + {"E",[{"I","1.0.0",[]}]}]}, + {"B", "1.0.0", [{"F",[]}, {"G",[]}]}, - {"C", "2", [{"H",[]}]} + {"C", "2.0.0", [{"H",[]}]} ], ["C"], - {"A", [{"A","1"}, "D", "J", "E", - {"B","1"}, "F", "G", - {"C","1"}, "H", {"I","2"}]}}; + {"A", [{"A","1.0.0"}, "D", "J", "E", + {"B","1.0.0"}, "F", "G", + {"C","1.0.0"}, "H", {"I","2.0.0"}]}}; upgrades(tree_b) -> - {[{"A", "1", [{"D",[{"J",[]}]}, - {"E",[{"I","1",[]}]}]}, - {"B", "1", [{"F",[]}, + {[{"A", "1.0.0", [{"D",[{"J",[]}]}, + {"E",[{"I","1.0.0",[]}]}]}, + {"B", "1.0.0", [{"F",[]}, {"G",[]}]}, - {"C", "1", [{"H",[]}, - {"I","2",[]}]} + {"C", "1.0.0", [{"H",[]}, + {"I","2.0.0",[]}]} ], - [{"A", "1", [{"D",[{"J",[]}]}, - {"E",[{"I","1",[]}]}]}, - {"B", "1", [{"F",[]}, + [{"A", "1.0.0", [{"D",[{"J",[]}]}, + {"E",[{"I","1.0.0",[]}]}]}, + {"B", "1.0.0", [{"F",[]}, {"G",[]}]}, - {"C", "2", [{"H",[]}]} + {"C", "2.0.0", [{"H",[]}]} ], ["C"], - {"B", [{"A","1"}, "D", "J", "E", - {"B","1"}, "F", "G", - {"C","1"}, "H", {"I","2"}]}}; + {"B", [{"A","1.0.0"}, "D", "J", "E", + {"B","1.0.0"}, "F", "G", + {"C","1.0.0"}, "H", {"I","2.0.0"}]}}; upgrades(tree_c) -> - {[{"A", "1", [{"D",[{"J",[]}]}, - {"E",[{"I","1",[]}]}]}, - {"B", "1", [{"F",[]}, + {[{"A", "1.0.0", [{"D",[{"J",[]}]}, + {"E",[{"I","1.0.0",[]}]}]}, + {"B", "1.0.0", [{"F",[]}, {"G",[]}]}, - {"C", "1", [{"H",[]}, - {"I","2",[]}]} + {"C", "1.0.0", [{"H",[]}, + {"I","2.0.0",[]}]} ], - [{"A", "1", [{"D",[{"J",[]}]}, - {"E",[{"I","1",[]}]}]}, - {"B", "1", [{"F",[]}, + [{"A", "1.0.0", [{"D",[{"J",[]}]}, + {"E",[{"I","1.0.0",[]}]}]}, + {"B", "1.0.0", [{"F",[]}, {"G",[]}]}, - {"C", "1", [{"H",[]}]} + {"C", "1.0.0", [{"H",[]}]} ], ["C","I"], - {"C", [{"A","1"}, "D", "J", "E", {"I","1"}, - {"B","1"}, "F", "G", - {"C","1"}, "H"]}}; + {"C", [{"A","1.0.0"}, "D", "J", "E", {"I","1.0.0"}, + {"B","1.0.0"}, "F", "G", + {"C","1.0.0"}, "H"]}}; upgrades(tree_c2) -> - {[{"A", "1", [{"D",[{"J",[]}]}, - {"E",[{"I","1",[]}]}]}, - {"B", "1", [{"F",[]}, + {[{"A", "1.0.0", [{"D",[{"J",[]}]}, + {"E",[{"I","1.0.0",[]}]}]}, + {"B", "1.0.0", [{"F",[]}, {"G",[]}]}, - {"C", "1", [{"H",[]}, - {"I","2",[]}]} + {"C", "1.0.0", [{"H",[]}, + {"I","2.0.0",[]}]} ], - [{"A", "1", [{"D",[{"J",[]}]}, - {"E",[{"I","1",[]}]}]}, - {"B", "1", [{"F",[]}, + [{"A", "1.0.0", [{"D",[{"J",[]}]}, + {"E",[{"I","1.0.0",[]}]}]}, + {"B", "1.0.0", [{"F",[]}, {"G",[]}]}, - {"C", "1", [{"H",[{"K",[]}]}, - {"I","2",[]}]} + {"C", "1.0.0", [{"H",[{"K",[]}]}, + {"I","2.0.0",[]}]} ], ["C", "H"], - {"C", [{"A","1"}, "D", "J", "E", - {"B","1"}, "F", "G", - {"C","1"}, "H", {"I", "2"}, "K"]}}; + {"C", [{"A","1.0.0"}, "D", "J", "E", + {"B","1.0.0"}, "F", "G", + {"C","1.0.0"}, "H", {"I", "2.0.0"}, "K"]}}; upgrades(tree_cj) -> - {[{"A", "1", [{"D",[{"J", "1",[]}]}, - {"E",[{"I","1",[]}]}]}, - {"B", "1", [{"F",[]}, + {[{"A", "1.0.0", [{"D",[{"J", "1.0.0",[]}]}, + {"E",[{"I","1.0.0",[]}]}]}, + {"B", "1.0.0", [{"F",[]}, {"G",[]}]}, - {"C", "1", [{"H",[]}, - {"I","1",[]}]} + {"C", "1.0.0", [{"H",[]}, + {"I","1.0.0",[]}]} ], - [{"A", "1", [{"D",[{"J", "2", []}]}, - {"E",[{"I","1",[]}]}]}, - {"B", "1", [{"F",[]}, + [{"A", "1.0.0", [{"D",[{"J", "2.0.0", []}]}, + {"E",[{"I","1.0.0",[]}]}]}, + {"B", "1.0.0", [{"F",[]}, {"G",[]}]}, - {"C", "1", [{"H",[]}, - {"I","1",[]}]} + {"C", "1.0.0", [{"H",[]}, + {"I","1.0.0",[]}]} ], ["C","J"], - {"C", [{"A","1"}, "D", {"J", "1"}, "E", {"I","1"}, - {"B","1"}, "F", "G", - {"C","1"}, "H"]}}; + {"C", [{"A","1.0.0"}, "D", {"J", "1.0.0"}, "E", {"I","1.0.0"}, + {"B","1.0.0"}, "F", "G", + {"C","1.0.0"}, "H"]}}; upgrades(tree_ac) -> - {[{"A", "1", [{"D",[{"J",[]}]}, - {"E",[{"I","1",[]}]}]}, - {"B", "1", [{"F",[]}, + {[{"A", "1.0.0", [{"D",[{"J",[]}]}, + {"E",[{"I","1.0.0",[]}]}]}, + {"B", "1.0.0", [{"F",[]}, {"G",[]}]}, - {"C", "1", [{"H",[]}, - {"I","2",[]}]} + {"C", "1.0.0", [{"H",[]}, + {"I","2.0.0",[]}]} ], - [{"A", "1", [{"D",[{"J",[]}]}, - {"E",[{"I","1",[]}]}]}, - {"B", "1", [{"F",[]}, + [{"A", "1.0.0", [{"D",[{"J",[]}]}, + {"E",[{"I","1.0.0",[]}]}]}, + {"B", "1.0.0", [{"F",[]}, {"G",[]}]}, - {"C", "1", [{"H",[]}]} + {"C", "1.0.0", [{"H",[]}]} ], ["C","I"], - {"C, A", [{"A","1"}, "D", "J", "E", {"I","1"}, - {"B","1"}, "F", "G", - {"C","1"}, "H"]}}; + {"C, A", [{"A","1.0.0"}, "D", "J", "E", {"I","1.0.0"}, + {"B","1.0.0"}, "F", "G", + {"C","1.0.0"}, "H"]}}; upgrades(tree_all) -> - {[{"A", "1", [{"D",[{"J",[]}]}, - {"E",[{"I","1",[]}]}]}, - {"B", "1", [{"F",[]}, + {[{"A", "1.0.0", [{"D",[{"J",[]}]}, + {"E",[{"I","1.0.0",[]}]}]}, + {"B", "1.0.0", [{"F",[]}, {"G",[]}]}, - {"C", "1", [{"H",[]}, - {"I","2",[]}]} + {"C", "1.0.0", [{"H",[]}, + {"I","2.0.0",[]}]} ], - [{"A", "1", [{"D",[{"J",[]}]}, - {"E",[{"I","1",[]}]}]}, - {"B", "1", [{"F",[]}, + [{"A", "1.0.0", [{"D",[{"J",[]}]}, + {"E",[{"I","1.0.0",[]}]}]}, + {"B", "1.0.0", [{"F",[]}, {"G",[]}]}, - {"C", "1", [{"H",[]}]} + {"C", "1.0.0", [{"H",[]}]} ], ["C","I"], - {"", [{"A","1"}, "D", "J", "E", {"I","1"}, - {"B","1"}, "F", "G", - {"C","1"}, "H"]}}; + {"", [{"A","1.0.0"}, "D", "J", "E", {"I","1.0.0"}, + {"B","1.0.0"}, "F", "G", + {"C","1.0.0"}, "H"]}}; upgrades(delete_d) -> - {[{"A", "1", [{"B", [{"D", "1", []}]}, - {"C", [{"D", "2", []}]}]} + {[{"A", "1.0.0", [{"B", [{"D", "1.0.0", []}]}, + {"C", [{"D", "2.0.0", []}]}]} ], - [{"A", "2", [{"B", []}, + [{"A", "2.0.0", [{"B", []}, {"C", []}]} ], ["A","B", "C"], %% upgrade vs. new tree - {"", [{"A","2"}, "B", "C"]}}; + {"", [{"A","2.0.0"}, "B", "C"]}}; upgrades(promote) -> - {[{"A", "1", [{"C", "1", []}]}, - {"B", "1", [{"D", "1", []}]} + {[{"A", "1.0.0", [{"C", "1.0.0", []}]}, + {"B", "1.0.0", [{"D", "1.0.0", []}]} ], - [{"A", "2", [{"C", "2", []}]}, - {"B", "2", [{"D", "2", []}]}, - {"C", "3", []} + [{"A", "2.0.0", [{"C", "2.0.0", []}]}, + {"B", "2.0.0", [{"D", "2.0.0", []}]}, + {"C", "3.0.0", []} ], ["A","B","C","D"], - {"C", [{"A","1"},{"C","3"},{"B","1"},{"D","1"}]}}; + {"C", [{"A","1.0.0"},{"C","3.0.0"},{"B","1.0.0"},{"D","1.0.0"}]}}; upgrades(stable_lock) -> - {[{"A", "1", [{"C", "1", []}]}, - {"B", "1", [{"D", "1", []}]} + {[{"A", "1.0.0", [{"C", "1.0.0", []}]}, + {"B", "1.0.0", [{"D", "1.0.0", []}]} ], % lock after this - [{"A", "2", [{"C", "2", []}]}, - {"B", "2", [{"D", "2", []}]} + [{"A", "2.0.0", [{"C", "2.0.0", []}]}, + {"B", "2.0.0", [{"D", "2.0.0", []}]} ], [], %% Run a regular lock and no app should be upgraded - {"any", [{"A","1"},{"C","1"},{"B","1"},{"D","1"}]}}; + {"any", [{"A","1.0.0"},{"C","1.0.0"},{"B","1.0.0"},{"D","1.0.0"}]}}; upgrades(fwd_lock) -> - {[{"A", "1", [{"C", "1", []}]}, - {"B", "1", [{"D", "1", []}]} + {[{"A", "1.0.0", [{"C", "1.0.0", []}]}, + {"B", "1.0.0", [{"D", "1.0.0", []}]} ], - [{"A", "2", [{"C", "2", []}]}, - {"B", "2", [{"D", "2", []}]} + [{"A", "2.0.0", [{"C", "2.0.0", []}]}, + {"B", "2.0.0", [{"D", "2.0.0", []}]} ], ["A","B","C","D"], %% For this one, we should build, rewrite the lock %% file to include the result post-upgrade, and then %% run a regular lock to see that the lock file is respected %% in deps. - {"any", [{"A","2"},{"C","2"},{"B","2"},{"D","2"}]}}; + {"any", [{"A","2.0.0"},{"C","2.0.0"},{"B","2.0.0"},{"D","2.0.0"}]}}; upgrades(compile_upgrade_parity) -> - {[{"A", "1", [{"D",[{"J",[]}]}, - {"E",[{"I","1",[]}]}]}, - {"B", "1", [{"F",[]}, + {[{"A", "1.0.0", [{"D",[{"J",[]}]}, + {"E",[{"I","1.0.0",[]}]}]}, + {"B", "1.0.0", [{"F",[]}, {"G",[]}]}, - {"C", "1", [{"H",[]}, - {"I","2",[]}]} + {"C", "1.0.0", [{"H",[]}, + {"I","2.0.0",[]}]} ], [], [], - {"", [{"A","1"}, "D", "J", "E", {"I","1"}, - {"B","1"}, "F", "G", - {"C","1"}, "H"]}}; + {"", [{"A","1.0.0"}, "D", "J", "E", {"I","1.0.0"}, + {"B","1.0.0"}, "F", "G", + {"C","1.0.0"}, "H"]}}; upgrades(umbrella_config) -> - {[{"A", "1", []}], - [{"A", "2", []}], + {[{"A", "1.0.0", []}], + [{"A", "2.0.0", []}], ["A"], - {"A", [{"A","2"}]}}. + {"A", [{"A","2.0.0"}]}}; +upgrades(profiles) -> + %% Ensure that we can unlock deps under a given profile; + %% B and C should both be in a custom profile + %% and must not be locked. + {[{"A", "1.0.0", [{"D",[]}, + {"E","3.0.0",[]}]}, + {"B", "1.0.0", [{"F","1.0.0",[]}, + {"G",[]}]}, + {"C", "0.0.0", [{"H","3.0.0",[]}, + {"I",[]}]}], + [{"A", "2.0.0", [{"D",[]}, + {"E","2.0.0",[]}]}, + {"B", "2.0.0", [{"F","2.0.0",[]}, + {"G",[]}]}, + {"C", "1.0.0", [{"H","4.0.0",[]}, + {"I",[]}]}], + ["A","B","C","E","F","H"], + {"C", [{"A","1.0.0"}, "D", {"E","3.0.0"}, + {"B","2.0.0"}, {"F","2.0.0"}, "G", + {"C","1.0.0"}, {"H","4.0.0"}, "I"]}}; +upgrades(profiles_exclusion) -> + %% Ensure that we can unlock deps under a given profile; + %% B and C should both be in a custom profile + %% and must not be locked. + {[{"A", "1.0.0", [{"D",[]}, + {"E","3.0.0",[]}]}, + {"B", "1.0.0", [{"F","1.0.0",[]}, + {"G",[]}]}, + {"C", "0.0.0", [{"H","3.0.0",[]}, + {"I",[]}]}], + [{"A", "2.0.0", [{"D",[]}, + {"E","2.0.0",[]}]}, + {"B", "2.0.0", [{"F","2.0.0",[]}, + {"G",[]}]}, + {"C", "1.0.0", [{"H","4.0.0",[]}, + {"I",[]}]}], + ["A","B","C","E","F","H"], + {"A", [{"A","1.0.0"}, "D", {"E","3.0.0"}, + {"B","2.0.0"}, {"F","2.0.0"}, "G", + {"C","1.0.0"}, {"H","4.0.0"}, "I"]}}. %% TODO: add a test that verifies that unlocking files and then %% running the upgrade code is enough to properly upgrade things. @@ -613,6 +671,37 @@ umbrella_config(Config) -> ), meck:unload(rebar_prv_upgrade). +profiles(Config) -> + apply(?config(mock, Config), []), + {ok, TopConfig} = file:consult(?config(rebarconfig, Config)), + %% Install dependencies before re-mocking for an upgrade + rebar_test_utils:run_and_check(Config, TopConfig, ["lock"], {ok, []}), + %% Install test deps along with them + rebar_test_utils:run_and_check(Config, TopConfig, ["as","fake","lock"], {ok, []}), + {App, Unlocks} = ?config(expected, Config), + ct:pal("Upgrades: ~p -> ~p", [App, Unlocks]), + Expectation = case Unlocks of + {error, Term} -> {error, Term}; + _ -> {ok, [T || T <- Unlocks, + element(1,T) == dep orelse + lists:member(element(2,T), ["A","D","E"])]} + end, + + meck:new(rebar_prv_app_discovery, [passthrough]), + meck:expect(rebar_prv_app_discovery, do, fun(S) -> + apply(?config(mock_update, Config), []), + meck:passthrough([S]) + end), + NewRebarConf = rebar_test_utils:create_config(?config(apps, Config), + ?config(next_config, Config)), + {ok, NewRebarConfig} = file:consult(NewRebarConf), + rebar_test_utils:run_and_check( + Config, NewRebarConfig, ["as","fake","upgrade", App], Expectation + ), + meck:unload(rebar_prv_app_discovery). + +profiles_exclusion(Config) -> profiles(Config). + run(Config) -> apply(?config(mock, Config), []), ConfigPath = ?config(rebarconfig, Config), diff --git a/test/rebar_utils_SUITE.erl b/test/rebar_utils_SUITE.erl index b32992d..233fcff 100644 --- a/test/rebar_utils_SUITE.erl +++ b/test/rebar_utils_SUITE.erl @@ -31,7 +31,9 @@ nonblacklisted_otp_version/1, blacklisted_otp_version/1, sh_does_not_miss_messages/1, - tup_merge/1]). + tup_merge/1, + proxy_auth/1, + is_list_of_strings/1]). -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -46,7 +48,8 @@ end_per_testcase(_, _Config) -> all() -> [{group, args_to_tasks}, sh_does_not_miss_messages, - tup_merge]. + tup_merge, + proxy_auth, is_list_of_strings]. groups() -> [{args_to_tasks, [], [empty_arglist, @@ -272,3 +275,47 @@ tup_merge(_Config) -> rebar_utils:tup_sort([{a,a},{a,a,a},a,{b,a,a},b,{z,a},{z,a,a},{b,a},z]) ) ). + +proxy_auth(Config) -> + proxy_auth(Config, "http://", "http_proxy"), + proxy_auth(Config, "https://", "https_proxy"), + proxy_auth(Config, "", "http_proxy"), + proxy_auth(Config, "", "https_proxy"). + +proxy_auth(_Config, Schema, ProxyEnvKey) -> + Host = "host:", + Port = "1234", + + %% remember current proxy specification + OldProxySpec = os:getenv(ProxyEnvKey), + + %% proxy auth not set + application:unset_env(rebar, proxy_auth), + ?assertEqual([], rebar_utils:get_proxy_auth()), + + %% proxy auth with regular username/password + os:putenv(ProxyEnvKey, Schema++"Username:Password@" ++ Host ++ Port), + rebar_utils:set_httpc_options(), + ?assertEqual([{proxy_auth, {"Username", "Password"}}], + rebar_utils:get_proxy_auth()), + + %% proxy auth with username missing and url encoded password + os:putenv(ProxyEnvKey, Schema++":%3F!abc%23%24@" ++ Host ++ Port), + rebar_utils:set_httpc_options(), + ?assertEqual([{proxy_auth, {"", "?!abc#$"}}], + rebar_utils:get_proxy_auth()), + + %% restore original proxy specification if any + restore_proxy_env(ProxyEnvKey, OldProxySpec), + application:unset_env(rebar, proxy_auth). + +restore_proxy_env(ProxyEnvKey, false) -> + os:putenv(ProxyEnvKey, ""); +restore_proxy_env(ProxyEnvKey, ProxySpec) -> + os:putenv(ProxyEnvKey, ProxySpec). + +is_list_of_strings(_Config) -> + ?assert(rebar_utils:is_list_of_strings(["foo"])), + ?assert(rebar_utils:is_list_of_strings([])), + ?assert(rebar_utils:is_list_of_strings("")), + ?assert(rebar_utils:is_list_of_strings("foo") == false). diff --git a/test/rebar_xref_SUITE.erl b/test/rebar_xref_SUITE.erl index 75d6786..9f4bc7d 100644 --- a/test/rebar_xref_SUITE.erl +++ b/test/rebar_xref_SUITE.erl @@ -9,7 +9,9 @@ end_per_testcase/2, all/0, xref_test/1, - xref_ignore_test/1]). + xref_ignore_test/1, + xref_dep_hook/1, + xref_undef_behaviour/1]). -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -28,6 +30,15 @@ init_per_suite(Config) -> end_per_suite(_Config) -> ok. +init_per_testcase(xref_dep_hook, Config) -> + Src = filename:join([?config(data_dir, Config), "recursive"]), + Dst = filename:join([?config(priv_dir, Config), "recursive"]), + ok = rebar_file_utils:cp_r([Src], Dst), + GlobalDir = filename:join([?config(priv_dir, Config), "cache"]), + State = rebar_state:new([{base_dir, filename:join([Dst, "_build"])} + ,{global_rebar_dir, GlobalDir} + ,{root_dir, Dst}]), + [{apps, Dst}, {state, State} | Config]; init_per_testcase(Case, Config) -> UpdConfig = rebar_test_utils:init_rebar_state(Config), AppDir = ?config(apps, UpdConfig), @@ -35,9 +46,11 @@ init_per_testcase(Case, Config) -> Name = rebar_test_utils:create_random_name("xrefapp_"), Vsn = rebar_test_utils:create_random_vsn(), rebar_test_utils:create_empty_app(AppDir, Name, Vsn, [kernel, stdlib]), - AppModules = [behaviour1, behaviour2, mymod, othermod], + AppModules = [behaviour1, behaviour2, mymod, othermod, ignoremod, ignoremod2], [write_src_file(AppDir, Name, Module, ignore_xref(Case)) || Module <- AppModules], + IgnoreMod = list_to_atom(Name ++ "_" ++ "ignoremod"), RebarConfig = [{erl_opts, [debug_info]}, + {xref_ignores, [IgnoreMod]}, {xref_checks, [deprecated_function_calls,deprecated_functions, undefined_function_calls,undefined_functions, exports_not_used,locals_not_used]}], @@ -48,7 +61,7 @@ end_per_testcase(_, _Config) -> ok. all() -> - [xref_test, xref_ignore_test]. + [xref_test, xref_ignore_test, xref_dep_hook, xref_undef_behaviour]. %% =================================================================== %% Test cases @@ -70,6 +83,20 @@ xref_ignore_test(Config) -> Result = rebar3:run(rebar_state:new(State, RebarConfig, AppDir), ["xref"]), verify_results(xref_ignore_test, Name, Result). +xref_dep_hook(Config) -> + rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, []}). + +xref_undef_behaviour(Config) -> + AppDir = ?config(apps, Config), + State = ?config(state, Config), + Name = ?config(app_name, Config), + RebarConfig = ?config(rebar_config, Config), + %% delete one of the behaviours, which should create new warnings + delete_src_file(AppDir, Name, behaviour1), + %% just ensure this does not crash + Result = rebar3:run(rebar_state:new(State, RebarConfig, AppDir), ["xref"]), + verify_results(xref_undef_behaviour, Name, Result). + %% =================================================================== %% Helper functions %% =================================================================== @@ -110,9 +137,35 @@ verify_test_results(xref_test, AppName, XrefResults, _QueryResults) -> ?assertNot(lists:member({MyMod, bh2_a, 1}, ExportsNotUsed)), ?assertNot(lists:member({MyMod, bh2_b, 1}, ExportsNotUsed)), ok; +verify_test_results(xref_undef_behaviour, AppName, XrefResults, _QueryResults) -> + AppModules = ["behaviour2", "mymod", "othermod", "somemod"], + [Behaviour2Mod, MyMod, OtherMod, SomeMod] = + [list_to_atom(AppName ++ "_" ++ Mod) || Mod <- AppModules], + UndefFuns = proplists:get_value(undefined_functions, XrefResults), + UndefFunCalls = proplists:get_value(undefined_function_calls, XrefResults), + LocalsNotUsed = proplists:get_value(locals_not_used, XrefResults), + ExportsNotUsed = proplists:get_value(exports_not_used, XrefResults), + DeprecatedFuns = proplists:get_value(deprecated_functions, XrefResults), + DeprecatedFunCalls = proplists:get_value(deprecated_function_calls, XrefResults), + ?assert(lists:member({SomeMod, notavailable, 1}, UndefFuns)), + ?assert(lists:member({{OtherMod, somefunc, 0}, {SomeMod, notavailable, 1}}, + UndefFunCalls)), + ?assert(lists:member({MyMod, fdeprecated, 0}, DeprecatedFuns)), + ?assert(lists:member({{OtherMod, somefunc, 0}, {MyMod, fdeprecated, 0}}, + DeprecatedFunCalls)), + ?assert(lists:member({MyMod, localfunc2, 0}, LocalsNotUsed)), + ?assert(lists:member({Behaviour2Mod, behaviour_info, 1}, ExportsNotUsed)), + ?assert(lists:member({MyMod, other2, 1}, ExportsNotUsed)), + ?assert(lists:member({OtherMod, somefunc, 0}, ExportsNotUsed)), + ?assert(lists:member({MyMod, bh1_a, 1}, ExportsNotUsed)), + ?assert(lists:member({MyMod, bh1_b, 1}, ExportsNotUsed)), + ?assertNot(lists:member({MyMod, bh2_a, 1}, ExportsNotUsed)), + ?assertNot(lists:member({MyMod, bh2_b, 1}, ExportsNotUsed)), + ok; verify_test_results(xref_ignore_test, AppName, XrefResults, _QueryResults) -> - AppModules = ["behaviour1", "behaviour2", "mymod", "othermod", "somemod"], - [_Behaviour1Mod, _Behaviour2Mod, _MyMod, _OtherMod, SomeMod] = + AppModules = ["behaviour1", "behaviour2", "mymod", "othermod", "somemod", + "ignoremod", "ignoremod2"], + [_Behaviour1Mod, _Behaviour2Mod, _MyMod, _OtherMod, SomeMod, IgnoreMod, IgnoreMod2] = [list_to_atom(AppName ++ "_" ++ Mod) || Mod <- AppModules], UndefFuns = proplists:get_value(undefined_functions, XrefResults), ?assertNot(lists:keymember(undefined_function_calls, 1, XrefResults)), @@ -120,6 +173,8 @@ verify_test_results(xref_ignore_test, AppName, XrefResults, _QueryResults) -> ?assertNot(lists:keymember(exports_not_used, 1, XrefResults)), ?assertNot(lists:keymember(deprecated_functions, 1, XrefResults)), ?assertNot(lists:keymember(deprecated_function_calls, 1, XrefResults)), + ?assertNot(lists:member({IgnoreMod, notavailable, 1}, UndefFuns)), + ?assertNot(lists:member({IgnoreMod2, notavailable, 1}, UndefFuns)), ?assert(lists:member({SomeMod, notavailable, 1}, UndefFuns)), ok. @@ -128,19 +183,25 @@ write_src_file(Dir, AppName, Module, IgnoreXref) -> ok = filelib:ensure_dir(Erl), ok = ec_file:write(Erl, get_module_body(Module, AppName, IgnoreXref)). +delete_src_file(Dir, AppName, Module) -> + Erl = filename:join([Dir, "src", module_name(AppName, Module)]), + ok = file:delete(Erl). + module_name(AppName, Module) -> lists:flatten([AppName, "_", atom_to_list(Module), ".erl"]). get_module_body(behaviour1, AppName, IgnoreXref) -> ["-module(", AppName, "_behaviour1).\n", "-export([behaviour_info/1]).\n", - ["-ignore_xref({behaviour_info,1}).\n" || X <- [IgnoreXref], X =:= true], + ["-ignore_xref([ignoremod,{behaviour_info,1}]).\n" + || X <- [IgnoreXref], X =:= true], "behaviour_info(callbacks) -> [{bh1_a,1},{bh1_b,1}];\n", "behaviour_info(_Other) -> undefined.\n"]; get_module_body(behaviour2, AppName, IgnoreXref) -> ["-module(", AppName, "_behaviour2).\n", "-export([behaviour_info/1]).\n", - ["-ignore_xref({behaviour_info,1}).\n" || X <- [IgnoreXref], X =:= true], + ["-ignore_xref({behaviour_info,1}).\n" + || X <- [IgnoreXref], X =:= true], "behaviour_info(callbacks) -> [{bh2_a,1},{bh2_b,1}];\n", "behaviour_info(_Other) -> undefined.\n"]; get_module_body(mymod, AppName, IgnoreXref) -> @@ -150,7 +211,7 @@ get_module_body(mymod, AppName, IgnoreXref) -> ["-ignore_xref([{other2,1},{localfunc2,0},{fdeprecated,0}]).\n" || X <- [IgnoreXref], X =:= true], "-behaviour(", AppName, "_behaviour1).\n", % 2 behaviours - "-behaviour(", AppName, "_behaviour2).\n", + "-behavior(", AppName, "_behaviour2).\n", "-deprecated({fdeprecated,0}).\n", % deprecated function "bh1_a(A) -> localfunc1(bh1_a, A).\n", % behaviour functions "bh1_b(A) -> localfunc1(bh1_b, A).\n", @@ -162,6 +223,26 @@ get_module_body(mymod, AppName, IgnoreXref) -> "localfunc2() -> ok.\n", % unused local "fdeprecated() -> ok.\n" % deprecated function ]; +get_module_body(ignoremod, AppName, IgnoreXref) -> + ["-module(", AppName, "_ignoremod).\n", + "-export([]).\n", + [["-ignore_xref(", AppName, "_ignoremod).\n"] + || X <- [IgnoreXref], X =:= true], + "localfunc1(A, B) -> {A, B}.\n", % used local + "localfunc2() -> ok.\n", % unused local + "fdeprecated() -> ok.\n" % deprecated function + + ]; +get_module_body(ignoremod2, AppName, IgnoreXref) -> + ["-module(", AppName, "_ignoremod2).\n", + "-export([]).\n", + [["-ignore_xref(", AppName, "_ignoremod2).\n"] + || X <- [IgnoreXref], X =:= true], + "localfunc1(A, B) -> {A, B}.\n", % used local + "localfunc2() -> ok.\n", % unused local + "fdeprecated() -> ok.\n" % deprecated function + + ]; get_module_body(othermod, AppName, IgnoreXref) -> ["-module(", AppName, "_othermod).\n", "-export([somefunc/0]).\n", @@ -171,4 +252,5 @@ get_module_body(othermod, AppName, IgnoreXref) -> "somefunc() ->\n", " ", AppName, "_mymod:other1(arg),\n", " ", AppName, "_somemod:notavailable(arg),\n", - " ", AppName, "_mymod:fdeprecated().\n"]. + " ", AppName, "_mymod:fdeprecated(),\n", + " ", AppName, "_ignoremod:notavailable().\n"]. diff --git a/test/rebar_xref_SUITE_data/recursive/apps/rebar_issue1/rebar.config b/test/rebar_xref_SUITE_data/recursive/apps/rebar_issue1/rebar.config new file mode 100644 index 0000000..cf48edf --- /dev/null +++ b/test/rebar_xref_SUITE_data/recursive/apps/rebar_issue1/rebar.config @@ -0,0 +1,12 @@ +{erl_opts, [debug_info]}. +{deps, []}. + +{xref_checks,[ + undefined_function_calls, + undefined_functions, + locals_not_used, + deprecated_function_calls, + deprecated_functions +]}. + +{provider_hooks, [{post, [{compile, xref}]}]}. diff --git a/test/rebar_xref_SUITE_data/recursive/apps/rebar_issue1/src/rebar_issue1.app.src b/test/rebar_xref_SUITE_data/recursive/apps/rebar_issue1/src/rebar_issue1.app.src new file mode 100644 index 0000000..b935082 --- /dev/null +++ b/test/rebar_xref_SUITE_data/recursive/apps/rebar_issue1/src/rebar_issue1.app.src @@ -0,0 +1,16 @@ +{application, rebar_issue1, + [{description, "An OTP application"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, { rebar_issue1_app, []}}, + {applications, + [kernel, + stdlib + ]}, + {env,[]}, + {modules, []}, + + {maintainers, []}, + {licenses, []}, + {links, []} + ]}. diff --git a/test/rebar_xref_SUITE_data/recursive/apps/rebar_issue1/src/rebar_issue1_app.erl b/test/rebar_xref_SUITE_data/recursive/apps/rebar_issue1/src/rebar_issue1_app.erl new file mode 100644 index 0000000..78c88c1 --- /dev/null +++ b/test/rebar_xref_SUITE_data/recursive/apps/rebar_issue1/src/rebar_issue1_app.erl @@ -0,0 +1,26 @@ +%%%------------------------------------------------------------------- +%% @doc rebar_issue1 public API +%% @end +%%%------------------------------------------------------------------- + +-module(rebar_issue1_app). + +-behaviour(application). + +%% Application callbacks +-export([start/2, stop/1]). + +%%==================================================================== +%% API +%%==================================================================== + +start(_StartType, _StartArgs) -> + rebar_issue1_sup:start_link(). + +%%-------------------------------------------------------------------- +stop(_State) -> + ok. + +%%==================================================================== +%% Internal functions +%%==================================================================== diff --git a/test/rebar_xref_SUITE_data/recursive/apps/rebar_issue1/src/rebar_issue1_sup.erl b/test/rebar_xref_SUITE_data/recursive/apps/rebar_issue1/src/rebar_issue1_sup.erl new file mode 100644 index 0000000..6e5a9f8 --- /dev/null +++ b/test/rebar_xref_SUITE_data/recursive/apps/rebar_issue1/src/rebar_issue1_sup.erl @@ -0,0 +1,35 @@ +%%%------------------------------------------------------------------- +%% @doc rebar_issue1 top level supervisor. +%% @end +%%%------------------------------------------------------------------- + +-module(rebar_issue1_sup). + +-behaviour(supervisor). + +%% API +-export([start_link/0]). + +%% Supervisor callbacks +-export([init/1]). + +-define(SERVER, ?MODULE). + +%%==================================================================== +%% API functions +%%==================================================================== + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +%%==================================================================== +%% Supervisor callbacks +%%==================================================================== + +%% Child :: {Id,StartFunc,Restart,Shutdown,Type,Modules} +init([]) -> + {ok, { {one_for_all, 0, 1}, []} }. + +%%==================================================================== +%% Internal functions +%%==================================================================== diff --git a/test/rebar_xref_SUITE_data/recursive/apps/rebar_issue2/src/rebar_issue2.app.src b/test/rebar_xref_SUITE_data/recursive/apps/rebar_issue2/src/rebar_issue2.app.src new file mode 100644 index 0000000..59ffa35 --- /dev/null +++ b/test/rebar_xref_SUITE_data/recursive/apps/rebar_issue2/src/rebar_issue2.app.src @@ -0,0 +1,17 @@ +{application, rebar_issue2, + [{description, "An OTP application"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, { rebar_issue2_app, []}}, + {applications, + [kernel, + stdlib, + rebar_issue1 + ]}, + {env,[]}, + {modules, []}, + + {maintainers, []}, + {licenses, []}, + {links, []} + ]}. diff --git a/test/rebar_xref_SUITE_data/recursive/apps/rebar_issue2/src/rebar_issue2_app.erl b/test/rebar_xref_SUITE_data/recursive/apps/rebar_issue2/src/rebar_issue2_app.erl new file mode 100644 index 0000000..968966c --- /dev/null +++ b/test/rebar_xref_SUITE_data/recursive/apps/rebar_issue2/src/rebar_issue2_app.erl @@ -0,0 +1,26 @@ +%%%------------------------------------------------------------------- +%% @doc rebar_issue2 public API +%% @end +%%%------------------------------------------------------------------- + +-module(rebar_issue2_app). + +-behaviour(application). + +%% Application callbacks +-export([start/2, stop/1]). + +%%==================================================================== +%% API +%%==================================================================== + +start(_StartType, _StartArgs) -> + rebar_issue2_sup:start_link(). + +%%-------------------------------------------------------------------- +stop(_State) -> + ok. + +%%==================================================================== +%% Internal functions +%%==================================================================== diff --git a/test/rebar_xref_SUITE_data/recursive/apps/rebar_issue2/src/rebar_issue2_sup.erl b/test/rebar_xref_SUITE_data/recursive/apps/rebar_issue2/src/rebar_issue2_sup.erl new file mode 100644 index 0000000..3673548 --- /dev/null +++ b/test/rebar_xref_SUITE_data/recursive/apps/rebar_issue2/src/rebar_issue2_sup.erl @@ -0,0 +1,35 @@ +%%%------------------------------------------------------------------- +%% @doc rebar_issue2 top level supervisor. +%% @end +%%%------------------------------------------------------------------- + +-module(rebar_issue2_sup). + +-behaviour(supervisor). + +%% API +-export([start_link/0]). + +%% Supervisor callbacks +-export([init/1]). + +-define(SERVER, ?MODULE). + +%%==================================================================== +%% API functions +%%==================================================================== + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +%%==================================================================== +%% Supervisor callbacks +%%==================================================================== + +%% Child :: {Id,StartFunc,Restart,Shutdown,Type,Modules} +init([]) -> + {ok, { {one_for_all, 0, 1}, []} }. + +%%==================================================================== +%% Internal functions +%%==================================================================== |