#!/usr/bin/env escript %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- %% ex: ft=erlang ts=4 sw=4 et main(_) -> application:start(crypto), application:start(asn1), application:start(public_key), application:start(ssl), inets:start(), inets:start(httpc, [{profile, rebar}]), set_httpc_options(), %% Clear directories for builds since bootstrapping may require %% a changed structure from an older one rm_rf("_build/bootstrap"), %% Fetch and build deps required to build rebar3 BaseDeps = [{providers, []} ,{getopt, []} ,{cf, []} ,{erlware_commons, ["ec_dictionary.erl", "ec_vsn.erl"]} ,{parse_trans, ["parse_trans.erl", "parse_trans_pp.erl", "parse_trans_codegen.erl"]} ,{certifi, []} ,{hex_core, []}], Deps = get_deps(), [fetch_and_compile(Dep, Deps) || Dep <- BaseDeps], %% Build rebar3 modules with compile:file bootstrap_rebar3(), %% Build rebar.app from rebar.app.src {ok, App} = rebar_app_info:new(rebar, "3.8.0", filename:absname("_build/default/lib/rebar/")), rebar_otp_app:compile(rebar_state:new(), App), %% Because we are compiling files that are loaded already we want to silence %% not_purged errors in rebar_erlc_compiler:opts_changed/1 error_logger:tty(false), setup_env(), os:putenv("REBAR_PROFILE", "bootstrap"), {ok, State} = rebar3:run(["compile"]), reset_env(), os:putenv("REBAR_PROFILE", ""), DepsPaths = rebar_state:code_paths(State, all_deps), code:add_pathsa(DepsPaths), rebar3:run(["clean", "-a"]), rebar3:run(["as", "prod", "escriptize"]), %% Done with compile, can turn back on error logger error_logger:tty(true). fetch_and_compile({Name, ErlFirstFiles}, Deps) -> case lists:keyfind(Name, 1, Deps) of {Name, Vsn} -> ok = fetch({pkg, atom_to_binary(Name, utf8), list_to_binary(Vsn)}, Name); {Name, _, Source} -> ok = fetch(Source, Name) end, %% Hack: erlware_commons depends on a .script file to check if it is being built with %% rebar2 or rebar3. But since rebar3 isn't built yet it can't get the vsn with get_key. %% So we simply make sure that file is deleted before compiling file:delete("_build/default/lib/erlware_commons/rebar.config.script"), compile(Name, ErlFirstFiles). fetch({pkg, Name, Vsn}, App) -> Dir = filename:join([filename:absname("_build/default/lib/"), App]), case filelib:is_dir(Dir) of false -> CDN = "https://repo.hex.pm/tarballs", Package = binary_to_list(<>), Url = join([CDN, Package], "/"), case request(Url) of {ok, Binary} -> {ok, Contents} = extract(Binary), ok = erl_tar:extract({binary, Contents}, [{cwd, Dir}, compressed]); {error, {Reason, _}} -> ReasonText = re:replace(atom_to_list(Reason), "_", " ", [global,{return,list}]), io:format("Error: Unable to fetch package ~s ~s: ~s~n", [Name, Vsn, ReasonText]) end; true -> io:format("Dependency ~s already exists~n", [Name]) end. extract(Binary) -> {ok, Files} = erl_tar:extract({binary, Binary}, [memory]), {"contents.tar.gz", Contents} = lists:keyfind("contents.tar.gz", 1, Files), {ok, Contents}. request(Url) -> HttpOptions = [{relaxed, true} | get_proxy_auth()], case httpc:request(get, {Url, []}, HttpOptions, [{body_format, binary}], rebar) of {ok, {{_Version, 200, _Reason}, _Headers, Body}} -> {ok, Body}; Error -> Error end. get_rebar_config() -> {ok, [[Home]]} = init:get_argument(home), ConfDir = filename:join(Home, ".config/rebar3"), case file:consult(filename:join(ConfDir, "rebar.config")) of {ok, Config} -> Config; _ -> [] end. get_http_vars(Scheme) -> OS = case os:getenv(atom_to_list(Scheme)) of Str when is_list(Str) -> Str; _ -> [] end, proplists:get_value(Scheme, get_rebar_config(), OS). set_httpc_options() -> set_httpc_options(https_proxy, get_http_vars(https_proxy)), set_httpc_options(proxy, get_http_vars(http_proxy)). set_httpc_options(_, []) -> ok; set_httpc_options(Scheme, Proxy) -> {ok, {_, UserInfo, Host, Port, _, _}} = http_uri:parse(Proxy), httpc:set_options([{Scheme, {{Host, Port}, []}}], rebar), set_proxy_auth(UserInfo). compile(App, FirstFiles) -> Dir = filename:join(filename:absname("_build/default/lib/"), App), filelib:ensure_dir(filename:join([Dir, "ebin", "dummy.beam"])), code:add_path(filename:join(Dir, "ebin")), FirstFilesPaths = [filename:join([Dir, "src", Module]) || Module <- FirstFiles], LeexFiles = filelib:wildcard(filename:join([Dir, "src", "*.xrl"])), [compile_xrl_file(X) || X <- LeexFiles], YeccFiles = filelib:wildcard(filename:join([Dir, "src", "*.yrl"])), [compile_yrl_file(X) || X <- YeccFiles], Sources = FirstFilesPaths ++ filelib:wildcard(filename:join([Dir, "src", "*.erl"])), [compile_erl_file(X, [{i, filename:join(Dir, "include")} ,debug_info ,{outdir, filename:join(Dir, "ebin")} ,return | additional_defines()]) || X <- Sources]. compile_xrl_file(File) -> {ok, _} = leex:file(File). compile_yrl_file(File) -> {ok, _} = yecc:file(File). compile_erl_file(File, Opts) -> case compile:file(File, Opts) of {ok, _Mod} -> ok; {ok, _Mod, []} -> ok; {ok, _Mod, Ws} -> io:format("~s~n", [format_warnings(File, Ws)]), halt(1); {error, Es, Ws} -> io:format("~s ~s~n", [format_errors(File, Es), format_warnings(File, Ws)]), halt(1) end. bootstrap_rebar3() -> filelib:ensure_dir("_build/default/lib/rebar/ebin/dummy.beam"), code:add_path("_build/default/lib/rebar/ebin/"), Res = symlink_or_copy(filename:absname("src"), filename:absname("_build/default/lib/rebar/src")), true = Res == ok orelse Res == exists, Sources = ["src/rebar_resource_v2.erl", "src/rebar_resource.erl" | filelib:wildcard("src/*.erl")], [compile_erl_file(X, [{outdir, "_build/default/lib/rebar/ebin/"} ,return | additional_defines()]) || X <- Sources], code:add_patha(filename:absname("_build/default/lib/rebar/ebin")). %%rebar.hrl -define(FMT(Str, Args), lists:flatten(io_lib:format(Str, Args))). %%/rebar.hrl %%rebar_file_utils -include_lib("kernel/include/file.hrl"). symlink_or_copy(Source, Target) -> Link = case os:type() of {win32, _} -> Source; _ -> make_relative_path(Source, Target) end, case file:make_symlink(Link, Target) of ok -> ok; {error, eexist} -> exists; {error, _} -> case os:type() of {win32, _} -> S = unicode:characters_to_list(Source), T = unicode:characters_to_list(Target), case filelib:is_dir(S) of true -> win32_symlink_or_copy(S, T); false -> cp_r([S], T) end; _ -> case filelib:is_dir(Target) of true -> ok; false -> cp_r([Source], Target) end end end. -spec rm_rf(string()) -> 'ok'. rm_rf(Target) -> case os:type() of {unix, _} -> EscTarget = escape_chars(Target), {ok, []} = sh(?FMT("rm -rf ~ts", [EscTarget]), [{use_stdout, false}, abort_on_error]), ok; {win32, _} -> Filelist = filelib:wildcard(Target), Dirs = [F || F <- Filelist, filelib:is_dir(F)], Files = Filelist -- Dirs, ok = delete_each(Files), ok = delete_each_dir_win32(Dirs), ok end. -spec cp_r(list(string()), file:filename()) -> 'ok'. cp_r([], _Dest) -> ok; cp_r(Sources, Dest) -> case os:type() of {unix, _} -> EscSources = [escape_chars(Src) || Src <- Sources], SourceStr = join(EscSources, " "), {ok, []} = sh(?FMT("cp -Rp ~ts \"~ts\"", [SourceStr, escape_double_quotes(Dest)]), [{use_stdout, false}, abort_on_error]), ok; {win32, _} -> lists:foreach(fun(Src) -> ok = cp_r_win32(Src,Dest) end, Sources), ok end. %% @private Compatibility function for windows win32_symlink_or_copy(Source, Target) -> Res = sh(?FMT("cmd /c mklink /j \"~ts\" \"~ts\"", [escape_double_quotes(filename:nativename(Target)), escape_double_quotes(filename:nativename(Source))]), [{use_stdout, false}, return_on_error]), case win32_mklink_ok(Res, Target) of true -> ok; false -> cp_r_win32(Source, drop_last_dir_from_path(Target)) end. cp_r_win32({true, SourceDir}, {true, DestDir}) -> %% from directory to directory ok = case file:make_dir(DestDir) of {error, eexist} -> ok; Other -> Other end, ok = xcopy_win32(SourceDir, DestDir); cp_r_win32({false, Source} = S,{true, DestDir}) -> %% from file to directory cp_r_win32(S, {false, filename:join(DestDir, filename:basename(Source))}); cp_r_win32({false, Source},{false, Dest}) -> %% from file to file {ok,_} = file:copy(Source, Dest), ok; cp_r_win32({true, SourceDir}, {false, DestDir}) -> case filelib:is_regular(DestDir) of true -> %% From directory to file? This shouldn't happen {error, lists:flatten( io_lib:format("Cannot copy dir (~p) to file (~p)\n", [SourceDir, DestDir]))}; false -> %% Specifying a target directory that doesn't currently exist. %% So let's attempt to create this directory case filelib:ensure_dir(filename:join(DestDir, "dummy")) of ok -> ok = xcopy_win32(SourceDir, DestDir); {error, Reason} -> {error, lists:flatten( io_lib:format("Unable to create dir ~p: ~p\n", [DestDir, Reason]))} end end; cp_r_win32(Source,Dest) -> Dst = {filelib:is_dir(Dest), Dest}, lists:foreach(fun(Src) -> ok = cp_r_win32({filelib:is_dir(Src), Src}, Dst) end, filelib:wildcard(Source)), ok. %% drops the last 'node' of the filename, presumably the last dir such as 'src' %% this is because cp_r_win32/2 automatically adds the dir name, to appease %% robocopy and be more uniform with POSIX drop_last_dir_from_path([]) -> []; drop_last_dir_from_path(Path) -> case lists:droplast(filename:split(Path)) of [] -> []; Dirs -> filename:join(Dirs) end. %% @private specifically pattern match against the output %% of the windows 'mklink' shell call; different values from %% what win32_ok/1 handles win32_mklink_ok({ok, _}, _) -> true; win32_mklink_ok({error,{1,"Local NTFS volumes are required to complete the operation.\n"}}, _) -> false; win32_mklink_ok({error,{1,"Cannot create a file when that file already exists.\n"}}, Target) -> % File or dir is already in place; find if it is already a symlink (true) or % if it is a directory (copy-required; false) is_symlink(Target); win32_mklink_ok(_, _) -> false. xcopy_win32(Source,Dest)-> %% "xcopy \"~ts\" \"~ts\" /q /y /e 2> nul", Changed to robocopy to %% handle long names. May have issues with older windows. Cmd = case filelib:is_dir(Source) of true -> %% For robocopy, copying /a/b/c/ to /d/e/f/ recursively does not %% create /d/e/f/c/*, but rather copies all files to /d/e/f/*. %% The usage we make here expects the former, not the later, so we %% must manually add the last fragment of a directory to the `Dest` %% in order to properly replicate POSIX platforms NewDest = filename:join([Dest, filename:basename(Source)]), ?FMT("robocopy \"~ts\" \"~ts\" /e 1> nul", [escape_double_quotes(filename:nativename(Source)), escape_double_quotes(filename:nativename(NewDest))]); false -> ?FMT("robocopy \"~ts\" \"~ts\" \"~ts\" /e 1> nul", [escape_double_quotes(filename:nativename(filename:dirname(Source))), escape_double_quotes(filename:nativename(Dest)), escape_double_quotes(filename:basename(Source))]) end, Res = sh(Cmd, [{use_stdout, false}, return_on_error]), case win32_ok(Res) of true -> ok; false -> {error, lists:flatten( io_lib:format("Failed to copy ~ts to ~ts~n", [Source, Dest]))} end. is_symlink(Filename) -> {ok, Info} = file:read_link_info(Filename), Info#file_info.type == symlink. win32_ok({ok, _}) -> true; win32_ok({error, {Rc, _}}) when Rc<9; Rc=:=16 -> true; win32_ok(_) -> false. %% @private windows rm_rf helpers delete_each([]) -> ok; delete_each([File | Rest]) -> case file:delete(File) of ok -> delete_each(Rest); {error, enoent} -> delete_each(Rest); {error, Reason} -> io:format("Failed to delete file ~ts: ~p\n", [File, Reason]), error end. delete_each_dir_win32([]) -> ok; delete_each_dir_win32([Dir | Rest]) -> {ok, []} = sh(?FMT("rd /q /s \"~ts\"", [escape_double_quotes(filename:nativename(Dir))]), [{use_stdout, false}, return_on_error]), delete_each_dir_win32(Rest). %%/rebar_file_utils %%rebar_utils %% escape\ as\ a\ shell\? escape_chars(Str) when is_atom(Str) -> escape_chars(atom_to_list(Str)); escape_chars(Str) -> re:replace(Str, "([ ()?`!$&;\"\'])", "\\\\&", [global, {return, list}, unicode]). %% "escape inside these" escape_double_quotes(Str) -> re:replace(Str, "([\"\\\\`!$&*;])", "\\\\&", [global, {return, list}, unicode]). sh(Command0, Options0) -> DefaultOptions = [{use_stdout, false}], Options = [expand_sh_flag(V) || V <- proplists:compact(Options0 ++ DefaultOptions)], ErrorHandler = proplists:get_value(error_handler, Options), OutputHandler = proplists:get_value(output_handler, Options), Command = lists:flatten(patch_on_windows(Command0, proplists:get_value(env, Options, []))), PortSettings = proplists:get_all_values(port_settings, Options) ++ [exit_status, {line, 16384}, use_stdio, stderr_to_stdout, hide, eof], Port = open_port({spawn, Command}, PortSettings), try case sh_loop(Port, OutputHandler, []) of {ok, _Output} = Ok -> Ok; {error, {_Rc, _Output}=Err} -> ErrorHandler(Command, Err) end after port_close(Port) end. sh_loop(Port, Fun, Acc) -> receive {Port, {data, {eol, Line}}} -> sh_loop(Port, Fun, Fun(Line ++ "\n", Acc)); {Port, {data, {noeol, Line}}} -> sh_loop(Port, Fun, Fun(Line, Acc)); {Port, eof} -> Data = lists:flatten(lists:reverse(Acc)), receive {Port, {exit_status, 0}} -> {ok, Data}; {Port, {exit_status, Rc}} -> {error, {Rc, Data}} end end. expand_sh_flag(return_on_error) -> {error_handler, fun(_Command, Err) -> {error, Err} end}; expand_sh_flag(abort_on_error) -> {error_handler, %% moved log_and_abort/2 here because some versions somehow had trouble %% interpreting it and thought `fun log_and_abort/2' was in `erl_eval' fun(Command, {Rc, Output}) -> io:format("sh(~ts)~n" "failed with return code ~w and the following output:~n" "~ts", [Command, Rc, Output]), throw(bootstrap_abort) end}; expand_sh_flag({use_stdout, false}) -> {output_handler, fun(Line, Acc) -> [Line | Acc] end}; expand_sh_flag({cd, _CdArg} = Cd) -> {port_settings, Cd}; expand_sh_flag({env, _EnvArg} = Env) -> {port_settings, Env}. %% We do the shell variable substitution ourselves on Windows and hope that the %% command doesn't use any other shell magic. patch_on_windows(Cmd, Env) -> case os:type() of {win32,nt} -> Cmd1 = "cmd /q /c " ++ lists:foldl(fun({Key, Value}, Acc) -> expand_env_variable(Acc, Key, Value) end, Cmd, Env), %% Remove left-over vars re:replace(Cmd1, "\\\$\\w+|\\\${\\w+}", "", [global, {return, list}, unicode]); _ -> Cmd end. %% @doc Given env. variable `FOO' we want to expand all references to %% it in `InStr'. References can have two forms: `$FOO' and `${FOO}' %% The end of form `$FOO' is delimited with whitespace or EOL -spec expand_env_variable(string(), string(), term()) -> string(). expand_env_variable(InStr, VarName, RawVarValue) -> case chr(InStr, $$) of 0 -> %% No variables to expand InStr; _ -> ReOpts = [global, unicode, {return, list}], VarValue = re:replace(RawVarValue, "\\\\", "\\\\\\\\", ReOpts), %% Use a regex to match/replace: %% Given variable "FOO": match $FOO\s | $FOOeol | ${FOO} RegEx = io_lib:format("\\\$(~ts(\\W|$)|{~ts})", [VarName, VarName]), re:replace(InStr, RegEx, [VarValue, "\\2"], ReOpts) end. %%/rebar_utils %%rebar_dir make_relative_path(Source, Target) -> AbsSource = make_normalized_path(Source), AbsTarget = make_normalized_path(Target), do_make_relative_path(filename:split(AbsSource), filename:split(AbsTarget)). %% @private based on fragments of paths, replace the number of common %% segments by `../' bits, and add the rest of the source alone after it -spec do_make_relative_path([string()], [string()]) -> file:filename(). do_make_relative_path([H|T1], [H|T2]) -> do_make_relative_path(T1, T2); do_make_relative_path(Source, Target) -> Base = lists:duplicate(max(length(Target) - 1, 0), ".."), filename:join(Base ++ Source). make_normalized_path(Path) -> AbsPath = make_absolute_path(Path), Components = filename:split(AbsPath), make_normalized_path(Components, []). make_absolute_path(Path) -> case filename:pathtype(Path) of absolute -> Path; relative -> {ok, Dir} = file:get_cwd(), filename:join([Dir, Path]); volumerelative -> Volume = hd(filename:split(Path)), {ok, Dir} = file:get_cwd(Volume), filename:join([Dir, Path]) end. -spec make_normalized_path([string()], [string()]) -> file:filename(). make_normalized_path([], NormalizedPath) -> filename:join(lists:reverse(NormalizedPath)); make_normalized_path([H|T], NormalizedPath) -> case H of "." when NormalizedPath == [], T == [] -> make_normalized_path(T, ["."]); "." -> make_normalized_path(T, NormalizedPath); ".." when NormalizedPath == [] -> make_normalized_path(T, [".."]); ".." when hd(NormalizedPath) =/= ".." -> make_normalized_path(T, tl(NormalizedPath)); _ -> make_normalized_path(T, [H|NormalizedPath]) end. %%/rebar_dir setup_env() -> %% We don't need or want relx providers loaded yet application:load(rebar), {ok, Providers} = application:get_env(rebar, providers), Providers1 = Providers -- [rebar_prv_release, rebar_prv_relup, rebar_prv_tar], application:set_env(rebar, providers, Providers1). reset_env() -> %% Reset the env so we get all providers application:unset_env(rebar, providers), application:unload(rebar), application:load(rebar). get_deps() -> case file:consult("rebar.lock") of {ok, [[]]} -> %% Something went wrong in a previous build, lock file shouldn't be empty io:format("Empty list in lock file, deleting rebar.lock~n"), ok = file:delete("rebar.lock"), {ok, Config} = file:consult("rebar.config"), proplists:get_value(deps, Config); {ok, [Deps]} -> [{binary_to_atom(Name, utf8), "", Source} || {Name, Source, _Level} <- Deps]; _ -> {ok, Config} = file:consult("rebar.config"), proplists:get_value(deps, Config) end. format_errors(Source, Errors) -> format_errors(Source, "", Errors). format_warnings(Source, Warnings) -> format_warnings(Source, Warnings, []). format_warnings(Source, Warnings, Opts) -> Prefix = case lists:member(warnings_as_errors, Opts) of true -> ""; false -> "Warning: " end, format_errors(Source, Prefix, Warnings). format_errors(_MainSource, Extra, Errors) -> [begin [format_error(Source, Extra, Desc) || Desc <- Descs] end || {Source, Descs} <- Errors]. format_error(AbsSource, Extra, {{Line, Column}, Mod, Desc}) -> ErrorDesc = Mod:format_error(Desc), io_lib:format("~s:~w:~w: ~s~s~n", [AbsSource, Line, Column, Extra, ErrorDesc]); format_error(AbsSource, Extra, {Line, Mod, Desc}) -> ErrorDesc = Mod:format_error(Desc), io_lib:format("~s:~w: ~s~s~n", [AbsSource, Line, Extra, ErrorDesc]); format_error(AbsSource, Extra, {Mod, Desc}) -> ErrorDesc = Mod:format_error(Desc), io_lib:format("~s: ~s~s~n", [AbsSource, Extra, ErrorDesc]). additional_defines() -> [{d, D} || {Re, D} <- [{"^[0-9]+", namespaced_types}, {"^R1[4|5]", deprecated_crypto}, {"^2", unicode_str}, {"^(R|1|20)", fun_stacktrace}, {"^((1[8|9])|2)", rand_module}], is_otp_release(Re)]. is_otp_release(ArchRegex) -> case re:run(otp_release(), ArchRegex, [{capture, none}]) of match -> true; nomatch -> false end. otp_release() -> otp_release1(erlang:system_info(otp_release)). %% If OTP <= R16, otp_release is already what we want. otp_release1([$R,N|_]=Rel) when is_integer(N) -> Rel; %% If OTP >= 17.x, erlang:system_info(otp_release) returns just the %% major version number, we have to read the full version from %% a file. See http://www.erlang.org/doc/system_principles/versions.html %% Read vsn string from the 'OTP_VERSION' file and return as list without %% the "\n". otp_release1(Rel) -> File = filename:join([code:root_dir(), "releases", Rel, "OTP_VERSION"]), case file:read_file(File) of {error, _} -> Rel; {ok, Vsn} -> %% It's fine to rely on the binary module here because we can %% be sure that it's available when the otp_release string does %% not begin with $R. Size = byte_size(Vsn), %% The shortest vsn string consists of at least two digits %% followed by "\n". Therefore, it's safe to assume Size >= 3. case binary:part(Vsn, {Size, -3}) of <<"**\n">> -> %% The OTP documentation mentions that a system patched %% using the otp_patch_apply tool available to licensed %% customers will leave a '**' suffix in the version as a %% flag saying the system consists of application versions %% from multiple OTP versions. We ignore this flag and %% drop the suffix, given for all intents and purposes, we %% cannot obtain relevant information from it as far as %% tooling is concerned. binary:bin_to_list(Vsn, {0, Size - 3}); _ -> binary:bin_to_list(Vsn, {0, Size - 1}) end end. set_proxy_auth([]) -> ok; set_proxy_auth(UserInfo) -> [Username, Password] = re:split(UserInfo, ":", [{return, list}, {parts,2}, unicode]), %% password may contain url encoded characters, need to decode them first put(proxy_auth, [{proxy_auth, {Username, http_uri:decode(Password)}}]). get_proxy_auth() -> case get(proxy_auth) of undefined -> []; ProxyAuth -> ProxyAuth end. %% string:join/2 copy; string:join/2 is getting obsoleted %% and replaced by lists:join/2, but lists:join/2 is too new %% for version support (only appeared in 19.0) so it cannot be %% used. Instead we just adopt join/2 locally and hope it works %% for most unicode use cases anyway. join([], Sep) when is_list(Sep) -> []; join([H|T], Sep) -> H ++ lists:append([Sep ++ X || X <- T]). %% Same for chr; no non-deprecated equivalent in OTP20+ chr(S, C) when is_integer(C) -> chr(S, C, 1). chr([C|_Cs], C, I) -> I; chr([_|Cs], C, I) -> chr(Cs, C, I+1); chr([], _C, _I) -> 0.