diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/rebar_app_info.erl | 6 | ||||
-rw-r--r-- | src/rebar_erlc_compiler.erl | 10 | ||||
-rw-r--r-- | src/rebar_fetch.erl | 27 | ||||
-rw-r--r-- | src/rebar_git_resource.erl | 18 | ||||
-rw-r--r-- | src/rebar_packages.erl | 18 | ||||
-rw-r--r-- | src/rebar_pkg_resource.erl | 8 | ||||
-rw-r--r-- | src/rebar_prv_install_deps.erl | 47 | ||||
-rw-r--r-- | src/rebar_prv_update.erl | 64 | ||||
-rw-r--r-- | src/rebar_resource.erl | 4 |
9 files changed, 122 insertions, 80 deletions
diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index dca5aa8..9d779be 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -79,7 +79,7 @@ new(AppName, Vsn) -> new(AppName, Vsn, Dir) -> {ok, #app_info_t{name=ec_cnv:to_binary(AppName), original_vsn=Vsn, - dir=Dir}}. + dir=ec_cnv:to_list(Dir)}}. %% @doc build a complete version of the app info with all fields set. -spec new(atom() | binary() | string(), binary() | string(), file:name(), list()) -> @@ -87,7 +87,7 @@ new(AppName, Vsn, Dir) -> new(AppName, Vsn, Dir, Deps) -> {ok, #app_info_t{name=ec_cnv:to_binary(AppName), original_vsn=Vsn, - dir=Dir, + dir=ec_cnv:to_list(Dir), deps=Deps}}. %% @doc discover a complete version of the app info with all fields set. @@ -200,7 +200,7 @@ dir(#app_info_t{dir=Dir}) -> -spec dir(t(), file:name()) -> t(). dir(AppInfo=#app_info_t{}, Dir) -> - AppInfo#app_info_t{dir=Dir}. + AppInfo#app_info_t{dir=ec_cnv:to_list(Dir)}. -spec ebin_dir(t()) -> file:name(). ebin_dir(#app_info_t{dir=Dir}) -> diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index b096705..cb342a8 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -144,17 +144,13 @@ doterl_compile(Config, Dir, MoreSources, ErlOpts) -> filename:join(Dir, X) end, rebar_dir:src_dirs(proplists:append_values(src_dirs, ErlOpts))), AllErlFiles = gather_src(SrcDirs, []) ++ MoreSources, - %% NOTE: If and when erl_first_files is not inherited anymore - %% (rebar_state:get instead of rebar_state:get_list), consider - %% logging a warning message for any file listed in erl_first_files which - %% wasn't found via gather_src. - + %% Issue: rebar/rebar3#140 (fix matching based on same path + order of %% erl_first_files) ErlFirstFiles = get_erl_first_files(ErlFirstFilesConf, AllErlFiles), RestErls = [ File || File <- AllErlFiles, not lists:member(File, ErlFirstFiles) ], - + %% Make sure that ebin/ exists and is on the path ok = filelib:ensure_dir(filename:join([Dir, "ebin", "dummy.beam"])), CurrPath = code:get_path(), @@ -634,5 +630,3 @@ log_files(Prefix, Files) -> _ -> ?DEBUG("~s:~n~p", [Prefix, Files]) end. - - diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index 088a49a..4dfd790 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -8,12 +8,13 @@ -module(rebar_fetch). -export([lock_source/2, - download_source/2, + download_source/3, needs_update/2]). -export([format_error/1]). -include("rebar.hrl"). +-include_lib("providers/include/providers.hrl"). %% map short versions of resources to module names -define(RESOURCES, [{git, rebar_git_resource}, {pkg, rebar_pkg_resource}]). @@ -24,13 +25,14 @@ lock_source(AppDir, Source) -> Module = get_resource_type(Source), Module:lock(AppDir, Source). --spec download_source(file:filename_all(), rebar_resource:resource()) -> true | {error, any()}. -download_source(AppDir, Source) -> +-spec download_source(file:filename_all(), rebar_resource:resource(), rebar_state:t()) -> + true | {error, any()}. +download_source(AppDir, Source, State) -> try Module = get_resource_type(Source), TmpDir = ec_file:insecure_mkdtemp(), AppDir1 = ec_cnv:to_list(AppDir), - case Module:download(TmpDir, Source) of + case Module:download(TmpDir, Source, State) of {ok, _} -> ec_file:mkdir_p(AppDir1), code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), @@ -38,20 +40,21 @@ download_source(AppDir, Source) -> ok = ec_file:copy(TmpDir, filename:absname(AppDir1), [recursive]), true; {tarball, File} -> + Contents = filename:join(TmpDir, "contents"), ec_file:mkdir_p(AppDir1), - ok = erl_tar:extract(File, [{cwd, TmpDir} - ,compressed]), - BaseName = filename:basename(AppDir1), - [FromDir] = filelib:wildcard(filename:join(TmpDir, BaseName++"-*")), + ec_file:mkdir_p(Contents), + ok = erl_tar:extract(File, [{cwd, TmpDir}]), + ok = erl_tar:extract(filename:join(TmpDir, "contents.tar.gz"), + [{cwd, Contents}, compressed]), code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), ec_file:remove(filename:absname(AppDir1), [recursive]), - ok = ec_file:copy(FromDir, filename:absname(AppDir1), [recursive]), - true = code:add_patha(filename:join(AppDir1, "ebin")), + ok = ec_file:copy(Contents, filename:absname(AppDir1), [recursive]), true end catch - _:_ -> - {error, {rebar_fetch, {fetch_fail, Source}}} + C:T -> + ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, erlang:get_stacktrace()]), + throw(?PRV_ERROR({fetch_fail, Source})) end. -spec needs_update(file:filename_all(), rebar_resource:resource()) -> boolean() | {error, string()}. diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl index bd0a978..785c362 100644 --- a/src/rebar_git_resource.erl +++ b/src/rebar_git_resource.erl @@ -5,7 +5,7 @@ -behaviour(rebar_resource). -export([lock/2 - ,download/2 + ,download/3 ,needs_update/2 ,make_vsn/1]). @@ -75,28 +75,28 @@ parse_git_url("https://" ++ HostPath) -> [Host | Path] = string:tokens(HostPath, "/"), {Host, filename:rootname(filename:join(Path), ".git")}. -download(Dir, {git, Url}) -> +download(Dir, {git, Url}, State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), - download(Dir, {git, Url, {branch, "master"}}); -download(Dir, {git, Url, ""}) -> + download(Dir, {git, Url, {branch, "master"}}, State); +download(Dir, {git, Url, ""}, State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), - download(Dir, {git, Url, {branch, "master"}}); -download(Dir, {git, Url, {branch, Branch}}) -> + download(Dir, {git, Url, {branch, "master"}}, State); +download(Dir, {git, Url, {branch, Branch}}, _State) -> ok = filelib:ensure_dir(Dir), rebar_utils:sh(?FMT("git clone ~s ~s -b ~s --single-branch", [Url, filename:basename(Dir), Branch]), [{cd, filename:dirname(Dir)}]); -download(Dir, {git, Url, {tag, Tag}}) -> +download(Dir, {git, Url, {tag, Tag}}, _State) -> ok = filelib:ensure_dir(Dir), rebar_utils:sh(?FMT("git clone ~s ~s -b ~s --single-branch", [Url, filename:basename(Dir), Tag]), [{cd, filename:dirname(Dir)}]); -download(Dir, {git, Url, {ref, Ref}}) -> +download(Dir, {git, Url, {ref, Ref}}, _State) -> ok = filelib:ensure_dir(Dir), rebar_utils:sh(?FMT("git clone -n ~s ~s", [Url, filename:basename(Dir)]), [{cd, filename:dirname(Dir)}]), rebar_utils:sh(?FMT("git checkout -q ~s", [Ref]), [{cd, Dir}]); -download(Dir, {git, Url, Rev}) -> +download(Dir, {git, Url, Rev}, _State) -> ?WARN("WARNING: It is recommended to use {branch, Name}, {tag, Tag} or {ref, Ref}, otherwise updating the dep may not work as expected.", []), ok = filelib:ensure_dir(Dir), rebar_utils:sh(?FMT("git clone -n ~s ~s", [Url, filename:basename(Dir)]), diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 30d11c9..5400544 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -13,13 +13,21 @@ -spec get_packages(rebar_state:t()) -> {rebar_dict(), rebar_digraph()}. get_packages(State) -> RebarDir = rebar_dir:global_config_dir(State), - PackagesFile = filename:join(RebarDir, "packages"), - case ec_file:exists(PackagesFile) of + RegistryDir = filename:join(RebarDir, "packages"), + DictFile = filename:join(RegistryDir, "dict"), + Edges = filename:join(RegistryDir, "edges"), + Vertices = filename:join(RegistryDir, "vertices"), + Neighbors = filename:join(RegistryDir, "neighbors"), + + case lists:all(fun(X) -> filelib:is_file(X) end, [DictFile, Edges, Vertices, Neighbors]) of true -> try - {ok, Binary} = file:read_file(PackagesFile), - {Dict, Graph} = binary_to_term(Binary), - {Dict, rebar_digraph:restore_graph(Graph)} + {ok, DictBinary} = file:read_file(DictFile), + Dict = binary_to_term(DictBinary), + {ok, EdgesTab} = ets:file2tab(Edges), + {ok, VerticesTab} = ets:file2tab(Vertices), + {ok, NeighborsTab} = ets:file2tab(Neighbors), + {Dict, {digraph, EdgesTab, VerticesTab, NeighborsTab, true}} catch _:_ -> ?ERROR("Bad packages index, try to fix with `rebar3 update`", []), diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 5cd6fc8..0508e2c 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -5,7 +5,7 @@ -behaviour(rebar_resource). -export([lock/2 - ,download/2 + ,download/3 ,needs_update/2 ,make_vsn/1]). @@ -23,9 +23,11 @@ needs_update(Dir, {pkg, _Name, Vsn, _Url}) -> true end. -download(Dir, {pkg, _Name, _Vsn, Url}) -> +download(Dir, {pkg, Name, Vsn}, State) -> TmpFile = filename:join(Dir, "package.tar.gz"), - {ok, saved_to_file} = httpc:request(get, {binary_to_list(Url), []}, [], [{stream, TmpFile}]), + CDN = rebar_state:get(State, rebar_packages_cdn, "https://s3.amazonaws.com/s3.hex.pm/tarballs"), + Url = string:join([CDN, binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>)], "/"), + {ok, saved_to_file} = httpc:request(get, {Url, []}, [], [{stream, TmpFile}]), {tarball, TmpFile}. make_vsn(_) -> diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index dab4c24..61fbd70 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -75,19 +75,19 @@ do(State) -> Profiles = rebar_state:current_profiles(State), ProjectApps = rebar_state:project_apps(State), - {SrcApps, State1} = - lists:foldl(fun(Profile, {SrcAppsAcc, StateAcc}) -> + {Apps, State1} = + lists:foldl(fun(Profile, {AppsAcc, StateAcc}) -> Locks = rebar_state:get(StateAcc, {locks, Profile}, []), - {ok, NewSrcApps, NewState} = + {ok, NewApps, NewState} = handle_deps(Profile ,StateAcc ,rebar_state:get(StateAcc, {deps, Profile}, []) ,Locks), - {NewSrcApps++SrcAppsAcc, NewState} + {NewApps++AppsAcc, NewState} end, {[], State}, lists:reverse(Profiles)), - Source = ProjectApps ++ SrcApps, - case find_cycles(Source ++ rebar_state:all_deps(State1)) of + Source = ProjectApps ++ Apps, + case find_cycles(Source) of {cycles, Cycles} -> ?PRV_ERROR({cycles, Cycles}); {error, Error} -> @@ -117,7 +117,9 @@ find_cycles(Apps) -> -spec format_error(any()) -> iolist(). format_error({parse_dep, Dep}) -> - io_lib:format("Failed parsing dep ~p~n", [Dep]); + io_lib:format("Failed parsing dep ~p", [Dep]); +format_error({missing_package, Package, Version}) -> + io_lib:format("Package not found in registry: ~s-~s", [Package, Version]); format_error({cycles, Cycles}) -> Prints = [["applications: ", [io_lib:format("~s ", [Dep]) || Dep <- Cycle], @@ -178,7 +180,7 @@ handle_deps(Profile, State, Deps, Upgrade, Locks) -> %% Sort all apps to build order State3 = rebar_state:all_deps(State2, AllDeps), - {ok, SrcApps, State3}. + {ok, AllDeps, State3}. %% =================================================================== %% Internal functions @@ -193,7 +195,7 @@ update_pkg_deps(Profile, Pkgs, Packages, Upgrade, Seen, State) -> ,Packages ,Pkg), {SeenAcc1, StateAcc1} = maybe_lock(Profile, AppInfo, SeenAcc, StateAcc, 0), - case maybe_fetch(AppInfo, Upgrade, SeenAcc1) of + case maybe_fetch(AppInfo, Upgrade, SeenAcc1, State) of true -> {[AppInfo | Acc], SeenAcc1, StateAcc1}; false -> @@ -227,15 +229,14 @@ maybe_lock(Profile, AppInfo, Seen, State, Level) -> package_to_app(DepsDir, Packages, {Name, Vsn}) -> case dict:find({Name, Vsn}, Packages) of error -> - {error, missing_package}; + throw(?PRV_ERROR({missing_package, Name, Vsn})); {ok, P} -> PkgDeps = [{PkgName, PkgVsn} || {PkgName,PkgVsn} <- proplists:get_value(<<"deps">>, P, [])], - Link = proplists:get_value(<<"link">>, P, ""), {ok, AppInfo} = rebar_app_info:new(Name, Vsn), AppInfo1 = rebar_app_info:deps(AppInfo, PkgDeps), AppInfo2 = rebar_app_info:dir(AppInfo1, rebar_dir:deps_dir(DepsDir, Name)), - rebar_app_info:source(AppInfo2, {pkg, Name, Vsn, Link}) + rebar_app_info:source(AppInfo2, {pkg, Name, Vsn}) end. -spec update_src_deps(atom(), non_neg_integer(), list(), list(), list(), rebar_state:t(), boolean(), sets:set(binary()), list()) -> {rebar_state:t(), list(), list(), sets:set(binary())}. @@ -282,7 +283,7 @@ update_src_deps(Profile, Level, SrcDeps, PkgDeps, SrcApps, State, Upgrade, Seen, ,StateAcc1 ,LocksAcc); _ -> - maybe_fetch(AppInfo, false, SeenAcc), + maybe_fetch(AppInfo, false, SeenAcc, StateAcc), handle_dep(AppInfo ,SrcDepsAcc ,PkgDepsAcc @@ -306,7 +307,7 @@ handle_upgrade(AppInfo, SrcDeps, PkgDeps, SrcApps, Level, State, Locks) -> Name = rebar_app_info:name(AppInfo), case lists:keyfind(Name, 1, Locks) of false -> - case maybe_fetch(AppInfo, true, sets:new()) of + case maybe_fetch(AppInfo, true, sets:new(), State) of true -> handle_dep(AppInfo ,SrcDeps @@ -357,8 +358,8 @@ handle_dep(State, DepsDir, AppInfo, Locks, Level) -> {AppInfo2, SrcDeps, PkgDeps, Locks++NewLocks}. -spec maybe_fetch(rebar_app_info:t(), boolean() | {true, binary(), integer()}, - sets:set(binary())) -> boolean(). -maybe_fetch(AppInfo, Upgrade, Seen) -> + sets:set(binary()), rebar_state:t()) -> boolean(). +maybe_fetch(AppInfo, Upgrade, Seen, State) -> AppDir = ec_cnv:to_list(rebar_app_info:dir(AppInfo)), Apps = rebar_app_discover:find_apps(["_checkouts"], all), case rebar_app_utils:find(rebar_app_info:name(AppInfo), Apps) of @@ -368,13 +369,13 @@ maybe_fetch(AppInfo, Upgrade, Seen) -> error -> case not app_exists(AppDir) of true -> - fetch_app(AppInfo, AppDir); + fetch_app(AppInfo, AppDir, State); false -> case sets:is_element(rebar_app_info:name(AppInfo), Seen) of true -> false; false -> - maybe_upgrade(AppInfo, AppDir, Upgrade) + maybe_upgrade(AppInfo, AppDir, Upgrade, State) end end end. @@ -457,25 +458,25 @@ app_exists(AppDir) -> end end. -fetch_app(AppInfo, AppDir) -> +fetch_app(AppInfo, AppDir, State) -> ?INFO("Fetching ~s (~p)", [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)]), Source = rebar_app_info:source(AppInfo), - case rebar_fetch:download_source(AppDir, Source) of + case rebar_fetch:download_source(AppDir, Source, State) of {error, Reason} -> throw(Reason); Result -> Result end. -maybe_upgrade(AppInfo, AppDir, false) -> +maybe_upgrade(AppInfo, AppDir, false, _State) -> Source = rebar_app_info:source(AppInfo), rebar_fetch:needs_update(AppDir, Source); -maybe_upgrade(AppInfo, AppDir, true) -> +maybe_upgrade(AppInfo, AppDir, true, State) -> Source = rebar_app_info:source(AppInfo), case rebar_fetch:needs_update(AppDir, Source) of true -> ?INFO("Updating ~s", [rebar_app_info:name(AppInfo)]), - case rebar_fetch:download_source(AppDir, Source) of + case rebar_fetch:download_source(AppDir, Source, State) of {error, Reason} -> throw(Reason); Result -> diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index 524a684..655e5a3 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -34,18 +34,22 @@ init(State) -> do(State) -> ?INFO("Updating package index...", []), try - Url = url(State), + Url = rebar_state:get(State, rebar_packages_cdn, "https://s3.amazonaws.com/s3.hex.pm/registry.ets.gz"), TmpDir = ec_file:insecure_mkdtemp(), - TmpFile = filename:join(TmpDir, "packages"), - Home = rebar_dir:home_dir(), - PackagesFile = filename:join([Home, ?CONFIG_DIR, "packages"]), - filelib:ensure_dir(PackagesFile), - {ok, _RequestId} = httpc:request(get, {Url, [{"Accept", "application/erlang"}]}, - [], [{stream, TmpFile}, {sync, true}]), - ok = ec_file:copy(TmpFile, PackagesFile) + TmpFile = filename:join(TmpDir, "packages.gz"), + HexFile = filename:join(TmpDir, "packages"), + {ok, _RequestId} = httpc:request(get, {Url, []}, + [], [{stream, TmpFile}, {sync, true}]), + {ok, Data} = file:read_file(TmpFile), + Unzipped = zlib:gunzip(Data), + ok = file:write_file(HexFile, Unzipped), + {Dict, Graph} = hex_to_graph(HexFile), + write_registry(Dict, Graph, State), + ok catch - _:_ -> - {error, {?MODULE, package_index_write}} + E:C -> + io:format("E C ~p ~p~n", [E, C]), + throw({error, {?MODULE, package_index_write}}) end, {ok, State}. @@ -54,9 +58,39 @@ do(State) -> format_error(package_index_write) -> "Failed to write package index.". -url(State) -> - ErtsVsn = erlang:system_info(version), +write_registry(Dict, {digraph, Edges, Vertices, Neighbors, _}, State) -> + Dir = rebar_dir:global_config_dir(State), + RegistryDir = filename:join(Dir, "packages"), + filelib:ensure_dir(filename:join(RegistryDir, "dummy")), + ets:tab2file(Edges, filename:join(RegistryDir, "edges")), + ets:tab2file(Vertices, filename:join(RegistryDir, "vertices")), + ets:tab2file(Neighbors, filename:join(RegistryDir, "neighbors")), + file:write_file(filename:join(RegistryDir, "dict"), term_to_binary(Dict)). - Qs = [io_lib:format("~s=~s", [X, Y]) || {X, Y} <- [{"erts", ErtsVsn}]], - Url = rebar_state:get(State, rebar_packages_url, "http://packages.rebar3.org/packages"), - Url ++ "?" ++ string:join(Qs, "&"). +hex_to_graph(Filename) -> + {ok, T} = ets:file2tab(Filename), + Graph = digraph:new(), + ets:foldl(fun({Pkg, [Versions]}, ok) when is_binary(Pkg), is_list(Versions) -> + lists:foreach(fun(Version) -> + digraph:add_vertex(Graph, {Pkg, Version}, 1) + end, Versions); + (_, ok) -> + ok + end, ok, T), + + Dict1 = ets:foldl(fun({{Pkg, PkgVsn}, [Deps | _]}, Dict) -> + DepsList = lists:foldl(fun([Dep, DepVsn, _, _], DepsListAcc) -> + case DepVsn of + <<"~> ", Vsn/binary>> -> + digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, Vsn}), + [{Dep, DepVsn} | DepsListAcc]; + Vsn -> + digraph:add_edge(Graph, {Pkg, PkgVsn}, {Dep, Vsn}), + [{Dep, Vsn} | DepsListAcc] + end + end, [], Deps), + dict:store({Pkg, PkgVsn}, DepsList, Dict); + (_, Dict) -> + Dict + end, dict:new(), T), + {Dict1, Graph}. diff --git a/src/rebar_resource.erl b/src/rebar_resource.erl index 2bc28e2..7c58135 100644 --- a/src/rebar_resource.erl +++ b/src/rebar_resource.erl @@ -23,7 +23,7 @@ -spec behaviour_info(atom()) -> [{atom(), arity()}] | undefined. behaviour_info(callbacks) -> [{lock, 2}, - {download, 2}, + {download, 3}, {needs_update,2}, {make_vsn, 1}]; behaviour_info(_) -> @@ -33,7 +33,7 @@ behaviour_info(_) -> -callback lock(file:filename_all(), tuple()) -> rebar_resource:resource(). --callback download(file:filename_all(), tuple()) -> +-callback download(file:filename_all(), tuple(), rebar_state:t()) -> {tarball, file:filename_all()} | {ok, any()} | {error, any()}. -callback needs_update(file:filename_all(), tuple()) -> boolean(). |