From 391959dae8540a96c1945082f3b62fbe46914800 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sat, 22 Aug 2015 19:58:20 -0500 Subject: use 'default' for default hex repo path in cache and include in info messages --- src/rebar_file_utils.erl | 6 +++++- src/rebar_packages.erl | 27 ++++++++++++++++++++------- src/rebar_pkg_resource.erl | 4 ++-- src/rebar_prv_update.erl | 13 ++++++++----- test/rebar_pkg_SUITE.erl | 3 ++- 5 files changed, 37 insertions(+), 16 deletions(-) diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 07c63f1..4f8eff5 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -27,6 +27,7 @@ -module(rebar_file_utils). -export([try_consult/1, + replace_home_dir/1, format_error/1, symlink_or_copy/2, rm_rf/1, @@ -59,6 +60,10 @@ try_consult(File) -> throw(?PRV_ERROR({bad_term_file, File, Reason})) end. +replace_home_dir(Dir) -> + HomeDir = rebar_dir:home_dir(), + re:replace(Dir, [$^ | HomeDir], "~", [{return, list}]). + format_error({bad_term_file, AppFile, Reason}) -> io_lib:format("Error reading file ~s: ~s", [AppFile, file:format_error(Reason)]). @@ -318,4 +323,3 @@ cp_r_win32(Source,Dest) -> ok = cp_r_win32({filelib:is_dir(Src), Src}, Dst) end, filelib:wildcard(Source)), ok. - diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index f6993c5..3352323 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -4,6 +4,7 @@ ,close_packages/0 ,load_and_verify_version/1 ,deps/3 + ,registry_dir/1 ,package_dir/1 ,registry_checksum/2 ,find_highest_matching/4 @@ -34,7 +35,7 @@ close_packages() -> catch ets:delete(?PACKAGE_TABLE). load_and_verify_version(State) -> - RegistryDir = package_dir(State), + RegistryDir = registry_dir(State), case ets:file2tab(filename:join(RegistryDir, ?INDEX_FILE)) of {ok, _} -> case ets:lookup_element(?PACKAGE_TABLE, package_index_version, 2) of @@ -57,13 +58,25 @@ deps(Name, Vsn, State) -> throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)})) end. -package_dir(State) -> +registry_dir(State) -> CacheDir = rebar_dir:global_cache_dir(State), - CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), - {ok, {_, _, Host, _, Path, _}} = http_uri:parse(CDN), - CDNHostPath = lists:reverse(string:tokens(Host, ".")), - CDNPath = tl(filename:split(Path)), - PackageDir = filename:join([CacheDir, "hex"] ++ CDNHostPath ++ CDNPath ++ ["packages"]), + case rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN) of + ?DEFAULT_CDN -> + RegistryDir = filename:join([CacheDir, "hex", "default"]), + ok = filelib:ensure_dir(filename:join(RegistryDir, "placeholder")), + RegistryDir; + CDN -> + {ok, {_, _, Host, _, Path, _}} = http_uri:parse(CDN), + CDNHostPath = lists:reverse(string:tokens(Host, ".")), + CDNPath = tl(filename:split(Path)), + RegistryDir = filename:join([CacheDir, "hex"] ++ CDNHostPath ++ CDNPath), + ok = filelib:ensure_dir(filename:join(RegistryDir, "placeholder")), + RegistryDir + end. + +package_dir(State) -> + RegistryDir = registry_dir(State), + PackageDir = filename:join([RegistryDir, "packages"]), ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")), PackageDir. diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index fdc69e2..3430e81 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -34,7 +34,7 @@ download(TmpDir, Pkg={pkg, Name, Vsn}, State) -> Url = string:join([CDN, Package], "/"), cached_download(TmpDir, CachePath, Pkg, Url, etag(CachePath), State). -cached_download(TmpDir, CachePath, Pkg, Url, ETag, State) -> +cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn}, Url, ETag, State) -> case request(Url, ETag) of {ok, cached} -> serve_from_cache(TmpDir, CachePath, Pkg, State); @@ -44,7 +44,7 @@ cached_download(TmpDir, CachePath, Pkg, Url, ETag, State) -> ?DEBUG("Download ~s error, using ~s from cache", [Url, CachePath]), serve_from_cache(TmpDir, CachePath, Pkg, State); error -> - request_failed + {fetch_fail, Name, Vsn} end. serve_from_cache(TmpDir, CachePath, Pkg, State) -> diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index 4b3a155..a4628b2 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -35,11 +35,11 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - ?INFO("Updating package index...", []), try - RegistryDir = rebar_packages:package_dir(State), + RegistryDir = rebar_packages:registry_dir(State), filelib:ensure_dir(filename:join(RegistryDir, "dummy")), HexFile = filename:join(RegistryDir, "registry"), + ?INFO("Updating package registry...", []), TmpDir = ec_file:insecure_mkdtemp(), TmpFile = filename:join(TmpDir, "packages.gz"), @@ -50,7 +50,7 @@ do(State) -> {ok, Data} = file:read_file(TmpFile), Unzipped = zlib:gunzip(Data), ok = file:write_file(HexFile, Unzipped), - + ?INFO("Writing registry to ~s", [rebar_file_utils:replace_home_dir(HexFile)]), hex_to_index(State), ok catch @@ -71,11 +71,13 @@ is_supported(<<"rebar3">>) -> true; is_supported(_) -> false. hex_to_index(State) -> - RegistryDir = rebar_packages:package_dir(State), + RegistryDir = rebar_packages:registry_dir(State), HexFile = filename:join(RegistryDir, "registry"), try ets:file2tab(HexFile) of {ok, Registry} -> try + PackageIndex = filename:join(RegistryDir, "packages.idx"), + ?INFO("Generating package index...", []), (catch ets:delete(?PACKAGE_TABLE)), ets:new(?PACKAGE_TABLE, [named_table, public]), ets:foldl(fun({{Pkg, PkgVsn}, [Deps, Checksum, BuildTools | _]}, _) when is_list(BuildTools) -> @@ -93,7 +95,8 @@ hex_to_index(State) -> end, true, Registry), ets:insert(?PACKAGE_TABLE, {package_index_version, ?PACKAGE_INDEX_VERSION}), - ets:tab2file(?PACKAGE_TABLE, filename:join(RegistryDir, "packages.idx")), + ?INFO("Writing index to ~s", [rebar_file_utils:replace_home_dir(PackageIndex)]), + ets:tab2file(?PACKAGE_TABLE, PackageIndex), true after catch ets:delete(Registry) diff --git a/test/rebar_pkg_SUITE.erl b/test/rebar_pkg_SUITE.erl index 3cd3a67..b0b7eda 100644 --- a/test/rebar_pkg_SUITE.erl +++ b/test/rebar_pkg_SUITE.erl @@ -148,7 +148,7 @@ bad_disconnect(Config) -> Tmp = ?config(tmp_dir, Config), {Pkg,Vsn} = ?config(pkg, Config), State = ?config(state, Config), - ?assertEqual(request_failed, + ?assertEqual({fetch_fail, Pkg, Vsn}, rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn}, State)). @@ -183,6 +183,7 @@ mock_config(Name, Config) -> meck:expect(rebar_dir, global_cache_dir, fun(_) -> CacheRoot end), meck:new(rebar_packages, [passthrough]), + meck:expect(rebar_packages, registry_dir, fun(_) -> CacheDir end), meck:expect(rebar_packages, package_dir, fun(_) -> CacheDir end), rebar_prv_update:hex_to_index(rebar_state:new()), -- cgit v1.1