summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/rebar.hrl1
-rw-r--r--src/rebar_packages.erl18
-rw-r--r--src/rebar_pkg_resource.erl26
-rw-r--r--src/rebar_prv_update.erl6
-rw-r--r--test/rebar_pkg_SUITE.erl100
-rw-r--r--test/rebar_pkg_SUITE_data/badindexchk-1.0.0.tarbin0 -> 10240 bytes
-rw-r--r--test/rebar_pkg_SUITE_data/badpkg-1.0.0.tarbin0 -> 10240 bytes
-rw-r--r--test/rebar_pkg_SUITE_data/goodpkg-1.0.0.tarbin0 -> 10240 bytes
8 files changed, 125 insertions, 26 deletions
diff --git a/src/rebar.hrl b/src/rebar.hrl
index 1f051d7..4540b1a 100644
--- a/src/rebar.hrl
+++ b/src/rebar.hrl
@@ -22,6 +22,7 @@
-define(DEFAULT_TEST_DEPS_DIR, "test/lib").
-define(DEFAULT_RELEASE_DIR, "rel").
-define(DEFAULT_CONFIG_FILE, "rebar.config").
+-define(DEFAULT_CDN, "https://s3.amazonaws.com/s3.hex.pm/tarballs").
-define(LOCK_FILE, "rebar.lock").
-ifdef(namespaced_types).
diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl
index a344328..e21f1fd 100644
--- a/src/rebar_packages.erl
+++ b/src/rebar_packages.erl
@@ -2,6 +2,7 @@
-export([get_packages/1
,registry/1
+ ,package_dir/1
,check_registry/3
,registry_checksum/2
,find_highest_matching/3]).
@@ -16,8 +17,7 @@
-spec get_packages(rebar_state:t()) -> {rebar_dict(), rebar_digraph()}.
get_packages(State) ->
- RebarDir = rebar_dir:global_cache_dir(State),
- RegistryDir = filename:join(RebarDir, "packages"),
+ RegistryDir = package_dir(State),
DictFile = filename:join(RegistryDir, "dict"),
Edges = filename:join(RegistryDir, "edges"),
Vertices = filename:join(RegistryDir, "vertices"),
@@ -43,8 +43,7 @@ get_packages(State) ->
end.
registry(State) ->
- Dir = rebar_dir:global_cache_dir(State),
- RegistryDir = filename:join(Dir, "packages"),
+ RegistryDir = package_dir(State),
HexFile = filename:join(RegistryDir, "registry"),
case ets:file2tab(HexFile) of
{ok, T} ->
@@ -54,6 +53,17 @@ registry(State) ->
error
end.
+package_dir(State) ->
+ CacheDir = rebar_dir:global_cache_dir(State),
+ CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN),
+ {ok, {_, _, Host, _, Path, _}} = http_uri:parse(CDN),
+ CDNHostPath = lists:reverse(string:tokens(Host, ".")),
+ CDNPath = tl(filename:split(Path)),
+ PackageDir = filename:join([CacheDir, "hex"] ++ CDNHostPath ++ CDNPath ++ ["packages"]),
+ ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")),
+ PackageDir.
+
+
check_registry(Pkg, Vsn, State) ->
case rebar_state:registry(State) of
{ok, T} ->
diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl
index f6bb29b..59ce0dc 100644
--- a/src/rebar_pkg_resource.erl
+++ b/src/rebar_pkg_resource.erl
@@ -11,8 +11,6 @@
-include("rebar.hrl").
--define(DEFAULT_CDN, "https://s3.amazonaws.com/s3.hex.pm/tarballs").
-
lock(_AppDir, Source) ->
Source.
@@ -27,7 +25,7 @@ needs_update(Dir, {pkg, _Name, Vsn}) ->
download(TmpDir, Pkg={pkg, Name, Vsn}, State) ->
CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN),
- PackageDir = hex_package_dir(CDN, State),
+ PackageDir = rebar_packages:package_dir(State),
Package = binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>),
CachePath = filename:join(PackageDir, Package),
Url = string:join([CDN, Package], "/"),
@@ -53,10 +51,13 @@ serve_from_cache(TmpDir, CachePath, Pkg, State) ->
ok = erl_tar:extract({binary, Contents}, [{cwd, TmpDir}, compressed]),
{ok, true};
{_Bin, Chk, Chk} ->
+ ?DEBUG("Checksums: registry: ~p, pkg: ~p", [Chk, _Bin]),
{failed_extract, CachePath};
{Chk, _Reg, Chk} ->
+ ?DEBUG("Checksums: registry: ~p, pkg: ~p", [_Reg, Chk]),
{bad_registry_checksum, CachePath};
{_Bin, _Reg, _Tar} ->
+ ?DEBUG("Checksums: registry: ~p, pkg: ~p, meta: ~p", [_Reg, _Bin, _Tar]),
{bad_checksum, CachePath}
end.
@@ -67,7 +68,7 @@ serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State) ->
ETag ->
serve_from_cache(TmpDir, CachePath, Package, State);
FileETag ->
- ?DEBUG("Download ETag ~s doesn't match returned ETag ~s", [ETag, FileETag]),
+ ?DEBUG("Downloaded file ~s ETag ~s doesn't match returned ETag ~s", [CachePath, ETag, FileETag]),
{bad_download, CachePath}
end.
@@ -84,28 +85,17 @@ checksums(Pkg, Files, Contents, Version, Meta, State) ->
Blob = <<Version/binary, Meta/binary, Contents/binary>>,
<<X:256/big-unsigned>> = crypto:hash(sha256, Blob),
BinChecksum = list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))),
- RegistryChecksum = rebar_packages:registry_sum(Pkg, State),
+ RegistryChecksum = rebar_packages:registry_checksum(Pkg, State),
{"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files),
{BinChecksum, RegistryChecksum, TarChecksum}.
make_vsn(_) ->
{error, "Replacing version of type pkg not supported."}.
-%% Use the shared hex package directory unless a non-default package repo is used
-hex_package_dir(?DEFAULT_CDN, _) ->
- filename:join([rebar_dir:home_dir(), ".hex", "packages"]);
-hex_package_dir(CDN, State) ->
- CacheDir = rebar_dir:global_cache_dir(State),
- {ok, {_, _, Host, _, _, _}} = http_uri:parse(CDN),
- CDNPath = filename:join(lists:reverse(string:tokens(Host, "."))),
- PackageDir = filename:join([CacheDir, "hex", CDNPath, "packages"]),
- ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")),
- PackageDir.
-
request(Url, ETag) ->
case httpc:request(get, {Url, [{"if-none-match", ETag} || ETag =/= false]},
- [{relaxed, true}],
- [{body_format, binary}]) of
+ [{relaxed, true}],
+ [{body_format, binary}]) of
{ok, {{_Version, 200, _Reason}, Headers, Body}} ->
?DEBUG("Successfully downloaded ~s", [Url]),
{"etag", ETag1} = lists:keyfind("etag", 1, Headers),
diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl
index 942b386..dfb719a 100644
--- a/src/rebar_prv_update.erl
+++ b/src/rebar_prv_update.erl
@@ -35,8 +35,7 @@ init(State) ->
do(State) ->
?INFO("Updating package index...", []),
try
- Dir = rebar_dir:global_cache_dir(State),
- RegistryDir = filename:join(Dir, "packages"),
+ RegistryDir = rebar_packages:package_dir(State),
filelib:ensure_dir(filename:join(RegistryDir, "dummy")),
HexFile = filename:join(RegistryDir, "registry"),
TmpDir = ec_file:insecure_mkdtemp(),
@@ -64,8 +63,7 @@ format_error(package_index_write) ->
"Failed to write package index.".
write_registry(Dict, {digraph, Edges, Vertices, Neighbors, _}, State) ->
- Dir = rebar_dir:global_cache_dir(State),
- RegistryDir = filename:join(Dir, "packages"),
+ RegistryDir = rebar_packages:package_dir(State),
filelib:ensure_dir(filename:join(RegistryDir, "dummy")),
ets:tab2file(Edges, filename:join(RegistryDir, "edges")),
ets:tab2file(Vertices, filename:join(RegistryDir, "vertices")),
diff --git a/test/rebar_pkg_SUITE.erl b/test/rebar_pkg_SUITE.erl
new file mode 100644
index 0000000..19c4bd0
--- /dev/null
+++ b/test/rebar_pkg_SUITE.erl
@@ -0,0 +1,100 @@
+%% Test suite for the rebar pkg index caching and decompression
+%% mechanisms.
+-module(rebar_pkg_SUITE).
+-compile(export_all).
+-include_lib("common_test/include/ct.hrl").
+-include_lib("eunit/include/eunit.hrl").
+
+-define(bad_etag, "abcdef").
+-define(good_etag, "22e1d7387c9085a462340088a2a8ba67").
+-define(bad_checksum, <<"D576B442A68C7B92BACDE1EFE9C6E54D8D6C74BDB71D8175B9D3C6EC8C7B62A7">>).
+-define(good_checksum, <<"1C6CE379D191FBAB41B7905075E0BF87CBBE23C77CECE775C5A0B786B2244C35">>).
+
+all() -> [good_uncached].
+
+
+init_per_suite(Config) ->
+ application:start(meck),
+ Config.
+
+end_per_suite(_Config) ->
+ application:stop(meck).
+
+init_per_testcase(good_uncached=Name, Config0) ->
+ Config = [{good_cache, false},
+ {pkg, {<<"goodpkg">>, <<"1.0.0">>}}
+ | Config0],
+ mock_config(Name, Config);
+init_per_testcase(good_cached=Name, Config0) ->
+ Pkg = {<<"goodpkg">>, <<"1.0.0">>},
+ Config1 = [{good_cache, true},
+ {pkg, Pkg}
+ | Config0],
+ Config = mock_config(Name, Config1),
+ copy_to_cache(Pkg, Config),
+ Config.
+
+
+end_per_testcase(_, Config) ->
+ unmock_config(Config),
+ Config.
+
+mock_config(Name, Config) ->
+ Priv = ?config(priv_dir, Config),
+ CacheRoot = filename:join([Priv, "cache", atom_to_list(Name)]),
+ TmpDir = filename:join([Priv, "tmp", atom_to_list(Name)]),
+ T = ets:new(fake_registry, [public]),
+ ets:insert_new(T, [
+ {{<<"badindexchk">>,<<"1.0.0">>}, [[], ?bad_checksum]},
+ {{<<"goodpkg">>,<<"1.0.0">>}, [[], ?good_checksum]},
+ {{<<"badpkg">>,<<"1.0.0">>}, [[], ?good_checksum]}
+ ]),
+ CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]),
+ filelib:ensure_dir(filename:join([CacheDir, "registry"])),
+ ok = ets:tab2file(T, filename:join([CacheDir, "registry"])),
+ %% The state returns us a fake registry
+ meck:new(rebar_state, [passthrough]),
+ meck:expect(rebar_state, registry,
+ fun(_State) -> {ok, fake_registry} end),
+ meck:expect(rebar_state, get,
+ fun(_State, rebar_packages_cdn, _Default) ->
+ "http://test.com/"
+ end),
+ meck:new(rebar_dir, [passthrough]),
+ meck:expect(rebar_dir, global_cache_dir, fun(_) -> CacheRoot end),
+ %% Cache fetches are mocked -- we assume the server and clients are
+ %% correctly used.
+ GoodCache = ?config(good_cache, Config),
+ {Pkg,Vsn} = ?config(pkg, Config),
+ PkgFile = <<Pkg/binary, "-", Vsn/binary, ".tar">>,
+ {ok, PkgContents} = file:read_file(filename:join(?config(data_dir, Config), PkgFile)),
+ meck:new(httpc, [passthrough, unsticky]),
+ meck:expect(httpc, request,
+ fun(get, {_Url, _Opts}, _, _) when GoodCache ->
+ {ok, {{Vsn, 304, <<"Not Modified">>}, [{"etag", ?good_etag}], <<>>}};
+ (get, {_Url, _Opts}, _, _) ->
+ {ok, {{Vsn, 200, <<"OK">>}, [{"etag", ?good_etag}], PkgContents}}
+ end),
+ [{cache_root, CacheRoot},
+ {cache_dir, CacheDir},
+ {tmp_dir, TmpDir},
+ {mock_table, T} | Config].
+
+unmock_config(Config) ->
+ meck:unload(),
+ ets:delete(?config(mock_table, Config)).
+
+copy_to_cache({Pkg,Vsn}, Config) ->
+ Name = <<Pkg/binary, "-", Vsn/binary, ".tar">>,
+ Source = filename:join(?config(data_dir, Config), Name),
+ Dest = filename:join(?config(cache_dir, Config), Name),
+ ec_file:copy(Source, Dest).
+
+good_uncached(Config) ->
+ Tmp = ?config(tmp_dir, Config),
+ {Pkg,Vsn} = ?config(pkg, Config),
+ State = ?config(state, Config),
+ ?assertEqual({ok, true},
+ rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn}, State)),
+ Cache = ?config(cache_dir, Config),
+ ?assert(filelib:is_regular(filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>))).
diff --git a/test/rebar_pkg_SUITE_data/badindexchk-1.0.0.tar b/test/rebar_pkg_SUITE_data/badindexchk-1.0.0.tar
new file mode 100644
index 0000000..e5b963f
--- /dev/null
+++ b/test/rebar_pkg_SUITE_data/badindexchk-1.0.0.tar
Binary files differ
diff --git a/test/rebar_pkg_SUITE_data/badpkg-1.0.0.tar b/test/rebar_pkg_SUITE_data/badpkg-1.0.0.tar
new file mode 100644
index 0000000..4930cd2
--- /dev/null
+++ b/test/rebar_pkg_SUITE_data/badpkg-1.0.0.tar
Binary files differ
diff --git a/test/rebar_pkg_SUITE_data/goodpkg-1.0.0.tar b/test/rebar_pkg_SUITE_data/goodpkg-1.0.0.tar
new file mode 100644
index 0000000..e5b963f
--- /dev/null
+++ b/test/rebar_pkg_SUITE_data/goodpkg-1.0.0.tar
Binary files differ
generated by cgit v1.1 at 2024-12-25 06:12:18 +0000