summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/rebar.hrl1
-rw-r--r--src/rebar_fetch.erl64
-rw-r--r--src/rebar_packages.erl28
-rw-r--r--src/rebar_pkg_resource.erl99
-rw-r--r--src/rebar_prv_common_test.erl2
-rw-r--r--src/rebar_prv_install_deps.erl33
-rw-r--r--src/rebar_prv_update.erl6
-rw-r--r--src/rebar_resource.erl19
8 files changed, 172 insertions, 80 deletions
diff --git a/src/rebar.hrl b/src/rebar.hrl
index 1f051d7..4540b1a 100644
--- a/src/rebar.hrl
+++ b/src/rebar.hrl
@@ -22,6 +22,7 @@
-define(DEFAULT_TEST_DEPS_DIR, "test/lib").
-define(DEFAULT_RELEASE_DIR, "rel").
-define(DEFAULT_CONFIG_FILE, "rebar.config").
+-define(DEFAULT_CDN, "https://s3.amazonaws.com/s3.hex.pm/tarballs").
-define(LOCK_FILE, "rebar.lock").
-ifdef(namespaced_types).
diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl
index 20bf46b..0aca308 100644
--- a/src/rebar_fetch.erl
+++ b/src/rebar_fetch.erl
@@ -26,42 +26,34 @@ lock_source(AppDir, Source, State) ->
-spec download_source(file:filename_all(), rebar_resource:resource(), rebar_state:t()) ->
true | {error, any()}.
download_source(AppDir, Source, State) ->
- try
- Resources = rebar_state:resources(State),
- Module = get_resource_type(Source, Resources),
- TmpDir = ec_file:insecure_mkdtemp(),
- AppDir1 = ec_cnv:to_list(AppDir),
- case Module:download(TmpDir, Source, State) of
- {ok, _} ->
- ec_file:mkdir_p(AppDir1),
- code:del_path(filename:absname(filename:join(AppDir1, "ebin"))),
- ec_file:remove(filename:absname(AppDir1), [recursive]),
- ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]),
- ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)),
- true;
- {tarball, File} ->
- Contents = filename:join(TmpDir, "contents"),
- ec_file:mkdir_p(AppDir1),
- ec_file:mkdir_p(Contents),
- ok = erl_tar:extract(File, [{cwd, TmpDir}]),
- ok = erl_tar:extract(filename:join(TmpDir, "contents.tar.gz"),
- [{cwd, Contents}, compressed]),
- code:del_path(filename:absname(filename:join(AppDir1, "ebin"))),
- ec_file:remove(filename:absname(AppDir1), [recursive]),
-
- ?DEBUG("Moving contents ~p to ~p", [Contents, filename:absname(AppDir1)]),
- ok = rebar_file_utils:mv(Contents, filename:absname(AppDir1)),
-
- ?DEBUG("Removing tmp dir ~p", [TmpDir]),
- ec_file:remove(TmpDir, [recursive]),
- true
- end
+ try download_source_(AppDir, Source, State) of
+ true ->
+ true;
+ Error ->
+ throw(?PRV_ERROR(Error))
catch
C:T ->
?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, erlang:get_stacktrace()]),
throw(?PRV_ERROR({fetch_fail, Source}))
end.
+download_source_(AppDir, Source, State) ->
+ Resources = rebar_state:resources(State),
+ Module = get_resource_type(Source, Resources),
+ TmpDir = ec_file:insecure_mkdtemp(),
+ AppDir1 = ec_cnv:to_list(AppDir),
+ case Module:download(TmpDir, Source, State) of
+ {ok, _} ->
+ ec_file:mkdir_p(AppDir1),
+ code:del_path(filename:absname(filename:join(AppDir1, "ebin"))),
+ ec_file:remove(filename:absname(AppDir1), [recursive]),
+ ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]),
+ ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)),
+ true;
+ Error ->
+ Error
+ end.
+
-spec needs_update(file:filename_all(), rebar_resource:resource(), rebar_state:t()) -> boolean() | {error, string()}.
needs_update(AppDir, Source, State) ->
Resources = rebar_state:resources(State),
@@ -73,8 +65,18 @@ needs_update(AppDir, Source, State) ->
true
end.
+format_error({bad_download, CachePath}) ->
+ io_lib:format("Download of package does not match md5sum from server: ~s", [CachePath]);
+format_error({failed_extract, CachePath}) ->
+ io_lib:format("Failed to extract package: ~s", [CachePath]);
+format_error({bad_etag, Source}) ->
+ io_lib:format("MD5 Checksum comparison failed for: ~s", [Source]);
format_error({fetch_fail, Source}) ->
- io_lib:format("Failed to fetch and copy dep: ~p", [Source]).
+ io_lib:format("Failed to fetch and copy dep: ~s", [Source]);
+format_error({bad_checksum, File}) ->
+ io_lib:format("Checksum mismatch against tarball in ~s", [File]);
+format_error({bad_registry_checksum, File}) ->
+ io_lib:format("Checksum mismatch against registry in ~s", [File]).
get_resource_type({Type, Location}, Resources) ->
find_resource_module(Type, Location, Resources);
diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl
index 4ab5f9f..e21f1fd 100644
--- a/src/rebar_packages.erl
+++ b/src/rebar_packages.erl
@@ -2,7 +2,9 @@
-export([get_packages/1
,registry/1
+ ,package_dir/1
,check_registry/3
+ ,registry_checksum/2
,find_highest_matching/3]).
-export_type([package/0]).
@@ -15,8 +17,7 @@
-spec get_packages(rebar_state:t()) -> {rebar_dict(), rebar_digraph()}.
get_packages(State) ->
- RebarDir = rebar_dir:global_cache_dir(State),
- RegistryDir = filename:join(RebarDir, "packages"),
+ RegistryDir = package_dir(State),
DictFile = filename:join(RegistryDir, "dict"),
Edges = filename:join(RegistryDir, "edges"),
Vertices = filename:join(RegistryDir, "vertices"),
@@ -42,8 +43,7 @@ get_packages(State) ->
end.
registry(State) ->
- Dir = rebar_dir:global_cache_dir(State),
- RegistryDir = filename:join(Dir, "packages"),
+ RegistryDir = package_dir(State),
HexFile = filename:join(RegistryDir, "registry"),
case ets:file2tab(HexFile) of
{ok, T} ->
@@ -53,6 +53,17 @@ registry(State) ->
error
end.
+package_dir(State) ->
+ CacheDir = rebar_dir:global_cache_dir(State),
+ CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN),
+ {ok, {_, _, Host, _, Path, _}} = http_uri:parse(CDN),
+ CDNHostPath = lists:reverse(string:tokens(Host, ".")),
+ CDNPath = tl(filename:split(Path)),
+ PackageDir = filename:join([CacheDir, "hex"] ++ CDNHostPath ++ CDNPath ++ ["packages"]),
+ ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")),
+ PackageDir.
+
+
check_registry(Pkg, Vsn, State) ->
case rebar_state:registry(State) of
{ok, T} ->
@@ -66,6 +77,15 @@ check_registry(Pkg, Vsn, State) ->
false
end.
+registry_checksum({pkg, Name, Vsn}, State) ->
+ {ok, Registry} = registry(State),
+ case ets:lookup(Registry, {Name, Vsn}) of
+ [{{_, _}, [_, Checksum | _]}] ->
+ Checksum;
+ [] ->
+ none
+ end.
+
%% Hex supports use of ~> to specify the version required for a dependency.
%% Since rebar3 requires exact versions to choose from we find the highest
%% available version of the dep that passes the constraint.
diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl
index 3b44fc8..59ce0dc 100644
--- a/src/rebar_pkg_resource.erl
+++ b/src/rebar_pkg_resource.erl
@@ -23,12 +23,99 @@ needs_update(Dir, {pkg, _Name, Vsn}) ->
true
end.
-download(Dir, {pkg, Name, Vsn}, State) ->
- TmpFile = filename:join(Dir, "package.tar.gz"),
- CDN = rebar_state:get(State, rebar_packages_cdn, "https://s3.amazonaws.com/s3.hex.pm/tarballs"),
- Url = string:join([CDN, binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>)], "/"),
- {ok, saved_to_file} = httpc:request(get, {Url, []}, [], [{stream, TmpFile}]),
- {tarball, TmpFile}.
+download(TmpDir, Pkg={pkg, Name, Vsn}, State) ->
+ CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN),
+ PackageDir = rebar_packages:package_dir(State),
+ Package = binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>),
+ CachePath = filename:join(PackageDir, Package),
+ Url = string:join([CDN, Package], "/"),
+ cached_download(TmpDir, CachePath, Pkg, Url, etag(CachePath), State).
+
+cached_download(TmpDir, CachePath, Pkg, Url, ETag, State) ->
+ case request(Url, ETag) of
+ {ok, cached} ->
+ serve_from_cache(TmpDir, CachePath, Pkg, State);
+ {ok, Body, NewETag} ->
+ serve_from_download(TmpDir, CachePath, Pkg, NewETag, Body, State);
+ error when ETag =/= false ->
+ ?DEBUG("Download ~s error, using ~s from cache", [Url, CachePath]),
+ serve_from_cache(TmpDir, CachePath, Pkg, State);
+ error ->
+ request_failed
+ end.
+
+serve_from_cache(TmpDir, CachePath, Pkg, State) ->
+ {Files, Contents, Version, Meta} = extract(TmpDir, CachePath),
+ case checksums(Pkg, Files, Contents, Version, Meta, State) of
+ {Chk, Chk, Chk} ->
+ ok = erl_tar:extract({binary, Contents}, [{cwd, TmpDir}, compressed]),
+ {ok, true};
+ {_Bin, Chk, Chk} ->
+ ?DEBUG("Checksums: registry: ~p, pkg: ~p", [Chk, _Bin]),
+ {failed_extract, CachePath};
+ {Chk, _Reg, Chk} ->
+ ?DEBUG("Checksums: registry: ~p, pkg: ~p", [_Reg, Chk]),
+ {bad_registry_checksum, CachePath};
+ {_Bin, _Reg, _Tar} ->
+ ?DEBUG("Checksums: registry: ~p, pkg: ~p, meta: ~p", [_Reg, _Bin, _Tar]),
+ {bad_checksum, CachePath}
+ end.
+
+serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State) ->
+ ?DEBUG("Writing ~p to cache at ~s", [Package, CachePath]),
+ file:write_file(CachePath, Binary),
+ case etag(CachePath) of
+ ETag ->
+ serve_from_cache(TmpDir, CachePath, Package, State);
+ FileETag ->
+ ?DEBUG("Downloaded file ~s ETag ~s doesn't match returned ETag ~s", [CachePath, ETag, FileETag]),
+ {bad_download, CachePath}
+ end.
+
+
+extract(TmpDir, CachePath) ->
+ ec_file:mkdir_p(TmpDir),
+ {ok, Files} = erl_tar:extract(CachePath, [memory]),
+ {"contents.tar.gz", Contents} = lists:keyfind("contents.tar.gz", 1, Files),
+ {"VERSION", Version} = lists:keyfind("VERSION", 1, Files),
+ {"metadata.config", Meta} = lists:keyfind("metadata.config", 1, Files),
+ {Files, Contents, Version, Meta}.
+
+checksums(Pkg, Files, Contents, Version, Meta, State) ->
+ Blob = <<Version/binary, Meta/binary, Contents/binary>>,
+ <<X:256/big-unsigned>> = crypto:hash(sha256, Blob),
+ BinChecksum = list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))),
+ RegistryChecksum = rebar_packages:registry_checksum(Pkg, State),
+ {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files),
+ {BinChecksum, RegistryChecksum, TarChecksum}.
make_vsn(_) ->
{error, "Replacing version of type pkg not supported."}.
+
+request(Url, ETag) ->
+ case httpc:request(get, {Url, [{"if-none-match", ETag} || ETag =/= false]},
+ [{relaxed, true}],
+ [{body_format, binary}]) of
+ {ok, {{_Version, 200, _Reason}, Headers, Body}} ->
+ ?DEBUG("Successfully downloaded ~s", [Url]),
+ {"etag", ETag1} = lists:keyfind("etag", 1, Headers),
+ {ok, Body, string:strip(ETag1, both, $")};
+ {ok, {{_Version, 304, _Reason}, _Headers, _Body}} ->
+ ?DEBUG("Cached copy of ~s still valid", [Url]),
+ {ok, cached};
+ {ok, {{_Version, Code, _Reason}, _Headers, _Body}} ->
+ ?DEBUG("Request to ~p failed: status code ~p", [Url, Code]),
+ error;
+ {error, Reason} ->
+ ?DEBUG("Request to ~p failed: ~p", [Url, Reason]),
+ error
+ end.
+
+etag(Path) ->
+ case file:read_file(Path) of
+ {ok, Binary} ->
+ <<X:128/big-unsigned-integer>> = crypto:hash(md5, Binary),
+ string:to_lower(lists:flatten(io_lib:format("~32.16.0b", [X])));
+ {error, _} ->
+ false
+ end.
diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl
index d4370de..eb51d8d 100644
--- a/src/rebar_prv_common_test.erl
+++ b/src/rebar_prv_common_test.erl
@@ -303,7 +303,7 @@ copy(State, Dir) ->
compile_dir(State, Dir, OutDir) ->
NewState = replace_src_dirs(State, [Dir]),
- ok = rebar_erlc_compiler:compile(NewState, rebar_dir:base_dir(State), OutDir),
+ ok = rebar_erlc_compiler:compile(NewState, rebar_state:dir(State), OutDir),
ok = maybe_cover_compile(State, Dir),
OutDir.
diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl
index bc0a113..ba49532 100644
--- a/src/rebar_prv_install_deps.erl
+++ b/src/rebar_prv_install_deps.erl
@@ -151,7 +151,7 @@ handle_deps(Profile, State0, Deps, Upgrade, Locks) ->
,{Packages, Graph}),
update_pkg_deps(Profile, Packages, PkgDeps1
- ,Graph, Upgrade, Seen, State2)
+ ,Graph, Upgrade, Seen, State2, Locks)
end,
AllDeps = lists:ukeymerge(2
@@ -184,7 +184,7 @@ find_cycles(Apps) ->
cull_compile(TopSortedDeps, ProjectApps) ->
lists:dropwhile(fun not_needs_compile/1, TopSortedDeps -- ProjectApps).
-update_pkg_deps(Profile, Packages, PkgDeps, Graph, Upgrade, Seen, State) ->
+update_pkg_deps(Profile, Packages, PkgDeps, Graph, Upgrade, Seen, State, Locks) ->
case PkgDeps of
[] -> %% No pkg deps
{[], State};
@@ -196,13 +196,16 @@ update_pkg_deps(Profile, Packages, PkgDeps, Graph, Upgrade, Seen, State) ->
{ok, Solution, []} ->
Solution;
{ok, Solution, Discarded} ->
- [warn_skip_pkg(Pkg, State) || Pkg <- Discarded],
+ [warn_skip_pkg(Pkg, State) || Pkg <- Discarded, not(pkg_locked(Pkg, Locks))],
Solution
end,
- update_pkg_deps(Profile, S, Packages, Upgrade, Seen, State)
+ update_pkg_deps(Profile, S, Packages, Upgrade, Seen, State, Locks)
end.
-update_pkg_deps(Profile, Pkgs, Packages, Upgrade, Seen, State) ->
+pkg_locked({Name, _}, Locks) ->
+ false =/= lists:keyfind(Name, 1, Locks).
+
+update_pkg_deps(Profile, Pkgs, Packages, Upgrade, Seen, State, _Locks) ->
%% Create app_info record for each pkg dep
DepsDir = profile_dep_dir(State, Profile),
{Solved, _, State1}
@@ -563,10 +566,10 @@ fetch_app(AppInfo, AppDir, State) ->
?INFO("Fetching ~s (~p)", [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)]),
Source = rebar_app_info:source(AppInfo),
case rebar_fetch:download_source(AppDir, Source, State) of
- {error, Reason} ->
- throw(Reason);
- Result ->
- Result
+ true ->
+ true;
+ Error ->
+ throw(Error)
end.
update_app_info(AppInfo) ->
@@ -587,10 +590,10 @@ maybe_upgrade(AppInfo, AppDir, true, State) ->
true ->
?INFO("Upgrading ~s", [rebar_app_info:name(AppInfo)]),
case rebar_fetch:download_source(AppDir, Source, State) of
- {error, Reason} ->
- throw(Reason);
- Result ->
- Result
+ true ->
+ true;
+ Error ->
+ throw(Error)
end;
false ->
?INFO("No upgrade needed for ~s", [rebar_app_info:name(AppInfo)]),
@@ -610,7 +613,7 @@ parse_goal(Name, Constraint) ->
warn_skip_deps(AppInfo, State) ->
Msg = "Skipping ~s (from ~p) as an app of the same name "
- "has already been fetched~n",
+ "has already been fetched",
Args = [rebar_app_info:name(AppInfo),
rebar_app_info:source(AppInfo)],
case rebar_state:get(State, deps_error_on_conflict, false) of
@@ -620,7 +623,7 @@ warn_skip_deps(AppInfo, State) ->
warn_skip_pkg({Name, Source}, State) ->
Msg = "Skipping ~s (version ~s from package index) as an app of the same "
- "name has already been fetched~n",
+ "name has already been fetched",
Args = [Name, Source],
case rebar_state:get(State, deps_error_on_conflict, false) of
false -> ?WARN(Msg, Args);
diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl
index 942b386..dfb719a 100644
--- a/src/rebar_prv_update.erl
+++ b/src/rebar_prv_update.erl
@@ -35,8 +35,7 @@ init(State) ->
do(State) ->
?INFO("Updating package index...", []),
try
- Dir = rebar_dir:global_cache_dir(State),
- RegistryDir = filename:join(Dir, "packages"),
+ RegistryDir = rebar_packages:package_dir(State),
filelib:ensure_dir(filename:join(RegistryDir, "dummy")),
HexFile = filename:join(RegistryDir, "registry"),
TmpDir = ec_file:insecure_mkdtemp(),
@@ -64,8 +63,7 @@ format_error(package_index_write) ->
"Failed to write package index.".
write_registry(Dict, {digraph, Edges, Vertices, Neighbors, _}, State) ->
- Dir = rebar_dir:global_cache_dir(State),
- RegistryDir = filename:join(Dir, "packages"),
+ RegistryDir = rebar_packages:package_dir(State),
filelib:ensure_dir(filename:join(RegistryDir, "dummy")),
ets:tab2file(Edges, filename:join(RegistryDir, "edges")),
ets:tab2file(Vertices, filename:join(RegistryDir, "vertices")),
diff --git a/src/rebar_resource.erl b/src/rebar_resource.erl
index 7c58135..cdce7a8 100644
--- a/src/rebar_resource.erl
+++ b/src/rebar_resource.erl
@@ -14,23 +14,6 @@
-type location() :: string().
-type ref() :: any().
--ifdef(no_callback_support).
-
-%% In the case where R14 or lower is being used to compile the system
-%% we need to export a behaviour info
--export([behaviour_info/1]).
-
--spec behaviour_info(atom()) -> [{atom(), arity()}] | undefined.
-behaviour_info(callbacks) ->
- [{lock, 2},
- {download, 3},
- {needs_update,2},
- {make_vsn, 1}];
-behaviour_info(_) ->
- undefined.
-
--else.
-
-callback lock(file:filename_all(), tuple()) ->
rebar_resource:resource().
-callback download(file:filename_all(), tuple(), rebar_state:t()) ->
@@ -39,5 +22,3 @@ behaviour_info(_) ->
boolean().
-callback make_vsn(file:filename_all()) ->
{plain, string()} | {error, string()}.
-
--endif.
generated by cgit v1.1 at 2024-12-26 16:38:19 +0000