Update from Upstream

1 files changed, 243 insertions, 171 deletions

diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl
index 5817817..823b7fc 100644
--- a/src/rebar_pkg_resource.erl
+++ b/src/rebar_pkg_resource.erl
@@ -2,208 +2,280 @@
 %% ex: ts=4 sw=4 et
 -module(rebar_pkg_resource).
 
--behaviour(rebar_resource).
+-behaviour(rebar_resource_v2).
 
--export([lock/2
-        ,download/3
-        ,needs_update/2
-        ,make_vsn/1]).
+-export([init/2,
+         lock/2,
+         download/4,
+         download/5,
+         needs_update/2,
+         make_vsn/2,
+         format_error/1]).
 
--export([request/2
-        ,etag/1
-        ,ssl_opts/1]).
+-ifdef(TEST).
+%% exported for test purposes
+-export([store_etag_in_cache/2]).
+-endif.
 
 -include("rebar.hrl").
--include_lib("public_key/include/OTP-PUB-KEY.hrl").
+-include_lib("providers/include/providers.hrl").
 
-lock(_AppDir, Source) ->
-    Source.
+-type package() :: {pkg, binary(), binary(), binary(), rebar_hex_repos:repo()}.
 
-needs_update(Dir, {pkg, _Name, Vsn, _Hash}) ->
-    [AppInfo] = rebar_app_discover:find_apps([Dir], all),
-    case rebar_app_info:original_vsn(AppInfo) =:= ec_cnv:to_list(Vsn) of
+%%==============================================================================
+%% Public API
+%%==============================================================================
+
+-spec init(atom(), rebar_state:t()) -> {ok, rebar_resource_v2:resource()}.
+init(Type, State) ->
+    {ok, Vsn} = application:get_key(rebar, vsn),
+    BaseConfig = #{http_adapter => hex_http_httpc,
+                   http_user_agent_fragment =>
+                       <<"(rebar3/", (list_to_binary(Vsn))/binary, ") (httpc)">>,
+                   http_adapter_config => #{profile => rebar}},
+    Repos = rebar_hex_repos:from_state(BaseConfig, State),
+    Resource = rebar_resource_v2:new(Type, ?MODULE, #{repos => Repos,
+                                                      base_config => BaseConfig}),
+    {ok, Resource}.
+
+
+
+-spec lock(AppInfo, ResourceState) -> Res when
+      AppInfo :: rebar_app_info:t(),
+      ResourceState :: rebar_resource_v2:resource_state(),
+      Res :: {atom(), string(), any(), binary()}.
+lock(AppInfo, _) ->
+    {pkg, Name, Vsn, Hash, _RepoConfig} = rebar_app_info:source(AppInfo),
+    {pkg, Name, Vsn, Hash}.
+
+%%------------------------------------------------------------------------------
+%% @doc
+%% Return true if the stored version of the pkg is older than the current
+%% version.
+%% @end
+%%------------------------------------------------------------------------------
+-spec needs_update(AppInfo, ResourceState) -> Res when
+      AppInfo :: rebar_app_info:t(),
+      ResourceState :: rebar_resource_v2:resource_state(),
+      Res :: boolean().
+needs_update(AppInfo, _) ->
+    {pkg, _Name, Vsn, _Hash, _} = rebar_app_info:source(AppInfo),
+    case rebar_utils:to_binary(rebar_app_info:original_vsn(AppInfo)) =:= rebar_utils:to_binary(Vsn) of
         true ->
             false;
         false ->
             true
     end.
 
-download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State) ->
-    CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN),
-    {ok, PackageDir} = rebar_packages:package_dir(State),
-    Package = binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>),
-    CachePath = filename:join(PackageDir, Package),
-    case rebar_utils:url_append_path(CDN, filename:join(?REMOTE_PACKAGE_DIR, Package)) of
-        {ok, Url} ->
-            cached_download(TmpDir, CachePath, Pkg, Url, etag(CachePath), State);
-        _ ->
-            {fetch_fail, Name, Vsn}
+%%------------------------------------------------------------------------------
+%% @doc
+%% Download the given pkg.
+%% @end
+%%------------------------------------------------------------------------------
+-spec download(TmpDir, AppInfo, State, ResourceState) -> Res when
+      TmpDir :: file:name(),
+      AppInfo :: rebar_app_info:t(),
+      ResourceState :: rebar_resource_v2:resource_state(),
+      State :: rebar_state:t(),
+      Res :: ok | {error,_}.
+download(TmpDir, AppInfo, State, ResourceState) ->
+    case download(TmpDir, rebar_app_info:source(AppInfo), State, ResourceState, true) of
+        ok ->
+            ok;
+        Error ->
+            {error, Error}
     end.
 
-cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, Url, ETag, State) ->
-    case request(Url, ETag) of
-        {ok, cached} ->
-            ?INFO("Version cached at ~s is up to date, reusing it", [CachePath]),
-            serve_from_cache(TmpDir, CachePath, Pkg, State);
-        {ok, Body, NewETag} ->
-            ?INFO("Downloaded package, caching at ~s", [CachePath]),
-            serve_from_download(TmpDir, CachePath, Pkg, NewETag, Body, State);
-        error when ETag =/= false ->
-            ?INFO("Download error, using cached file at ~s", [CachePath]),
-            serve_from_cache(TmpDir, CachePath, Pkg, State);
-        error ->
-            {fetch_fail, Name, Vsn}
-    end.
-
-serve_from_cache(TmpDir, CachePath, Pkg, State) ->
-    {Files, Contents, Version, Meta} = extract(TmpDir, CachePath),
-    case checksums(Pkg, Files, Contents, Version, Meta, State) of
-        {Chk, Chk, Chk, Chk} ->
-            ok = erl_tar:extract({binary, Contents}, [{cwd, TmpDir}, compressed]),
-            {ok, true};
-        {_Hash, Chk, Chk, Chk} ->
-            ?DEBUG("Expected hash ~p does not match checksums ~p", [_Hash, Chk]),
-            {unexpected_hash, CachePath, _Hash, Chk};
-        {Chk, _Bin, Chk, Chk} ->
-            ?DEBUG("Checksums: registry: ~p, pkg: ~p", [Chk, _Bin]),
-            {failed_extract, CachePath};
-        {Chk, Chk, _Reg, Chk} ->
-            ?DEBUG("Checksums: registry: ~p, pkg: ~p", [_Reg, Chk]),
-            {bad_registry_checksum, CachePath};
-        {_Hash, _Bin, _Reg, _Tar} ->
-            ?DEBUG("Checksums: expected: ~p, registry: ~p, pkg: ~p, meta: ~p", [_Hash, _Reg, _Bin, _Tar]),
-            {bad_checksum, CachePath}
-    end.
-
-serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State) ->
-    ?DEBUG("Writing ~p to cache at ~s", [Package, CachePath]),
-    file:write_file(CachePath, Binary),
-    case etag(CachePath) of
-        ETag ->
-            serve_from_cache(TmpDir, CachePath, Package, State);
-        FileETag ->
-            ?DEBUG("Downloaded file ~s ETag ~s doesn't match returned ETag ~s", [CachePath, ETag, FileETag]),
-            {bad_download, CachePath}
+%%------------------------------------------------------------------------------
+%% @doc
+%% Download the given pkg. The etag belonging to the pkg file will be updated
+%% only if the UpdateEtag is true and the ETag returned from the hexpm server
+%% is different.
+%% @end
+%%------------------------------------------------------------------------------
+-spec download(TmpDir, Pkg, State, ResourceState, UpdateETag) -> Res when
+      TmpDir :: file:name(),
+      Pkg :: package(),
+      State :: rebar_state:t(),
+      ResourceState:: rebar_resource_v2:resource_state(),
+      UpdateETag :: boolean(),
+      Res :: ok | {error,_} | {unexpected_hash, string(), integer(), integer()} |
+             {fetch_fail, binary(), binary()}.
+download(TmpDir, Pkg={pkg, Name, Vsn, _Hash, Repo}, State, _ResourceState, UpdateETag) ->
+    {ok, PackageDir} = rebar_packages:package_dir(Repo, State),
+    Package = binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>),
+    ETagFile = binary_to_list(<<Name/binary, "-", Vsn/binary, ".etag">>),
+    CachePath = filename:join(PackageDir, Package),
+    ETagPath = filename:join(PackageDir, ETagFile),
+    case cached_download(TmpDir, CachePath, Pkg, etag(CachePath, ETagPath), ETagPath, UpdateETag) of
+        {bad_registry_checksum, Expected, Found} ->
+            %% checksum comparison failed. in case this is from a modified cached package
+            %% overwrite the etag if it exists so it is not relied on again
+            store_etag_in_cache(ETagPath, <<>>),
+            ?PRV_ERROR({bad_registry_checksum, Name, Vsn, Expected, Found});
+        Result ->
+            Result
     end.
 
-
-extract(TmpDir, CachePath) ->
-    ec_file:mkdir_p(TmpDir),
-    {ok, Files} = erl_tar:extract(CachePath, [memory]),
-    {"contents.tar.gz", Contents} = lists:keyfind("contents.tar.gz", 1, Files),
-    {"VERSION", Version} = lists:keyfind("VERSION", 1, Files),
-    {"metadata.config", Meta} = lists:keyfind("metadata.config", 1, Files),
-    {Files, Contents, Version, Meta}.
-
-checksums(Pkg={pkg, _Name, _Vsn, Hash}, Files, Contents, Version, Meta, State) ->
-    Blob = <<Version/binary, Meta/binary, Contents/binary>>,
-    <<X:256/big-unsigned>> = crypto:hash(sha256, Blob),
-    BinChecksum = list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))),
-    RegistryChecksum = rebar_packages:registry_checksum(Pkg, State),
-    {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files),
-    {Hash, BinChecksum, RegistryChecksum, TarChecksum}.
-
-make_vsn(_) ->
+%%------------------------------------------------------------------------------
+%% @doc
+%% Implementation of rebar_resource make_vsn callback.
+%% Returns {error, string()} as this operation is not supported for pkg sources.
+%% @end
+%%------------------------------------------------------------------------------
+-spec make_vsn(AppInfo, ResourceState) -> Res when
+      AppInfo :: rebar_app_info:t(),
+      ResourceState :: rebar_resource_v2:resource_state(),
+      Res :: {'error', string()}.
+make_vsn(_, _) ->
     {error, "Replacing version of type pkg not supported."}.
 
-request(Url, ETag) ->
-    case httpc:request(get, {Url, [{"if-none-match", ETag} || ETag =/= false]++[{"User-Agent", rebar_utils:user_agent()}]},
-                       [{ssl, ssl_opts(Url)}, {relaxed, true}],
-                       [{body_format, binary}],
-                       rebar) of
-        {ok, {{_Version, 200, _Reason}, Headers, Body}} ->
-            ?DEBUG("Successfully downloaded ~s", [Url]),
-            {"etag", ETag1} = lists:keyfind("etag", 1, Headers),
-            {ok, Body, string:strip(ETag1, both, $")};
-        {ok, {{_Version, 304, _Reason}, _Headers, _Body}} ->
-            ?DEBUG("Cached copy of ~s still valid", [Url]),
+format_error({bad_registry_checksum, Name, Vsn, Expected, Found}) ->
+    io_lib:format("The checksum for package at ~ts-~ts (~ts) does not match the "
+                  "checksum expected from the registry (~ts). "
+                  "Run `rebar3 do unlock ~ts, update` and then try again.",
+                  [Name, Vsn, Found, Expected, Name]).
+
+%%------------------------------------------------------------------------------
+%% @doc
+%% Download the pkg belonging to the given address. If the etag of the pkg
+%% is the same what we stored in the etag file previously return {ok, cached},
+%% if the file has changed (so the etag is not the same anymore) return
+%% {ok, Contents, NewEtag}, otherwise if some error occured return error.
+%% @end
+%%------------------------------------------------------------------------------
+-spec request(rebar_hex_repos:repo(), binary(), binary(), false | binary())
+             -> {ok, cached} | {ok, binary(), binary()} | error.
+request(Config, Name, Version, ETag) ->
+    Config1 = Config#{http_etag => ETag},
+    try hex_repo:get_tarball(Config1, Name, Version) of
+        {ok, {200, #{<<"etag">> := ETag1}, Tarball}} ->
+            {ok, Tarball, ETag1};
+        {ok, {304, _Headers, _}} ->
             {ok, cached};
-        {ok, {{_Version, Code, _Reason}, _Headers, _Body}} ->
-            ?DEBUG("Request to ~p failed: status code ~p", [Url, Code]),
+        {ok, {Code, _Headers, _Body}} ->
+            ?DEBUG("Request for package ~s-~s failed: status code ~p", [Name, Version, Code]),
             error;
         {error, Reason} ->
-            ?DEBUG("Request to ~p failed: ~p", [Url, Reason]),
+            ?DEBUG("Request for package ~s-~s failed: ~p", [Name, Version, Reason]),
+            error
+    catch
+        _:Exception ->
+            ?DEBUG("hex_repo:get_tarball failed: ~p", [Exception]),
             error
     end.
 
-etag(Path) ->
-    case file:read_file(Path) of
-        {ok, Binary} ->
-            <<X:128/big-unsigned-integer>> = crypto:hash(md5, Binary),
-            string:to_lower(lists:flatten(io_lib:format("~32.16.0b", [X])));
+%%------------------------------------------------------------------------------
+%% @doc
+%% Read the etag belonging to the pkg file from the cache directory. The etag
+%% is stored in a separate file when the etag belonging to the package is
+%% returned from the hexpm server. The name is package-vsn.etag.
+%% @end
+%%------------------------------------------------------------------------------
+-spec etag(PackagePath, ETagPath) -> Res when
+      PackagePath :: file:name(),
+      ETagPath :: file:name(),
+      Res :: binary().
+etag(PackagePath, ETagPath) ->
+    case file:read_file(ETagPath) of
+        {ok, Bin} ->
+            %% just in case a user deleted a cached package but not its etag
+            %% verify the package is also there, and if not, ignore the etag
+            case filelib:is_file(PackagePath) of
+                true ->
+                    Bin;
+                false ->
+                    <<>>
+            end;
         {error, _} ->
-            false
+            <<>>
     end.
 
-ssl_opts(Url) ->
-    case get_ssl_config() of
-        ssl_verify_enabled ->
-            ssl_opts(ssl_verify_enabled, Url);
-        ssl_verify_disabled ->
-            [{verify, verify_none}]
-    end.
-
-ssl_opts(ssl_verify_enabled, Url) ->
-    case check_ssl_version() of
-        true ->
-            {ok, {_, _, Hostname, _, _, _}} = http_uri:parse(ec_cnv:to_list(Url)),
-            VerifyFun = {fun ssl_verify_hostname:verify_fun/3, [{check_hostname, Hostname}]},
-            CACerts = certifi:cacerts(),
-            [{verify, verify_peer}, {depth, 2}, {cacerts, CACerts}
-            ,{partial_chain, fun partial_chain/1}, {verify_fun, VerifyFun}];
-        false ->
-            ?WARN("Insecure HTTPS request (peer verification disabled), please update to OTP 17.4 or later", []),
-            [{verify, verify_none}]
-    end.
+%%------------------------------------------------------------------------------
+%% @doc
+%% Store the given etag in the .cache folder. The name is pakckage-vsn.etag.
+%% @end
+%%------------------------------------------------------------------------------
+-spec store_etag_in_cache(File, ETag) -> Res when
+      File :: file:name(),
+      ETag :: binary(),
+      Res :: ok.
+store_etag_in_cache(Path, ETag) ->
+    _ = file:write_file(Path, ETag).
 
-partial_chain(Certs) ->
-    Certs1 = [{Cert, public_key:pkix_decode_cert(Cert, otp)} || Cert <- Certs],
-    CACerts = certifi:cacerts(),
-    CACerts1 = [public_key:pkix_decode_cert(Cert, otp) || Cert <- CACerts],
-
-    case ec_lists:find(fun({_, Cert}) ->
-                              check_cert(CACerts1, Cert)
-                       end, Certs1) of
-        {ok, Trusted} ->
-            {trusted_ca, element(1, Trusted)};
-        _ ->
-            unknown_ca
+%%%=============================================================================
+%%% Private functions
+%%%=============================================================================
+-spec cached_download(TmpDir, CachePath, Pkg, ETag, ETagPath, UpdateETag) -> Res when
+      TmpDir :: file:name(),
+      CachePath :: file:name(),
+      Pkg :: package(),
+      ETag :: binary(),
+      ETagPath :: file:name(),
+      UpdateETag :: boolean(),
+      Res :: ok | {unexpected_hash, integer(), integer()} | {fetch_fail, binary(), binary()}.
+cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash, RepoConfig}, ETag,
+                ETagPath, UpdateETag) ->
+    case request(RepoConfig, Name, Vsn, ETag) of
+        {ok, cached} ->
+            ?INFO("Version cached at ~ts is up to date, reusing it", [CachePath]),
+            serve_from_cache(TmpDir, CachePath, Pkg);
+        {ok, Body, NewETag} ->
+            ?INFO("Downloaded package, caching at ~ts", [CachePath]),
+            maybe_store_etag_in_cache(UpdateETag, ETagPath, NewETag),
+            serve_from_download(TmpDir, CachePath, Pkg, Body);
+        error when ETag =/= <<>> ->
+            store_etag_in_cache(ETagPath, ETag),
+            ?INFO("Download error, using cached file at ~ts", [CachePath]),
+            serve_from_cache(TmpDir, CachePath, Pkg);
+        error ->
+            {fetch_fail, Name, Vsn}
     end.
 
-extract_public_key_info(Cert) ->
-    ((Cert#'OTPCertificate'.tbsCertificate)#'OTPTBSCertificate'.subjectPublicKeyInfo).
-
-check_cert(CACerts, Cert) ->
-    lists:any(fun(CACert) ->
-                      extract_public_key_info(CACert) == extract_public_key_info(Cert)
-              end, CACerts).
+-spec serve_from_cache(TmpDir, CachePath, Pkg) -> Res when
+      TmpDir :: file:name(),
+      CachePath :: file:name(),
+      Pkg :: package(),
+      Res :: ok | {error,_} | {bad_registry_checksum, integer(), integer()}.
+serve_from_cache(TmpDir, CachePath, Pkg) ->
+    {ok, Binary} = file:read_file(CachePath),
+    serve_from_memory(TmpDir, Binary, Pkg).
 
-check_ssl_version() ->
-    case application:get_key(ssl, vsn) of
-        {ok, Vsn} ->
-            parse_vsn(Vsn) >= {5, 3, 6};
-        _ ->
-            false
+-spec serve_from_memory(TmpDir, Tarball, Package) -> Res when
+      TmpDir :: file:name(),
+      Tarball :: binary(),
+      Package :: package(),
+      Res :: ok | {error,_} | {bad_registry_checksum, integer(), integer()}.
+serve_from_memory(TmpDir, Binary, {pkg, _Name, _Vsn, Hash, _RepoConfig}) ->
+    RegistryChecksum = list_to_integer(binary_to_list(Hash), 16),
+    case hex_tarball:unpack(Binary, TmpDir) of
+        {ok, #{checksum := <<Checksum:256/big-unsigned>>}} when RegistryChecksum =/= Checksum ->
+            ?DEBUG("Expected hash ~64.16.0B does not match checksum of fetched package ~64.16.0B",
+                   [RegistryChecksum, Checksum]),
+            {bad_registry_checksum, RegistryChecksum, Checksum};
+        {ok, #{checksum := <<RegistryChecksum:256/big-unsigned>>}} ->
+            ok;
+        {error, Reason} ->
+            {error, {hex_tarball, Reason}}
     end.
 
-get_ssl_config() ->
-    GlobalConfigFile = rebar_dir:global_config(),
-    Config = rebar_config:consult_file(GlobalConfigFile),
-    case proplists:get_value(ssl_verify, Config, []) of
-        false ->
-            ssl_verify_disabled;
-        _ ->
-            ssl_verify_enabled
-    end.
+-spec serve_from_download(TmpDir, CachePath, Package, Binary) -> Res when
+      TmpDir :: file:name(),
+      CachePath :: file:name(),
+      Package :: package(),
+      Binary :: binary(),
+      Res :: ok | {error,_}.
+serve_from_download(TmpDir, CachePath, Package, Binary) ->
+    ?DEBUG("Writing ~p to cache at ~ts", [Package, CachePath]),
+    file:write_file(CachePath, Binary),
+    serve_from_memory(TmpDir, Binary, Package).
 
-parse_vsn(Vsn) ->
-    version_pad(string:tokens(Vsn, ".-")).
-
-version_pad([Major]) ->
-    {list_to_integer(Major), 0, 0};
-version_pad([Major, Minor]) ->
-    {list_to_integer(Major), list_to_integer(Minor), 0};
-version_pad([Major, Minor, Patch]) ->
-    {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)};
-version_pad([Major, Minor, Patch | _]) ->
-    {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}.
+-spec maybe_store_etag_in_cache(UpdateETag, Path, ETag) -> Res when
+      UpdateETag :: boolean(),
+      Path :: file:name(),
+      ETag :: binary(),
+      Res :: ok.
+maybe_store_etag_in_cache(false = _UpdateETag, _Path, _ETag) ->
+    ok;
+maybe_store_etag_in_cache(true = _UpdateETag, Path, ETag) ->
+    store_etag_in_cache(Path, ETag).