From 3da4cc222197e01886d3baaeca7a380e02ff3125 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 25 Nov 2016 21:32:43 -0500 Subject: Type specifications and edocs improvements Includes improvments and function documentation for all modules (in alphabetical order) up to rebar_core, and may have included more in other modules as I saw fit to dig and understand more of the internals. --- src/rebar_pkg_resource.erl | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src/rebar_pkg_resource.erl') diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 5817817..8a84ce3 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -135,6 +135,9 @@ etag(Path) -> false end. +%% @doc returns the SSL options adequate for the project based on +%% its configuration, including for validation of certs. +-spec ssl_opts(string()) -> [term()]. ssl_opts(Url) -> case get_ssl_config() of ssl_verify_enabled -> @@ -143,6 +146,9 @@ ssl_opts(Url) -> [{verify, verify_none}] end. +%% @doc returns the SSL options adequate for the project based on +%% its configuration, including for validation of certs. +-spec ssl_opts(atom(), string()) -> [term()]. ssl_opts(ssl_verify_enabled, Url) -> case check_ssl_version() of true -> -- cgit v1.1 From ffc2cf98d390a35cf3f61c7c0bf5f26e3552fd2c Mon Sep 17 00:00:00 2001 From: Artem Pervin Date: Wed, 30 Nov 2016 13:45:06 -0500 Subject: 1394: added fix for rebar_utils, moved setting of http_options into init_config, added unit tests --- src/rebar_pkg_resource.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src/rebar_pkg_resource.erl') diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 5817817..8e1713d 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -107,8 +107,10 @@ make_vsn(_) -> {error, "Replacing version of type pkg not supported."}. request(Url, ETag) -> + HttpOptions = [{ssl, ssl_opts(Url)}, {relaxed, true} | rebar_utils:get_proxy_auth()], + case httpc:request(get, {Url, [{"if-none-match", ETag} || ETag =/= false]++[{"User-Agent", rebar_utils:user_agent()}]}, - [{ssl, ssl_opts(Url)}, {relaxed, true}], + HttpOptions, [{body_format, binary}], rebar) of {ok, {{_Version, 200, _Reason}, Headers, Body}} -> -- cgit v1.1 From 963c49f5eb9ab5b34e1843fb43305743720917ac Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sun, 6 Aug 2017 07:26:21 -0400 Subject: Unicode support in all the places This is done through 3 main change groups: - replacing `~s` by `~ts` in format strings, so that strings that contain unicode are properly printed rather than crashing - adding the `unicode` argument to all function of the `re` module to ensure transformations on strings containing unicode data are valid instead of crashing (see issue #1302) - replacing `ec_cnv:to_binary/1` and `ec_cnv:to_list/1` with matching functions in `rebar_utils`. The last point has been done, rather than modifying and updating erlware commons, because binary and list conversions can be a contentious subject. For example, if what is being handled is actually bytes from a given binary stream, then forcing a byte-oriented interpretation of the data can corrupt it. As such, it does not appear safe to modify erlware commons' conversion functions since it may not be safe for all its users. Instead, rebar3 reimplements a subset of them (only converting atoms and chardata, ignoring numbers) with the explicit purpose of handling unicode string data. Tests were left as unchanged as possible. This may impact the ability to run rebar3's own suites in a unicode path, but respects a principle of least change for such a large patch. --- src/rebar_pkg_resource.erl | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'src/rebar_pkg_resource.erl') diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 88419bd..d588f24 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -21,7 +21,7 @@ lock(_AppDir, Source) -> needs_update(Dir, {pkg, _Name, Vsn, _Hash}) -> [AppInfo] = rebar_app_discover:find_apps([Dir], all), - case rebar_app_info:original_vsn(AppInfo) =:= ec_cnv:to_list(Vsn) of + case rebar_app_info:original_vsn(AppInfo) =:= rebar_utils:to_list(Vsn) of true -> false; false -> @@ -43,13 +43,13 @@ download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State) -> cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, Url, ETag, State) -> case request(Url, ETag) of {ok, cached} -> - ?INFO("Version cached at ~s is up to date, reusing it", [CachePath]), + ?INFO("Version cached at ~ts is up to date, reusing it", [CachePath]), serve_from_cache(TmpDir, CachePath, Pkg, State); {ok, Body, NewETag} -> - ?INFO("Downloaded package, caching at ~s", [CachePath]), + ?INFO("Downloaded package, caching at ~ts", [CachePath]), serve_from_download(TmpDir, CachePath, Pkg, NewETag, Body, State); error when ETag =/= false -> - ?INFO("Download error, using cached file at ~s", [CachePath]), + ?INFO("Download error, using cached file at ~ts", [CachePath]), serve_from_cache(TmpDir, CachePath, Pkg, State); error -> {fetch_fail, Name, Vsn} @@ -76,13 +76,13 @@ serve_from_cache(TmpDir, CachePath, Pkg, State) -> end. serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State) -> - ?DEBUG("Writing ~p to cache at ~s", [Package, CachePath]), + ?DEBUG("Writing ~p to cache at ~ts", [Package, CachePath]), file:write_file(CachePath, Binary), case etag(CachePath) of ETag -> serve_from_cache(TmpDir, CachePath, Package, State); FileETag -> - ?DEBUG("Downloaded file ~s ETag ~s doesn't match returned ETag ~s", [CachePath, ETag, FileETag]), + ?DEBUG("Downloaded file ~ts ETag ~ts doesn't match returned ETag ~ts", [CachePath, ETag, FileETag]), {bad_download, CachePath} end. @@ -114,11 +114,11 @@ request(Url, ETag) -> [{body_format, binary}], rebar) of {ok, {{_Version, 200, _Reason}, Headers, Body}} -> - ?DEBUG("Successfully downloaded ~s", [Url]), + ?DEBUG("Successfully downloaded ~ts", [Url]), {"etag", ETag1} = lists:keyfind("etag", 1, Headers), {ok, Body, string:strip(ETag1, both, $")}; {ok, {{_Version, 304, _Reason}, _Headers, _Body}} -> - ?DEBUG("Cached copy of ~s still valid", [Url]), + ?DEBUG("Cached copy of ~ts still valid", [Url]), {ok, cached}; {ok, {{_Version, Code, _Reason}, _Headers, _Body}} -> ?DEBUG("Request to ~p failed: status code ~p", [Url, Code]), @@ -154,7 +154,7 @@ ssl_opts(Url) -> ssl_opts(ssl_verify_enabled, Url) -> case check_ssl_version() of true -> - {ok, {_, _, Hostname, _, _, _}} = http_uri:parse(ec_cnv:to_list(Url)), + {ok, {_, _, Hostname, _, _, _}} = http_uri:parse(rebar_utils:to_list(Url)), VerifyFun = {fun ssl_verify_hostname:verify_fun/3, [{check_hostname, Hostname}]}, CACerts = certifi:cacerts(), [{verify, verify_peer}, {depth, 2}, {cacerts, CACerts} -- cgit v1.1 From 2d5cd9c00cfa4e58066b48beee4057fdd52cc7be Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 1 Nov 2017 19:38:03 -0400 Subject: OTP-21 readiness, Full Unicode support This replaces all deprecated function usage by alternative ones based on a version switch enacted at compile time, preventing all warnings. This will likely introduce some possible runtime errors in using a Rebar3 compiled on OTP-20 or OTP-21 back in versions 19 and earlier, but we can't really work around that. A bunch of dependencies have been updated to support OTP-21 without warnings as well. --- src/rebar_pkg_resource.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src/rebar_pkg_resource.erl') diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index d588f24..60ad8f9 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -98,7 +98,7 @@ extract(TmpDir, CachePath) -> checksums(Pkg={pkg, _Name, _Vsn, Hash}, Files, Contents, Version, Meta, State) -> Blob = <>, <> = crypto:hash(sha256, Blob), - BinChecksum = list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))), + BinChecksum = list_to_binary(rebar_string:uppercase(lists:flatten(io_lib:format("~64.16.0b", [X])))), RegistryChecksum = rebar_packages:registry_checksum(Pkg, State), {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files), {Hash, BinChecksum, RegistryChecksum, TarChecksum}. @@ -116,7 +116,7 @@ request(Url, ETag) -> {ok, {{_Version, 200, _Reason}, Headers, Body}} -> ?DEBUG("Successfully downloaded ~ts", [Url]), {"etag", ETag1} = lists:keyfind("etag", 1, Headers), - {ok, Body, string:strip(ETag1, both, $")}; + {ok, Body, rebar_string:trim(ETag1, both, [$"])}; {ok, {{_Version, 304, _Reason}, _Headers, _Body}} -> ?DEBUG("Cached copy of ~ts still valid", [Url]), {ok, cached}; @@ -132,7 +132,7 @@ etag(Path) -> case file:read_file(Path) of {ok, Binary} -> <> = crypto:hash(md5, Binary), - string:to_lower(lists:flatten(io_lib:format("~32.16.0b", [X]))); + rebar_string:lowercase(lists:flatten(io_lib:format("~32.16.0b", [X]))); {error, _} -> false end. @@ -205,7 +205,7 @@ get_ssl_config() -> end. parse_vsn(Vsn) -> - version_pad(string:tokens(Vsn, ".-")). + version_pad(rebar_string:lexemes(Vsn, ".-")). version_pad([Major]) -> {list_to_integer(Major), 0, 0}; -- cgit v1.1 From e754621e73db9533f9670124206b91869b2b35b6 Mon Sep 17 00:00:00 2001 From: tothlac Date: Mon, 9 Apr 2018 17:45:51 +0200 Subject: (#1741): Fix quotes in etag values --- src/rebar_pkg_resource.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/rebar_pkg_resource.erl') diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 60ad8f9..7d42ccd 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -109,7 +109,8 @@ make_vsn(_) -> request(Url, ETag) -> HttpOptions = [{ssl, ssl_opts(Url)}, {relaxed, true} | rebar_utils:get_proxy_auth()], - case httpc:request(get, {Url, [{"if-none-match", ETag} || ETag =/= false]++[{"User-Agent", rebar_utils:user_agent()}]}, + case httpc:request(get, {Url, [{"if-none-match", "\"" ++ ETag ++ "\""} || ETag =/= false]++ + [{"User-Agent", rebar_utils:user_agent()}]}, HttpOptions, [{body_format, binary}], rebar) of -- cgit v1.1 From a03da56fb39bddcf6b5ef75cb5b5e40b5b5ab1d9 Mon Sep 17 00:00:00 2001 From: Laszlo Toth Date: Wed, 11 Apr 2018 14:03:57 +0200 Subject: (#1743): store the etags in cache files --- src/rebar_pkg_resource.erl | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) (limited to 'src/rebar_pkg_resource.erl') diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 7d42ccd..f77a81a 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -6,6 +6,7 @@ -export([lock/2 ,download/3 + ,download/4 ,needs_update/2 ,make_vsn/1]). @@ -13,6 +14,8 @@ ,etag/1 ,ssl_opts/1]). +-export([store_etag_in_cache/2]). + -include("rebar.hrl"). -include_lib("public_key/include/OTP-PUB-KEY.hrl"). @@ -28,27 +31,34 @@ needs_update(Dir, {pkg, _Name, Vsn, _Hash}) -> true end. -download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State) -> +download(TmpDir, Pkg, State) -> + download(TmpDir, Pkg, State, true). + +download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State, UpdateETag) -> CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), {ok, PackageDir} = rebar_packages:package_dir(State), Package = binary_to_list(<>), + ETagFile = binary_to_list(<>), CachePath = filename:join(PackageDir, Package), + ETagPath = filename:join(PackageDir, ETagFile), case rebar_utils:url_append_path(CDN, filename:join(?REMOTE_PACKAGE_DIR, Package)) of {ok, Url} -> - cached_download(TmpDir, CachePath, Pkg, Url, etag(CachePath), State); + cached_download(TmpDir, CachePath, Pkg, Url, etag(ETagPath), State, ETagPath, UpdateETag); _ -> {fetch_fail, Name, Vsn} end. -cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, Url, ETag, State) -> +cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, Url, ETag, State, ETagPath, UpdateETag) -> case request(Url, ETag) of {ok, cached} -> ?INFO("Version cached at ~ts is up to date, reusing it", [CachePath]), serve_from_cache(TmpDir, CachePath, Pkg, State); {ok, Body, NewETag} -> ?INFO("Downloaded package, caching at ~ts", [CachePath]), - serve_from_download(TmpDir, CachePath, Pkg, NewETag, Body, State); + maybe_store_etag_in_cache(UpdateETag, ETagPath, NewETag), + serve_from_download(TmpDir, CachePath, Pkg, NewETag, Body, State, ETagPath); error when ETag =/= false -> + store_etag_in_cache(ETagPath, ETag), ?INFO("Download error, using cached file at ~ts", [CachePath]), serve_from_cache(TmpDir, CachePath, Pkg, State); error -> @@ -75,10 +85,10 @@ serve_from_cache(TmpDir, CachePath, Pkg, State) -> {bad_checksum, CachePath} end. -serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State) -> +serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State, ETagPath) -> ?DEBUG("Writing ~p to cache at ~ts", [Package, CachePath]), file:write_file(CachePath, Binary), - case etag(CachePath) of + case etag(ETagPath) of ETag -> serve_from_cache(TmpDir, CachePath, Package, State); FileETag -> @@ -86,7 +96,6 @@ serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State) -> {bad_download, CachePath} end. - extract(TmpDir, CachePath) -> ec_file:mkdir_p(TmpDir), {ok, Files} = erl_tar:extract(CachePath, [memory]), @@ -131,9 +140,8 @@ request(Url, ETag) -> etag(Path) -> case file:read_file(Path) of - {ok, Binary} -> - <> = crypto:hash(md5, Binary), - rebar_string:lowercase(lists:flatten(io_lib:format("~32.16.0b", [X]))); + {ok, Bin} -> + binary_to_list(Bin); {error, _} -> false end. @@ -216,3 +224,11 @@ version_pad([Major, Minor, Patch]) -> {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}; version_pad([Major, Minor, Patch | _]) -> {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}. + +maybe_store_etag_in_cache(false = _UpdateETag, _Path, _ETag) -> + ok; +maybe_store_etag_in_cache(true = _UpdateETag, Path, ETag) -> + store_etag_in_cache(Path, ETag). + +store_etag_in_cache(Path, ETag) -> + ok = file:write_file(Path, ETag). -- cgit v1.1 From 97c48e86bb7ca8969fd19c999f55259e88d4a6db Mon Sep 17 00:00:00 2001 From: Laszlo Toth Date: Wed, 11 Apr 2018 17:03:34 +0200 Subject: (#1743): Add specs for dialyzer --- src/rebar_pkg_resource.erl | 121 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 118 insertions(+), 3 deletions(-) (limited to 'src/rebar_pkg_resource.erl') diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index f77a81a..8806c8b 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -19,9 +19,26 @@ -include("rebar.hrl"). -include_lib("public_key/include/OTP-PUB-KEY.hrl"). +-type cached_result() :: {'bad_checksum',string()} | + {'bad_registry_checksum',string()} | + {'failed_extract',string()} | + {'ok','true'} | + {'unexpected_hash',string(),_,binary()}. + +-type download_result() :: {bad_download, binary() | string()} | + {fetch_fail, _, _} | cached_result(). + +-spec lock(AppDir, Source) -> Res when + AppDir :: file:name(), + Source :: tuple(), + Res :: {atom(), string(), any()}. lock(_AppDir, Source) -> Source. +-spec needs_update(Dir, Pkg) -> Res when + Dir :: file:name(), + Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + Res :: boolean(). needs_update(Dir, {pkg, _Name, Vsn, _Hash}) -> [AppInfo] = rebar_app_discover:find_apps([Dir], all), case rebar_app_info:original_vsn(AppInfo) =:= rebar_utils:to_list(Vsn) of @@ -31,9 +48,20 @@ needs_update(Dir, {pkg, _Name, Vsn, _Hash}) -> true end. +-spec download(TmpDir, Pkg, State) -> Res when + TmpDir :: file:name(), + Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + State :: rebar_state:t(), + Res :: {'error',_} | {'ok',_} | {'tarball',binary() | string()}. download(TmpDir, Pkg, State) -> download(TmpDir, Pkg, State, true). +-spec download(TmpDir, Pkg, State, UpdateETag) -> Res when + TmpDir :: file:name(), + Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + State :: rebar_state:t(), + UpdateETag :: boolean(), + Res :: download_result(). download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State, UpdateETag) -> CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), {ok, PackageDir} = rebar_packages:package_dir(State), @@ -48,6 +76,16 @@ download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State, UpdateETag) -> {fetch_fail, Name, Vsn} end. +-spec cached_download(TmpDir, CachePath, Pkg, Url, ETag, State, ETagPath, UpdateETag) -> Res when + TmpDir :: file:name(), + CachePath :: file:name(), + Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + Url :: string(), + ETag :: false | string(), + State :: rebar_state:t(), + ETagPath :: file:name(), + UpdateETag :: boolean(), + Res :: download_result(). cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, Url, ETag, State, ETagPath, UpdateETag) -> case request(Url, ETag) of {ok, cached} -> @@ -65,6 +103,12 @@ cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, Url, ETag, State {fetch_fail, Name, Vsn} end. +-spec serve_from_cache(TmpDir, CachePath, Pkg, State) -> Res when + TmpDir :: file:name(), + CachePath :: file:name(), + Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + State :: rebar_state:t(), + Res :: cached_result(). serve_from_cache(TmpDir, CachePath, Pkg, State) -> {Files, Contents, Version, Meta} = extract(TmpDir, CachePath), case checksums(Pkg, Files, Contents, Version, Meta, State) of @@ -85,6 +129,15 @@ serve_from_cache(TmpDir, CachePath, Pkg, State) -> {bad_checksum, CachePath} end. +-spec serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State, ETagPath) -> Res when + TmpDir :: file:name(), + CachePath :: file:name(), + Package :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + ETag :: string(), + Binary :: binary(), + State :: rebar_state:t(), + ETagPath :: file:name(), + Res :: download_result(). serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State, ETagPath) -> ?DEBUG("Writing ~p to cache at ~ts", [Package, CachePath]), file:write_file(CachePath, Binary), @@ -96,6 +149,14 @@ serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State, ETagPath) - {bad_download, CachePath} end. +-spec extract(TmpDir, CachePath) -> Res when + TmpDir :: file:name(), + CachePath :: file:name(), + Res :: {Files, Contents, Version, Meta}, + Files :: list({file:name(), binary()}), + Contents :: binary(), + Version :: binary(), + Meta :: binary(). extract(TmpDir, CachePath) -> ec_file:mkdir_p(TmpDir), {ok, Files} = erl_tar:extract(CachePath, [memory]), @@ -104,6 +165,18 @@ extract(TmpDir, CachePath) -> {"metadata.config", Meta} = lists:keyfind("metadata.config", 1, Files), {Files, Contents, Version, Meta}. +-spec checksums(Pkg, Files, Contents, Version, Meta, State) -> Res when + Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + Files :: list({file:name(), binary()}), + Contents :: binary(), + Version :: binary(), + Meta :: binary(), + State :: rebar_state:t(), + Res :: {Hash, BinChecksum, RegistryChecksum, TarChecksum}, + Hash :: binary(), + BinChecksum :: binary(), + RegistryChecksum :: any(), + TarChecksum :: binary(). checksums(Pkg={pkg, _Name, _Vsn, Hash}, Files, Contents, Version, Meta, State) -> Blob = <>, <> = crypto:hash(sha256, Blob), @@ -112,12 +185,18 @@ checksums(Pkg={pkg, _Name, _Vsn, Hash}, Files, Contents, Version, Meta, State) - {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files), {Hash, BinChecksum, RegistryChecksum, TarChecksum}. +-spec make_vsn(Vsn) -> Res when + Vsn :: any(), + Res :: {'error',[1..255,...]}. make_vsn(_) -> {error, "Replacing version of type pkg not supported."}. +-spec request(Url, ETag) -> Res when + Url :: string(), + ETag :: false | string(), + Res :: 'error' | {ok, cached} | {ok, any(), string()}. request(Url, ETag) -> HttpOptions = [{ssl, ssl_opts(Url)}, {relaxed, true} | rebar_utils:get_proxy_auth()], - case httpc:request(get, {Url, [{"if-none-match", "\"" ++ ETag ++ "\""} || ETag =/= false]++ [{"User-Agent", rebar_utils:user_agent()}]}, HttpOptions, @@ -138,6 +217,9 @@ request(Url, ETag) -> error end. +-spec etag(Path) -> Res when + Path :: file:name(), + Res :: false | string(). etag(Path) -> case file:read_file(Path) of {ok, Bin} -> @@ -148,7 +230,9 @@ etag(Path) -> %% @doc returns the SSL options adequate for the project based on %% its configuration, including for validation of certs. --spec ssl_opts(string()) -> [term()]. +-spec ssl_opts(Url) -> Res when + Url :: string(), + Res :: proplists:proplist(). ssl_opts(Url) -> case get_ssl_config() of ssl_verify_enabled -> @@ -159,7 +243,10 @@ ssl_opts(Url) -> %% @doc returns the SSL options adequate for the project based on %% its configuration, including for validation of certs. --spec ssl_opts(atom(), string()) -> [term()]. +-spec ssl_opts(Enabled, Url) -> Res when + Enabled :: atom(), + Url :: string(), + Res :: proplists:proplist(). ssl_opts(ssl_verify_enabled, Url) -> case check_ssl_version() of true -> @@ -173,6 +260,9 @@ ssl_opts(ssl_verify_enabled, Url) -> [{verify, verify_none}] end. +-spec partial_chain(Certs) -> Res when + Certs :: list(any()), + Res :: unknown_ca | {trusted_ca, any()}. partial_chain(Certs) -> Certs1 = [{Cert, public_key:pkix_decode_cert(Cert, otp)} || Cert <- Certs], CACerts = certifi:cacerts(), @@ -187,14 +277,23 @@ partial_chain(Certs) -> unknown_ca end. +-spec extract_public_key_info(Cert) -> Res when + Cert :: #'OTPCertificate'{tbsCertificate::#'OTPTBSCertificate'{}}, + Res :: any(). extract_public_key_info(Cert) -> ((Cert#'OTPCertificate'.tbsCertificate)#'OTPTBSCertificate'.subjectPublicKeyInfo). +-spec check_cert(CACerts, Cert) -> Res when + CACerts :: list(any()), + Cert :: any(), + Res :: boolean(). check_cert(CACerts, Cert) -> lists:any(fun(CACert) -> extract_public_key_info(CACert) == extract_public_key_info(Cert) end, CACerts). +-spec check_ssl_version() -> + boolean(). check_ssl_version() -> case application:get_key(ssl, vsn) of {ok, Vsn} -> @@ -203,6 +302,8 @@ check_ssl_version() -> false end. +-spec get_ssl_config() -> + ssl_verify_disabled | ssl_verify_enabled. get_ssl_config() -> GlobalConfigFile = rebar_dir:global_config(), Config = rebar_config:consult_file(GlobalConfigFile), @@ -213,9 +314,14 @@ get_ssl_config() -> ssl_verify_enabled end. +-spec parse_vsn(Vsn) -> Res when + Vsn :: string(), + Res :: {integer(), integer(), integer()}. parse_vsn(Vsn) -> version_pad(rebar_string:lexemes(Vsn, ".-")). +-spec version_pad(list(nonempty_string())) -> Res when + Res :: {integer(), integer(), integer()}. version_pad([Major]) -> {list_to_integer(Major), 0, 0}; version_pad([Major, Minor]) -> @@ -225,10 +331,19 @@ version_pad([Major, Minor, Patch]) -> version_pad([Major, Minor, Patch | _]) -> {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}. +-spec maybe_store_etag_in_cache(UpdateETag, Path, ETag) -> Res when + UpdateETag :: boolean(), + Path :: file:name(), + ETag :: string(), + Res :: ok. maybe_store_etag_in_cache(false = _UpdateETag, _Path, _ETag) -> ok; maybe_store_etag_in_cache(true = _UpdateETag, Path, ETag) -> store_etag_in_cache(Path, ETag). +-spec store_etag_in_cache(File, ETag) -> Res when + File :: file:name(), + ETag :: string(), + Res :: ok. store_etag_in_cache(Path, ETag) -> ok = file:write_file(Path, ETag). -- cgit v1.1 From 66143a65e0adb3ef9b4c46a4a5389940aa182494 Mon Sep 17 00:00:00 2001 From: Laszlo Toth Date: Thu, 12 Apr 2018 09:22:51 +0200 Subject: (#1743): Refactor rebar_pkg_resource, add documentation --- src/rebar_pkg_resource.erl | 233 +++++++++++++++++++++++++++++---------------- 1 file changed, 152 insertions(+), 81 deletions(-) (limited to 'src/rebar_pkg_resource.erl') diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 8806c8b..16ec83f 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -14,6 +14,7 @@ ,etag/1 ,ssl_opts/1]). +%% Exported for ct -export([store_etag_in_cache/2]). -include("rebar.hrl"). @@ -28,6 +29,9 @@ -type download_result() :: {bad_download, binary() | string()} | {fetch_fail, _, _} | cached_result(). +%%============================================================================== +%% Public API +%%============================================================================== -spec lock(AppDir, Source) -> Res when AppDir :: file:name(), Source :: tuple(), @@ -35,6 +39,12 @@ lock(_AppDir, Source) -> Source. +%%------------------------------------------------------------------------------ +%% @doc +%% Return true if the stored version of the pkg is older than the current +%% version. +%% @end +%%------------------------------------------------------------------------------ -spec needs_update(Dir, Pkg) -> Res when Dir :: file:name(), Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, @@ -48,6 +58,11 @@ needs_update(Dir, {pkg, _Name, Vsn, _Hash}) -> true end. +%%------------------------------------------------------------------------------ +%% @doc +%% Download the given pkg. +%% @end +%%------------------------------------------------------------------------------ -spec download(TmpDir, Pkg, State) -> Res when TmpDir :: file:name(), Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, @@ -56,6 +71,13 @@ needs_update(Dir, {pkg, _Name, Vsn, _Hash}) -> download(TmpDir, Pkg, State) -> download(TmpDir, Pkg, State, true). +%%------------------------------------------------------------------------------ +%% @doc +%% Download the given pkg. The etag belonging to the pkg file will be updated +%% only if the UpdateEtag is true and the ETag returned from the hexpm server +%% is different. +%% @end +%%------------------------------------------------------------------------------ -spec download(TmpDir, Pkg, State, UpdateETag) -> Res when TmpDir :: file:name(), Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, @@ -69,14 +91,113 @@ download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State, UpdateETag) -> ETagFile = binary_to_list(<>), CachePath = filename:join(PackageDir, Package), ETagPath = filename:join(PackageDir, ETagFile), - case rebar_utils:url_append_path(CDN, filename:join(?REMOTE_PACKAGE_DIR, Package)) of + case rebar_utils:url_append_path(CDN, filename:join(?REMOTE_PACKAGE_DIR, + Package)) of {ok, Url} -> - cached_download(TmpDir, CachePath, Pkg, Url, etag(ETagPath), State, ETagPath, UpdateETag); + cached_download(TmpDir, CachePath, Pkg, Url, etag(ETagPath), State, + ETagPath, UpdateETag); _ -> {fetch_fail, Name, Vsn} end. --spec cached_download(TmpDir, CachePath, Pkg, Url, ETag, State, ETagPath, UpdateETag) -> Res when +%%------------------------------------------------------------------------------ +%% @doc +%% Implementation of rebar_resource make_vsn callback. +%% Returns {error, string()} as this operation is not supported for pkg sources. +%% @end +%%------------------------------------------------------------------------------ +-spec make_vsn(Vsn) -> Res when + Vsn :: any(), + Res :: {'error',[1..255,...]}. +make_vsn(_) -> + {error, "Replacing version of type pkg not supported."}. + +%%------------------------------------------------------------------------------ +%% @doc +%% Download the pkg belonging to the given address. If the etag of the pkg +%% is the same what we stored in the etag file previously return {ok, cached}, +%% if the file has changed (so the etag is not the same anymore) return +%% {ok, Contents, NewEtag}, otherwise if some error occured return error. +%% @end +%%------------------------------------------------------------------------------ +-spec request(Url, ETag) -> Res when + Url :: string(), + ETag :: false | string(), + Res :: 'error' | {ok, cached} | {ok, any(), string()}. +request(Url, ETag) -> + HttpOptions = [{ssl, ssl_opts(Url)}, + {relaxed, true} | rebar_utils:get_proxy_auth()], + case httpc:request(get, {Url, [{"if-none-match", "\"" ++ ETag ++ "\""} + || ETag =/= false] ++ + [{"User-Agent", rebar_utils:user_agent()}]}, + HttpOptions, [{body_format, binary}], rebar) of + {ok, {{_Version, 200, _Reason}, Headers, Body}} -> + ?DEBUG("Successfully downloaded ~ts", [Url]), + {"etag", ETag1} = lists:keyfind("etag", 1, Headers), + {ok, Body, rebar_string:trim(ETag1, both, [$"])}; + {ok, {{_Version, 304, _Reason}, _Headers, _Body}} -> + ?DEBUG("Cached copy of ~ts still valid", [Url]), + {ok, cached}; + {ok, {{_Version, Code, _Reason}, _Headers, _Body}} -> + ?DEBUG("Request to ~p failed: status code ~p", [Url, Code]), + error; + {error, Reason} -> + ?DEBUG("Request to ~p failed: ~p", [Url, Reason]), + error + end. + +%%------------------------------------------------------------------------------ +%% @doc +%% Read the etag belonging to the pkg file from the cache directory. The etag +%% is stored in a separate file when the etag belonging to the package is +%% returned from the hexpm server. The name is package-vsn.etag. +%% @end +%%------------------------------------------------------------------------------ +-spec etag(Path) -> Res when + Path :: file:name(), + Res :: false | string(). +etag(Path) -> + case file:read_file(Path) of + {ok, Bin} -> + binary_to_list(Bin); + {error, _} -> + false + end. + +%%------------------------------------------------------------------------------ +%% @doc +%% Return the SSL options adequate for the project based on +%% its configuration, including for validation of certs. +%% @end +%%------------------------------------------------------------------------------ +-spec ssl_opts(Url) -> Res when + Url :: string(), + Res :: proplists:proplist(). +ssl_opts(Url) -> + case get_ssl_config() of + ssl_verify_enabled -> + ssl_opts(ssl_verify_enabled, Url); + ssl_verify_disabled -> + [{verify, verify_none}] + end. + +%%------------------------------------------------------------------------------ +%% @doc +%% Store the given etag in the .cache folder. The name is pakckage-vsn.etag. +%% @end +%%------------------------------------------------------------------------------ +-spec store_etag_in_cache(File, ETag) -> Res when + File :: file:name(), + ETag :: string(), + Res :: ok. +store_etag_in_cache(Path, ETag) -> + ok = file:write_file(Path, ETag). + +%%%============================================================================= +%%% Private functions +%%%============================================================================= +-spec cached_download(TmpDir, CachePath, Pkg, Url, ETag, State, ETagPath, + UpdateETag) -> Res when TmpDir :: file:name(), CachePath :: file:name(), Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, @@ -86,7 +207,8 @@ download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State, UpdateETag) -> ETagPath :: file:name(), UpdateETag :: boolean(), Res :: download_result(). -cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, Url, ETag, State, ETagPath, UpdateETag) -> +cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, Url, ETag, + State, ETagPath, UpdateETag) -> case request(Url, ETag) of {ok, cached} -> ?INFO("Version cached at ~ts is up to date, reusing it", [CachePath]), @@ -94,7 +216,8 @@ cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, Url, ETag, State {ok, Body, NewETag} -> ?INFO("Downloaded package, caching at ~ts", [CachePath]), maybe_store_etag_in_cache(UpdateETag, ETagPath, NewETag), - serve_from_download(TmpDir, CachePath, Pkg, NewETag, Body, State, ETagPath); + serve_from_download(TmpDir, CachePath, Pkg, NewETag, Body, State, + ETagPath); error when ETag =/= false -> store_etag_in_cache(ETagPath, ETag), ?INFO("Download error, using cached file at ~ts", [CachePath]), @@ -125,11 +248,13 @@ serve_from_cache(TmpDir, CachePath, Pkg, State) -> ?DEBUG("Checksums: registry: ~p, pkg: ~p", [_Reg, Chk]), {bad_registry_checksum, CachePath}; {_Hash, _Bin, _Reg, _Tar} -> - ?DEBUG("Checksums: expected: ~p, registry: ~p, pkg: ~p, meta: ~p", [_Hash, _Reg, _Bin, _Tar]), + ?DEBUG("Checksums: expected: ~p, registry: ~p, pkg: ~p, meta: ~p", + [_Hash, _Reg, _Bin, _Tar]), {bad_checksum, CachePath} end. --spec serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State, ETagPath) -> Res when +-spec serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State, + ETagPath) -> Res when TmpDir :: file:name(), CachePath :: file:name(), Package :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, @@ -145,7 +270,8 @@ serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State, ETagPath) - ETag -> serve_from_cache(TmpDir, CachePath, Package, State); FileETag -> - ?DEBUG("Downloaded file ~ts ETag ~ts doesn't match returned ETag ~ts", [CachePath, ETag, FileETag]), + ?DEBUG("Downloaded file ~ts ETag ~ts doesn't match returned ETag ~ts", + [CachePath, ETag, FileETag]), {bad_download, CachePath} end. @@ -180,69 +306,19 @@ extract(TmpDir, CachePath) -> checksums(Pkg={pkg, _Name, _Vsn, Hash}, Files, Contents, Version, Meta, State) -> Blob = <>, <> = crypto:hash(sha256, Blob), - BinChecksum = list_to_binary(rebar_string:uppercase(lists:flatten(io_lib:format("~64.16.0b", [X])))), + BinChecksum = list_to_binary( + rebar_string:uppercase( + lists:flatten(io_lib:format("~64.16.0b", [X])))), RegistryChecksum = rebar_packages:registry_checksum(Pkg, State), {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files), {Hash, BinChecksum, RegistryChecksum, TarChecksum}. --spec make_vsn(Vsn) -> Res when - Vsn :: any(), - Res :: {'error',[1..255,...]}. -make_vsn(_) -> - {error, "Replacing version of type pkg not supported."}. - --spec request(Url, ETag) -> Res when - Url :: string(), - ETag :: false | string(), - Res :: 'error' | {ok, cached} | {ok, any(), string()}. -request(Url, ETag) -> - HttpOptions = [{ssl, ssl_opts(Url)}, {relaxed, true} | rebar_utils:get_proxy_auth()], - case httpc:request(get, {Url, [{"if-none-match", "\"" ++ ETag ++ "\""} || ETag =/= false]++ - [{"User-Agent", rebar_utils:user_agent()}]}, - HttpOptions, - [{body_format, binary}], - rebar) of - {ok, {{_Version, 200, _Reason}, Headers, Body}} -> - ?DEBUG("Successfully downloaded ~ts", [Url]), - {"etag", ETag1} = lists:keyfind("etag", 1, Headers), - {ok, Body, rebar_string:trim(ETag1, both, [$"])}; - {ok, {{_Version, 304, _Reason}, _Headers, _Body}} -> - ?DEBUG("Cached copy of ~ts still valid", [Url]), - {ok, cached}; - {ok, {{_Version, Code, _Reason}, _Headers, _Body}} -> - ?DEBUG("Request to ~p failed: status code ~p", [Url, Code]), - error; - {error, Reason} -> - ?DEBUG("Request to ~p failed: ~p", [Url, Reason]), - error - end. - --spec etag(Path) -> Res when - Path :: file:name(), - Res :: false | string(). -etag(Path) -> - case file:read_file(Path) of - {ok, Bin} -> - binary_to_list(Bin); - {error, _} -> - false - end. - -%% @doc returns the SSL options adequate for the project based on -%% its configuration, including for validation of certs. --spec ssl_opts(Url) -> Res when - Url :: string(), - Res :: proplists:proplist(). -ssl_opts(Url) -> - case get_ssl_config() of - ssl_verify_enabled -> - ssl_opts(ssl_verify_enabled, Url); - ssl_verify_disabled -> - [{verify, verify_none}] - end. - -%% @doc returns the SSL options adequate for the project based on +%%------------------------------------------------------------------------------ +%% @doc +%% Return the SSL options adequate for the project based on %% its configuration, including for validation of certs. +%% @end +%%------------------------------------------------------------------------------ -spec ssl_opts(Enabled, Url) -> Res when Enabled :: atom(), Url :: string(), @@ -250,13 +326,16 @@ ssl_opts(Url) -> ssl_opts(ssl_verify_enabled, Url) -> case check_ssl_version() of true -> - {ok, {_, _, Hostname, _, _, _}} = http_uri:parse(rebar_utils:to_list(Url)), - VerifyFun = {fun ssl_verify_hostname:verify_fun/3, [{check_hostname, Hostname}]}, + {ok, {_, _, Hostname, _, _, _}} = + http_uri:parse(rebar_utils:to_list(Url)), + VerifyFun = {fun ssl_verify_hostname:verify_fun/3, + [{check_hostname, Hostname}]}, CACerts = certifi:cacerts(), - [{verify, verify_peer}, {depth, 2}, {cacerts, CACerts} - ,{partial_chain, fun partial_chain/1}, {verify_fun, VerifyFun}]; + [{verify, verify_peer}, {depth, 2}, {cacerts, CACerts}, + {partial_chain, fun partial_chain/1}, {verify_fun, VerifyFun}]; false -> - ?WARN("Insecure HTTPS request (peer verification disabled), please update to OTP 17.4 or later", []), + ?WARN("Insecure HTTPS request (peer verification disabled), " + "please update to OTP 17.4 or later", []), [{verify, verify_none}] end. @@ -267,9 +346,8 @@ partial_chain(Certs) -> Certs1 = [{Cert, public_key:pkix_decode_cert(Cert, otp)} || Cert <- Certs], CACerts = certifi:cacerts(), CACerts1 = [public_key:pkix_decode_cert(Cert, otp) || Cert <- CACerts], - case ec_lists:find(fun({_, Cert}) -> - check_cert(CACerts1, Cert) + check_cert(CACerts1, Cert) end, Certs1) of {ok, Trusted} -> {trusted_ca, element(1, Trusted)}; @@ -293,7 +371,7 @@ check_cert(CACerts, Cert) -> end, CACerts). -spec check_ssl_version() -> - boolean(). + boolean(). check_ssl_version() -> case application:get_key(ssl, vsn) of {ok, Vsn} -> @@ -340,10 +418,3 @@ maybe_store_etag_in_cache(false = _UpdateETag, _Path, _ETag) -> ok; maybe_store_etag_in_cache(true = _UpdateETag, Path, ETag) -> store_etag_in_cache(Path, ETag). - --spec store_etag_in_cache(File, ETag) -> Res when - File :: file:name(), - ETag :: string(), - Res :: ok. -store_etag_in_cache(Path, ETag) -> - ok = file:write_file(Path, ETag). -- cgit v1.1 From b46f56a158b2a02b05881e4ac3b53be7f1d46011 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 20 Apr 2018 18:48:12 -0400 Subject: Do not die on a pkg etag cache write fail We can probably still move ahead without a cache if we must --- src/rebar_pkg_resource.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/rebar_pkg_resource.erl') diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index f77a81a..2fb1868 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -231,4 +231,4 @@ maybe_store_etag_in_cache(true = _UpdateETag, Path, ETag) -> store_etag_in_cache(Path, ETag). store_etag_in_cache(Path, ETag) -> - ok = file:write_file(Path, ETag). + _ = file:write_file(Path, ETag). -- cgit v1.1 From 56b7d88975aa8da6446857cfd92de0825024bf63 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 13 Sep 2018 19:36:00 -0600 Subject: support for hex v2, multiple repository fetching, private organizations (#1884) * update to hex_core for hex-v2 repo support (#1865) * update to hex_core for hex-v2 repo support This patch adds only single repo hex-v2 support through hex_core. Packages no longer filtered out by buildtool metadata and the package index is updated per-package instead of fetched as one large ets dump. * tell travis to also build hex_core branch * support list of repos for hex packages (#1866) * support list of repos for hex packages repos are defined under the hex key in rebar configs. They can be defined at the top level of a project or globally, but not in profiles and the repos configured in dependencies are also ignored. Searching for packages involves first checking for a match in the local repo index cache, in the order repos are defined. If not found each repo is checked through the hex api for any known versions of the package and the first repo with a version that fits the constraint is used. * add {repos, replace, []} for overriding the global & default repos * add hex auth handling for repos (#1874) auth token are kept in a hex.config file that is modified by the rebar3 hex plugin. Repo names that have a : separating a parent and child are considered organizations. The parent repo's auth will be included with the child. So an organization named hexpm:rebar3_test will include any hexpm auth tokens found in the rebar3_test organization's configuration. * move packages to top level of of hexpm cache dir (#1876) * move packages to top level of of hexpm cache dir * append organization name to parent's repo_url when parsing repos * only eval config scripts and apply overrides once per app (#1879) * only eval config scripts and apply overrides once per app * move new resource behaviour to rebar_resource_v2 and keep v1 * cleanup use of rebar_resource module and unused functions * cleanup error messages and unused code * when discovering apps support mix packages as unbuilt apps (#1882) * use hex_core tarball unpacking support in pkg resource (#1883) * use hex_core tarball unpacking support in pkg resource * ignore etag if package doesn't exist and delete if checksum fails * add back tests for bad package checksums * improve bad registry checksum error message --- src/rebar_pkg_resource.erl | 437 ++++++++++++++++----------------------------- 1 file changed, 149 insertions(+), 288 deletions(-) (limited to 'src/rebar_pkg_resource.erl') diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 2cf167e..97eabb6 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -2,42 +2,51 @@ %% ex: ts=4 sw=4 et -module(rebar_pkg_resource). --behaviour(rebar_resource). +-behaviour(rebar_resource_v2). --export([lock/2 - ,download/3 - ,download/4 - ,needs_update/2 - ,make_vsn/1]). +-export([init/2, + lock/2, + download/4, + download/5, + needs_update/2, + make_vsn/2, + format_error/1]). --export([request/2 - ,etag/1 - ,ssl_opts/1]). - -%% Exported for ct +-ifdef(TEST). +%% exported for test purposes -export([store_etag_in_cache/2]). +-endif. -include("rebar.hrl"). --include_lib("public_key/include/OTP-PUB-KEY.hrl"). - --type cached_result() :: {'bad_checksum',string()} | - {'bad_registry_checksum',string()} | - {'failed_extract',string()} | - {'ok','true'} | - {'unexpected_hash',string(),_,binary()}. +-include_lib("providers/include/providers.hrl"). --type download_result() :: {bad_download, binary() | string()} | - {fetch_fail, _, _} | cached_result(). +-type package() :: {pkg, binary(), binary(), binary(), rebar_hex_repos:repo()}. %%============================================================================== %% Public API %%============================================================================== --spec lock(AppDir, Source) -> Res when - AppDir :: file:name(), - Source :: tuple(), - Res :: {atom(), string(), any()}. -lock(_AppDir, Source) -> - Source. + +-spec init(atom(), rebar_state:t()) -> {ok, rebar_resource_v2:resource()}. +init(Type, State) -> + {ok, Vsn} = application:get_key(rebar, vsn), + BaseConfig = #{http_adapter => hex_http_httpc, + http_user_agent_fragment => + <<"(rebar3/", (list_to_binary(Vsn))/binary, ") (httpc)">>, + http_adapter_config => #{profile => rebar}}, + Repos = rebar_hex_repos:from_state(BaseConfig, State), + Resource = rebar_resource_v2:new(Type, ?MODULE, #{repos => Repos, + base_config => BaseConfig}), + {ok, Resource}. + + + +-spec lock(AppInfo, ResourceState) -> Res when + AppInfo :: rebar_app_info:t(), + ResourceState :: rebar_resource_v2:resource_state(), + Res :: {atom(), string(), any(), binary()}. +lock(AppInfo, _) -> + {pkg, Name, Vsn, Hash, _RepoConfig} = rebar_app_info:source(AppInfo), + {pkg, Name, Vsn, Hash}. %%------------------------------------------------------------------------------ %% @doc @@ -45,13 +54,13 @@ lock(_AppDir, Source) -> %% version. %% @end %%------------------------------------------------------------------------------ --spec needs_update(Dir, Pkg) -> Res when - Dir :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, +-spec needs_update(AppInfo, ResourceState) -> Res when + AppInfo :: rebar_app_info:t(), + ResourceState :: rebar_resource_v2:resource_state(), Res :: boolean(). -needs_update(Dir, {pkg, _Name, Vsn, _Hash}) -> - [AppInfo] = rebar_app_discover:find_apps([Dir], all), - case rebar_app_info:original_vsn(AppInfo) =:= rebar_utils:to_list(Vsn) of +needs_update(AppInfo, _) -> + {pkg, _Name, Vsn, _Hash, _} = rebar_app_info:source(AppInfo), + case rebar_app_info:original_vsn(AppInfo) =:= rebar_utils:to_binary(Vsn) of true -> false; false -> @@ -63,13 +72,19 @@ needs_update(Dir, {pkg, _Name, Vsn, _Hash}) -> %% Download the given pkg. %% @end %%------------------------------------------------------------------------------ --spec download(TmpDir, Pkg, State) -> Res when +-spec download(TmpDir, AppInfo, State, ResourceState) -> Res when TmpDir :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + AppInfo :: rebar_app_info:t(), + ResourceState :: rebar_resource_v2:resource_state(), State :: rebar_state:t(), - Res :: {'error',_} | {'ok',_} | {'tarball',binary() | string()}. -download(TmpDir, Pkg, State) -> - download(TmpDir, Pkg, State, true). + Res :: ok | {error,_}. +download(TmpDir, AppInfo, State, ResourceState) -> + case download(TmpDir, rebar_app_info:source(AppInfo), State, ResourceState, true) of + ok -> + ok; + Error -> + {error, Error} + end. %%------------------------------------------------------------------------------ %% @doc @@ -78,26 +93,28 @@ download(TmpDir, Pkg, State) -> %% is different. %% @end %%------------------------------------------------------------------------------ --spec download(TmpDir, Pkg, State, UpdateETag) -> Res when +-spec download(TmpDir, Pkg, State, ResourceState, UpdateETag) -> Res when TmpDir :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, + Pkg :: package(), State :: rebar_state:t(), + ResourceState:: rebar_resource_v2:resource_state(), UpdateETag :: boolean(), - Res :: download_result(). -download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State, UpdateETag) -> - CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), - {ok, PackageDir} = rebar_packages:package_dir(State), + Res :: ok | {error,_} | {unexpected_hash, string(), integer(), integer()} | + {fetch_fail, binary(), binary()}. +download(TmpDir, Pkg={pkg, Name, Vsn, _Hash, Repo}, State, _ResourceState, UpdateETag) -> + {ok, PackageDir} = rebar_packages:package_dir(Repo, State), Package = binary_to_list(<>), ETagFile = binary_to_list(<>), CachePath = filename:join(PackageDir, Package), ETagPath = filename:join(PackageDir, ETagFile), - case rebar_utils:url_append_path(CDN, filename:join(?REMOTE_PACKAGE_DIR, - Package)) of - {ok, Url} -> - cached_download(TmpDir, CachePath, Pkg, Url, etag(ETagPath), State, - ETagPath, UpdateETag); - _ -> - {fetch_fail, Name, Vsn} + case cached_download(TmpDir, CachePath, Pkg, etag(CachePath, ETagPath), ETagPath, UpdateETag) of + {bad_registry_checksum, Expected, Found} -> + %% checksum comparison failed. in case this is from a modified cached package + %% overwrite the etag if it exists so it is not relied on again + store_etag_in_cache(ETagPath, <<>>), + ?PRV_ERROR({bad_registry_checksum, Name, Vsn, Expected, Found}); + Result -> + Result end. %%------------------------------------------------------------------------------ @@ -106,12 +123,19 @@ download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State, UpdateETag) -> %% Returns {error, string()} as this operation is not supported for pkg sources. %% @end %%------------------------------------------------------------------------------ --spec make_vsn(Vsn) -> Res when - Vsn :: any(), - Res :: {'error',[1..255,...]}. -make_vsn(_) -> +-spec make_vsn(AppInfo, ResourceState) -> Res when + AppInfo :: rebar_app_info:t(), + ResourceState :: rebar_resource_v2:resource_state(), + Res :: {'error', string()}. +make_vsn(_, _) -> {error, "Replacing version of type pkg not supported."}. +format_error({bad_registry_checksum, Name, Vsn, Expected, Found}) -> + io_lib:format("The checksum for package at ~ts-~ts (~ts) does not match the " + "checksum expected from the registry (~ts). " + "Run `rebar3 do unlock ~ts, update` and then try again.", + [Name, Vsn, Found, Expected, Name]). + %%------------------------------------------------------------------------------ %% @doc %% Download the pkg belonging to the given address. If the etag of the pkg @@ -120,29 +144,24 @@ make_vsn(_) -> %% {ok, Contents, NewEtag}, otherwise if some error occured return error. %% @end %%------------------------------------------------------------------------------ --spec request(Url, ETag) -> Res when - Url :: string(), - ETag :: false | string(), - Res :: 'error' | {ok, cached} | {ok, any(), string()}. -request(Url, ETag) -> - HttpOptions = [{ssl, ssl_opts(Url)}, - {relaxed, true} | rebar_utils:get_proxy_auth()], - case httpc:request(get, {Url, [{"if-none-match", "\"" ++ ETag ++ "\""} - || ETag =/= false] ++ - [{"User-Agent", rebar_utils:user_agent()}]}, - HttpOptions, [{body_format, binary}], rebar) of - {ok, {{_Version, 200, _Reason}, Headers, Body}} -> - ?DEBUG("Successfully downloaded ~ts", [Url]), - {"etag", ETag1} = lists:keyfind("etag", 1, Headers), - {ok, Body, rebar_string:trim(ETag1, both, [$"])}; - {ok, {{_Version, 304, _Reason}, _Headers, _Body}} -> - ?DEBUG("Cached copy of ~ts still valid", [Url]), +-spec request(rebar_hex_repos:repo(), binary(), binary(), false | binary()) + -> {ok, cached} | {ok, binary(), binary()} | error. +request(Config, Name, Version, ETag) -> + Config1 = Config#{http_etag => ETag}, + try hex_repo:get_tarball(Config1, Name, Version) of + {ok, {200, #{<<"etag">> := ETag1}, Tarball}} -> + {ok, Tarball, ETag1}; + {ok, {304, _Headers, _}} -> {ok, cached}; - {ok, {{_Version, Code, _Reason}, _Headers, _Body}} -> - ?DEBUG("Request to ~p failed: status code ~p", [Url, Code]), + {ok, {Code, _Headers, _Body}} -> + ?DEBUG("Request for package ~s-~s failed: status code ~p", [Name, Version, Code]), error; {error, Reason} -> - ?DEBUG("Request to ~p failed: ~p", [Url, Reason]), + ?DEBUG("Request for package ~s-~s failed: ~p", [Name, Version, Reason]), + error + catch + _:Exception -> + ?DEBUG("hex_repo:get_tarball failed: ~p", [Exception]), error end. @@ -153,32 +172,23 @@ request(Url, ETag) -> %% returned from the hexpm server. The name is package-vsn.etag. %% @end %%------------------------------------------------------------------------------ --spec etag(Path) -> Res when - Path :: file:name(), - Res :: false | string(). -etag(Path) -> - case file:read_file(Path) of +-spec etag(PackagePath, ETagPath) -> Res when + PackagePath :: file:name(), + ETagPath :: file:name(), + Res :: binary(). +etag(PackagePath, ETagPath) -> + case file:read_file(ETagPath) of {ok, Bin} -> - binary_to_list(Bin); + %% just in case a user deleted a cached package but not its etag + %% verify the package is also there, and if not, ignore the etag + case filelib:is_file(PackagePath) of + true -> + Bin; + false -> + <<>> + end; {error, _} -> - false - end. - -%%------------------------------------------------------------------------------ -%% @doc -%% Return the SSL options adequate for the project based on -%% its configuration, including for validation of certs. -%% @end -%%------------------------------------------------------------------------------ --spec ssl_opts(Url) -> Res when - Url :: string(), - Res :: proplists:proplist(). -ssl_opts(Url) -> - case get_ssl_config() of - ssl_verify_enabled -> - ssl_opts(ssl_verify_enabled, Url); - ssl_verify_disabled -> - [{verify, verify_none}] + <<>> end. %%------------------------------------------------------------------------------ @@ -188,7 +198,7 @@ ssl_opts(Url) -> %%------------------------------------------------------------------------------ -spec store_etag_in_cache(File, ETag) -> Res when File :: file:name(), - ETag :: string(), + ETag :: binary(), Res :: ok. store_etag_in_cache(Path, ETag) -> _ = file:write_file(Path, ETag). @@ -196,223 +206,74 @@ store_etag_in_cache(Path, ETag) -> %%%============================================================================= %%% Private functions %%%============================================================================= --spec cached_download(TmpDir, CachePath, Pkg, Url, ETag, State, ETagPath, - UpdateETag) -> Res when +-spec cached_download(TmpDir, CachePath, Pkg, ETag, ETagPath, UpdateETag) -> Res when TmpDir :: file:name(), CachePath :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, - Url :: string(), - ETag :: false | string(), - State :: rebar_state:t(), + Pkg :: package(), + ETag :: binary(), ETagPath :: file:name(), UpdateETag :: boolean(), - Res :: download_result(). -cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, Url, ETag, - State, ETagPath, UpdateETag) -> - case request(Url, ETag) of + Res :: ok | {unexpected_hash, integer(), integer()} | {fetch_fail, binary(), binary()}. +cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash, RepoConfig}, ETag, + ETagPath, UpdateETag) -> + case request(RepoConfig, Name, Vsn, ETag) of {ok, cached} -> ?INFO("Version cached at ~ts is up to date, reusing it", [CachePath]), - serve_from_cache(TmpDir, CachePath, Pkg, State); + serve_from_cache(TmpDir, CachePath, Pkg); {ok, Body, NewETag} -> ?INFO("Downloaded package, caching at ~ts", [CachePath]), maybe_store_etag_in_cache(UpdateETag, ETagPath, NewETag), - serve_from_download(TmpDir, CachePath, Pkg, NewETag, Body, State, - ETagPath); - error when ETag =/= false -> + serve_from_download(TmpDir, CachePath, Pkg, Body); + error when ETag =/= <<>> -> store_etag_in_cache(ETagPath, ETag), ?INFO("Download error, using cached file at ~ts", [CachePath]), - serve_from_cache(TmpDir, CachePath, Pkg, State); + serve_from_cache(TmpDir, CachePath, Pkg); error -> {fetch_fail, Name, Vsn} end. --spec serve_from_cache(TmpDir, CachePath, Pkg, State) -> Res when +-spec serve_from_cache(TmpDir, CachePath, Pkg) -> Res when TmpDir :: file:name(), CachePath :: file:name(), - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, - State :: rebar_state:t(), - Res :: cached_result(). -serve_from_cache(TmpDir, CachePath, Pkg, State) -> - {Files, Contents, Version, Meta} = extract(TmpDir, CachePath), - case checksums(Pkg, Files, Contents, Version, Meta, State) of - {Chk, Chk, Chk, Chk} -> - ok = erl_tar:extract({binary, Contents}, [{cwd, TmpDir}, compressed]), - {ok, true}; - {_Hash, Chk, Chk, Chk} -> - ?DEBUG("Expected hash ~p does not match checksums ~p", [_Hash, Chk]), - {unexpected_hash, CachePath, _Hash, Chk}; - {Chk, _Bin, Chk, Chk} -> - ?DEBUG("Checksums: registry: ~p, pkg: ~p", [Chk, _Bin]), - {failed_extract, CachePath}; - {Chk, Chk, _Reg, Chk} -> - ?DEBUG("Checksums: registry: ~p, pkg: ~p", [_Reg, Chk]), - {bad_registry_checksum, CachePath}; - {_Hash, _Bin, _Reg, _Tar} -> - ?DEBUG("Checksums: expected: ~p, registry: ~p, pkg: ~p, meta: ~p", - [_Hash, _Reg, _Bin, _Tar]), - {bad_checksum, CachePath} + Pkg :: package(), + Res :: ok | {error,_} | {bad_registry_checksum, integer(), integer()}. +serve_from_cache(TmpDir, CachePath, Pkg) -> + {ok, Binary} = file:read_file(CachePath), + serve_from_memory(TmpDir, Binary, Pkg). + +-spec serve_from_memory(TmpDir, Tarball, Package) -> Res when + TmpDir :: file:name(), + Tarball :: binary(), + Package :: package(), + Res :: ok | {error,_} | {bad_registry_checksum, integer(), integer()}. +serve_from_memory(TmpDir, Binary, {pkg, _Name, _Vsn, Hash, _RepoConfig}) -> + RegistryChecksum = list_to_integer(binary_to_list(Hash), 16), + case hex_tarball:unpack(Binary, TmpDir) of + {ok, #{checksum := <>}} when RegistryChecksum =/= Checksum -> + ?DEBUG("Expected hash ~64.16.0B does not match checksum of fetched package ~64.16.0B", + [RegistryChecksum, Checksum]), + {bad_registry_checksum, RegistryChecksum, Checksum}; + {ok, #{checksum := <>}} -> + ok; + {error, Reason} -> + {error, {hex_tarball, Reason}} end. --spec serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State, - ETagPath) -> Res when +-spec serve_from_download(TmpDir, CachePath, Package, Binary) -> Res when TmpDir :: file:name(), CachePath :: file:name(), - Package :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, - ETag :: string(), + Package :: package(), Binary :: binary(), - State :: rebar_state:t(), - ETagPath :: file:name(), - Res :: download_result(). -serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State, ETagPath) -> + Res :: ok | {error,_}. +serve_from_download(TmpDir, CachePath, Package, Binary) -> ?DEBUG("Writing ~p to cache at ~ts", [Package, CachePath]), file:write_file(CachePath, Binary), - case etag(ETagPath) of - ETag -> - serve_from_cache(TmpDir, CachePath, Package, State); - FileETag -> - ?DEBUG("Downloaded file ~ts ETag ~ts doesn't match returned ETag ~ts", - [CachePath, ETag, FileETag]), - {bad_download, CachePath} - end. - --spec extract(TmpDir, CachePath) -> Res when - TmpDir :: file:name(), - CachePath :: file:name(), - Res :: {Files, Contents, Version, Meta}, - Files :: list({file:name(), binary()}), - Contents :: binary(), - Version :: binary(), - Meta :: binary(). -extract(TmpDir, CachePath) -> - ec_file:mkdir_p(TmpDir), - {ok, Files} = erl_tar:extract(CachePath, [memory]), - {"contents.tar.gz", Contents} = lists:keyfind("contents.tar.gz", 1, Files), - {"VERSION", Version} = lists:keyfind("VERSION", 1, Files), - {"metadata.config", Meta} = lists:keyfind("metadata.config", 1, Files), - {Files, Contents, Version, Meta}. - --spec checksums(Pkg, Files, Contents, Version, Meta, State) -> Res when - Pkg :: {pkg, Name :: binary(), Vsn :: binary(), Hash :: binary()}, - Files :: list({file:name(), binary()}), - Contents :: binary(), - Version :: binary(), - Meta :: binary(), - State :: rebar_state:t(), - Res :: {Hash, BinChecksum, RegistryChecksum, TarChecksum}, - Hash :: binary(), - BinChecksum :: binary(), - RegistryChecksum :: any(), - TarChecksum :: binary(). -checksums(Pkg={pkg, _Name, _Vsn, Hash}, Files, Contents, Version, Meta, State) -> - Blob = <>, - <> = crypto:hash(sha256, Blob), - BinChecksum = list_to_binary( - rebar_string:uppercase( - lists:flatten(io_lib:format("~64.16.0b", [X])))), - RegistryChecksum = rebar_packages:registry_checksum(Pkg, State), - {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files), - {Hash, BinChecksum, RegistryChecksum, TarChecksum}. - -%%------------------------------------------------------------------------------ -%% @doc -%% Return the SSL options adequate for the project based on -%% its configuration, including for validation of certs. -%% @end -%%------------------------------------------------------------------------------ --spec ssl_opts(Enabled, Url) -> Res when - Enabled :: atom(), - Url :: string(), - Res :: proplists:proplist(). -ssl_opts(ssl_verify_enabled, Url) -> - case check_ssl_version() of - true -> - {ok, {_, _, Hostname, _, _, _}} = - http_uri:parse(rebar_utils:to_list(Url)), - VerifyFun = {fun ssl_verify_hostname:verify_fun/3, - [{check_hostname, Hostname}]}, - CACerts = certifi:cacerts(), - [{verify, verify_peer}, {depth, 2}, {cacerts, CACerts}, - {partial_chain, fun partial_chain/1}, {verify_fun, VerifyFun}]; - false -> - ?WARN("Insecure HTTPS request (peer verification disabled), " - "please update to OTP 17.4 or later", []), - [{verify, verify_none}] - end. - --spec partial_chain(Certs) -> Res when - Certs :: list(any()), - Res :: unknown_ca | {trusted_ca, any()}. -partial_chain(Certs) -> - Certs1 = [{Cert, public_key:pkix_decode_cert(Cert, otp)} || Cert <- Certs], - CACerts = certifi:cacerts(), - CACerts1 = [public_key:pkix_decode_cert(Cert, otp) || Cert <- CACerts], - case ec_lists:find(fun({_, Cert}) -> - check_cert(CACerts1, Cert) - end, Certs1) of - {ok, Trusted} -> - {trusted_ca, element(1, Trusted)}; - _ -> - unknown_ca - end. - --spec extract_public_key_info(Cert) -> Res when - Cert :: #'OTPCertificate'{tbsCertificate::#'OTPTBSCertificate'{}}, - Res :: any(). -extract_public_key_info(Cert) -> - ((Cert#'OTPCertificate'.tbsCertificate)#'OTPTBSCertificate'.subjectPublicKeyInfo). - --spec check_cert(CACerts, Cert) -> Res when - CACerts :: list(any()), - Cert :: any(), - Res :: boolean(). -check_cert(CACerts, Cert) -> - lists:any(fun(CACert) -> - extract_public_key_info(CACert) == extract_public_key_info(Cert) - end, CACerts). - --spec check_ssl_version() -> - boolean(). -check_ssl_version() -> - case application:get_key(ssl, vsn) of - {ok, Vsn} -> - parse_vsn(Vsn) >= {5, 3, 6}; - _ -> - false - end. - --spec get_ssl_config() -> - ssl_verify_disabled | ssl_verify_enabled. -get_ssl_config() -> - GlobalConfigFile = rebar_dir:global_config(), - Config = rebar_config:consult_file(GlobalConfigFile), - case proplists:get_value(ssl_verify, Config, []) of - false -> - ssl_verify_disabled; - _ -> - ssl_verify_enabled - end. - --spec parse_vsn(Vsn) -> Res when - Vsn :: string(), - Res :: {integer(), integer(), integer()}. -parse_vsn(Vsn) -> - version_pad(rebar_string:lexemes(Vsn, ".-")). - --spec version_pad(list(nonempty_string())) -> Res when - Res :: {integer(), integer(), integer()}. -version_pad([Major]) -> - {list_to_integer(Major), 0, 0}; -version_pad([Major, Minor]) -> - {list_to_integer(Major), list_to_integer(Minor), 0}; -version_pad([Major, Minor, Patch]) -> - {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}; -version_pad([Major, Minor, Patch | _]) -> - {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}. + serve_from_memory(TmpDir, Binary, Package). -spec maybe_store_etag_in_cache(UpdateETag, Path, ETag) -> Res when UpdateETag :: boolean(), Path :: file:name(), - ETag :: string(), + ETag :: binary(), Res :: ok. maybe_store_etag_in_cache(false = _UpdateETag, _Path, _ETag) -> ok; -- cgit v1.1 From 86519cf743204eab1d922ca8133fbf00c66f9ee8 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sun, 14 Oct 2018 10:50:16 -0600 Subject: fix resolving versions from vcs (#1915) app_info was turning any vsn into a binary which the vcs_vsn function interprets as being an actual version and not something like <<"git">>. original_vsn shouldn't be converted as it is then not longer "original". --- src/rebar_pkg_resource.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/rebar_pkg_resource.erl') diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 97eabb6..823b7fc 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -60,7 +60,7 @@ lock(AppInfo, _) -> Res :: boolean(). needs_update(AppInfo, _) -> {pkg, _Name, Vsn, _Hash, _} = rebar_app_info:source(AppInfo), - case rebar_app_info:original_vsn(AppInfo) =:= rebar_utils:to_binary(Vsn) of + case rebar_utils:to_binary(rebar_app_info:original_vsn(AppInfo)) =:= rebar_utils:to_binary(Vsn) of true -> false; false -> -- cgit v1.1