From 47db2e91fce39da12ec91742b41d80f142b30a4b Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 17 May 2017 10:47:46 +0200 Subject: Use rm_rf instead of ec_file:delete. In contrast to `ec_file:delete`, `rebar_file_utils:rm_rf` will also delete write-protected files on Windows which is needed for git object files. Fixes #1483. --- src/rebar_fetch.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/rebar_fetch.erl') diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index 47bfe1d..f5e5db5 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -46,7 +46,7 @@ download_source_(AppDir, Source, State) -> {ok, _} -> ec_file:mkdir_p(AppDir1), code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), - ec_file:remove(filename:absname(AppDir1), [recursive]), + ok = rebar_file_utils:rm_rf(filename:absname(AppDir1)), ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]), ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)), true; -- cgit v1.1 From 963c49f5eb9ab5b34e1843fb43305743720917ac Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sun, 6 Aug 2017 07:26:21 -0400 Subject: Unicode support in all the places This is done through 3 main change groups: - replacing `~s` by `~ts` in format strings, so that strings that contain unicode are properly printed rather than crashing - adding the `unicode` argument to all function of the `re` module to ensure transformations on strings containing unicode data are valid instead of crashing (see issue #1302) - replacing `ec_cnv:to_binary/1` and `ec_cnv:to_list/1` with matching functions in `rebar_utils`. The last point has been done, rather than modifying and updating erlware commons, because binary and list conversions can be a contentious subject. For example, if what is being handled is actually bytes from a given binary stream, then forcing a byte-oriented interpretation of the data can corrupt it. As such, it does not appear safe to modify erlware commons' conversion functions since it may not be safe for all its users. Instead, rebar3 reimplements a subset of them (only converting atoms and chardata, ignoring numbers) with the explicit purpose of handling unicode string data. Tests were left as unchanged as possible. This may impact the ability to run rebar3's own suites in a unicode path, but respects a principle of least change for such a large patch. --- src/rebar_fetch.erl | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'src/rebar_fetch.erl') diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index f5e5db5..f68a54d 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -41,7 +41,7 @@ download_source_(AppDir, Source, State) -> Resources = rebar_state:resources(State), Module = get_resource_type(Source, Resources), TmpDir = ec_file:insecure_mkdtemp(), - AppDir1 = ec_cnv:to_list(AppDir), + AppDir1 = rebar_utils:to_list(AppDir), case Module:download(TmpDir, Source, State) of {ok, _} -> ec_file:mkdir_p(AppDir1), @@ -66,25 +66,25 @@ needs_update(AppDir, Source, State) -> end. format_error({bad_download, CachePath}) -> - io_lib:format("Download of package does not match md5sum from server: ~s", [CachePath]); + io_lib:format("Download of package does not match md5sum from server: ~ts", [CachePath]); format_error({unexpected_hash, CachePath, Expected, Found}) -> - io_lib:format("The checksum for package at ~s (~s) does not match the " - "checksum previously locked (~s). Either unlock or " + io_lib:format("The checksum for package at ~ts (~ts) does not match the " + "checksum previously locked (~ts). Either unlock or " "upgrade the package, or make sure you fetched it from " "the same index from which it was initially fetched.", [CachePath, Found, Expected]); format_error({failed_extract, CachePath}) -> - io_lib:format("Failed to extract package: ~s", [CachePath]); + io_lib:format("Failed to extract package: ~ts", [CachePath]); format_error({bad_etag, Source}) -> - io_lib:format("MD5 Checksum comparison failed for: ~s", [Source]); + io_lib:format("MD5 Checksum comparison failed for: ~ts", [Source]); format_error({fetch_fail, Name, Vsn}) -> - io_lib:format("Failed to fetch and copy dep: ~s-~s", [Name, Vsn]); + io_lib:format("Failed to fetch and copy dep: ~ts-~ts", [Name, Vsn]); format_error({fetch_fail, Source}) -> io_lib:format("Failed to fetch and copy dep: ~p", [Source]); format_error({bad_checksum, File}) -> - io_lib:format("Checksum mismatch against tarball in ~s", [File]); + io_lib:format("Checksum mismatch against tarball in ~ts", [File]); format_error({bad_registry_checksum, File}) -> - io_lib:format("Checksum mismatch against registry in ~s", [File]). + io_lib:format("Checksum mismatch against registry in ~ts", [File]). get_resource_type({Type, Location}, Resources) -> find_resource_module(Type, Location, Resources); @@ -100,7 +100,7 @@ find_resource_module(Type, Location, Resources) -> false -> case code:which(Type) of non_existing -> - {error, io_lib:format("Cannot handle dependency ~s.~n" + {error, io_lib:format("Cannot handle dependency ~ts.~n" " No module for resource type ~p", [Location, Type])}; _ -> Type -- cgit v1.1 From 5f9b4293bc029e2132d7c442cb5b4480915ea0e4 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 3 May 2018 07:12:11 -0400 Subject: Work around OTP-21 deprecation of get_stacktrace() Based off a macro by @okeuday at https://github.com/erlang/otp/pull/1783 --- src/rebar_fetch.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/rebar_fetch.erl') diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index f68a54d..d2c7706 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -32,8 +32,8 @@ download_source(AppDir, Source, State) -> Error -> throw(?PRV_ERROR(Error)) catch - C:T -> - ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, erlang:get_stacktrace()]), + ?WITH_STACKTRACE(C,T,S) + ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, S]), throw(?PRV_ERROR({fetch_fail, Source})) end. -- cgit v1.1 From 56b7d88975aa8da6446857cfd92de0825024bf63 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 13 Sep 2018 19:36:00 -0600 Subject: support for hex v2, multiple repository fetching, private organizations (#1884) * update to hex_core for hex-v2 repo support (#1865) * update to hex_core for hex-v2 repo support This patch adds only single repo hex-v2 support through hex_core. Packages no longer filtered out by buildtool metadata and the package index is updated per-package instead of fetched as one large ets dump. * tell travis to also build hex_core branch * support list of repos for hex packages (#1866) * support list of repos for hex packages repos are defined under the hex key in rebar configs. They can be defined at the top level of a project or globally, but not in profiles and the repos configured in dependencies are also ignored. Searching for packages involves first checking for a match in the local repo index cache, in the order repos are defined. If not found each repo is checked through the hex api for any known versions of the package and the first repo with a version that fits the constraint is used. * add {repos, replace, []} for overriding the global & default repos * add hex auth handling for repos (#1874) auth token are kept in a hex.config file that is modified by the rebar3 hex plugin. Repo names that have a : separating a parent and child are considered organizations. The parent repo's auth will be included with the child. So an organization named hexpm:rebar3_test will include any hexpm auth tokens found in the rebar3_test organization's configuration. * move packages to top level of of hexpm cache dir (#1876) * move packages to top level of of hexpm cache dir * append organization name to parent's repo_url when parsing repos * only eval config scripts and apply overrides once per app (#1879) * only eval config scripts and apply overrides once per app * move new resource behaviour to rebar_resource_v2 and keep v1 * cleanup use of rebar_resource module and unused functions * cleanup error messages and unused code * when discovering apps support mix packages as unbuilt apps (#1882) * use hex_core tarball unpacking support in pkg resource (#1883) * use hex_core tarball unpacking support in pkg resource * ignore etag if package doesn't exist and delete if checksum fails * add back tests for bad package checksums * improve bad registry checksum error message --- src/rebar_fetch.erl | 108 +++++++++++++++++++--------------------------------- 1 file changed, 39 insertions(+), 69 deletions(-) (limited to 'src/rebar_fetch.erl') diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index d2c7706..9c76e0e 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -7,104 +7,74 @@ %% ------------------------------------------------------------------- -module(rebar_fetch). --export([lock_source/3, - download_source/3, - needs_update/3]). +-export([lock_source/2, + download_source/2, + needs_update/2]). -export([format_error/1]). -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). --spec lock_source(file:filename_all(), rebar_resource:resource(), rebar_state:t()) -> - rebar_resource:resource() | {error, string()}. -lock_source(AppDir, Source, State) -> - Resources = rebar_state:resources(State), - Module = get_resource_type(Source, Resources), - Module:lock(AppDir, Source). +-spec lock_source(rebar_app_info:t(), rebar_state:t()) + -> rebar_resource_v2:source() | {error, string()}. +lock_source(AppInfo, State) -> + rebar_resource_v2:lock(AppInfo, State). --spec download_source(file:filename_all(), rebar_resource:resource(), rebar_state:t()) -> - true | {error, any()}. -download_source(AppDir, Source, State) -> - try download_source_(AppDir, Source, State) of - true -> - true; - Error -> - throw(?PRV_ERROR(Error)) +-spec download_source(rebar_app_info:t(), rebar_state:t()) + -> rebar_app_info:t() | {error, any()}. +download_source(AppInfo, State) -> + AppDir = rebar_app_info:dir(AppInfo), + try download_source_(AppInfo, State) of + ok -> + %% freshly downloaded, update the app info opts to reflect the new config + Config = rebar_config:consult(AppDir), + AppInfo1 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), Config), + case rebar_app_discover:find_app(AppInfo1, AppDir, all) of + {true, AppInfo2} -> + rebar_app_info:is_available(AppInfo2, true); + false -> + throw(?PRV_ERROR({dep_app_not_found, rebar_app_info:name(AppInfo1)})) + end; + {error, Reason} -> + throw(?PRV_ERROR(Reason)) catch + throw:{no_resource, Type, Location} -> + throw(?PRV_ERROR({no_resource, Location, Type})); ?WITH_STACKTRACE(C,T,S) ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, S]), - throw(?PRV_ERROR({fetch_fail, Source})) + throw(?PRV_ERROR({fetch_fail, rebar_app_info:source(AppInfo)})) end. -download_source_(AppDir, Source, State) -> - Resources = rebar_state:resources(State), - Module = get_resource_type(Source, Resources), +download_source_(AppInfo, State) -> + AppDir = rebar_app_info:dir(AppInfo), TmpDir = ec_file:insecure_mkdtemp(), AppDir1 = rebar_utils:to_list(AppDir), - case Module:download(TmpDir, Source, State) of - {ok, _} -> + case rebar_resource_v2:download(TmpDir, AppInfo, State) of + ok -> ec_file:mkdir_p(AppDir1), code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), ok = rebar_file_utils:rm_rf(filename:absname(AppDir1)), ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]), - ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)), - true; + rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)); Error -> Error end. --spec needs_update(file:filename_all(), rebar_resource:resource(), rebar_state:t()) -> boolean() | {error, string()}. -needs_update(AppDir, Source, State) -> - Resources = rebar_state:resources(State), - Module = get_resource_type(Source, Resources), +-spec needs_update(rebar_app_info:t(), rebar_state:t()) + -> boolean() | {error, string()}. +needs_update(AppInfo, State) -> try - Module:needs_update(AppDir, Source) + rebar_resource_v2:needs_update(AppInfo, State) catch _:_ -> true end. -format_error({bad_download, CachePath}) -> - io_lib:format("Download of package does not match md5sum from server: ~ts", [CachePath]); -format_error({unexpected_hash, CachePath, Expected, Found}) -> - io_lib:format("The checksum for package at ~ts (~ts) does not match the " - "checksum previously locked (~ts). Either unlock or " - "upgrade the package, or make sure you fetched it from " - "the same index from which it was initially fetched.", - [CachePath, Found, Expected]); -format_error({failed_extract, CachePath}) -> - io_lib:format("Failed to extract package: ~ts", [CachePath]); -format_error({bad_etag, Source}) -> - io_lib:format("MD5 Checksum comparison failed for: ~ts", [Source]); format_error({fetch_fail, Name, Vsn}) -> io_lib:format("Failed to fetch and copy dep: ~ts-~ts", [Name, Vsn]); format_error({fetch_fail, Source}) -> io_lib:format("Failed to fetch and copy dep: ~p", [Source]); -format_error({bad_checksum, File}) -> - io_lib:format("Checksum mismatch against tarball in ~ts", [File]); -format_error({bad_registry_checksum, File}) -> - io_lib:format("Checksum mismatch against registry in ~ts", [File]). - -get_resource_type({Type, Location}, Resources) -> - find_resource_module(Type, Location, Resources); -get_resource_type({Type, Location, _}, Resources) -> - find_resource_module(Type, Location, Resources); -get_resource_type({Type, _, _, Location}, Resources) -> - find_resource_module(Type, Location, Resources); -get_resource_type(_, _) -> - rebar_pkg_resource. - -find_resource_module(Type, Location, Resources) -> - case lists:keyfind(Type, 1, Resources) of - false -> - case code:which(Type) of - non_existing -> - {error, io_lib:format("Cannot handle dependency ~ts.~n" - " No module for resource type ~p", [Location, Type])}; - _ -> - Type - end; - {Type, Module} -> - Module - end. +format_error({dep_app_not_found, AppName}) -> + io_lib:format("Dependency failure: source for ~ts does not contain a " + "recognizable project and can not be built", [AppName]). -- cgit v1.1