From 7905a9f19cdb0ca2ff70567613784084e3e0a2fd Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 8 May 2015 20:06:43 -0500 Subject: store hex package downloads in shared cache dir --- src/rebar_fetch.erl | 15 ++++------ src/rebar_pkg_resource.erl | 70 ++++++++++++++++++++++++++++++++++++++++++---- src/rebar_resource.erl | 19 ------------- 3 files changed, 69 insertions(+), 35 deletions(-) (limited to 'src') diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index 20bf46b..ec16089 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -40,20 +40,15 @@ download_source(AppDir, Source, State) -> ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)), true; {tarball, File} -> - Contents = filename:join(TmpDir, "contents"), ec_file:mkdir_p(AppDir1), - ec_file:mkdir_p(Contents), - ok = erl_tar:extract(File, [{cwd, TmpDir}]), - ok = erl_tar:extract(filename:join(TmpDir, "contents.tar.gz"), - [{cwd, Contents}, compressed]), + {ok, Files} = erl_tar:extract(File, [memory]), + code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), ec_file:remove(filename:absname(AppDir1), [recursive]), - ?DEBUG("Moving contents ~p to ~p", [Contents, filename:absname(AppDir1)]), - ok = rebar_file_utils:mv(Contents, filename:absname(AppDir1)), - - ?DEBUG("Removing tmp dir ~p", [TmpDir]), - ec_file:remove(TmpDir, [recursive]), + {"contents.tar.gz", Binary} = lists:keyfind("contents.tar.gz", 1, Files), + ok = erl_tar:extract({binary, Binary}, + [{cwd, filename:absname(AppDir1)}, compressed]), true end catch diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 3b44fc8..92e6cd4 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -9,8 +9,11 @@ ,needs_update/2 ,make_vsn/1]). +-include_lib("providers/include/providers.hrl"). -include("rebar.hrl"). +-define(DEFAULT_CDN, "https://s3.amazonaws.com/s3.hex.pm/tarballs"). + lock(_AppDir, Source) -> Source. @@ -23,12 +26,67 @@ needs_update(Dir, {pkg, _Name, Vsn}) -> true end. -download(Dir, {pkg, Name, Vsn}, State) -> - TmpFile = filename:join(Dir, "package.tar.gz"), - CDN = rebar_state:get(State, rebar_packages_cdn, "https://s3.amazonaws.com/s3.hex.pm/tarballs"), - Url = string:join([CDN, binary_to_list(<>)], "/"), - {ok, saved_to_file} = httpc:request(get, {Url, []}, [], [{stream, TmpFile}]), - {tarball, TmpFile}. +download(_Dir, {pkg, Name, Vsn}, State) -> + CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), + PackageDir = hex_package_dir(CDN, State), + + Package = binary_to_list(<>), + Path = filename:join(PackageDir, Package), + Url = string:join([CDN, Package], "/"), + + case request(Url, etag(Path)) of + {ok, cached} -> + {tarball, Path}; + {ok, Binary} -> + file:write_file(Path, Binary), + {tarball, Path}; + error -> + case filelib:is_regular(Path) of + true -> + ?DEBUG("Download ~s error, using ~s since it exists", [Url, Path]), + {tarball, Path}; + false -> + throw(request_failed) + end + end. make_vsn(_) -> {error, "Replacing version of type pkg not supported."}. + +%% Use the shared hex package directory unless a non-default package repo is used +hex_package_dir(?DEFAULT_CDN, _) -> + filename:join([rebar_dir:home_dir(), ".hex", "packages"]); +hex_package_dir(CDN, State) -> + CacheDir = rebar_dir:global_cache_dir(State), + {ok, {_, _, Host, _, _, _}} = http_uri:parse(CDN), + CDNPath = filename:join(lists:reverse(string:tokens(Host, "."))), + PackageDir = filename:join([CacheDir, "hex", CDNPath, "packages"]), + ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")), + PackageDir. + +request(Url, ETag) -> + case httpc:request(get, {Url, [{"if-none-match", ETag} || ETag =/= false]}, + [{relaxed, true}], + [{body_format, binary}]) of + {ok, {{_Version, 200, _Reason}, _Headers, Body}} -> + ?DEBUG("Successfully downloaded ~s", [Url]), + {ok, Body}; + {ok, {{_Version, 304, _Reason}, _Headers, _Body}} -> + ?DEBUG("Cached copy of ~s still valid", [Url]), + {ok, cached}; + {ok, {{_Version, Code, _Reason}, _Headers, _Body}} -> + ?DEBUG("Request to ~p failed: status code ~p", [Url, Code]), + error; + {error, Reason} -> + ?DEBUG("Request to ~p failed: ~p", [Url, Reason]), + error + end. + +etag(Path) -> + case file:read_file(Path) of + {ok, Binary} -> + <> = crypto:hash(md5, Binary), + string:to_lower(lists:flatten(io_lib:format("~32.16.0b", [X]))); + {error, _} -> + false + end. diff --git a/src/rebar_resource.erl b/src/rebar_resource.erl index 7c58135..cdce7a8 100644 --- a/src/rebar_resource.erl +++ b/src/rebar_resource.erl @@ -14,23 +14,6 @@ -type location() :: string(). -type ref() :: any(). --ifdef(no_callback_support). - -%% In the case where R14 or lower is being used to compile the system -%% we need to export a behaviour info --export([behaviour_info/1]). - --spec behaviour_info(atom()) -> [{atom(), arity()}] | undefined. -behaviour_info(callbacks) -> - [{lock, 2}, - {download, 3}, - {needs_update,2}, - {make_vsn, 1}]; -behaviour_info(_) -> - undefined. - --else. - -callback lock(file:filename_all(), tuple()) -> rebar_resource:resource(). -callback download(file:filename_all(), tuple(), rebar_state:t()) -> @@ -39,5 +22,3 @@ behaviour_info(_) -> boolean(). -callback make_vsn(file:filename_all()) -> {plain, string()} | {error, string()}. - --endif. -- cgit v1.1 From 819d0fb06fbaaec25b1929e971964d5757ba59ff Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sat, 9 May 2015 23:01:17 -0500 Subject: verify checksums of hex packages --- src/rebar_fetch.erl | 48 ++++++++++++++++++++++++++++++++---------- src/rebar_packages.erl | 10 +++++++++ src/rebar_prv_install_deps.erl | 16 +++++++------- 3 files changed, 55 insertions(+), 19 deletions(-) (limited to 'src') diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index ec16089..16840eb 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -40,16 +40,7 @@ download_source(AppDir, Source, State) -> ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)), true; {tarball, File} -> - ec_file:mkdir_p(AppDir1), - {ok, Files} = erl_tar:extract(File, [memory]), - - code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), - ec_file:remove(filename:absname(AppDir1), [recursive]), - - {"contents.tar.gz", Binary} = lists:keyfind("contents.tar.gz", 1, Files), - ok = erl_tar:extract({binary, Binary}, - [{cwd, filename:absname(AppDir1)}, compressed]), - true + verify_and_extract(File, Source, AppDir1, State) end catch C:T -> @@ -69,7 +60,11 @@ needs_update(AppDir, Source, State) -> end. format_error({fetch_fail, Source}) -> - io_lib:format("Failed to fetch and copy dep: ~p", [Source]). + io_lib:format("Failed to fetch and copy dep: ~p", [Source]); +format_error({bad_checksum, File}) -> + io_lib:format("Checksum mismatch against tarball in ~s", [File]); +format_error({bad_registry_checksum, File}) -> + io_lib:format("Checksum mismatch against registry in ~s", [File]). get_resource_type({Type, Location}, Resources) -> find_resource_module(Type, Location, Resources); @@ -93,3 +88,34 @@ find_resource_module(Type, Location, Resources) -> {Type, Module} -> Module end. + +verify_and_extract(File, Source, AppDir, State) -> + ec_file:mkdir_p(AppDir), + {ok, Files} = erl_tar:extract(File, [memory]), + + code:del_path(filename:absname(filename:join(AppDir, "ebin"))), + ec_file:remove(filename:absname(AppDir), [recursive]), + + {"contents.tar.gz", Contents} = lists:keyfind("contents.tar.gz", 1, Files), + {"VERSION", Version} = lists:keyfind("VERSION", 1, Files), + {"metadata.config", Meta} = lists:keyfind("metadata.config", 1, Files), + + Checksum = checksum(Contents, Version, Meta), + RegistryChecksum = rebar_packages:registry_checksum(Source, State), + {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files), + + if + Checksum =/= TarChecksum -> + ?PRV_ERROR({bad_checksum, File}); + Checksum =/= RegistryChecksum -> + ?PRV_ERROR({bad_registry_checksum, File}); + true -> + ok = erl_tar:extract({binary, Contents}, + [{cwd, filename:absname(AppDir)}, compressed]), + true + end. + +checksum(Contents, Version, Meta) -> + Blob = <>, + <> = crypto:hash(sha256, Blob), + list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))). diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 4ab5f9f..a344328 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -3,6 +3,7 @@ -export([get_packages/1 ,registry/1 ,check_registry/3 + ,registry_checksum/2 ,find_highest_matching/3]). -export_type([package/0]). @@ -66,6 +67,15 @@ check_registry(Pkg, Vsn, State) -> false end. +registry_checksum({pkg, Name, Vsn}, State) -> + {ok, Registry} = registry(State), + case ets:lookup(Registry, {Name, Vsn}) of + [{{_, _}, [_, Checksum | _]}] -> + Checksum; + [] -> + none + end. + %% Hex supports use of ~> to specify the version required for a dependency. %% Since rebar3 requires exact versions to choose from we find the highest %% available version of the dep that passes the constraint. diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index bc0a113..2570bc0 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -563,10 +563,10 @@ fetch_app(AppInfo, AppDir, State) -> ?INFO("Fetching ~s (~p)", [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)]), Source = rebar_app_info:source(AppInfo), case rebar_fetch:download_source(AppDir, Source, State) of - {error, Reason} -> - throw(Reason); - Result -> - Result + true -> + true; + Error -> + throw(Error) end. update_app_info(AppInfo) -> @@ -587,10 +587,10 @@ maybe_upgrade(AppInfo, AppDir, true, State) -> true -> ?INFO("Upgrading ~s", [rebar_app_info:name(AppInfo)]), case rebar_fetch:download_source(AppDir, Source, State) of - {error, Reason} -> - throw(Reason); - Result -> - Result + true -> + true; + Error -> + throw(Error) end; false -> ?INFO("No upgrade needed for ~s", [rebar_app_info:name(AppInfo)]), -- cgit v1.1 From f30e5063caf954f39c80e54ed8ae7882592441e6 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sat, 9 May 2015 23:32:06 -0500 Subject: don't print skip message if dep is locked for pkgs as well --- src/rebar_prv_install_deps.erl | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index 2570bc0..ba49532 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -151,7 +151,7 @@ handle_deps(Profile, State0, Deps, Upgrade, Locks) -> ,{Packages, Graph}), update_pkg_deps(Profile, Packages, PkgDeps1 - ,Graph, Upgrade, Seen, State2) + ,Graph, Upgrade, Seen, State2, Locks) end, AllDeps = lists:ukeymerge(2 @@ -184,7 +184,7 @@ find_cycles(Apps) -> cull_compile(TopSortedDeps, ProjectApps) -> lists:dropwhile(fun not_needs_compile/1, TopSortedDeps -- ProjectApps). -update_pkg_deps(Profile, Packages, PkgDeps, Graph, Upgrade, Seen, State) -> +update_pkg_deps(Profile, Packages, PkgDeps, Graph, Upgrade, Seen, State, Locks) -> case PkgDeps of [] -> %% No pkg deps {[], State}; @@ -196,13 +196,16 @@ update_pkg_deps(Profile, Packages, PkgDeps, Graph, Upgrade, Seen, State) -> {ok, Solution, []} -> Solution; {ok, Solution, Discarded} -> - [warn_skip_pkg(Pkg, State) || Pkg <- Discarded], + [warn_skip_pkg(Pkg, State) || Pkg <- Discarded, not(pkg_locked(Pkg, Locks))], Solution end, - update_pkg_deps(Profile, S, Packages, Upgrade, Seen, State) + update_pkg_deps(Profile, S, Packages, Upgrade, Seen, State, Locks) end. -update_pkg_deps(Profile, Pkgs, Packages, Upgrade, Seen, State) -> +pkg_locked({Name, _}, Locks) -> + false =/= lists:keyfind(Name, 1, Locks). + +update_pkg_deps(Profile, Pkgs, Packages, Upgrade, Seen, State, _Locks) -> %% Create app_info record for each pkg dep DepsDir = profile_dep_dir(State, Profile), {Solved, _, State1} @@ -610,7 +613,7 @@ parse_goal(Name, Constraint) -> warn_skip_deps(AppInfo, State) -> Msg = "Skipping ~s (from ~p) as an app of the same name " - "has already been fetched~n", + "has already been fetched", Args = [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)], case rebar_state:get(State, deps_error_on_conflict, false) of @@ -620,7 +623,7 @@ warn_skip_deps(AppInfo, State) -> warn_skip_pkg({Name, Source}, State) -> Msg = "Skipping ~s (version ~s from package index) as an app of the same " - "name has already been fetched~n", + "name has already been fetched", Args = [Name, Source], case rebar_state:get(State, deps_error_on_conflict, false) of false -> ?WARN(Msg, Args); -- cgit v1.1 From b636822d73410795cdc46d4722f7225c0f463fd3 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 12 May 2015 09:38:22 -0500 Subject: check md5sum of package against that sent by s3 --- src/rebar_fetch.erl | 4 ++++ src/rebar_pkg_resource.erl | 17 +++++++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index 16840eb..75970ed 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -43,6 +43,8 @@ download_source(AppDir, Source, State) -> verify_and_extract(File, Source, AppDir1, State) end catch + _:bad_etag -> + throw(?PRV_ERROR({bad_etag, Source})); C:T -> ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, erlang:get_stacktrace()]), throw(?PRV_ERROR({fetch_fail, Source})) @@ -59,6 +61,8 @@ needs_update(AppDir, Source, State) -> true end. +format_error({bad_etag, Source}) -> + io_lib:format("MD5 Checksum comparison failed for: ~p", [Source]); format_error({fetch_fail, Source}) -> io_lib:format("Failed to fetch and copy dep: ~p", [Source]); format_error({bad_checksum, File}) -> diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 92e6cd4..9b430d2 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -37,9 +37,17 @@ download(_Dir, {pkg, Name, Vsn}, State) -> case request(Url, etag(Path)) of {ok, cached} -> {tarball, Path}; - {ok, Binary} -> + {ok, Binary, EtagHeader} -> file:write_file(Path, Binary), - {tarball, Path}; + Etag = etag(Path), + case EtagHeader =:= Etag of + true -> + {tarball, Path}; + false -> + ?DEBUG("Bad md5sum for ~s of ~s comparing to ~s sent by server", + [Path, Etag, EtagHeader]), + throw(bad_etag) + end; error -> case filelib:is_regular(Path) of true -> @@ -68,9 +76,10 @@ request(Url, ETag) -> case httpc:request(get, {Url, [{"if-none-match", ETag} || ETag =/= false]}, [{relaxed, true}], [{body_format, binary}]) of - {ok, {{_Version, 200, _Reason}, _Headers, Body}} -> + {ok, {{_Version, 200, _Reason}, Headers, Body}} -> + {"etag", ETag1} = lists:keyfind("etag", 1, Headers), ?DEBUG("Successfully downloaded ~s", [Url]), - {ok, Body}; + {ok, Body, string:strip(ETag1, both, $")}; {ok, {{_Version, 304, _Reason}, _Headers, _Body}} -> ?DEBUG("Cached copy of ~s still valid", [Url]), {ok, cached}; -- cgit v1.1 From 56c925b75b86a6304da75635874722348ee21351 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Tue, 12 May 2015 16:58:38 +0000 Subject: Ad-hoc attempt at restructuring pkg cache --- src/rebar_fetch.erl | 51 ++++-------------------------- src/rebar_pkg_resource.erl | 79 ++++++++++++++++++++++++++++++++-------------- 2 files changed, 62 insertions(+), 68 deletions(-) (limited to 'src') diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index 75970ed..235aa03 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -31,20 +31,14 @@ download_source(AppDir, Source, State) -> Module = get_resource_type(Source, Resources), TmpDir = ec_file:insecure_mkdtemp(), AppDir1 = ec_cnv:to_list(AppDir), - case Module:download(TmpDir, Source, State) of - {ok, _} -> - ec_file:mkdir_p(AppDir1), - code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), - ec_file:remove(filename:absname(AppDir1), [recursive]), - ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]), - ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)), - true; - {tarball, File} -> - verify_and_extract(File, Source, AppDir1, State) - end + {ok, _} = Module:download(TmpDir, Source, State), + ec_file:mkdir_p(AppDir1), + code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), + ec_file:remove(filename:absname(AppDir1), [recursive]), + ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]), + ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)), + true catch - _:bad_etag -> - throw(?PRV_ERROR({bad_etag, Source})); C:T -> ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, erlang:get_stacktrace()]), throw(?PRV_ERROR({fetch_fail, Source})) @@ -92,34 +86,3 @@ find_resource_module(Type, Location, Resources) -> {Type, Module} -> Module end. - -verify_and_extract(File, Source, AppDir, State) -> - ec_file:mkdir_p(AppDir), - {ok, Files} = erl_tar:extract(File, [memory]), - - code:del_path(filename:absname(filename:join(AppDir, "ebin"))), - ec_file:remove(filename:absname(AppDir), [recursive]), - - {"contents.tar.gz", Contents} = lists:keyfind("contents.tar.gz", 1, Files), - {"VERSION", Version} = lists:keyfind("VERSION", 1, Files), - {"metadata.config", Meta} = lists:keyfind("metadata.config", 1, Files), - - Checksum = checksum(Contents, Version, Meta), - RegistryChecksum = rebar_packages:registry_checksum(Source, State), - {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files), - - if - Checksum =/= TarChecksum -> - ?PRV_ERROR({bad_checksum, File}); - Checksum =/= RegistryChecksum -> - ?PRV_ERROR({bad_registry_checksum, File}); - true -> - ok = erl_tar:extract({binary, Contents}, - [{cwd, filename:absname(AppDir)}, compressed]), - true - end. - -checksum(Contents, Version, Meta) -> - Blob = <>, - <> = crypto:hash(sha256, Blob), - list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))). diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 9b430d2..56e08fc 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -26,38 +26,69 @@ needs_update(Dir, {pkg, _Name, Vsn}) -> true end. -download(_Dir, {pkg, Name, Vsn}, State) -> +download(TmpDir, Pkg={pkg, Name, Vsn}, State) -> CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), PackageDir = hex_package_dir(CDN, State), - Package = binary_to_list(<>), - Path = filename:join(PackageDir, Package), + CachePath = filename:join(PackageDir, Package), Url = string:join([CDN, Package], "/"), + cached_download(TmpDir, CachePath, Pkg, Url, etag(CachePath), State). - case request(Url, etag(Path)) of +cached_download(TmpDir, CachePath, Pkg, Url, ETag, State) -> + case request(Url, ETag) of {ok, cached} -> - {tarball, Path}; - {ok, Binary, EtagHeader} -> - file:write_file(Path, Binary), - Etag = etag(Path), - case EtagHeader =:= Etag of - true -> - {tarball, Path}; - false -> - ?DEBUG("Bad md5sum for ~s of ~s comparing to ~s sent by server", - [Path, Etag, EtagHeader]), - throw(bad_etag) - end; + serve_from_cache(TmpDir, CachePath, Pkg, State); + {ok, Body, NewETag} -> + serve_from_download(TmpDir, CachePath, Pkg, NewETag, Body, State); + error when ETag =/= false -> + ?DEBUG("Download ~s error, using ~s from cache", [Url, CachePath]), + serve_from_cache(TmpDir, CachePath, Pkg, State); error -> - case filelib:is_regular(Path) of - true -> - ?DEBUG("Download ~s error, using ~s since it exists", [Url, Path]), - {tarball, Path}; - false -> - throw(request_failed) - end + throw(request_failed) + end. + +serve_from_cache(TmpDir, CachePath, Pkg, State) -> + {Files, Contents, Version, Meta} = extract(TmpDir, CachePath), + case checksums(Pkg, Files, Contents, Version, Meta, State) of + {Chk, Chk, Chk} -> + ok = erl_tar:extract({binary, Contents}, [{cwd, TmpDir}, compressed]), + {ok, true}; + {_Bin, Chk, Chk} -> + ?PRV_ERROR({failed_extract, CachePath}); + {Chk, _Reg, Chk} -> + ?PRV_ERROR({bad_registry_checksum, CachePath}); + {_Bin, _Reg, _Tar} -> + ?PRV_ERROR({bad_checksum, CachePath}) + end. + +serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State) -> + ?DEBUG("Writing ~p to cache at ~s", [Package, CachePath]), + file:write_file(CachePath, Binary), + case etag(CachePath) of + ETag -> + serve_from_cache(TmpDir, CachePath, Package, State); + FileETag -> + ?DEBUG("Download ETag ~s doesn't match cached ETag ~s", [ETag, FileETag]), + ?PRV_ERROR({bad_download, CachePath}) end. + +extract(TmpDir, CachePath) -> + ec_file:mkdir_p(TmpDir), + {ok, Files} = erl_tar:extract(CachePath, [memory]), + {"contents.tar.gz", Contents} = lists:keyfind("contents.tar.gz", 1, Files), + {"VERSION", Version} = lists:keyfind("VERSION", 1, Files), + {"metadata.config", Meta} = lists:keyfind("metadata.config", 1, Files), + {Files, Contents, Version, Meta}. + +checksums(Pkg, Files, Contents, Version, Meta, State) -> + Blob = <>, + <> = crypto:hash(sha256, Blob), + BinChecksum = list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))), + RegistryChecksum = rebar_packages:registry_sum(Pkg, State), + {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files), + {BinChecksum, RegistryChecksum, TarChecksum}. + make_vsn(_) -> {error, "Replacing version of type pkg not supported."}. @@ -77,8 +108,8 @@ request(Url, ETag) -> [{relaxed, true}], [{body_format, binary}]) of {ok, {{_Version, 200, _Reason}, Headers, Body}} -> - {"etag", ETag1} = lists:keyfind("etag", 1, Headers), ?DEBUG("Successfully downloaded ~s", [Url]), + {"etag", ETag1} = lists:keyfind("etag", 1, Headers), {ok, Body, string:strip(ETag1, both, $")}; {ok, {{_Version, 304, _Reason}, _Headers, _Body}} -> ?DEBUG("Cached copy of ~s still valid", [Url]), -- cgit v1.1 From 1fe93136c426a0c742134f392cc7f4d4d7ef7b3b Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 12 May 2015 19:23:37 -0500 Subject: throw errors returned by resource:download/3 --- src/rebar_fetch.erl | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) (limited to 'src') diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index 235aa03..96bbff1 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -26,24 +26,34 @@ lock_source(AppDir, Source, State) -> -spec download_source(file:filename_all(), rebar_resource:resource(), rebar_state:t()) -> true | {error, any()}. download_source(AppDir, Source, State) -> - try - Resources = rebar_state:resources(State), - Module = get_resource_type(Source, Resources), - TmpDir = ec_file:insecure_mkdtemp(), - AppDir1 = ec_cnv:to_list(AppDir), - {ok, _} = Module:download(TmpDir, Source, State), - ec_file:mkdir_p(AppDir1), - code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), - ec_file:remove(filename:absname(AppDir1), [recursive]), - ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]), - ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)), - true + try download_source_(AppDir, Source, State) of + true -> + true; + Error -> + throw(Error) catch C:T -> ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, erlang:get_stacktrace()]), throw(?PRV_ERROR({fetch_fail, Source})) end. +download_source_(AppDir, Source, State) -> + Resources = rebar_state:resources(State), + Module = get_resource_type(Source, Resources), + TmpDir = ec_file:insecure_mkdtemp(), + AppDir1 = ec_cnv:to_list(AppDir), + case Module:download(TmpDir, Source, State) of + {ok, _} -> + ec_file:mkdir_p(AppDir1), + code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), + ec_file:remove(filename:absname(AppDir1), [recursive]), + ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]), + ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)), + true; + Error -> + Error + end. + -spec needs_update(file:filename_all(), rebar_resource:resource(), rebar_state:t()) -> boolean() | {error, string()}. needs_update(AppDir, Source, State) -> Resources = rebar_state:resources(State), -- cgit v1.1 From 1643f4a1fc8a480c20ce972d9df47669111fff22 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 12 May 2015 19:32:24 -0500 Subject: wrap fetch errors in rebar_fetch PRV_ERROR --- src/rebar_fetch.erl | 8 +++++--- src/rebar_pkg_resource.erl | 11 +++++------ 2 files changed, 10 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index 96bbff1..0bb2270 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -30,7 +30,7 @@ download_source(AppDir, Source, State) -> true -> true; Error -> - throw(Error) + throw(?PRV_ERROR(Error)) catch C:T -> ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, erlang:get_stacktrace()]), @@ -65,10 +65,12 @@ needs_update(AppDir, Source, State) -> true end. +format_error({failed_extract, CachePath}) -> + io_lib:format("Failed to extract package: ~s", [CachePath]); format_error({bad_etag, Source}) -> - io_lib:format("MD5 Checksum comparison failed for: ~p", [Source]); + io_lib:format("MD5 Checksum comparison failed for: ~s", [Source]); format_error({fetch_fail, Source}) -> - io_lib:format("Failed to fetch and copy dep: ~p", [Source]); + io_lib:format("Failed to fetch and copy dep: ~s", [Source]); format_error({bad_checksum, File}) -> io_lib:format("Checksum mismatch against tarball in ~s", [File]); format_error({bad_registry_checksum, File}) -> diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 56e08fc..8bbd963 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -9,7 +9,6 @@ ,needs_update/2 ,make_vsn/1]). --include_lib("providers/include/providers.hrl"). -include("rebar.hrl"). -define(DEFAULT_CDN, "https://s3.amazonaws.com/s3.hex.pm/tarballs"). @@ -44,7 +43,7 @@ cached_download(TmpDir, CachePath, Pkg, Url, ETag, State) -> ?DEBUG("Download ~s error, using ~s from cache", [Url, CachePath]), serve_from_cache(TmpDir, CachePath, Pkg, State); error -> - throw(request_failed) + request_failed end. serve_from_cache(TmpDir, CachePath, Pkg, State) -> @@ -54,11 +53,11 @@ serve_from_cache(TmpDir, CachePath, Pkg, State) -> ok = erl_tar:extract({binary, Contents}, [{cwd, TmpDir}, compressed]), {ok, true}; {_Bin, Chk, Chk} -> - ?PRV_ERROR({failed_extract, CachePath}); + {failed_extract, CachePath}; {Chk, _Reg, Chk} -> - ?PRV_ERROR({bad_registry_checksum, CachePath}); + {bad_registry_checksum, CachePath}; {_Bin, _Reg, _Tar} -> - ?PRV_ERROR({bad_checksum, CachePath}) + {bad_checksum, CachePath} end. serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State) -> @@ -69,7 +68,7 @@ serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State) -> serve_from_cache(TmpDir, CachePath, Package, State); FileETag -> ?DEBUG("Download ETag ~s doesn't match cached ETag ~s", [ETag, FileETag]), - ?PRV_ERROR({bad_download, CachePath}) + {bad_download, CachePath} end. -- cgit v1.1 From 86fbaaeb08e7fc35ee87daea62be7febacd14ea9 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 12 May 2015 19:36:10 -0500 Subject: add bad_download error --- src/rebar_fetch.erl | 2 ++ src/rebar_pkg_resource.erl | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index 0bb2270..0aca308 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -65,6 +65,8 @@ needs_update(AppDir, Source, State) -> true end. +format_error({bad_download, CachePath}) -> + io_lib:format("Download of package does not match md5sum from server: ~s", [CachePath]); format_error({failed_extract, CachePath}) -> io_lib:format("Failed to extract package: ~s", [CachePath]); format_error({bad_etag, Source}) -> diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 8bbd963..f6bb29b 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -67,7 +67,7 @@ serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State) -> ETag -> serve_from_cache(TmpDir, CachePath, Package, State); FileETag -> - ?DEBUG("Download ETag ~s doesn't match cached ETag ~s", [ETag, FileETag]), + ?DEBUG("Download ETag ~s doesn't match returned ETag ~s", [ETag, FileETag]), {bad_download, CachePath} end. -- cgit v1.1 From ca038d39f70ea60ca4eb1c22a47e9b3bfcfe51a1 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 13 May 2015 20:16:03 +0000 Subject: Proper custom pkg index support, some tests - The rebar package index files have been moved off the default path and will require a new `rebar3 update` - Caching of downloaded packages automatically takes place in a path relative to the CDN used - The cache path is not shared with hex as we now write and modify data in there arbitrarily - Basic tests plus the working set for more of them is included --- src/rebar.hrl | 1 + src/rebar_packages.erl | 18 ++++++++++++++---- src/rebar_pkg_resource.erl | 26 ++++++++------------------ src/rebar_prv_update.erl | 6 ++---- 4 files changed, 25 insertions(+), 26 deletions(-) (limited to 'src') diff --git a/src/rebar.hrl b/src/rebar.hrl index 1f051d7..4540b1a 100644 --- a/src/rebar.hrl +++ b/src/rebar.hrl @@ -22,6 +22,7 @@ -define(DEFAULT_TEST_DEPS_DIR, "test/lib"). -define(DEFAULT_RELEASE_DIR, "rel"). -define(DEFAULT_CONFIG_FILE, "rebar.config"). +-define(DEFAULT_CDN, "https://s3.amazonaws.com/s3.hex.pm/tarballs"). -define(LOCK_FILE, "rebar.lock"). -ifdef(namespaced_types). diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index a344328..e21f1fd 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -2,6 +2,7 @@ -export([get_packages/1 ,registry/1 + ,package_dir/1 ,check_registry/3 ,registry_checksum/2 ,find_highest_matching/3]). @@ -16,8 +17,7 @@ -spec get_packages(rebar_state:t()) -> {rebar_dict(), rebar_digraph()}. get_packages(State) -> - RebarDir = rebar_dir:global_cache_dir(State), - RegistryDir = filename:join(RebarDir, "packages"), + RegistryDir = package_dir(State), DictFile = filename:join(RegistryDir, "dict"), Edges = filename:join(RegistryDir, "edges"), Vertices = filename:join(RegistryDir, "vertices"), @@ -43,8 +43,7 @@ get_packages(State) -> end. registry(State) -> - Dir = rebar_dir:global_cache_dir(State), - RegistryDir = filename:join(Dir, "packages"), + RegistryDir = package_dir(State), HexFile = filename:join(RegistryDir, "registry"), case ets:file2tab(HexFile) of {ok, T} -> @@ -54,6 +53,17 @@ registry(State) -> error end. +package_dir(State) -> + CacheDir = rebar_dir:global_cache_dir(State), + CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), + {ok, {_, _, Host, _, Path, _}} = http_uri:parse(CDN), + CDNHostPath = lists:reverse(string:tokens(Host, ".")), + CDNPath = tl(filename:split(Path)), + PackageDir = filename:join([CacheDir, "hex"] ++ CDNHostPath ++ CDNPath ++ ["packages"]), + ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")), + PackageDir. + + check_registry(Pkg, Vsn, State) -> case rebar_state:registry(State) of {ok, T} -> diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index f6bb29b..59ce0dc 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -11,8 +11,6 @@ -include("rebar.hrl"). --define(DEFAULT_CDN, "https://s3.amazonaws.com/s3.hex.pm/tarballs"). - lock(_AppDir, Source) -> Source. @@ -27,7 +25,7 @@ needs_update(Dir, {pkg, _Name, Vsn}) -> download(TmpDir, Pkg={pkg, Name, Vsn}, State) -> CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), - PackageDir = hex_package_dir(CDN, State), + PackageDir = rebar_packages:package_dir(State), Package = binary_to_list(<>), CachePath = filename:join(PackageDir, Package), Url = string:join([CDN, Package], "/"), @@ -53,10 +51,13 @@ serve_from_cache(TmpDir, CachePath, Pkg, State) -> ok = erl_tar:extract({binary, Contents}, [{cwd, TmpDir}, compressed]), {ok, true}; {_Bin, Chk, Chk} -> + ?DEBUG("Checksums: registry: ~p, pkg: ~p", [Chk, _Bin]), {failed_extract, CachePath}; {Chk, _Reg, Chk} -> + ?DEBUG("Checksums: registry: ~p, pkg: ~p", [_Reg, Chk]), {bad_registry_checksum, CachePath}; {_Bin, _Reg, _Tar} -> + ?DEBUG("Checksums: registry: ~p, pkg: ~p, meta: ~p", [_Reg, _Bin, _Tar]), {bad_checksum, CachePath} end. @@ -67,7 +68,7 @@ serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State) -> ETag -> serve_from_cache(TmpDir, CachePath, Package, State); FileETag -> - ?DEBUG("Download ETag ~s doesn't match returned ETag ~s", [ETag, FileETag]), + ?DEBUG("Downloaded file ~s ETag ~s doesn't match returned ETag ~s", [CachePath, ETag, FileETag]), {bad_download, CachePath} end. @@ -84,28 +85,17 @@ checksums(Pkg, Files, Contents, Version, Meta, State) -> Blob = <>, <> = crypto:hash(sha256, Blob), BinChecksum = list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))), - RegistryChecksum = rebar_packages:registry_sum(Pkg, State), + RegistryChecksum = rebar_packages:registry_checksum(Pkg, State), {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files), {BinChecksum, RegistryChecksum, TarChecksum}. make_vsn(_) -> {error, "Replacing version of type pkg not supported."}. -%% Use the shared hex package directory unless a non-default package repo is used -hex_package_dir(?DEFAULT_CDN, _) -> - filename:join([rebar_dir:home_dir(), ".hex", "packages"]); -hex_package_dir(CDN, State) -> - CacheDir = rebar_dir:global_cache_dir(State), - {ok, {_, _, Host, _, _, _}} = http_uri:parse(CDN), - CDNPath = filename:join(lists:reverse(string:tokens(Host, "."))), - PackageDir = filename:join([CacheDir, "hex", CDNPath, "packages"]), - ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")), - PackageDir. - request(Url, ETag) -> case httpc:request(get, {Url, [{"if-none-match", ETag} || ETag =/= false]}, - [{relaxed, true}], - [{body_format, binary}]) of + [{relaxed, true}], + [{body_format, binary}]) of {ok, {{_Version, 200, _Reason}, Headers, Body}} -> ?DEBUG("Successfully downloaded ~s", [Url]), {"etag", ETag1} = lists:keyfind("etag", 1, Headers), diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index 942b386..dfb719a 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -35,8 +35,7 @@ init(State) -> do(State) -> ?INFO("Updating package index...", []), try - Dir = rebar_dir:global_cache_dir(State), - RegistryDir = filename:join(Dir, "packages"), + RegistryDir = rebar_packages:package_dir(State), filelib:ensure_dir(filename:join(RegistryDir, "dummy")), HexFile = filename:join(RegistryDir, "registry"), TmpDir = ec_file:insecure_mkdtemp(), @@ -64,8 +63,7 @@ format_error(package_index_write) -> "Failed to write package index.". write_registry(Dict, {digraph, Edges, Vertices, Neighbors, _}, State) -> - Dir = rebar_dir:global_cache_dir(State), - RegistryDir = filename:join(Dir, "packages"), + RegistryDir = rebar_packages:package_dir(State), filelib:ensure_dir(filename:join(RegistryDir, "dummy")), ets:tab2file(Edges, filename:join(RegistryDir, "edges")), ets:tab2file(Vertices, filename:join(RegistryDir, "vertices")), -- cgit v1.1