From 56c925b75b86a6304da75635874722348ee21351 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Tue, 12 May 2015 16:58:38 +0000 Subject: Ad-hoc attempt at restructuring pkg cache --- src/rebar_fetch.erl | 51 ++++-------------------------- src/rebar_pkg_resource.erl | 79 ++++++++++++++++++++++++++++++++-------------- 2 files changed, 62 insertions(+), 68 deletions(-) (limited to 'src') diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index 75970ed..235aa03 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -31,20 +31,14 @@ download_source(AppDir, Source, State) -> Module = get_resource_type(Source, Resources), TmpDir = ec_file:insecure_mkdtemp(), AppDir1 = ec_cnv:to_list(AppDir), - case Module:download(TmpDir, Source, State) of - {ok, _} -> - ec_file:mkdir_p(AppDir1), - code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), - ec_file:remove(filename:absname(AppDir1), [recursive]), - ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]), - ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)), - true; - {tarball, File} -> - verify_and_extract(File, Source, AppDir1, State) - end + {ok, _} = Module:download(TmpDir, Source, State), + ec_file:mkdir_p(AppDir1), + code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), + ec_file:remove(filename:absname(AppDir1), [recursive]), + ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]), + ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)), + true catch - _:bad_etag -> - throw(?PRV_ERROR({bad_etag, Source})); C:T -> ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, erlang:get_stacktrace()]), throw(?PRV_ERROR({fetch_fail, Source})) @@ -92,34 +86,3 @@ find_resource_module(Type, Location, Resources) -> {Type, Module} -> Module end. - -verify_and_extract(File, Source, AppDir, State) -> - ec_file:mkdir_p(AppDir), - {ok, Files} = erl_tar:extract(File, [memory]), - - code:del_path(filename:absname(filename:join(AppDir, "ebin"))), - ec_file:remove(filename:absname(AppDir), [recursive]), - - {"contents.tar.gz", Contents} = lists:keyfind("contents.tar.gz", 1, Files), - {"VERSION", Version} = lists:keyfind("VERSION", 1, Files), - {"metadata.config", Meta} = lists:keyfind("metadata.config", 1, Files), - - Checksum = checksum(Contents, Version, Meta), - RegistryChecksum = rebar_packages:registry_checksum(Source, State), - {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files), - - if - Checksum =/= TarChecksum -> - ?PRV_ERROR({bad_checksum, File}); - Checksum =/= RegistryChecksum -> - ?PRV_ERROR({bad_registry_checksum, File}); - true -> - ok = erl_tar:extract({binary, Contents}, - [{cwd, filename:absname(AppDir)}, compressed]), - true - end. - -checksum(Contents, Version, Meta) -> - Blob = <>, - <> = crypto:hash(sha256, Blob), - list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))). diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 9b430d2..56e08fc 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -26,38 +26,69 @@ needs_update(Dir, {pkg, _Name, Vsn}) -> true end. -download(_Dir, {pkg, Name, Vsn}, State) -> +download(TmpDir, Pkg={pkg, Name, Vsn}, State) -> CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), PackageDir = hex_package_dir(CDN, State), - Package = binary_to_list(<>), - Path = filename:join(PackageDir, Package), + CachePath = filename:join(PackageDir, Package), Url = string:join([CDN, Package], "/"), + cached_download(TmpDir, CachePath, Pkg, Url, etag(CachePath), State). - case request(Url, etag(Path)) of +cached_download(TmpDir, CachePath, Pkg, Url, ETag, State) -> + case request(Url, ETag) of {ok, cached} -> - {tarball, Path}; - {ok, Binary, EtagHeader} -> - file:write_file(Path, Binary), - Etag = etag(Path), - case EtagHeader =:= Etag of - true -> - {tarball, Path}; - false -> - ?DEBUG("Bad md5sum for ~s of ~s comparing to ~s sent by server", - [Path, Etag, EtagHeader]), - throw(bad_etag) - end; + serve_from_cache(TmpDir, CachePath, Pkg, State); + {ok, Body, NewETag} -> + serve_from_download(TmpDir, CachePath, Pkg, NewETag, Body, State); + error when ETag =/= false -> + ?DEBUG("Download ~s error, using ~s from cache", [Url, CachePath]), + serve_from_cache(TmpDir, CachePath, Pkg, State); error -> - case filelib:is_regular(Path) of - true -> - ?DEBUG("Download ~s error, using ~s since it exists", [Url, Path]), - {tarball, Path}; - false -> - throw(request_failed) - end + throw(request_failed) + end. + +serve_from_cache(TmpDir, CachePath, Pkg, State) -> + {Files, Contents, Version, Meta} = extract(TmpDir, CachePath), + case checksums(Pkg, Files, Contents, Version, Meta, State) of + {Chk, Chk, Chk} -> + ok = erl_tar:extract({binary, Contents}, [{cwd, TmpDir}, compressed]), + {ok, true}; + {_Bin, Chk, Chk} -> + ?PRV_ERROR({failed_extract, CachePath}); + {Chk, _Reg, Chk} -> + ?PRV_ERROR({bad_registry_checksum, CachePath}); + {_Bin, _Reg, _Tar} -> + ?PRV_ERROR({bad_checksum, CachePath}) + end. + +serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State) -> + ?DEBUG("Writing ~p to cache at ~s", [Package, CachePath]), + file:write_file(CachePath, Binary), + case etag(CachePath) of + ETag -> + serve_from_cache(TmpDir, CachePath, Package, State); + FileETag -> + ?DEBUG("Download ETag ~s doesn't match cached ETag ~s", [ETag, FileETag]), + ?PRV_ERROR({bad_download, CachePath}) end. + +extract(TmpDir, CachePath) -> + ec_file:mkdir_p(TmpDir), + {ok, Files} = erl_tar:extract(CachePath, [memory]), + {"contents.tar.gz", Contents} = lists:keyfind("contents.tar.gz", 1, Files), + {"VERSION", Version} = lists:keyfind("VERSION", 1, Files), + {"metadata.config", Meta} = lists:keyfind("metadata.config", 1, Files), + {Files, Contents, Version, Meta}. + +checksums(Pkg, Files, Contents, Version, Meta, State) -> + Blob = <>, + <> = crypto:hash(sha256, Blob), + BinChecksum = list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))), + RegistryChecksum = rebar_packages:registry_sum(Pkg, State), + {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files), + {BinChecksum, RegistryChecksum, TarChecksum}. + make_vsn(_) -> {error, "Replacing version of type pkg not supported."}. @@ -77,8 +108,8 @@ request(Url, ETag) -> [{relaxed, true}], [{body_format, binary}]) of {ok, {{_Version, 200, _Reason}, Headers, Body}} -> - {"etag", ETag1} = lists:keyfind("etag", 1, Headers), ?DEBUG("Successfully downloaded ~s", [Url]), + {"etag", ETag1} = lists:keyfind("etag", 1, Headers), {ok, Body, string:strip(ETag1, both, $")}; {ok, {{_Version, 304, _Reason}, _Headers, _Body}} -> ?DEBUG("Cached copy of ~s still valid", [Url]), -- cgit v1.1