diff options
28 files changed, 406 insertions, 104 deletions
diff --git a/priv/templates/app.erl b/priv/templates/app.erl index 83eb9a3..2e7b909 100644 --- a/priv/templates/app.erl +++ b/priv/templates/app.erl @@ -3,7 +3,7 @@ %% @end %%%------------------------------------------------------------------- --module({{name}}_app). +-module('{{name}}_app'). -behaviour(application). @@ -16,7 +16,7 @@ %%==================================================================== start(_StartType, _StartArgs) -> - {{name}}_sup:start_link(). + '{{name}}_sup':start_link(). %%-------------------------------------------------------------------- stop(_State) -> diff --git a/priv/templates/app.template b/priv/templates/app.template index 4087e5e..d0bebd7 100644 --- a/priv/templates/app.template +++ b/priv/templates/app.template @@ -1,4 +1,4 @@ -{description, "OTP Application"}. +{description, "Complete OTP Application structure."}. {variables, [ {name, "mylib", "Name of the OTP application"}, {desc, "An OTP application", "Short description of the app"} diff --git a/priv/templates/cmake.template b/priv/templates/cmake.template index 2874c8a..227f5fa 100644 --- a/priv/templates/cmake.template +++ b/priv/templates/cmake.template @@ -1,2 +1,2 @@ -{description, "Makefile for building C/C++ in c_src"}. +{description, "Standalone Makefile for building C/C++ in c_src"}. {template, "Makefile", "c_src/Makefile"}. diff --git a/priv/templates/escript.template b/priv/templates/escript.template index c0afd3c..e41c313 100644 --- a/priv/templates/escript.template +++ b/priv/templates/escript.template @@ -1,4 +1,4 @@ -{description, "Escriptized application"}. +{description, "Complete escriptized application structure"}. {variables, [ {name, "mylib", "Name of the OTP application to be escriptized"}, {desc, "An escript", "Short description of the project"} diff --git a/priv/templates/escript_mod.erl b/priv/templates/escript_mod.erl index f8a779b..dd2feb8 100644 --- a/priv/templates/escript_mod.erl +++ b/priv/templates/escript_mod.erl @@ -1,4 +1,4 @@ --module({{name}}). +-module('{{name}}'). %% API exports -export([main/1]). diff --git a/priv/templates/escript_rebar.config b/priv/templates/escript_rebar.config index 196f835..c0a3301 100644 --- a/priv/templates/escript_rebar.config +++ b/priv/templates/escript_rebar.config @@ -2,9 +2,9 @@ {deps, []}. {escript_incl_apps, - [{{name}}]}. -{escript_top_level_app, {{name}}}. -{escript_name, {{name}}}. + ['{{name}}']}. +{escript_top_level_app, '{{name}}'}. +{escript_name, '{{name}}'}. {escript_emu_args, "%%! +sbtu +A0\n"}. %% Profiles diff --git a/priv/templates/lib.template b/priv/templates/lib.template index cbb3672..4d06901 100644 --- a/priv/templates/lib.template +++ b/priv/templates/lib.template @@ -1,4 +1,4 @@ -{description, "OTP Library application (no processes)"}. +{description, "Complete OTP Library application (no processes) structure"}. {variables, [ {name, "mylib", "Name of the OTP library application"}, {desc, "An OTP library", "Short description of the app"} diff --git a/priv/templates/mod.erl b/priv/templates/mod.erl index 2f5e09e..208307e 100644 --- a/priv/templates/mod.erl +++ b/priv/templates/mod.erl @@ -1,4 +1,4 @@ --module({{name}}). +-module('{{name}}'). %% API exports -export([]). diff --git a/priv/templates/otp_app.app.src b/priv/templates/otp_app.app.src index d5ea3d2..9ad7478 100644 --- a/priv/templates/otp_app.app.src +++ b/priv/templates/otp_app.app.src @@ -1,4 +1,4 @@ -{application, {{name}}, +{application, '{{name}}', [{description, "{{desc}}"}, {vsn, "0.1.0"}, {registered, []}, diff --git a/priv/templates/otp_lib.app.src b/priv/templates/otp_lib.app.src index 6af7640..3f4b56b 100644 --- a/priv/templates/otp_lib.app.src +++ b/priv/templates/otp_lib.app.src @@ -1,4 +1,4 @@ -{application, {{name}}, +{application, '{{name}}', [{description, "{{desc}}"}, {vsn, "0.1.0"}, {registered, []}, diff --git a/priv/templates/plugin.erl b/priv/templates/plugin.erl index abf4648..018dd0e 100644 --- a/priv/templates/plugin.erl +++ b/priv/templates/plugin.erl @@ -1,9 +1,9 @@ --module({{name}}). +-module('{{name}}'). -behaviour(provider). -export([init/1, do/1, format_error/1]). --define(PROVIDER, {{name}}). +-define(PROVIDER, '{{name}}'). -define(DEPS, [app_discovery]). %% =================================================================== diff --git a/priv/templates/plugin.template b/priv/templates/plugin.template index 0181bd2..811be0b 100644 --- a/priv/templates/plugin.template +++ b/priv/templates/plugin.template @@ -1,4 +1,4 @@ -{description, "Rebar3 plugin"}. +{description, "Rebar3 plugin project structure"}. {variables, [ {name, "myplugin", "Name of the plugin"}, {desc, "A rebar plugin", "Short description of the plugin's purpose"} diff --git a/priv/templates/relx_rebar.config b/priv/templates/relx_rebar.config index e7505eb..da32819 100644 --- a/priv/templates/relx_rebar.config +++ b/priv/templates/relx_rebar.config @@ -2,7 +2,7 @@ {deps, []}. {relx, [{release, {'{{name}}', "0.1.0"}, - [{{name}}, + ['{{name}}', sasl]}, {sys_config, "./config/sys.config"}, diff --git a/priv/templates/sup.erl b/priv/templates/sup.erl index a2e7209..cd9192a 100644 --- a/priv/templates/sup.erl +++ b/priv/templates/sup.erl @@ -3,7 +3,7 @@ %% @end %%%------------------------------------------------------------------- --module({{name}}_sup). +-module('{{name}}_sup'). -behaviour(supervisor). diff --git a/rebar.config b/rebar.config index 5eaf6de..504a462 100644 --- a/rebar.config +++ b/rebar.config @@ -30,9 +30,7 @@ {"rebar/include/*", "."}]}. {erl_opts, - [{platform_define, "R14", no_callback_support}, - {platform_define, "^[0-9]+", namespaced_types}, - {platform_define, "^R1[4|5]", deprecated_crypto}, + [{platform_define, "^[0-9]+", namespaced_types}, no_debug_info, warnings_as_errors]}. diff --git a/src/rebar.hrl b/src/rebar.hrl index 1f051d7..4540b1a 100644 --- a/src/rebar.hrl +++ b/src/rebar.hrl @@ -22,6 +22,7 @@ -define(DEFAULT_TEST_DEPS_DIR, "test/lib"). -define(DEFAULT_RELEASE_DIR, "rel"). -define(DEFAULT_CONFIG_FILE, "rebar.config"). +-define(DEFAULT_CDN, "https://s3.amazonaws.com/s3.hex.pm/tarballs"). -define(LOCK_FILE, "rebar.lock"). -ifdef(namespaced_types). diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index 20bf46b..0aca308 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -26,42 +26,34 @@ lock_source(AppDir, Source, State) -> -spec download_source(file:filename_all(), rebar_resource:resource(), rebar_state:t()) -> true | {error, any()}. download_source(AppDir, Source, State) -> - try - Resources = rebar_state:resources(State), - Module = get_resource_type(Source, Resources), - TmpDir = ec_file:insecure_mkdtemp(), - AppDir1 = ec_cnv:to_list(AppDir), - case Module:download(TmpDir, Source, State) of - {ok, _} -> - ec_file:mkdir_p(AppDir1), - code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), - ec_file:remove(filename:absname(AppDir1), [recursive]), - ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]), - ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)), - true; - {tarball, File} -> - Contents = filename:join(TmpDir, "contents"), - ec_file:mkdir_p(AppDir1), - ec_file:mkdir_p(Contents), - ok = erl_tar:extract(File, [{cwd, TmpDir}]), - ok = erl_tar:extract(filename:join(TmpDir, "contents.tar.gz"), - [{cwd, Contents}, compressed]), - code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), - ec_file:remove(filename:absname(AppDir1), [recursive]), - - ?DEBUG("Moving contents ~p to ~p", [Contents, filename:absname(AppDir1)]), - ok = rebar_file_utils:mv(Contents, filename:absname(AppDir1)), - - ?DEBUG("Removing tmp dir ~p", [TmpDir]), - ec_file:remove(TmpDir, [recursive]), - true - end + try download_source_(AppDir, Source, State) of + true -> + true; + Error -> + throw(?PRV_ERROR(Error)) catch C:T -> ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, erlang:get_stacktrace()]), throw(?PRV_ERROR({fetch_fail, Source})) end. +download_source_(AppDir, Source, State) -> + Resources = rebar_state:resources(State), + Module = get_resource_type(Source, Resources), + TmpDir = ec_file:insecure_mkdtemp(), + AppDir1 = ec_cnv:to_list(AppDir), + case Module:download(TmpDir, Source, State) of + {ok, _} -> + ec_file:mkdir_p(AppDir1), + code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), + ec_file:remove(filename:absname(AppDir1), [recursive]), + ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]), + ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)), + true; + Error -> + Error + end. + -spec needs_update(file:filename_all(), rebar_resource:resource(), rebar_state:t()) -> boolean() | {error, string()}. needs_update(AppDir, Source, State) -> Resources = rebar_state:resources(State), @@ -73,8 +65,18 @@ needs_update(AppDir, Source, State) -> true end. +format_error({bad_download, CachePath}) -> + io_lib:format("Download of package does not match md5sum from server: ~s", [CachePath]); +format_error({failed_extract, CachePath}) -> + io_lib:format("Failed to extract package: ~s", [CachePath]); +format_error({bad_etag, Source}) -> + io_lib:format("MD5 Checksum comparison failed for: ~s", [Source]); format_error({fetch_fail, Source}) -> - io_lib:format("Failed to fetch and copy dep: ~p", [Source]). + io_lib:format("Failed to fetch and copy dep: ~s", [Source]); +format_error({bad_checksum, File}) -> + io_lib:format("Checksum mismatch against tarball in ~s", [File]); +format_error({bad_registry_checksum, File}) -> + io_lib:format("Checksum mismatch against registry in ~s", [File]). get_resource_type({Type, Location}, Resources) -> find_resource_module(Type, Location, Resources); diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 4ab5f9f..e21f1fd 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -2,7 +2,9 @@ -export([get_packages/1 ,registry/1 + ,package_dir/1 ,check_registry/3 + ,registry_checksum/2 ,find_highest_matching/3]). -export_type([package/0]). @@ -15,8 +17,7 @@ -spec get_packages(rebar_state:t()) -> {rebar_dict(), rebar_digraph()}. get_packages(State) -> - RebarDir = rebar_dir:global_cache_dir(State), - RegistryDir = filename:join(RebarDir, "packages"), + RegistryDir = package_dir(State), DictFile = filename:join(RegistryDir, "dict"), Edges = filename:join(RegistryDir, "edges"), Vertices = filename:join(RegistryDir, "vertices"), @@ -42,8 +43,7 @@ get_packages(State) -> end. registry(State) -> - Dir = rebar_dir:global_cache_dir(State), - RegistryDir = filename:join(Dir, "packages"), + RegistryDir = package_dir(State), HexFile = filename:join(RegistryDir, "registry"), case ets:file2tab(HexFile) of {ok, T} -> @@ -53,6 +53,17 @@ registry(State) -> error end. +package_dir(State) -> + CacheDir = rebar_dir:global_cache_dir(State), + CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), + {ok, {_, _, Host, _, Path, _}} = http_uri:parse(CDN), + CDNHostPath = lists:reverse(string:tokens(Host, ".")), + CDNPath = tl(filename:split(Path)), + PackageDir = filename:join([CacheDir, "hex"] ++ CDNHostPath ++ CDNPath ++ ["packages"]), + ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")), + PackageDir. + + check_registry(Pkg, Vsn, State) -> case rebar_state:registry(State) of {ok, T} -> @@ -66,6 +77,15 @@ check_registry(Pkg, Vsn, State) -> false end. +registry_checksum({pkg, Name, Vsn}, State) -> + {ok, Registry} = registry(State), + case ets:lookup(Registry, {Name, Vsn}) of + [{{_, _}, [_, Checksum | _]}] -> + Checksum; + [] -> + none + end. + %% Hex supports use of ~> to specify the version required for a dependency. %% Since rebar3 requires exact versions to choose from we find the highest %% available version of the dep that passes the constraint. diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 3b44fc8..59ce0dc 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -23,12 +23,99 @@ needs_update(Dir, {pkg, _Name, Vsn}) -> true end. -download(Dir, {pkg, Name, Vsn}, State) -> - TmpFile = filename:join(Dir, "package.tar.gz"), - CDN = rebar_state:get(State, rebar_packages_cdn, "https://s3.amazonaws.com/s3.hex.pm/tarballs"), - Url = string:join([CDN, binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>)], "/"), - {ok, saved_to_file} = httpc:request(get, {Url, []}, [], [{stream, TmpFile}]), - {tarball, TmpFile}. +download(TmpDir, Pkg={pkg, Name, Vsn}, State) -> + CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), + PackageDir = rebar_packages:package_dir(State), + Package = binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>), + CachePath = filename:join(PackageDir, Package), + Url = string:join([CDN, Package], "/"), + cached_download(TmpDir, CachePath, Pkg, Url, etag(CachePath), State). + +cached_download(TmpDir, CachePath, Pkg, Url, ETag, State) -> + case request(Url, ETag) of + {ok, cached} -> + serve_from_cache(TmpDir, CachePath, Pkg, State); + {ok, Body, NewETag} -> + serve_from_download(TmpDir, CachePath, Pkg, NewETag, Body, State); + error when ETag =/= false -> + ?DEBUG("Download ~s error, using ~s from cache", [Url, CachePath]), + serve_from_cache(TmpDir, CachePath, Pkg, State); + error -> + request_failed + end. + +serve_from_cache(TmpDir, CachePath, Pkg, State) -> + {Files, Contents, Version, Meta} = extract(TmpDir, CachePath), + case checksums(Pkg, Files, Contents, Version, Meta, State) of + {Chk, Chk, Chk} -> + ok = erl_tar:extract({binary, Contents}, [{cwd, TmpDir}, compressed]), + {ok, true}; + {_Bin, Chk, Chk} -> + ?DEBUG("Checksums: registry: ~p, pkg: ~p", [Chk, _Bin]), + {failed_extract, CachePath}; + {Chk, _Reg, Chk} -> + ?DEBUG("Checksums: registry: ~p, pkg: ~p", [_Reg, Chk]), + {bad_registry_checksum, CachePath}; + {_Bin, _Reg, _Tar} -> + ?DEBUG("Checksums: registry: ~p, pkg: ~p, meta: ~p", [_Reg, _Bin, _Tar]), + {bad_checksum, CachePath} + end. + +serve_from_download(TmpDir, CachePath, Package, ETag, Binary, State) -> + ?DEBUG("Writing ~p to cache at ~s", [Package, CachePath]), + file:write_file(CachePath, Binary), + case etag(CachePath) of + ETag -> + serve_from_cache(TmpDir, CachePath, Package, State); + FileETag -> + ?DEBUG("Downloaded file ~s ETag ~s doesn't match returned ETag ~s", [CachePath, ETag, FileETag]), + {bad_download, CachePath} + end. + + +extract(TmpDir, CachePath) -> + ec_file:mkdir_p(TmpDir), + {ok, Files} = erl_tar:extract(CachePath, [memory]), + {"contents.tar.gz", Contents} = lists:keyfind("contents.tar.gz", 1, Files), + {"VERSION", Version} = lists:keyfind("VERSION", 1, Files), + {"metadata.config", Meta} = lists:keyfind("metadata.config", 1, Files), + {Files, Contents, Version, Meta}. + +checksums(Pkg, Files, Contents, Version, Meta, State) -> + Blob = <<Version/binary, Meta/binary, Contents/binary>>, + <<X:256/big-unsigned>> = crypto:hash(sha256, Blob), + BinChecksum = list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))), + RegistryChecksum = rebar_packages:registry_checksum(Pkg, State), + {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files), + {BinChecksum, RegistryChecksum, TarChecksum}. make_vsn(_) -> {error, "Replacing version of type pkg not supported."}. + +request(Url, ETag) -> + case httpc:request(get, {Url, [{"if-none-match", ETag} || ETag =/= false]}, + [{relaxed, true}], + [{body_format, binary}]) of + {ok, {{_Version, 200, _Reason}, Headers, Body}} -> + ?DEBUG("Successfully downloaded ~s", [Url]), + {"etag", ETag1} = lists:keyfind("etag", 1, Headers), + {ok, Body, string:strip(ETag1, both, $")}; + {ok, {{_Version, 304, _Reason}, _Headers, _Body}} -> + ?DEBUG("Cached copy of ~s still valid", [Url]), + {ok, cached}; + {ok, {{_Version, Code, _Reason}, _Headers, _Body}} -> + ?DEBUG("Request to ~p failed: status code ~p", [Url, Code]), + error; + {error, Reason} -> + ?DEBUG("Request to ~p failed: ~p", [Url, Reason]), + error + end. + +etag(Path) -> + case file:read_file(Path) of + {ok, Binary} -> + <<X:128/big-unsigned-integer>> = crypto:hash(md5, Binary), + string:to_lower(lists:flatten(io_lib:format("~32.16.0b", [X]))); + {error, _} -> + false + end. diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index bc0a113..ba49532 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -151,7 +151,7 @@ handle_deps(Profile, State0, Deps, Upgrade, Locks) -> ,{Packages, Graph}), update_pkg_deps(Profile, Packages, PkgDeps1 - ,Graph, Upgrade, Seen, State2) + ,Graph, Upgrade, Seen, State2, Locks) end, AllDeps = lists:ukeymerge(2 @@ -184,7 +184,7 @@ find_cycles(Apps) -> cull_compile(TopSortedDeps, ProjectApps) -> lists:dropwhile(fun not_needs_compile/1, TopSortedDeps -- ProjectApps). -update_pkg_deps(Profile, Packages, PkgDeps, Graph, Upgrade, Seen, State) -> +update_pkg_deps(Profile, Packages, PkgDeps, Graph, Upgrade, Seen, State, Locks) -> case PkgDeps of [] -> %% No pkg deps {[], State}; @@ -196,13 +196,16 @@ update_pkg_deps(Profile, Packages, PkgDeps, Graph, Upgrade, Seen, State) -> {ok, Solution, []} -> Solution; {ok, Solution, Discarded} -> - [warn_skip_pkg(Pkg, State) || Pkg <- Discarded], + [warn_skip_pkg(Pkg, State) || Pkg <- Discarded, not(pkg_locked(Pkg, Locks))], Solution end, - update_pkg_deps(Profile, S, Packages, Upgrade, Seen, State) + update_pkg_deps(Profile, S, Packages, Upgrade, Seen, State, Locks) end. -update_pkg_deps(Profile, Pkgs, Packages, Upgrade, Seen, State) -> +pkg_locked({Name, _}, Locks) -> + false =/= lists:keyfind(Name, 1, Locks). + +update_pkg_deps(Profile, Pkgs, Packages, Upgrade, Seen, State, _Locks) -> %% Create app_info record for each pkg dep DepsDir = profile_dep_dir(State, Profile), {Solved, _, State1} @@ -563,10 +566,10 @@ fetch_app(AppInfo, AppDir, State) -> ?INFO("Fetching ~s (~p)", [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)]), Source = rebar_app_info:source(AppInfo), case rebar_fetch:download_source(AppDir, Source, State) of - {error, Reason} -> - throw(Reason); - Result -> - Result + true -> + true; + Error -> + throw(Error) end. update_app_info(AppInfo) -> @@ -587,10 +590,10 @@ maybe_upgrade(AppInfo, AppDir, true, State) -> true -> ?INFO("Upgrading ~s", [rebar_app_info:name(AppInfo)]), case rebar_fetch:download_source(AppDir, Source, State) of - {error, Reason} -> - throw(Reason); - Result -> - Result + true -> + true; + Error -> + throw(Error) end; false -> ?INFO("No upgrade needed for ~s", [rebar_app_info:name(AppInfo)]), @@ -610,7 +613,7 @@ parse_goal(Name, Constraint) -> warn_skip_deps(AppInfo, State) -> Msg = "Skipping ~s (from ~p) as an app of the same name " - "has already been fetched~n", + "has already been fetched", Args = [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)], case rebar_state:get(State, deps_error_on_conflict, false) of @@ -620,7 +623,7 @@ warn_skip_deps(AppInfo, State) -> warn_skip_pkg({Name, Source}, State) -> Msg = "Skipping ~s (version ~s from package index) as an app of the same " - "name has already been fetched~n", + "name has already been fetched", Args = [Name, Source], case rebar_state:get(State, deps_error_on_conflict, false) of false -> ?WARN(Msg, Args); diff --git a/src/rebar_prv_unlock.erl b/src/rebar_prv_unlock.erl index dc9fe11..9790fcf 100644 --- a/src/rebar_prv_unlock.erl +++ b/src/rebar_prv_unlock.erl @@ -31,7 +31,7 @@ init(State) -> "their name can be listed in the command."}, {opts, [ {package, undefined, undefined, string, - "List of packages to upgrade. If not specified, all dependencies are upgraded."} + "List of packages to unlock. If not specified, all dependencies are unlocked."} ]} ]) ), diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl index 942b386..dfb719a 100644 --- a/src/rebar_prv_update.erl +++ b/src/rebar_prv_update.erl @@ -35,8 +35,7 @@ init(State) -> do(State) -> ?INFO("Updating package index...", []), try - Dir = rebar_dir:global_cache_dir(State), - RegistryDir = filename:join(Dir, "packages"), + RegistryDir = rebar_packages:package_dir(State), filelib:ensure_dir(filename:join(RegistryDir, "dummy")), HexFile = filename:join(RegistryDir, "registry"), TmpDir = ec_file:insecure_mkdtemp(), @@ -64,8 +63,7 @@ format_error(package_index_write) -> "Failed to write package index.". write_registry(Dict, {digraph, Edges, Vertices, Neighbors, _}, State) -> - Dir = rebar_dir:global_cache_dir(State), - RegistryDir = filename:join(Dir, "packages"), + RegistryDir = rebar_packages:package_dir(State), filelib:ensure_dir(filename:join(RegistryDir, "dummy")), ets:tab2file(Edges, filename:join(RegistryDir, "edges")), ets:tab2file(Vertices, filename:join(RegistryDir, "vertices")), diff --git a/src/rebar_resource.erl b/src/rebar_resource.erl index 7c58135..cdce7a8 100644 --- a/src/rebar_resource.erl +++ b/src/rebar_resource.erl @@ -14,23 +14,6 @@ -type location() :: string(). -type ref() :: any(). --ifdef(no_callback_support). - -%% In the case where R14 or lower is being used to compile the system -%% we need to export a behaviour info --export([behaviour_info/1]). - --spec behaviour_info(atom()) -> [{atom(), arity()}] | undefined. -behaviour_info(callbacks) -> - [{lock, 2}, - {download, 3}, - {needs_update,2}, - {make_vsn, 1}]; -behaviour_info(_) -> - undefined. - --else. - -callback lock(file:filename_all(), tuple()) -> rebar_resource:resource(). -callback download(file:filename_all(), tuple(), rebar_state:t()) -> @@ -39,5 +22,3 @@ behaviour_info(_) -> boolean(). -callback make_vsn(file:filename_all()) -> {plain, string()} | {error, string()}. - --endif. diff --git a/test/mock_pkg_resource.erl b/test/mock_pkg_resource.erl index 560caef..e94ea93 100644 --- a/test/mock_pkg_resource.erl +++ b/test/mock_pkg_resource.erl @@ -16,6 +16,7 @@ mock() -> mock([]). -spec mock(Opts) -> ok when Opts :: [Option], Option :: {update, [App]} + | {cache_dir, string()} | {default_vsn, Vsn} | {override_vsn, [{App, Vsn}]} | {not_in_index, [{App, Vsn}]} @@ -83,7 +84,8 @@ mock_download(Opts) -> [kernel, stdlib] ++ [element(1,D) || D <- AppDeps] ), rebar_test_utils:create_config(Dir, [{deps, AppDeps}]), - Tarball = filename:join([Dir, App++"-"++binary_to_list(Vsn)++".tar"]), + TarApp = App++"-"++binary_to_list(Vsn)++".tar", + Tarball = filename:join([Dir, TarApp]), Contents = filename:join([Dir, "contents.tar.gz"]), Files = all_files(rebar_app_info:dir(AppInfo)), ok = erl_tar:create(Contents, @@ -92,8 +94,11 @@ mock_download(Opts) -> ok = erl_tar:create(Tarball, [{"contents.tar.gz", Contents}], []), - [file:delete(F) || F <- Files], - {tarball, Tarball} + Cache = proplists:get_value(cache_dir, Opts, filename:join(Dir,"cache")), + Cached = filename:join([Cache, TarApp]), + filelib:ensure_dir(Cached), + rebar_file_utils:mv(Tarball, Cached), + {ok, true} end). %% @doc On top of the pkg resource mocking, we need to mock the package diff --git a/test/rebar_pkg_SUITE.erl b/test/rebar_pkg_SUITE.erl new file mode 100644 index 0000000..95eb6f6 --- /dev/null +++ b/test/rebar_pkg_SUITE.erl @@ -0,0 +1,207 @@ +%% Test suite for the rebar pkg index caching and decompression +%% mechanisms. +-module(rebar_pkg_SUITE). +-compile(export_all). +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-define(bad_etag, "abcdef"). +-define(good_etag, "22e1d7387c9085a462340088a2a8ba67"). +-define(bad_checksum, <<"D576B442A68C7B92BACDE1EFE9C6E54D8D6C74BDB71D8175B9D3C6EC8C7B62A7">>). +-define(good_checksum, <<"1C6CE379D191FBAB41B7905075E0BF87CBBE23C77CECE775C5A0B786B2244C35">>). + +all() -> [good_uncached, good_cached, badindexchk, badpkg, + bad_to_good, good_disconnect, bad_disconnect]. + +init_per_suite(Config) -> + application:start(meck), + Config. + +end_per_suite(_Config) -> + application:stop(meck). + +init_per_testcase(good_uncached=Name, Config0) -> + Config = [{good_cache, false}, + {pkg, {<<"goodpkg">>, <<"1.0.0">>}} + | Config0], + mock_config(Name, Config); +init_per_testcase(good_cached=Name, Config0) -> + Pkg = {<<"goodpkg">>, <<"1.0.0">>}, + Config1 = [{good_cache, true}, + {pkg, Pkg} + | Config0], + Config = mock_config(Name, Config1), + copy_to_cache(Pkg, Config), + Config; +init_per_testcase(badindexchk=Name, Config0) -> + Config = [{good_cache, false}, + {pkg, {<<"badindexchk">>, <<"1.0.0">>}} + | Config0], + mock_config(Name, Config); +init_per_testcase(badpkg=Name, Config0) -> + Config = [{good_cache, false}, + {pkg, {<<"badpkg">>, <<"1.0.0">>}} + | Config0], + mock_config(Name, Config); +init_per_testcase(bad_to_good=Name, Config0) -> + Config1 = [{good_cache, false}, + {pkg, {<<"goodpkg">>, <<"1.0.0">>}} + | Config0], + Config = mock_config(Name, Config1), + Source = filename:join(?config(data_dir, Config), <<"badpkg-1.0.0.tar">>), + Dest = filename:join(?config(cache_dir, Config), <<"goodpkg-1.0.0.tar">>), + ec_file:copy(Source, Dest), + Config; +init_per_testcase(good_disconnect=Name, Config0) -> + Pkg = {<<"goodpkg">>, <<"1.0.0">>}, + Config1 = [{good_cache, true}, + {pkg, Pkg} + | Config0], + Config = mock_config(Name, Config1), + copy_to_cache(Pkg, Config), + meck:unload(httpc), + meck:new(httpc, [passthrough, unsticky]), + meck:expect(httpc, request, fun(_, _, _, _) -> {error, econnrefused} end), + Config; +init_per_testcase(bad_disconnect=Name, Config0) -> + Pkg = {<<"goodpkg">>, <<"1.0.0">>}, + Config1 = [{good_cache, false}, + {pkg, Pkg} + | Config0], + Config = mock_config(Name, Config1), + meck:unload(httpc), + meck:new(httpc, [passthrough, unsticky]), + meck:expect(httpc, request, fun(_, _, _, _) -> {error, econnrefused} end), + Config. + +end_per_testcase(_, Config) -> + unmock_config(Config), + Config. + +good_uncached(Config) -> + Tmp = ?config(tmp_dir, Config), + {Pkg,Vsn} = ?config(pkg, Config), + State = ?config(state, Config), + ?assertEqual({ok, true}, + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn}, State)), + Cache = ?config(cache_dir, Config), + ?assert(filelib:is_regular(filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>))). + +good_cached(Config) -> + Tmp = ?config(tmp_dir, Config), + {Pkg,Vsn} = ?config(pkg, Config), + State = ?config(state, Config), + Cache = ?config(cache_dir, Config), + CachedFile = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>), + ?assert(filelib:is_regular(CachedFile)), + {ok, Content} = file:read_file(CachedFile), + ?assertEqual({ok, true}, + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn}, State)), + {ok, Content} = file:read_file(CachedFile). + +badindexchk(Config) -> + Tmp = ?config(tmp_dir, Config), + {Pkg,Vsn} = ?config(pkg, Config), + State = ?config(state, Config), + ?assertMatch({bad_registry_checksum, _Path}, + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn}, State)), + %% The cached file is there for forensic purposes + Cache = ?config(cache_dir, Config), + ?assert(filelib:is_regular(filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>))). + +badpkg(Config) -> + Tmp = ?config(tmp_dir, Config), + {Pkg,Vsn} = ?config(pkg, Config), + State = ?config(state, Config), + ?assertMatch({bad_download, _Path}, + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn}, State)), + %% The cached file is there for forensic purposes + Cache = ?config(cache_dir, Config), + ?assert(filelib:is_regular(filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>))). + +bad_to_good(Config) -> + Tmp = ?config(tmp_dir, Config), + {Pkg,Vsn} = ?config(pkg, Config), + State = ?config(state, Config), + Cache = ?config(cache_dir, Config), + CachedFile = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>), + ?assert(filelib:is_regular(CachedFile)), + {ok, Contents} = file:read_file(CachedFile), + ?assertEqual({ok, true}, + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn}, State)), + %% Cache has refreshed + ?assert({ok, Contents} =/= file:read_file(CachedFile)). + +good_disconnect(Config) -> + Tmp = ?config(tmp_dir, Config), + {Pkg,Vsn} = ?config(pkg, Config), + State = ?config(state, Config), + Cache = ?config(cache_dir, Config), + CachedFile = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>), + ?assert(filelib:is_regular(CachedFile)), + {ok, Content} = file:read_file(CachedFile), + ?assertEqual({ok, true}, + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn}, State)), + {ok, Content} = file:read_file(CachedFile). + +bad_disconnect(Config) -> + Tmp = ?config(tmp_dir, Config), + {Pkg,Vsn} = ?config(pkg, Config), + State = ?config(state, Config), + ?assertEqual(request_failed, + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn}, State)). + + +%%%%%%%%%%%%%%% +%%% Helpers %%% +%%%%%%%%%%%%%%% +mock_config(Name, Config) -> + Priv = ?config(priv_dir, Config), + CacheRoot = filename:join([Priv, "cache", atom_to_list(Name)]), + TmpDir = filename:join([Priv, "tmp", atom_to_list(Name)]), + T = ets:new(fake_registry, [public]), + ets:insert_new(T, [ + {{<<"badindexchk">>,<<"1.0.0">>}, [[], ?bad_checksum]}, + {{<<"goodpkg">>,<<"1.0.0">>}, [[], ?good_checksum]}, + {{<<"badpkg">>,<<"1.0.0">>}, [[], ?good_checksum]} + ]), + CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]), + filelib:ensure_dir(filename:join([CacheDir, "registry"])), + ok = ets:tab2file(T, filename:join([CacheDir, "registry"])), + %% The state returns us a fake registry + meck:new(rebar_state, [passthrough]), + meck:expect(rebar_state, registry, + fun(_State) -> {ok, fake_registry} end), + meck:expect(rebar_state, get, + fun(_State, rebar_packages_cdn, _Default) -> + "http://test.com/" + end), + meck:new(rebar_dir, [passthrough]), + meck:expect(rebar_dir, global_cache_dir, fun(_) -> CacheRoot end), + %% Cache fetches are mocked -- we assume the server and clients are + %% correctly used. + GoodCache = ?config(good_cache, Config), + {Pkg,Vsn} = ?config(pkg, Config), + PkgFile = <<Pkg/binary, "-", Vsn/binary, ".tar">>, + {ok, PkgContents} = file:read_file(filename:join(?config(data_dir, Config), PkgFile)), + meck:new(httpc, [passthrough, unsticky]), + meck:expect(httpc, request, + fun(get, {_Url, _Opts}, _, _) when GoodCache -> + {ok, {{Vsn, 304, <<"Not Modified">>}, [{"etag", ?good_etag}], <<>>}}; + (get, {_Url, _Opts}, _, _) -> + {ok, {{Vsn, 200, <<"OK">>}, [{"etag", ?good_etag}], PkgContents}} + end), + [{cache_root, CacheRoot}, + {cache_dir, CacheDir}, + {tmp_dir, TmpDir}, + {mock_table, T} | Config]. + +unmock_config(Config) -> + meck:unload(), + ets:delete(?config(mock_table, Config)). + +copy_to_cache({Pkg,Vsn}, Config) -> + Name = <<Pkg/binary, "-", Vsn/binary, ".tar">>, + Source = filename:join(?config(data_dir, Config), Name), + Dest = filename:join(?config(cache_dir, Config), Name), + ec_file:copy(Source, Dest). diff --git a/test/rebar_pkg_SUITE_data/badindexchk-1.0.0.tar b/test/rebar_pkg_SUITE_data/badindexchk-1.0.0.tar Binary files differnew file mode 100644 index 0000000..e5b963f --- /dev/null +++ b/test/rebar_pkg_SUITE_data/badindexchk-1.0.0.tar diff --git a/test/rebar_pkg_SUITE_data/badpkg-1.0.0.tar b/test/rebar_pkg_SUITE_data/badpkg-1.0.0.tar Binary files differnew file mode 100644 index 0000000..4930cd2 --- /dev/null +++ b/test/rebar_pkg_SUITE_data/badpkg-1.0.0.tar diff --git a/test/rebar_pkg_SUITE_data/goodpkg-1.0.0.tar b/test/rebar_pkg_SUITE_data/goodpkg-1.0.0.tar Binary files differnew file mode 100644 index 0000000..e5b963f --- /dev/null +++ b/test/rebar_pkg_SUITE_data/goodpkg-1.0.0.tar |