From 7c125d95f0583cb200c47c0c6af301c1182ce463 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sun, 26 Feb 2017 12:49:01 -0500 Subject: More fixes for bad calls to find_apps --- test/rebar_test_utils.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'test/rebar_test_utils.erl') diff --git a/test/rebar_test_utils.erl b/test/rebar_test_utils.erl index 8c177c9..3b8ffac 100644 --- a/test/rebar_test_utils.erl +++ b/test/rebar_test_utils.erl @@ -218,7 +218,7 @@ check_results(AppDir, Expected, ProfileRun) -> BuildDirs = filelib:wildcard(filename:join([AppDir, "_build", ProfileRun, "lib", "*"])), PluginDirs = filelib:wildcard(filename:join([AppDir, "_build", ProfileRun, "plugins", "*"])), GlobalPluginDirs = filelib:wildcard(filename:join([AppDir, "global", "plugins", "*"])), - CheckoutsDir = filename:join([AppDir, "_checkouts", "*"]), + CheckoutsDirs = filelib:wildcard(filename:join([AppDir, "_checkouts", "*"])), LockFile = filename:join([AppDir, "rebar.lock"]), Locks = lists:flatten(rebar_config:consult_lock_file(LockFile)), @@ -230,7 +230,7 @@ check_results(AppDir, Expected, ProfileRun) -> Deps = rebar_app_discover:find_apps(BuildDirs, all), DepsNames = [{ec_cnv:to_list(rebar_app_info:name(App)), App} || App <- Deps], - Checkouts = rebar_app_discover:find_apps([CheckoutsDir], all), + Checkouts = rebar_app_discover:find_apps(CheckoutsDirs, all), CheckoutsNames = [{ec_cnv:to_list(rebar_app_info:name(App)), App} || App <- Checkouts], Plugins = rebar_app_discover:find_apps(PluginDirs, all), PluginsNames = [{ec_cnv:to_list(rebar_app_info:name(App)), App} || App <- Plugins], -- cgit v1.1 From 963c49f5eb9ab5b34e1843fb43305743720917ac Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sun, 6 Aug 2017 07:26:21 -0400 Subject: Unicode support in all the places This is done through 3 main change groups: - replacing `~s` by `~ts` in format strings, so that strings that contain unicode are properly printed rather than crashing - adding the `unicode` argument to all function of the `re` module to ensure transformations on strings containing unicode data are valid instead of crashing (see issue #1302) - replacing `ec_cnv:to_binary/1` and `ec_cnv:to_list/1` with matching functions in `rebar_utils`. The last point has been done, rather than modifying and updating erlware commons, because binary and list conversions can be a contentious subject. For example, if what is being handled is actually bytes from a given binary stream, then forcing a byte-oriented interpretation of the data can corrupt it. As such, it does not appear safe to modify erlware commons' conversion functions since it may not be safe for all its users. Instead, rebar3 reimplements a subset of them (only converting atoms and chardata, ignoring numbers) with the explicit purpose of handling unicode string data. Tests were left as unchanged as possible. This may impact the ability to run rebar3's own suites in a unicode path, but respects a principle of least change for such a large patch. --- test/rebar_test_utils.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'test/rebar_test_utils.erl') diff --git a/test/rebar_test_utils.erl b/test/rebar_test_utils.erl index 3b8ffac..0ccec56 100644 --- a/test/rebar_test_utils.erl +++ b/test/rebar_test_utils.erl @@ -349,7 +349,7 @@ check_results(AppDir, Expected, ProfileRun) -> iolist_to_binary(LockVsn)) end ; ({release, Name, Vsn, ExpectedDevMode}) -> - ct:pal("Release: ~p-~s", [Name, Vsn]), + ct:pal("Release: ~p-~ts", [Name, Vsn]), {ok, Cwd} = file:get_cwd(), try file:set_cwd(AppDir), @@ -377,14 +377,14 @@ check_results(AppDir, Expected, ProfileRun) -> file:set_cwd(Cwd) end ; ({tar, Name, Vsn}) -> - ct:pal("Tarball: ~s-~s", [Name, Vsn]), + ct:pal("Tarball: ~ts-~ts", [Name, Vsn]), Tarball = filename:join([AppDir, "_build", "rel", Name, Name++"-"++Vsn++".tar.gz"]), ?assertNotEqual([], filelib:is_file(Tarball)) ; ({file, Filename}) -> - ct:pal("Filename: ~s", [Filename]), + ct:pal("Filename: ~ts", [Filename]), ?assert(filelib:is_file(Filename)) ; ({dir, Dirname}) -> - ct:pal("Directory: ~s", [Dirname]), + ct:pal("Directory: ~ts", [Dirname]), ?assert(filelib:is_dir(Dirname)) end, Expected). -- cgit v1.1 From 2d5cd9c00cfa4e58066b48beee4057fdd52cc7be Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 1 Nov 2017 19:38:03 -0400 Subject: OTP-21 readiness, Full Unicode support This replaces all deprecated function usage by alternative ones based on a version switch enacted at compile time, preventing all warnings. This will likely introduce some possible runtime errors in using a Rebar3 compiled on OTP-20 or OTP-21 back in versions 19 and earlier, but we can't really work around that. A bunch of dependencies have been updated to support OTP-21 without warnings as well. --- test/rebar_test_utils.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'test/rebar_test_utils.erl') diff --git a/test/rebar_test_utils.erl b/test/rebar_test_utils.erl index 0ccec56..c1e8b62 100644 --- a/test/rebar_test_utils.erl +++ b/test/rebar_test_utils.erl @@ -167,14 +167,14 @@ expand_deps(pkg, [{Name, Vsn, Deps} | Rest]) -> [{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)]; expand_deps(mixed, [{Name, Deps} | Rest]) -> Dep = if hd(Name) >= $a, hd(Name) =< $z -> - {pkg, string:to_upper(Name), "0.0.0", undefined} + {pkg, rebar_string:uppercase(Name), "0.0.0", undefined} ; hd(Name) >= $A, hd(Name) =< $Z -> {Name, ".*", {git, "https://example.org/user/"++Name++".git", "master"}} end, [{Dep, expand_deps(mixed, Deps)} | expand_deps(mixed, Rest)]; expand_deps(mixed, [{Name, Vsn, Deps} | Rest]) -> Dep = if hd(Name) >= $a, hd(Name) =< $z -> - {pkg, string:to_upper(Name), Vsn, undefined} + {pkg, rebar_string:uppercase(Name), Vsn, undefined} ; hd(Name) >= $A, hd(Name) =< $Z -> {Name, Vsn, {git, "https://example.org/user/"++Name++".git", {tag, Vsn}}} end, @@ -477,7 +477,7 @@ package_app(AppDir, DestDir, PkgName) -> {ok, Contents} = file:read_file(filename:join(DestDir, "contents.tar.gz")), Blob = <<"3who cares", Contents/binary>>, <> = crypto:hash(sha256, Blob), - BinChecksum = list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))), + BinChecksum = list_to_binary(rebar_string:uppercase(lists:flatten(io_lib:format("~64.16.0b", [X])))), ok = file:write_file(filename:join(DestDir, "CHECKSUM"), BinChecksum), PkgFiles = ["contents.tar.gz", "VERSION", "metadata.config", "CHECKSUM"], Archive = filename:join(DestDir, Name), @@ -485,5 +485,5 @@ package_app(AppDir, DestDir, PkgName) -> lists:zip(PkgFiles, [filename:join(DestDir,F) || F <- PkgFiles])), {ok, BinFull} = file:read_file(Archive), <> = crypto:hash(md5, BinFull), - Etag = string:to_lower(lists:flatten(io_lib:format("~32.16.0b", [E]))), + Etag = rebar_string:lowercase(lists:flatten(io_lib:format("~32.16.0b", [E]))), {BinChecksum, Etag}. -- cgit v1.1 From 6e672852a72f51d088e16f3662b0299744435127 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 1 Dec 2017 16:44:00 -0800 Subject: fix code path when validating plugins --- test/rebar_test_utils.erl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'test/rebar_test_utils.erl') diff --git a/test/rebar_test_utils.erl b/test/rebar_test_utils.erl index c1e8b62..b74aa2f 100644 --- a/test/rebar_test_utils.erl +++ b/test/rebar_test_utils.erl @@ -425,15 +425,16 @@ erl_src_file(Name) -> plugin_src_file(Name) -> io_lib:format("-module('~s').\n" - "-export([init/1]).\n" + "-export([init/1, do/1]).\n" "init(State) -> \n" "Provider = providers:create([\n" "{name, '~s'},\n" "{module, '~s'}\n" "]),\n" - "{ok, rebar_state:add_provider(State, Provider)}.\n", [filename:basename(Name, ".erl"), - filename:basename(Name, ".erl"), - filename:basename(Name, ".erl")]). + "{ok, rebar_state:add_provider(State, Provider)}.\n" + "do(State) -> {ok, State}.\n", [filename:basename(Name, ".erl"), + filename:basename(Name, ".erl"), + filename:basename(Name, ".erl")]). erl_eunitized_src_file(Name) -> io_lib:format("-module('~s').\n" -- cgit v1.1 From 56b7d88975aa8da6446857cfd92de0825024bf63 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 13 Sep 2018 19:36:00 -0600 Subject: support for hex v2, multiple repository fetching, private organizations (#1884) * update to hex_core for hex-v2 repo support (#1865) * update to hex_core for hex-v2 repo support This patch adds only single repo hex-v2 support through hex_core. Packages no longer filtered out by buildtool metadata and the package index is updated per-package instead of fetched as one large ets dump. * tell travis to also build hex_core branch * support list of repos for hex packages (#1866) * support list of repos for hex packages repos are defined under the hex key in rebar configs. They can be defined at the top level of a project or globally, but not in profiles and the repos configured in dependencies are also ignored. Searching for packages involves first checking for a match in the local repo index cache, in the order repos are defined. If not found each repo is checked through the hex api for any known versions of the package and the first repo with a version that fits the constraint is used. * add {repos, replace, []} for overriding the global & default repos * add hex auth handling for repos (#1874) auth token are kept in a hex.config file that is modified by the rebar3 hex plugin. Repo names that have a : separating a parent and child are considered organizations. The parent repo's auth will be included with the child. So an organization named hexpm:rebar3_test will include any hexpm auth tokens found in the rebar3_test organization's configuration. * move packages to top level of of hexpm cache dir (#1876) * move packages to top level of of hexpm cache dir * append organization name to parent's repo_url when parsing repos * only eval config scripts and apply overrides once per app (#1879) * only eval config scripts and apply overrides once per app * move new resource behaviour to rebar_resource_v2 and keep v1 * cleanup use of rebar_resource module and unused functions * cleanup error messages and unused code * when discovering apps support mix packages as unbuilt apps (#1882) * use hex_core tarball unpacking support in pkg resource (#1883) * use hex_core tarball unpacking support in pkg resource * ignore etag if package doesn't exist and delete if checksum fails * add back tests for bad package checksums * improve bad registry checksum error message --- test/rebar_test_utils.erl | 48 +++++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 22 deletions(-) (limited to 'test/rebar_test_utils.erl') diff --git a/test/rebar_test_utils.erl b/test/rebar_test_utils.erl index b74aa2f..8bcb6d1 100644 --- a/test/rebar_test_utils.erl +++ b/test/rebar_test_utils.erl @@ -4,8 +4,9 @@ -export([init_rebar_state/1, init_rebar_state/2, run_and_check/4, check_results/3]). -export([expand_deps/2, flat_deps/1, top_level_deps/1]). -export([create_app/4, create_plugin/4, create_eunit_app/4, create_empty_app/4, - create_config/2, create_config/3, package_app/3]). --export([create_random_name/1, create_random_vsn/0, write_src_file/2]). + create_config/2, create_config/3, package_app/4]). +-export([create_random_name/1, create_random_vsn/0, write_src_file/2, + random_element/1]). %% Pick the right random module -ifdef(rand_only). @@ -34,8 +35,10 @@ init_rebar_state(Config, Name) -> Verbosity = rebar3:log_level(), rebar_log:init(command_line, Verbosity), GlobalDir = filename:join([DataDir, "cache"]), + Repos = proplists:get_value(repos, Config, []), State = rebar_state:new([{base_dir, filename:join([AppsDir, "_build"])} ,{global_rebar_dir, GlobalDir} + ,{hex, [{repos, [#{name => R} || R <- Repos]}]} ,{root_dir, AppsDir}]), [{apps, AppsDir}, {checkouts, CheckoutsDir}, {state, State} | Config]. @@ -467,24 +470,25 @@ get_app_metadata(Name, Vsn, Deps) -> {registered, []}, {applications, Deps}]}. -package_app(AppDir, DestDir, PkgName) -> - Name = PkgName++".tar", - {ok, Fs} = rebar_utils:list_dir(AppDir), - ok = erl_tar:create(filename:join(DestDir, "contents.tar.gz"), - lists:zip(Fs, [filename:join(AppDir,F) || F <- Fs]), - [compressed]), - ok = file:write_file(filename:join(DestDir, "metadata.config"), "who cares"), - ok = file:write_file(filename:join(DestDir, "VERSION"), "3"), - {ok, Contents} = file:read_file(filename:join(DestDir, "contents.tar.gz")), - Blob = <<"3who cares", Contents/binary>>, - <> = crypto:hash(sha256, Blob), - BinChecksum = list_to_binary(rebar_string:uppercase(lists:flatten(io_lib:format("~64.16.0b", [X])))), - ok = file:write_file(filename:join(DestDir, "CHECKSUM"), BinChecksum), - PkgFiles = ["contents.tar.gz", "VERSION", "metadata.config", "CHECKSUM"], +package_app(AppDir, DestDir, PkgName, PkgVsn) -> + AppSrc = filename:join(AppDir, "src"), + {ok, Fs} = rebar_utils:list_dir(AppSrc), + Files = lists:zip([filename:join("src", F) || F <- Fs], [filename:join(AppSrc,F) || F <- Fs]), + Metadata = #{<<"app">> => list_to_binary(PkgName), + <<"version">> => list_to_binary(PkgVsn)}, + {ok, {Tarball, <>}} = hex_tarball:create(Metadata, Files), + + Name = PkgName++"-"++PkgVsn++".tar", Archive = filename:join(DestDir, Name), - ok = erl_tar:create(Archive, - lists:zip(PkgFiles, [filename:join(DestDir,F) || F <- PkgFiles])), - {ok, BinFull} = file:read_file(Archive), - <> = crypto:hash(md5, BinFull), - Etag = rebar_string:lowercase(lists:flatten(io_lib:format("~32.16.0b", [E]))), - {BinChecksum, Etag}. + file:write_file(Archive, Tarball), + + <> = crypto:hash(md5, Tarball), + + Checksum1 = list_to_binary( + rebar_string:uppercase( + lists:flatten(io_lib:format("~64.16.0b", [Checksum])))), + {Checksum1, E}. + +random_element(Repos) -> + Index = ?random:uniform(length(Repos)), + lists:nth(Index, Repos). -- cgit v1.1 From 8e0ef83de48968fed80114cc4518004ed59fa649 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Wed, 19 Sep 2018 08:24:38 -0600 Subject: add option to compiler provider to only build dependencies (#1888) --- test/rebar_test_utils.erl | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'test/rebar_test_utils.erl') diff --git a/test/rebar_test_utils.erl b/test/rebar_test_utils.erl index 8bcb6d1..efe7467 100644 --- a/test/rebar_test_utils.erl +++ b/test/rebar_test_utils.erl @@ -266,6 +266,14 @@ check_results(AppDir, Expected, ProfileRun) -> ok end ; ({dep_not_exist, Name}) -> + ct:pal("Dep Not Exist Name: ~p", [Name]), + case lists:keyfind(Name, 1, DepsNames) of + false -> + ok; + {Name, _App} -> + error({app_found, Name}) + end + ; ({app_not_exist, Name}) -> ct:pal("App Not Exist Name: ~p", [Name]), case lists:keyfind(Name, 1, DepsNames) of false -> -- cgit v1.1 From 2dfba204e4dea5d1c3821fd26d22bd7201595f6c Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 21 Sep 2018 10:32:57 -0600 Subject: properly support top level app erl_opts from REBAR_CONFIG os var (#1889) When REBAR_CONFIG was set it would not effect the top level app's configuration because app_discover was rereading the top level rebar.config which ignored REBAR_CONFIG. Instead this patch has it use the existing configuration from REBAR_CONFIG. --- test/rebar_test_utils.erl | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) (limited to 'test/rebar_test_utils.erl') diff --git a/test/rebar_test_utils.erl b/test/rebar_test_utils.erl index efe7467..944efa0 100644 --- a/test/rebar_test_utils.erl +++ b/test/rebar_test_utils.erl @@ -1,7 +1,7 @@ -module(rebar_test_utils). -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). --export([init_rebar_state/1, init_rebar_state/2, run_and_check/4, check_results/3]). +-export([init_rebar_state/1, init_rebar_state/2, run_and_check/3, run_and_check/4, check_results/3]). -export([expand_deps/2, flat_deps/1, top_level_deps/1]). -export([create_app/4, create_plugin/4, create_eunit_app/4, create_empty_app/4, create_config/2, create_config/3, package_app/4]). @@ -82,6 +82,33 @@ run_and_check(Config, RebarConfig, Command, Expect) -> rebar_abort when Expect =:= rebar_abort -> rebar_abort end. +run_and_check(Config, Command, Expect) -> + %% Assumes init_rebar_state has run first + AppDir = ?config(apps, Config), + {ok, Cwd} = file:get_cwd(), + try + ok = file:set_cwd(AppDir), + Res = rebar3:run(Command), + case Expect of + {error, Reason} -> + ?assertEqual({error, Reason}, Res); + {ok, Expected} -> + {ok, _} = Res, + check_results(AppDir, Expected, "*"), + Res; + {ok, Expected, ProfileRun} -> + {ok, _} = Res, + check_results(AppDir, Expected, ProfileRun), + Res; + return -> + Res + end + catch + rebar_abort when Expect =:= rebar_abort -> rebar_abort + after + ok = file:set_cwd(Cwd) + end. + %% @doc Creates a dummy application including: %% - src/.erl %% - src/.app.src -- cgit v1.1