From 124f121d38322a3db50458caeac47e84dccee45c Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 6 May 2016 23:18:55 -0400 Subject: Support package hashes in structure and lockfile - the internal representation for package locks moves from `{Name, {pkg, PkgName, Vsn}, Lvl}` to `{Name, {pkg, PkgName, Vsn, Hash}, Lvl}` - the internal representation for packages moves from `{pkg, PkgName, Vsn}` to `{pkg, PkgName, Vsn, Hash}` - the hash can be `undefined`, meaning no check will be done - no checking is done yet. --- src/rebar.hrl | 1 + src/rebar_app_utils.erl | 16 ++++++----- src/rebar_config.erl | 66 ++++++++++++++++++++++++++++++++++++---------- src/rebar_packages.erl | 2 +- src/rebar_pkg_resource.erl | 6 ++--- src/rebar_prv_deps.erl | 2 +- 6 files changed, 67 insertions(+), 26 deletions(-) (limited to 'src') diff --git a/src/rebar.hrl b/src/rebar.hrl index 0b7f0b1..4e1ec00 100644 --- a/src/rebar.hrl +++ b/src/rebar.hrl @@ -22,6 +22,7 @@ -define(DEFAULT_TEST_DEPS_DIR, "test/lib"). -define(DEFAULT_RELEASE_DIR, "rel"). -define(DEFAULT_CONFIG_FILE, "rebar.config"). +-define(CONFIG_VERSION, "1.1.0"). -define(DEFAULT_CDN, "https://repo.hex.pm/"). -define(REMOTE_PACKAGE_DIR, "tarballs"). -define(REMOTE_REGISTRY_FILE, "registry.ets.gz"). diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index d3ef841..7028c32 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -119,17 +119,17 @@ parse_dep(Dep, Parent, DepsDir, State, Locks, Level) -> parse_dep(Parent, {Name, Vsn, {pkg, PkgName}}, DepsDir, IsLock, State) -> {PkgName1, PkgVsn} = {ec_cnv:to_binary(PkgName), ec_cnv:to_binary(Vsn)}, - dep_to_app(Parent, DepsDir, Name, PkgVsn, {pkg, PkgName1, PkgVsn}, IsLock, State); + dep_to_app(Parent, DepsDir, Name, PkgVsn, {pkg, PkgName1, PkgVsn, undefined}, IsLock, State); parse_dep(Parent, {Name, {pkg, PkgName}}, DepsDir, IsLock, State) -> %% Package dependency with different package name from app name - dep_to_app(Parent, DepsDir, Name, undefined, {pkg, ec_cnv:to_binary(PkgName), undefined}, IsLock, State); + dep_to_app(Parent, DepsDir, Name, undefined, {pkg, ec_cnv:to_binary(PkgName), undefined, undefined}, IsLock, State); parse_dep(Parent, {Name, Vsn}, DepsDir, IsLock, State) when is_list(Vsn); is_binary(Vsn) -> %% Versioned Package dependency {PkgName, PkgVsn} = {ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}, - dep_to_app(Parent, DepsDir, PkgName, PkgVsn, {pkg, PkgName, PkgVsn}, IsLock, State); + dep_to_app(Parent, DepsDir, PkgName, PkgVsn, {pkg, PkgName, PkgVsn, undefined}, IsLock, State); parse_dep(Parent, Name, DepsDir, IsLock, State) when is_atom(Name); is_binary(Name) -> %% Unversioned package dependency - dep_to_app(Parent, DepsDir, ec_cnv:to_binary(Name), undefined, {pkg, ec_cnv:to_binary(Name), undefined}, IsLock, State); + dep_to_app(Parent, DepsDir, ec_cnv:to_binary(Name), undefined, {pkg, ec_cnv:to_binary(Name), undefined, undefined}, IsLock, State); parse_dep(Parent, {Name, Source}, DepsDir, IsLock, State) when is_tuple(Source) -> dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); parse_dep(Parent, {Name, _Vsn, Source}, DepsDir, IsLock, State) when is_tuple(Source) -> @@ -138,7 +138,9 @@ parse_dep(Parent, {Name, _Vsn, Source, Opts}, DepsDir, IsLock, State) when is_tu ?WARN("Dependency option list ~p in ~p is not supported and will be ignored", [Opts, Name]), dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); parse_dep(Parent, {Name, {pkg, PkgName, Vsn}, Level}, DepsDir, IsLock, State) when is_integer(Level) -> - dep_to_app(Parent, DepsDir, Name, Vsn, {pkg, PkgName, Vsn}, IsLock, State); + dep_to_app(Parent, DepsDir, Name, Vsn, {pkg, PkgName, Vsn, undefined}, IsLock, State); +parse_dep(Parent, {Name, {pkg, PkgName, Vsn, Hash}, Level}, DepsDir, IsLock, State) when is_integer(Level) -> + dep_to_app(Parent, DepsDir, Name, Vsn, {pkg, PkgName, Vsn, Hash}, IsLock, State); parse_dep(Parent, {Name, Source, Level}, DepsDir, IsLock, State) when is_tuple(Source) , is_integer(Level) -> dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); @@ -170,7 +172,7 @@ dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> AppInfo5 = rebar_app_info:profiles(AppInfo4, [default]), rebar_app_info:is_lock(AppInfo5, IsLock). -update_source(AppInfo, {pkg, PkgName, PkgVsn}, State) -> +update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) -> {PkgName1, PkgVsn1} = case PkgVsn of undefined -> get_package(PkgName, "0", State); @@ -180,7 +182,7 @@ update_source(AppInfo, {pkg, PkgName, PkgVsn}, State) -> _ -> {PkgName, PkgVsn} end, - AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName1, PkgVsn1}), + AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName1, PkgVsn1, Hash}), Deps = rebar_packages:deps(PkgName1 ,PkgVsn1 ,State), diff --git a/src/rebar_config.erl b/src/rebar_config.erl index 8d7bcf4..031df8b 100644 --- a/src/rebar_config.erl +++ b/src/rebar_config.erl @@ -56,35 +56,73 @@ consult_lock_file(File) -> [] -> []; [Locks] when is_list(Locks) -> % beta lock file - Locks; + read_attrs(beta, Locks, []); [{Vsn, Locks}|Attrs] when is_list(Locks) -> % versioned lock file - %% Make sure the warning above is to be shown whenever a version - %% newer than the current one is being used, as we can't parse - %% all the contents of the lock file properly. - ?WARN("Rebar3 detected a lock file from a newer version. " - "It will be loaded in compatibility mode, but important " - "information may be missing or lost. It is recommended to " - "upgrade Rebar3.", []), + case Vsn of + ?CONFIG_VERSION -> + ok; + _ -> + %% Make sure the warning below is to be shown whenever a version + %% newer than the current one is being used, as we can't parse + %% all the contents of the lock file properly. + ?WARN("Rebar3 detected a lock file from a newer version. " + "It will be loaded in compatibility mode, but important " + "information may be missing or lost. It is recommended to " + "upgrade Rebar3.", []) + end, read_attrs(Vsn, Locks, Attrs) end. write_lock_file(LockFile, Locks) -> - NewLocks = write_attrs(Locks), + {NewLocks, Attrs} = write_attrs(Locks), %% Write locks in the beta format, at least until it's been long %% enough we can start modifying the lock format. - file:write_file(LockFile, io_lib:format("~p.~n", [NewLocks])). + case Attrs of + [] -> % write the old beta copy to avoid changes + file:write_file(LockFile, io_lib:format("~p.~n", [NewLocks])); + _ -> + file:write_file(LockFile, + io_lib:format("{~p,~p}.~n~p.~n", + [?CONFIG_VERSION, NewLocks, Attrs])) + end. -read_attrs(_Vsn, Locks, _Attrs) -> +read_attrs(_Vsn, Locks, Attrs) -> %% Beta copy does not know how to expand attributes, but %% is ready to support it. - Locks. + expand_locks(Locks, extract_pkg_hashes(Attrs)). + +extract_pkg_hashes(Attrs) -> + Props = case Attrs of + [First|_] -> First; + [] -> [] + end, + proplists:get_value(pkg_hash, Props, []). + +expand_locks([], _Hashes) -> + []; +expand_locks([{Name, {pkg,PkgName,Vsn}, Lvl} | Locks], Hashes) -> + Hash = proplists:get_value(Name, Hashes), + [{Name, {pkg,PkgName,Vsn,Hash}, Lvl} | expand_locks(Locks, Hashes)]; +expand_locks([Lock|Locks], Hashes) -> + [Lock | expand_locks(Locks, Hashes)]. write_attrs(Locks) -> %% No attribute known that needs to be taken out of the structure, %% just return terms as is. - Locks. - + {NewLocks, Hashes} = split_locks(Locks, [], []), + case Hashes of + [] -> {NewLocks, []}; + _ -> {NewLocks, [{pkg_hash, lists:sort(Hashes)}]} + end. +split_locks([], Locks, Hashes) -> + {lists:reverse(Locks), Hashes}; +split_locks([{Name, {pkg,PkgName,Vsn,undefined}, Lvl} | Locks], LAcc, HAcc) -> + split_locks(Locks, [{Name,{pkg,PkgName,Vsn},Lvl}|LAcc], HAcc); +split_locks([{Name, {pkg,PkgName,Vsn,Hash}, Lvl} | Locks], LAcc, HAcc) -> + split_locks(Locks, [{Name,{pkg,PkgName,Vsn},Lvl}|LAcc], [{Name, Hash}|HAcc]); +split_locks([Lock|Locks], LAcc, HAcc) -> + split_locks(Locks, [Lock|LAcc], HAcc). consult_file(File) -> Terms = consult_file_(File), diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index d4b8a14..8b4611b 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -122,7 +122,7 @@ package_dir(State) -> Error end. -registry_checksum({pkg, Name, Vsn}, State) -> +registry_checksum({pkg, Name, Vsn, _Hash}, State) -> try ?MODULE:verify_table(State), ets:lookup_element(?PACKAGE_TABLE, {Name, Vsn}, 3) diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index ec7e09d..257df21 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -19,7 +19,7 @@ lock(_AppDir, Source) -> Source. -needs_update(Dir, {pkg, _Name, Vsn}) -> +needs_update(Dir, {pkg, _Name, Vsn, _Hash}) -> [AppInfo] = rebar_app_discover:find_apps([Dir], all), case rebar_app_info:original_vsn(AppInfo) =:= ec_cnv:to_list(Vsn) of true -> @@ -28,7 +28,7 @@ needs_update(Dir, {pkg, _Name, Vsn}) -> true end. -download(TmpDir, Pkg={pkg, Name, Vsn}, State) -> +download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State) -> CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), {ok, PackageDir} = rebar_packages:package_dir(State), Package = binary_to_list(<>), @@ -40,7 +40,7 @@ download(TmpDir, Pkg={pkg, Name, Vsn}, State) -> {fetch_fail, Name, Vsn} end. -cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn}, Url, ETag, State) -> +cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, Url, ETag, State) -> case request(Url, ETag) of {ok, cached} -> ?INFO("Version cached at ~s is up to date, reusing it", [CachePath]), diff --git a/src/rebar_prv_deps.erl b/src/rebar_prv_deps.erl index 9ff2bfa..c865276 100644 --- a/src/rebar_prv_deps.erl +++ b/src/rebar_prv_deps.erl @@ -97,7 +97,7 @@ display_dep(_State, {Name, _Vsn, Source}) when is_tuple(Source) -> display_dep(_State, {Name, _Vsn, Source, _Opts}) when is_tuple(Source) -> ?CONSOLE("~s* (~s source)", [ec_cnv:to_binary(Name), type(Source)]); %% Locked -display_dep(State, {Name, Source={pkg, _, Vsn}, Level}) when is_integer(Level) -> +display_dep(State, {Name, Source={pkg, _, Vsn, _}, Level}) when is_integer(Level) -> DepsDir = rebar_dir:deps_dir(State), AppDir = filename:join([DepsDir, ec_cnv:to_binary(Name)]), NeedsUpdate = case rebar_fetch:needs_update(AppDir, Source, State) of -- cgit v1.1 From 9a7ede0196c78224c15d1e9c8d13bed84743dacf Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Tue, 24 May 2016 20:43:38 -0400 Subject: Fetch hashes from index prior to fetching --- src/rebar_app_utils.erl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index 7028c32..f3b2962 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -182,7 +182,14 @@ update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) -> _ -> {PkgName, PkgVsn} end, - AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName1, PkgVsn1, Hash}), + %% store the expected hash for the dependency + Hash1 = case Hash of + undefined -> % unknown, define the hash since we know the dep + rebar_packages:registry_checksum({pkg, PkgName1, PkgVsn1, Hash}, State); + _ -> % keep as is + Hash + end, + AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName1, PkgVsn1, Hash1}), Deps = rebar_packages:deps(PkgName1 ,PkgVsn1 ,State), -- cgit v1.1 From 351b757a0eb03e2d2e8714d0f5f447531446aa68 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Tue, 24 May 2016 21:12:49 -0400 Subject: Validate checksums expected vs obtained --- src/rebar_fetch.erl | 6 ++++++ src/rebar_pkg_resource.erl | 17 ++++++++++------- 2 files changed, 16 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index b80c125..47bfe1d 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -67,6 +67,12 @@ needs_update(AppDir, Source, State) -> format_error({bad_download, CachePath}) -> io_lib:format("Download of package does not match md5sum from server: ~s", [CachePath]); +format_error({unexpected_hash, CachePath, Expected, Found}) -> + io_lib:format("The checksum for package at ~s (~s) does not match the " + "checksum previously locked (~s). Either unlock or " + "upgrade the package, or make sure you fetched it from " + "the same index from which it was initially fetched.", + [CachePath, Found, Expected]); format_error({failed_extract, CachePath}) -> io_lib:format("Failed to extract package: ~s", [CachePath]); format_error({bad_etag, Source}) -> diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 257df21..5817817 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -58,17 +58,20 @@ cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, Url, ETag, State serve_from_cache(TmpDir, CachePath, Pkg, State) -> {Files, Contents, Version, Meta} = extract(TmpDir, CachePath), case checksums(Pkg, Files, Contents, Version, Meta, State) of - {Chk, Chk, Chk} -> + {Chk, Chk, Chk, Chk} -> ok = erl_tar:extract({binary, Contents}, [{cwd, TmpDir}, compressed]), {ok, true}; - {_Bin, Chk, Chk} -> + {_Hash, Chk, Chk, Chk} -> + ?DEBUG("Expected hash ~p does not match checksums ~p", [_Hash, Chk]), + {unexpected_hash, CachePath, _Hash, Chk}; + {Chk, _Bin, Chk, Chk} -> ?DEBUG("Checksums: registry: ~p, pkg: ~p", [Chk, _Bin]), {failed_extract, CachePath}; - {Chk, _Reg, Chk} -> + {Chk, Chk, _Reg, Chk} -> ?DEBUG("Checksums: registry: ~p, pkg: ~p", [_Reg, Chk]), {bad_registry_checksum, CachePath}; - {_Bin, _Reg, _Tar} -> - ?DEBUG("Checksums: registry: ~p, pkg: ~p, meta: ~p", [_Reg, _Bin, _Tar]), + {_Hash, _Bin, _Reg, _Tar} -> + ?DEBUG("Checksums: expected: ~p, registry: ~p, pkg: ~p, meta: ~p", [_Hash, _Reg, _Bin, _Tar]), {bad_checksum, CachePath} end. @@ -92,13 +95,13 @@ extract(TmpDir, CachePath) -> {"metadata.config", Meta} = lists:keyfind("metadata.config", 1, Files), {Files, Contents, Version, Meta}. -checksums(Pkg, Files, Contents, Version, Meta, State) -> +checksums(Pkg={pkg, _Name, _Vsn, Hash}, Files, Contents, Version, Meta, State) -> Blob = <>, <> = crypto:hash(sha256, Blob), BinChecksum = list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))), RegistryChecksum = rebar_packages:registry_checksum(Pkg, State), {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files), - {BinChecksum, RegistryChecksum, TarChecksum}. + {Hash, BinChecksum, RegistryChecksum, TarChecksum}. make_vsn(_) -> {error, "Replacing version of type pkg not supported."}. -- cgit v1.1 From 760ffdc79d19ffff732915b2abaa81ecdbeeef91 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 25 May 2016 13:25:34 -0400 Subject: Hide the expected hash on fetch output --- src/rebar_prv_install_deps.erl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index 5e6aa4c..a8a7ea0 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -352,10 +352,14 @@ make_relative_to_root(State, Path) when is_list(Path) -> rebar_dir:make_relative_path(Path, Root). fetch_app(AppInfo, AppDir, State) -> - ?INFO("Fetching ~s (~p)", [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)]), + ?INFO("Fetching ~s (~p)", [rebar_app_info:name(AppInfo), + format_source(rebar_app_info:source(AppInfo))]), Source = rebar_app_info:source(AppInfo), true = rebar_fetch:download_source(AppDir, Source, State). +format_source({pkg, Name, Vsn, _Hash}) -> {pkg, Name, Vsn}; +format_source(Source) -> Source. + %% This is called after the dep has been downloaded and unpacked, if it hadn't been already. %% So this is the first time for newly downloaded apps that its .app/.app.src data can %% be read in an parsed. -- cgit v1.1 From 589eaf13e1de55695cea4f34719c22c4b6467734 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 9 Jun 2016 21:30:42 -0400 Subject: Hashes in lockfile are diff friendly This reworks the version and hash printing in the lockfile to minimize diff changes: - the version is on its own line so that the locks are mostly the same aside from the last line - the hashes are each printed on one line with the package name for simpler diffing too. --- src/rebar_config.erl | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar_config.erl b/src/rebar_config.erl index 031df8b..828c45d 100644 --- a/src/rebar_config.erl +++ b/src/rebar_config.erl @@ -82,10 +82,29 @@ write_lock_file(LockFile, Locks) -> file:write_file(LockFile, io_lib:format("~p.~n", [NewLocks])); _ -> file:write_file(LockFile, - io_lib:format("{~p,~p}.~n~p.~n", - [?CONFIG_VERSION, NewLocks, Attrs])) + io_lib:format("{~p,~n~p}.~n[~n~s~n].~n", + [?CONFIG_VERSION, NewLocks, + format_attrs(Attrs)])) end. +%% Attributes have a special formatting to ensure there's only one per +%% line in terms of pkg_hash, so we disturb source diffing as little +%% as possible. +format_attrs([]) -> []; +format_attrs([{pkg_hash, Vals}|T]) -> + [io_lib:format("{pkg_hash,[~n",[]), format_hashes(Vals), "]}", + maybe_comma(T) | format_attrs(T)]; +format_attrs([H|T]) -> + [io_lib:format("~p~s", [H, maybe_comma(T)]) | format_attrs(T)]. + +format_hashes([]) -> []; +format_hashes([{Pkg,Hash}|T]) -> + [" {", io_lib:format("~p",[Pkg]), ", ", io_lib:format("~p", [Hash]), "}", + maybe_comma(T) | format_hashes(T)]. + +maybe_comma([]) -> ""; +maybe_comma([_|_]) -> io_lib:format(",~n", []). + read_attrs(_Vsn, Locks, Attrs) -> %% Beta copy does not know how to expand attributes, but %% is ready to support it. -- cgit v1.1 From 71df9bf1411c04e2f7dae7e9f0352180664b9365 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 10 Jun 2016 08:43:13 -0400 Subject: Only display old version warning once This uses the env variable as a global store for variables. It's not the cleanest thing, but it sounded nicer than pdicts. --- src/rebar_config.erl | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/rebar_config.erl b/src/rebar_config.erl index 828c45d..9e13d46 100644 --- a/src/rebar_config.erl +++ b/src/rebar_config.erl @@ -58,6 +58,9 @@ consult_lock_file(File) -> [Locks] when is_list(Locks) -> % beta lock file read_attrs(beta, Locks, []); [{Vsn, Locks}|Attrs] when is_list(Locks) -> % versioned lock file + %% Because this is the first version of rebar3 to introduce a lock + %% file, all versionned lock files with a different versions have + %% to be newer. case Vsn of ?CONFIG_VERSION -> ok; @@ -65,14 +68,24 @@ consult_lock_file(File) -> %% Make sure the warning below is to be shown whenever a version %% newer than the current one is being used, as we can't parse %% all the contents of the lock file properly. - ?WARN("Rebar3 detected a lock file from a newer version. " - "It will be loaded in compatibility mode, but important " - "information may be missing or lost. It is recommended to " - "upgrade Rebar3.", []) + warn_vsn_once() end, read_attrs(Vsn, Locks, Attrs) end. +warn_vsn_once() -> + Warn = application:get_env(rebar, warn_config_vsn) =/= {ok, false}, + application:set_env(rebar, warn_config_vsn, false), + case Warn of + false -> ok; + true -> + ?WARN("Rebar3 detected a lock file from a newer version. " + "It will be loaded in compatibility mode, but important " + "information may be missing or lost. It is recommended to " + "upgrade Rebar3.", []) + end. + + write_lock_file(LockFile, Locks) -> {NewLocks, Attrs} = write_attrs(Locks), %% Write locks in the beta format, at least until it's been long -- cgit v1.1 From 391f3b3bec6283866fec8b16e128488de5205640 Mon Sep 17 00:00:00 2001 From: Jon Date: Sat, 18 Jun 2016 03:35:53 +0200 Subject: Let DEBUG="" and QUIET="" mean disable the option Setting DEBUG/QUIET environment variable to the empty string now acts the same as unsetting it. Unsetting is not always easy/possible. --- src/rebar3.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rebar3.erl b/src/rebar3.erl index 82b4472..d3ea15f 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -221,10 +221,10 @@ set_options(State, {Options, NonOptArgs}) -> %% log_level() -> case os:getenv("QUIET") of - false -> + Q when Q == false; Q == "" -> DefaultLevel = rebar_log:default_level(), case os:getenv("DEBUG") of - false -> + D when D == false; D == "" -> DefaultLevel; _ -> DefaultLevel + 3 -- cgit v1.1 From c7b7b9005f733ec867f99932b3fd50803a56cd99 Mon Sep 17 00:00:00 2001 From: Jon Date: Sat, 18 Jun 2016 06:04:42 +0200 Subject: Fix unicode rendering of deps tree (issue #1140) --- src/rebar_prv_deps_tree.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_deps_tree.erl b/src/rebar_prv_deps_tree.erl index cefa60a..c0c8bab 100644 --- a/src/rebar_prv_deps_tree.erl +++ b/src/rebar_prv_deps_tree.erl @@ -70,7 +70,7 @@ print_children(Prefix, [{Name, Vsn, Source} | Rest], Dict, Verbose) -> [Prefix, " "]; _ -> io:format("~ts~ts", [Prefix, <<226,148,156,226,148,128,32>>]), %Binary for ├─ utf8% - [Prefix, "│ "] + [Prefix, <<226,148,130,32,32>>] %Binary for │ utf8% end, io:format("~ts~ts~ts (~ts)~n", [Name, <<226,148,128>>, Vsn, type(Source, Verbose)]), %Binary for ─ utf8% case dict:find(Name, Dict) of -- cgit v1.1 From 2ffbb80cea3baf6763069319f089c078764edb59 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sat, 18 Jun 2016 16:59:13 -0400 Subject: Bump to 3.2.0 --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index bd0f871..020d152 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.2.0"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 7b56c41090d18e2fd97fc4e74bd986347ad845c7 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sat, 18 Jun 2016 17:02:18 -0400 Subject: go back to semver post-release --- src/rebar.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar.app.src b/src/rebar.app.src index 020d152..bd0f871 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.2.0"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, -- cgit v1.1 From 6bc8ccefd344ba1620b183c5c6810b6cd727d841 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sun, 26 Jun 2016 20:26:51 -0400 Subject: Escriptize based on configured apps only Prior to this patch, the escriptize command flat out selected all declared dependencies. This patch instead looks at the app files and only includes the dependencies of the top level app and the extra ones, avoiding to package more apps than required. This required a version bump on cth_readable as it mistakenly included 'syntax_lib' instead of 'syntax_tools' as a dependency. --- src/rebar_prv_escriptize.erl | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_escriptize.erl b/src/rebar_prv_escriptize.erl index 32f5508..d8704f6 100644 --- a/src/rebar_prv_escriptize.erl +++ b/src/rebar_prv_escriptize.erl @@ -90,9 +90,9 @@ escriptize(State0, App) -> %% Look for a list of other applications (dependencies) to include %% in the output file. We then use the .app files for each of these %% to pull in all the .beam files. - InclApps = lists:usort([ec_cnv:to_atom(AppName) | rebar_state:get(State, escript_incl_apps, []) - ++ all_deps(State)]), + TopInclApps = lists:usort([ec_cnv:to_atom(AppName) | rebar_state:get(State, escript_incl_apps, [])]), AllApps = rebar_state:all_deps(State)++rebar_state:project_apps(State), + InclApps = find_deps(TopInclApps, AllApps), InclBeams = get_apps_beams(InclApps, AllApps), %% Look for a list of extra files to include in the output file. @@ -219,9 +219,23 @@ usort(List) -> get_nonempty(Files) -> [{FName,FBin} || {FName,FBin} <- Files, FBin =/= <<>>]. -all_deps(State) -> - [list_to_existing_atom(binary_to_list(rebar_app_info:name(App))) - || App <- rebar_state:all_deps(State)]. +find_deps(AppNames, AllApps) -> + BinAppNames = [ec_cnv:to_binary(Name) || Name <- AppNames], + [ec_cnv:to_atom(Name) || + Name <- find_deps_of_deps(BinAppNames, AllApps, BinAppNames)]. + +%% Should look at the app files to find direct dependencies +find_deps_of_deps([], _, Acc) -> Acc; +find_deps_of_deps([Name|Names], Apps, Acc) -> + ?DEBUG("processing ~p", [Name]), + {ok, App} = rebar_app_utils:find(Name, Apps), + DepNames = proplists:get_value(applications, rebar_app_info:app_details(App), []), + BinDepNames = [ec_cnv:to_binary(Dep) || Dep <- DepNames, + %% ignore system libs; shouldn't include them. + not lists:prefix(code:root_dir(), code:lib_dir(Dep))] + -- ([Name|Names]++Acc), % avoid already seen deps + ?DEBUG("new deps of ~p found to be ~p", [Name, BinDepNames]), + find_deps_of_deps(BinDepNames ++ Names, Apps, BinDepNames ++ Acc). def(Rm, State, Key, Default) -> Value0 = rebar_state:get(State, Key, Default), -- cgit v1.1 From c86201b85b7f426f8eaca31d95ee39a71fab9faf Mon Sep 17 00:00:00 2001 From: Steve Strong Date: Tue, 28 Jun 2016 13:58:10 +0100 Subject: Fixes for windows environment: 1) copy empty directories when cloning (since git occasionally has a refs directory with no files in it - if the directory is not present, then git does not believe it is a git repo) and 2) change order of git rev-parse arguments to match git docs --- src/rebar_file_utils.erl | 4 ++-- src/rebar_git_resource.erl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 667be62..104c047 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -177,11 +177,11 @@ mv(Source, Dest) -> {win32, _} -> Cmd = case filelib:is_dir(Source) of true -> - ?FMT("robocopy /move /s \"~s\" \"~s\" 1> nul", + ?FMT("robocopy /move /e \"~s\" \"~s\" 1> nul", [rebar_utils:escape_double_quotes(filename:nativename(Source)), rebar_utils:escape_double_quotes(filename:nativename(Dest))]); false -> - ?FMT("robocopy /move /s \"~s\" \"~s\" \"~s\" 1> nul", + ?FMT("robocopy /move /e \"~s\" \"~s\" \"~s\" 1> nul", [rebar_utils:escape_double_quotes(filename:nativename(filename:dirname(Source))), rebar_utils:escape_double_quotes(filename:nativename(Dest)), rebar_utils:escape_double_quotes(filename:basename(Source))]) diff --git a/src/rebar_git_resource.erl b/src/rebar_git_resource.erl index f0f5f03..acb9ec0 100644 --- a/src/rebar_git_resource.erl +++ b/src/rebar_git_resource.erl @@ -22,7 +22,7 @@ lock(AppDir, {git, Url}) -> {ok, VsnString} = case os:type() of {win32, _} -> - rebar_utils:sh("git rev-parse --git-dir=\"" ++ Dir ++ "/.git\" --work-tree=\"" ++ Dir ++ "\" --verify HEAD", + rebar_utils:sh("git --git-dir=\"" ++ Dir ++ "/.git\" --work-tree=\"" ++ Dir ++ "\" rev-parse --verify HEAD", [{use_stdout, false}, {debug_abort_on_error, AbortMsg}]); _ -> rebar_utils:sh("git --git-dir=\"" ++ Dir ++ "/.git\" rev-parse --verify HEAD", -- cgit v1.1 From e334f211e373c9f019c68d5d35cc8b6aa0a1723a Mon Sep 17 00:00:00 2001 From: James Fish Date: Mon, 11 Jul 2016 14:28:38 +0100 Subject: Handle empty PLTs --- src/rebar_prv_dialyzer.erl | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_prv_dialyzer.erl b/src/rebar_prv_dialyzer.erl index 622ee60..f7e3f21 100644 --- a/src/rebar_prv_dialyzer.erl +++ b/src/rebar_prv_dialyzer.erl @@ -353,8 +353,19 @@ update_base_plt(State, BasePlt, Output, BaseFiles) -> build_plt(State, BasePlt, Output, BaseFiles) end. +build_plt(State, Plt, _, []) -> + ?INFO("Building with no files in ~p...", [Plt]), + Opts = [{get_warnings, false}, + {output_plt, Plt}, + {apps, [erts]}], + % Create a PLT with erts files and then remove erts files to be left with an + % empty PLT. Dialyzer will crash when trying to build a PLT with an empty + % file list. + _ = dialyzer:run([{analysis_type, plt_build} | Opts]), + _ = dialyzer:run([{analysis_type, plt_remove}, {init_plt, Plt} | Opts]), + {0, State}; build_plt(State, Plt, Output, Files) -> - ?INFO("Adding ~b files to ~p...", [length(Files), Plt]), + ?INFO("Building with ~b files in ~p...", [length(Files), Plt]), GetWarnings = get_config(State, get_warnings, false), Opts = [{analysis_type, plt_build}, {get_warnings, GetWarnings}, -- cgit v1.1 From 0fa2b501f05e700a35c272f98f1ab976a5bd6170 Mon Sep 17 00:00:00 2001 From: James Fish Date: Mon, 11 Jul 2016 14:41:19 +0100 Subject: Don't error when analyzing empty app --- src/rebar_prv_dialyzer.erl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_dialyzer.erl b/src/rebar_prv_dialyzer.erl index f7e3f21..ec9e81a 100644 --- a/src/rebar_prv_dialyzer.erl +++ b/src/rebar_prv_dialyzer.erl @@ -380,12 +380,15 @@ succ_typings(State, Plt, Output) -> {0, State}; _ -> Apps = rebar_state:project_apps(State), - succ_typings(State, Plt, Output, Apps) + ?INFO("Doing success typing analysis...", []), + Files = apps_to_files(Apps), + succ_typings(State, Plt, Output, Files) end. -succ_typings(State, Plt, Output, Apps) -> - ?INFO("Doing success typing analysis...", []), - Files = apps_to_files(Apps), +succ_typings(State, Plt, _, []) -> + ?INFO("Analyzing no files with ~p...", [Plt]), + {0, State}; +succ_typings(State, Plt, Output, Files) -> ?INFO("Analyzing ~b files with ~p...", [length(Files), Plt]), Opts = [{analysis_type, succ_typings}, {get_warnings, true}, -- cgit v1.1 From 941048dce7b0fab65ee8a07828bbc8c5e2034841 Mon Sep 17 00:00:00 2001 From: James Fish Date: Mon, 11 Jul 2016 18:01:33 +0100 Subject: Avoid PLT rebuild when files deleted on new dialyzer --- src/rebar_prv_dialyzer.erl | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rebar_prv_dialyzer.erl b/src/rebar_prv_dialyzer.erl index ec9e81a..82d2d07 100644 --- a/src/rebar_prv_dialyzer.erl +++ b/src/rebar_prv_dialyzer.erl @@ -249,10 +249,15 @@ ebin_files(EbinDir) -> File <- filelib:wildcard(Wildcard, EbinDir)]. read_plt(_State, Plt) -> - case dialyzer:plt_info(Plt) of - {ok, Info} -> - Files = proplists:get_value(files, Info, []), + Vsn = dialyzer_version(), + case plt_files(Plt) of + {ok, Files} when Vsn < {2, 9, 0} -> + % Before dialyzer-2.9 (OTP 18.3) removing a beam file from the PLT + % that no longer exists would crash. Therefore force a rebuild of + % PLT if any files no longer exist. read_plt_files(Plt, Files); + {ok, _} = Result when Vsn >= {2, 9, 0} -> + Result; {error, no_such_file} -> error; {error, read_error} -> @@ -260,6 +265,14 @@ read_plt(_State, Plt) -> throw({dialyzer_error, Error}) end. +plt_files(Plt) -> + case dialyzer:plt_info(Plt) of + {ok, Info} -> + {ok, proplists:get_value(files, Info, [])}; + {error, _} = Error -> + Error + end. + %% If any file no longer exists dialyzer will fail when updating the PLT. read_plt_files(Plt, Files) -> case [File || File <- Files, not filelib:is_file(File)] of -- cgit v1.1