Merge pull request #1207 from ferd/pkg-local-hash-lock

lock file contains expected hash for pkg dependencies

8 files changed, 125 insertions, 32 deletions

diff --git a/src/rebar.hrl b/src/rebar.hrl
index 4d69c7b..f96ed5e 100644
--- a/src/rebar.hrl
+++ b/src/rebar.hrl
@@ -23,6 +23,7 @@
 -define(DEFAULT_TEST_DEPS_DIR, "test/lib").
 -define(DEFAULT_RELEASE_DIR, "rel").
 -define(DEFAULT_CONFIG_FILE, "rebar.config").
+-define(CONFIG_VERSION, "1.1.0").
 -define(DEFAULT_CDN, "https://repo.hex.pm/").
 -define(REMOTE_PACKAGE_DIR, "tarballs").
 -define(REMOTE_REGISTRY_FILE, "registry.ets.gz").
diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl
index 957526e..d256cac 100644
--- a/src/rebar_app_utils.erl
+++ b/src/rebar_app_utils.erl
@@ -119,17 +119,17 @@ parse_dep(Dep, Parent, DepsDir, State, Locks, Level) ->
 
 parse_dep(Parent, {Name, Vsn, {pkg, PkgName}}, DepsDir, IsLock, State) ->
     {PkgName1, PkgVsn} = {ec_cnv:to_binary(PkgName), ec_cnv:to_binary(Vsn)},
-    dep_to_app(Parent, DepsDir, Name, PkgVsn, {pkg, PkgName1, PkgVsn}, IsLock, State);
+    dep_to_app(Parent, DepsDir, Name, PkgVsn, {pkg, PkgName1, PkgVsn, undefined}, IsLock, State);
 parse_dep(Parent, {Name, {pkg, PkgName}}, DepsDir, IsLock, State) ->
     %% Package dependency with different package name from app name
-    dep_to_app(Parent, DepsDir, Name, undefined, {pkg, ec_cnv:to_binary(PkgName), undefined}, IsLock, State);
+    dep_to_app(Parent, DepsDir, Name, undefined, {pkg, ec_cnv:to_binary(PkgName), undefined, undefined}, IsLock, State);
 parse_dep(Parent, {Name, Vsn}, DepsDir, IsLock, State) when is_list(Vsn); is_binary(Vsn) ->
     %% Versioned Package dependency
     {PkgName, PkgVsn} = {ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)},
-    dep_to_app(Parent, DepsDir, PkgName, PkgVsn, {pkg, PkgName, PkgVsn}, IsLock, State);
+    dep_to_app(Parent, DepsDir, PkgName, PkgVsn, {pkg, PkgName, PkgVsn, undefined}, IsLock, State);
 parse_dep(Parent, Name, DepsDir, IsLock, State) when is_atom(Name); is_binary(Name) ->
     %% Unversioned package dependency
-    dep_to_app(Parent, DepsDir, ec_cnv:to_binary(Name), undefined, {pkg, ec_cnv:to_binary(Name), undefined}, IsLock, State);
+    dep_to_app(Parent, DepsDir, ec_cnv:to_binary(Name), undefined, {pkg, ec_cnv:to_binary(Name), undefined, undefined}, IsLock, State);
 parse_dep(Parent, {Name, Source}, DepsDir, IsLock, State) when is_tuple(Source) ->
     dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State);
 parse_dep(Parent, {Name, _Vsn, Source}, DepsDir, IsLock, State) when is_tuple(Source) ->
@@ -143,7 +143,9 @@ parse_dep(Parent, {Name, Source, Opts}, DepsDir, IsLock, State) when is_tuple(So
     ?WARN("Dependency option list ~p in ~p is not supported and will be ignored", [Opts, Name]),
     dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State);
 parse_dep(Parent, {Name, {pkg, PkgName, Vsn}, Level}, DepsDir, IsLock, State) when is_integer(Level) ->
-    dep_to_app(Parent, DepsDir, Name, Vsn, {pkg, PkgName, Vsn}, IsLock, State);
+    dep_to_app(Parent, DepsDir, Name, Vsn, {pkg, PkgName, Vsn, undefined}, IsLock, State);
+parse_dep(Parent, {Name, {pkg, PkgName, Vsn, Hash}, Level}, DepsDir, IsLock, State) when is_integer(Level) ->
+    dep_to_app(Parent, DepsDir, Name, Vsn, {pkg, PkgName, Vsn, Hash}, IsLock, State);
 parse_dep(Parent, {Name, Source, Level}, DepsDir, IsLock, State) when is_tuple(Source)
                                                                     , is_integer(Level) ->
     dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State);
@@ -175,7 +177,7 @@ dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) ->
     AppInfo5 = rebar_app_info:profiles(AppInfo4, [default]),
     rebar_app_info:is_lock(AppInfo5, IsLock).
 
-update_source(AppInfo, {pkg, PkgName, PkgVsn}, State) ->
+update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) ->
     {PkgName1, PkgVsn1} = case PkgVsn of
                               undefined ->
                                   get_package(PkgName, "0", State);
@@ -185,7 +187,14 @@ update_source(AppInfo, {pkg, PkgName, PkgVsn}, State) ->
                               _ ->
                                   {PkgName, PkgVsn}
                           end,
-    AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName1, PkgVsn1}),
+    %% store the expected hash for the dependency
+    Hash1 = case Hash of
+        undefined -> % unknown, define the hash since we know the dep
+            rebar_packages:registry_checksum({pkg, PkgName1, PkgVsn1, Hash}, State);
+        _ -> % keep as is
+            Hash
+    end,
+    AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName1, PkgVsn1, Hash1}),
     Deps = rebar_packages:deps(PkgName1
                               ,PkgVsn1
                               ,State),
diff --git a/src/rebar_config.erl b/src/rebar_config.erl
index 836980e..b50c030 100644
--- a/src/rebar_config.erl
+++ b/src/rebar_config.erl
@@ -56,35 +56,105 @@ consult_lock_file(File) ->
         [] ->
             [];
         [Locks] when is_list(Locks) -> % beta lock file
-            Locks;
+            read_attrs(beta, Locks, []);
         [{Vsn, Locks}|Attrs] when is_list(Locks) -> % versioned lock file
-            %% Make sure the warning above is to be shown whenever a version
-            %% newer than the current one is being used, as we can't parse
-            %% all the contents of the lock file properly.
+            %% Because this is the first version of rebar3 to introduce a lock
+            %% file, all versionned lock files with a different versions have
+            %% to be newer.
+            case Vsn of
+                ?CONFIG_VERSION ->
+                    ok;
+                _ ->
+                    %% Make sure the warning below is to be shown whenever a version
+                    %% newer than the current one is being used, as we can't parse
+                    %% all the contents of the lock file properly.
+                    warn_vsn_once()
+            end,
+            read_attrs(Vsn, Locks, Attrs)
+    end.
+
+warn_vsn_once() ->
+    Warn = application:get_env(rebar, warn_config_vsn) =/= {ok, false},
+    application:set_env(rebar, warn_config_vsn, false),
+    case Warn of
+        false -> ok;
+        true ->
             ?WARN("Rebar3 detected a lock file from a newer version. "
                   "It will be loaded in compatibility mode, but important "
                   "information may be missing or lost. It is recommended to "
-                  "upgrade Rebar3.", []),
-            read_attrs(Vsn, Locks, Attrs)
+                  "upgrade Rebar3.", [])
     end.
 
+
 write_lock_file(LockFile, Locks) ->
-    NewLocks = write_attrs(Locks),
+    {NewLocks, Attrs} = write_attrs(Locks),
     %% Write locks in the beta format, at least until it's been long
     %% enough we can start modifying the lock format.
-    file:write_file(LockFile, io_lib:format("~p.~n", [NewLocks])).
+    case Attrs of
+        [] -> % write the old beta copy to avoid changes
+            file:write_file(LockFile, io_lib:format("~p.~n", [NewLocks]));
+        _ ->
+            file:write_file(LockFile,
+                            io_lib:format("{~p,~n~p}.~n[~n~s~n].~n",
+                                          [?CONFIG_VERSION, NewLocks,
+                                           format_attrs(Attrs)]))
+    end.
 
-read_attrs(_Vsn, Locks, _Attrs) ->
+%% Attributes have a special formatting to ensure there's only one per
+%% line in terms of pkg_hash, so we disturb source diffing as little
+%% as possible.
+format_attrs([]) -> [];
+format_attrs([{pkg_hash, Vals}|T]) ->
+    [io_lib:format("{pkg_hash,[~n",[]), format_hashes(Vals), "]}",
+     maybe_comma(T) | format_attrs(T)];
+format_attrs([H|T]) ->
+    [io_lib:format("~p~s", [H, maybe_comma(T)]) | format_attrs(T)].
+
+format_hashes([]) -> [];
+format_hashes([{Pkg,Hash}|T]) ->
+    [" {", io_lib:format("~p",[Pkg]), ", ", io_lib:format("~p", [Hash]), "}",
+     maybe_comma(T) | format_hashes(T)].
+
+maybe_comma([]) -> "";
+maybe_comma([_|_]) -> io_lib:format(",~n", []).
+
+read_attrs(_Vsn, Locks, Attrs) ->
     %% Beta copy does not know how to expand attributes, but
     %% is ready to support it.
-    Locks.
+    expand_locks(Locks, extract_pkg_hashes(Attrs)).
+
+extract_pkg_hashes(Attrs) ->
+    Props = case Attrs of
+                [First|_] -> First;
+                [] -> []
+            end,
+    proplists:get_value(pkg_hash, Props, []).
+
+expand_locks([], _Hashes) ->
+    [];
+expand_locks([{Name, {pkg,PkgName,Vsn}, Lvl} | Locks], Hashes) ->
+    Hash = proplists:get_value(Name, Hashes),
+    [{Name, {pkg,PkgName,Vsn,Hash}, Lvl} | expand_locks(Locks, Hashes)];
+expand_locks([Lock|Locks], Hashes) ->
+    [Lock | expand_locks(Locks, Hashes)].
 
 write_attrs(Locks) ->
     %% No attribute known that needs to be taken out of the structure,
     %% just return terms as is.
-    Locks.
-
+    {NewLocks, Hashes} = split_locks(Locks, [], []),
+    case Hashes of
+        [] -> {NewLocks, []};
+        _ -> {NewLocks, [{pkg_hash, lists:sort(Hashes)}]}
+    end.
 
+split_locks([], Locks, Hashes) ->
+    {lists:reverse(Locks), Hashes};
+split_locks([{Name, {pkg,PkgName,Vsn,undefined}, Lvl} | Locks], LAcc, HAcc) ->
+    split_locks(Locks, [{Name,{pkg,PkgName,Vsn},Lvl}|LAcc], HAcc);
+split_locks([{Name, {pkg,PkgName,Vsn,Hash}, Lvl} | Locks], LAcc, HAcc) ->
+    split_locks(Locks, [{Name,{pkg,PkgName,Vsn},Lvl}|LAcc], [{Name, Hash}|HAcc]);
+split_locks([Lock|Locks], LAcc, HAcc) ->
+    split_locks(Locks, [Lock|LAcc], HAcc).
 
 consult_file(File) ->
     Terms = consult_file_(File),
diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl
index b80c125..47bfe1d 100644
--- a/src/rebar_fetch.erl
+++ b/src/rebar_fetch.erl
@@ -67,6 +67,12 @@ needs_update(AppDir, Source, State) ->
 
 format_error({bad_download, CachePath}) ->
     io_lib:format("Download of package does not match md5sum from server: ~s", [CachePath]);
+format_error({unexpected_hash, CachePath, Expected, Found}) ->
+    io_lib:format("The checksum for package at ~s (~s) does not match the "
+                  "checksum previously locked (~s). Either unlock or "
+                  "upgrade the package, or make sure you fetched it from "
+                  "the same index from which it was initially fetched.",
+                  [CachePath, Found, Expected]);
 format_error({failed_extract, CachePath}) ->
     io_lib:format("Failed to extract package: ~s", [CachePath]);
 format_error({bad_etag, Source}) ->
diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl
index d4b8a14..8b4611b 100644
--- a/src/rebar_packages.erl
+++ b/src/rebar_packages.erl
@@ -122,7 +122,7 @@ package_dir(State) ->
             Error
     end.
 
-registry_checksum({pkg, Name, Vsn}, State) ->
+registry_checksum({pkg, Name, Vsn, _Hash}, State) ->
     try
         ?MODULE:verify_table(State),
         ets:lookup_element(?PACKAGE_TABLE, {Name, Vsn}, 3)
diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl
index ec7e09d..5817817 100644
--- a/src/rebar_pkg_resource.erl
+++ b/src/rebar_pkg_resource.erl
@@ -19,7 +19,7 @@
 lock(_AppDir, Source) ->
     Source.
 
-needs_update(Dir, {pkg, _Name, Vsn}) ->
+needs_update(Dir, {pkg, _Name, Vsn, _Hash}) ->
     [AppInfo] = rebar_app_discover:find_apps([Dir], all),
     case rebar_app_info:original_vsn(AppInfo) =:= ec_cnv:to_list(Vsn) of
         true ->
@@ -28,7 +28,7 @@ needs_update(Dir, {pkg, _Name, Vsn}) ->
             true
     end.
 
-download(TmpDir, Pkg={pkg, Name, Vsn}, State) ->
+download(TmpDir, Pkg={pkg, Name, Vsn, _Hash}, State) ->
     CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN),
     {ok, PackageDir} = rebar_packages:package_dir(State),
     Package = binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>),
@@ -40,7 +40,7 @@ download(TmpDir, Pkg={pkg, Name, Vsn}, State) ->
             {fetch_fail, Name, Vsn}
     end.
 
-cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn}, Url, ETag, State) ->
+cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash}, Url, ETag, State) ->
     case request(Url, ETag) of
         {ok, cached} ->
             ?INFO("Version cached at ~s is up to date, reusing it", [CachePath]),
@@ -58,17 +58,20 @@ cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn}, Url, ETag, State) ->
 serve_from_cache(TmpDir, CachePath, Pkg, State) ->
     {Files, Contents, Version, Meta} = extract(TmpDir, CachePath),
     case checksums(Pkg, Files, Contents, Version, Meta, State) of
-        {Chk, Chk, Chk} ->
+        {Chk, Chk, Chk, Chk} ->
             ok = erl_tar:extract({binary, Contents}, [{cwd, TmpDir}, compressed]),
             {ok, true};
-        {_Bin, Chk, Chk} ->
+        {_Hash, Chk, Chk, Chk} ->
+            ?DEBUG("Expected hash ~p does not match checksums ~p", [_Hash, Chk]),
+            {unexpected_hash, CachePath, _Hash, Chk};
+        {Chk, _Bin, Chk, Chk} ->
             ?DEBUG("Checksums: registry: ~p, pkg: ~p", [Chk, _Bin]),
             {failed_extract, CachePath};
-        {Chk, _Reg, Chk} ->
+        {Chk, Chk, _Reg, Chk} ->
             ?DEBUG("Checksums: registry: ~p, pkg: ~p", [_Reg, Chk]),
             {bad_registry_checksum, CachePath};
-        {_Bin, _Reg, _Tar} ->
-            ?DEBUG("Checksums: registry: ~p, pkg: ~p, meta: ~p", [_Reg, _Bin, _Tar]),
+        {_Hash, _Bin, _Reg, _Tar} ->
+            ?DEBUG("Checksums: expected: ~p, registry: ~p, pkg: ~p, meta: ~p", [_Hash, _Reg, _Bin, _Tar]),
             {bad_checksum, CachePath}
     end.
 
@@ -92,13 +95,13 @@ extract(TmpDir, CachePath) ->
     {"metadata.config", Meta} = lists:keyfind("metadata.config", 1, Files),
     {Files, Contents, Version, Meta}.
 
-checksums(Pkg, Files, Contents, Version, Meta, State) ->
+checksums(Pkg={pkg, _Name, _Vsn, Hash}, Files, Contents, Version, Meta, State) ->
     Blob = <<Version/binary, Meta/binary, Contents/binary>>,
     <<X:256/big-unsigned>> = crypto:hash(sha256, Blob),
     BinChecksum = list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))),
     RegistryChecksum = rebar_packages:registry_checksum(Pkg, State),
     {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files),
-    {BinChecksum, RegistryChecksum, TarChecksum}.
+    {Hash, BinChecksum, RegistryChecksum, TarChecksum}.
 
 make_vsn(_) ->
     {error, "Replacing version of type pkg not supported."}.
diff --git a/src/rebar_prv_deps.erl b/src/rebar_prv_deps.erl
index 9ff2bfa..c865276 100644
--- a/src/rebar_prv_deps.erl
+++ b/src/rebar_prv_deps.erl
@@ -97,7 +97,7 @@ display_dep(_State, {Name, _Vsn, Source}) when is_tuple(Source) ->
 display_dep(_State, {Name, _Vsn, Source, _Opts}) when is_tuple(Source) ->
     ?CONSOLE("~s* (~s source)", [ec_cnv:to_binary(Name), type(Source)]);
 %% Locked
-display_dep(State, {Name, Source={pkg, _, Vsn}, Level}) when is_integer(Level) ->
+display_dep(State, {Name, Source={pkg, _, Vsn, _}, Level}) when is_integer(Level) ->
     DepsDir = rebar_dir:deps_dir(State),
     AppDir = filename:join([DepsDir, ec_cnv:to_binary(Name)]),
     NeedsUpdate = case rebar_fetch:needs_update(AppDir, Source, State) of
diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl
index 5e6aa4c..a8a7ea0 100644
--- a/src/rebar_prv_install_deps.erl
+++ b/src/rebar_prv_install_deps.erl
@@ -352,10 +352,14 @@ make_relative_to_root(State, Path) when is_list(Path) ->
     rebar_dir:make_relative_path(Path, Root).
 
 fetch_app(AppInfo, AppDir, State) ->
-    ?INFO("Fetching ~s (~p)", [rebar_app_info:name(AppInfo), rebar_app_info:source(AppInfo)]),
+    ?INFO("Fetching ~s (~p)", [rebar_app_info:name(AppInfo),
+                               format_source(rebar_app_info:source(AppInfo))]),
     Source = rebar_app_info:source(AppInfo),
     true = rebar_fetch:download_source(AppDir, Source, State).
 
+format_source({pkg, Name, Vsn, _Hash}) -> {pkg, Name, Vsn};
+format_source(Source) -> Source.
+
 %% This is called after the dep has been downloaded and unpacked, if it hadn't been already.
 %% So this is the first time for newly downloaded apps that its .app/.app.src data can
 %% be read in an parsed.