Merge pull request #1986 from ericmj/emj/mirror-of

Update hex_core and add mirror_of repo config

7 files changed, 71 insertions, 39 deletions

diff --git a/bootstrap b/bootstrap
index ebbf35b..505963c 100755
--- a/bootstrap
+++ b/bootstrap
@@ -135,13 +135,23 @@ compile(App, FirstFiles) ->
     filelib:ensure_dir(filename:join([Dir, "ebin", "dummy.beam"])),
     code:add_path(filename:join(Dir, "ebin")),
     FirstFilesPaths = [filename:join([Dir, "src", Module]) || Module <- FirstFiles],
+    LeexFiles = filelib:wildcard(filename:join([Dir, "src", "*.xrl"])),
+    [compile_xrl_file(X) || X <- LeexFiles],
+    YeccFiles = filelib:wildcard(filename:join([Dir, "src", "*.yrl"])),
+    [compile_yrl_file(X) || X <- YeccFiles],
     Sources = FirstFilesPaths ++ filelib:wildcard(filename:join([Dir, "src", "*.erl"])),
-    [compile_file(X, [{i, filename:join(Dir, "include")}
+    [compile_erl_file(X, [{i, filename:join(Dir, "include")}
                      ,debug_info
                      ,{outdir,  filename:join(Dir, "ebin")}
                      ,return | additional_defines()]) || X <- Sources].
 
-compile_file(File, Opts) ->
+compile_xrl_file(File) ->
+    {ok, _} = leex:file(File).
+
+compile_yrl_file(File) ->
+    {ok, _} = yecc:file(File).
+
+compile_erl_file(File, Opts) ->
     case compile:file(File, Opts) of
         {ok, _Mod} ->
             ok;
@@ -162,7 +172,7 @@ bootstrap_rebar3() ->
                           filename:absname("_build/default/lib/rebar/src")),
     true = Res == ok orelse Res == exists,
     Sources = ["src/rebar_resource_v2.erl", "src/rebar_resource.erl" | filelib:wildcard("src/*.erl")],
-    [compile_file(X, [{outdir, "_build/default/lib/rebar/ebin/"}
+    [compile_erl_file(X, [{outdir, "_build/default/lib/rebar/ebin/"}
                      ,return | additional_defines()]) || X <- Sources],
     code:add_patha(filename:absname("_build/default/lib/rebar/ebin")).
 
@@ -471,7 +481,7 @@ make_normalized_path(Path) ->
     AbsPath = make_absolute_path(Path),
     Components = filename:split(AbsPath),
     make_normalized_path(Components, []).
-    
+
 make_absolute_path(Path) ->
     case filename:pathtype(Path) of
         absolute ->
@@ -636,7 +646,7 @@ join([], Sep) when is_list(Sep) ->
 join([H|T], Sep) ->
     H ++ lists:append([Sep ++ X || X <- T]).
 
-%% Same for chr; no non-deprecated equivalent in OTP20+ 
+%% Same for chr; no non-deprecated equivalent in OTP20+
 chr(S, C) when is_integer(C) -> chr(S, C, 1).
 chr([C|_Cs], C, I) -> I;
 chr([_|Cs], C, I) -> chr(Cs, C, I+1);
diff --git a/rebar.config b/rebar.config
index 4cfb71e..b8ad33e 100644
--- a/rebar.config
+++ b/rebar.config
@@ -11,7 +11,7 @@
         {relx,                "3.27.0"},
         {cf,                  "0.2.2"},
         {cth_readable,        "1.4.2"},
-        {hex_core,            "0.2.0"},
+        {hex_core,            "0.4.0"},
         {eunit_formatters,    "0.5.0"}]}.
 
 {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)",
diff --git a/rebar.lock b/rebar.lock
index 4fb5cc7..a0c6c9a 100644
--- a/rebar.lock
+++ b/rebar.lock
@@ -6,7 +6,7 @@
  {<<"erlware_commons">>,{pkg,<<"erlware_commons">>,<<"1.3.1">>},0},
  {<<"eunit_formatters">>,{pkg,<<"eunit_formatters">>,<<"0.5.0">>},0},
  {<<"getopt">>,{pkg,<<"getopt">>,<<"1.0.1">>},0},
- {<<"hex_core">>,{pkg,<<"hex_core">>,<<"0.2.0">>},0},
+ {<<"hex_core">>,{pkg,<<"hex_core">>,<<"0.4.0">>},0},
  {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.3.0">>},0},
  {<<"providers">>,{pkg,<<"providers">>,<<"1.7.0">>},0},
  {<<"relx">>,{pkg,<<"relx">>,<<"3.27.0">>},0},
@@ -20,7 +20,7 @@
  {<<"erlware_commons">>, <<"0CE192AD69BC6FD0880246D852D0ECE17631E234878011D1586E053641ED4C04">>},
  {<<"eunit_formatters">>, <<"6A9133943D36A465D804C1C5B6E6839030434B8879C5600D7DDB5B3BAD4CCB59">>},
  {<<"getopt">>, <<"C73A9FA687B217F2FF79F68A3B637711BB1936E712B521D8CE466B29CBF7808A">>},
- {<<"hex_core">>, <<"3A7EACCFB8ADD3FF05D950C10ED5BDB5D0C48C988EBBC5D7AE2A55498F0EFF1B">>},
+ {<<"hex_core">>, <<"6A0E0B1B519850344292298DA1BFA685E71534FDF4C69434993039F81078C7FA">>},
  {<<"parse_trans">>, <<"09765507A3C7590A784615CFD421D101AEC25098D50B89D7AA1D66646BC571C1">>},
  {<<"providers">>, <<"BBF730563914328EC2511D205E6477A94831DB7297DE313B3872A2B26C562EAB">>},
  {<<"relx">>, <<"96CC7663EDCC02A8117AB0C64FE6D15BE79760C08726ABEAD1DAACE11BFBF75D">>},
diff --git a/src/rebar_hex_repos.erl b/src/rebar_hex_repos.erl
index ebee191..57b5bc8 100644
--- a/src/rebar_hex_repos.erl
+++ b/src/rebar_hex_repos.erl
@@ -21,7 +21,8 @@
                   api_key => binary(),
                   repo_url => binary(),
                   repo_public_key => binary(),
-                  repo_verify => binary()}.
+                  repo_verify => binary(),
+                  repo_verify_origin => binary()}.
 
 from_state(BaseConfig, State) ->
     HexConfig = rebar_state:get(State, hex, []),
@@ -104,7 +105,13 @@ update_repo_list(R, []) ->
 
 default_repo() ->
     HexDefaultConfig = hex_core:default_config(),
-    HexDefaultConfig#{name => ?PUBLIC_HEX_REPO}.
+    HexDefaultConfig#{name => ?PUBLIC_HEX_REPO, repo_verify_origin => repo_verify_origin()}.
+
+repo_verify_origin() ->
+    case os:getenv("REBAR_NO_VERIFY_REPO_ORIGIN") of
+        "1" -> false;
+        _ -> true
+    end.
 
 repo_list([]) ->
     [];
diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl
index 757eb86..e92fbda 100644
--- a/src/rebar_packages.erl
+++ b/src/rebar_packages.erl
@@ -49,10 +49,10 @@ get(Config, Name) ->
 
 
 -spec get_all_names(rebar_state:t()) -> [binary()].
-get_all_names(State) ->    
+get_all_names(State) ->
     verify_table(State),
     lists:usort(ets:select(?PACKAGE_TABLE, [{#package{key={'$1', '_', '_'},
-                                                      _='_'}, 
+                                                      _='_'},
                                              [], ['$1']}])).
 
 -spec get_package_versions(unicode:unicode_binary(), ec_semver:semver(),
@@ -101,14 +101,14 @@ load_and_verify_version(State) ->
                     ?DEBUG("Package index version mismatch. Current version ~p, this rebar3 expecting ~p",
                            [V, ?PACKAGE_INDEX_VERSION]),
                     (catch ets:delete(?PACKAGE_TABLE)),
-                    new_package_table()                    
+                    new_package_table()
             end;
-        _ ->            
+        _ ->
             new_package_table()
     end.
 
 handle_missing_package(PkgKey, Repo, State, Fun) ->
-    Name = 
+    Name =
         case PkgKey of
             {N, Vsn, _Repo} ->
                 ?DEBUG("Package ~ts-~ts not found. Fetching registry updates for "
@@ -121,8 +121,8 @@ handle_missing_package(PkgKey, Repo, State, Fun) ->
         end,
 
     update_package(Name, Repo, State),
-    try 
-        Fun(State) 
+    try
+        Fun(State)
     catch
         _:_ ->
             %% Even after an update the package is still missing, time to error out
@@ -220,7 +220,7 @@ verify_table(State) ->
     ets:info(?PACKAGE_TABLE, named_table) =:= true orelse load_and_verify_version(State).
 
 parse_deps(Deps) ->
-    [{maps:get(app, D, Name), {pkg, Name, Constraint, undefined}} 
+    [{maps:get(app, D, Name), {pkg, Name, Constraint, undefined}}
      || D=#{package := Name,
             requirement := Constraint} <- Deps].
 
@@ -233,16 +233,19 @@ parse_checksum(Checksum) ->
 
 update_package(Name, RepoConfig=#{name := Repo}, State) ->
     ?MODULE:verify_table(State),
-    try hex_repo:get_package(RepoConfig#{repo_key => maps:get(read_key, RepoConfig, <<>>)}, Name) of
-        {ok, {200, _Headers, #{releases := Releases}}} ->
+    try hex_repo:get_package(get_package_repo_config(RepoConfig), Name) of
+        {ok, {200, _Headers, Releases}} ->
             _ = insert_releases(Name, Releases, Repo, ?PACKAGE_TABLE),
             {ok, RegistryDir} = rebar_packages:registry_dir(State),
             PackageIndex = filename:join(RegistryDir, ?INDEX_FILE),
             ok = ets:tab2file(?PACKAGE_TABLE, PackageIndex);
-        {ok, {403, _Headers, <<>>}} ->
+        {ok, {403, _Headers, _}} ->
             not_found;
         {ok, {404, _Headers, _}} ->
             not_found;
+        {error, unverified} ->
+            ?WARN(unverified_repo_message(), [Repo]),
+            fail;
         Error ->
             ?DEBUG("Hex get_package request failed: ~p", [Error]),
             %% TODO: add better log message. hex_core should export a format_error
@@ -254,6 +257,18 @@ update_package(Name, RepoConfig=#{name := Repo}, State) ->
             fail
     end.
 
+get_package_repo_config(RepoConfig=#{mirror_of := Repo}) ->
+    get_package_repo_config(maps:remove(mirror_of, RepoConfig#{name => Repo}));
+get_package_repo_config(RepoConfig=#{read_key := Key}) ->
+    get_package_repo_config(maps:remove(read_key, RepoConfig#{repo_key => Key}));
+get_package_repo_config(RepoConfig) ->
+    RepoConfig.
+
+unverified_repo_message() ->
+    "The registry repository ~ts uses a record format that has been deprecated for "
+    "security reasons. The repository should be updated in order to be safer. "
+    "You can disable this check by setting REBAR_NO_VERIFY_REPO_ORIGIN=1".
+
 insert_releases(Name, Releases, Repo, Table) ->
     [true = ets:insert(Table,
                        #package{key={Name, ec_semver:parse(Version), Repo},
diff --git a/test/rebar_pkg_SUITE.erl b/test/rebar_pkg_SUITE.erl
index ee74af5..ad06abd 100644
--- a/test/rebar_pkg_SUITE.erl
+++ b/test/rebar_pkg_SUITE.erl
@@ -99,7 +99,7 @@ init_per_testcase(bad_disconnect=Name, Config0) ->
     Config = mock_config(Name, Config1),
     meck:expect(hex_repo, get_tarball, fun(_, _, _) ->
                                                {error, econnrefused}
-                                       end),                
+                                       end),
     Config;
 init_per_testcase(Name, Config0) ->
     Config = [{good_cache, false},
@@ -252,7 +252,7 @@ mock_config(Name, Config) ->
     CacheRoot = filename:join([Priv, "cache", atom_to_list(Name)]),
     TmpDir = filename:join([Priv, "tmp", atom_to_list(Name)]),
     Tid = ets:new(registry_table, [public]),
-    AllDeps = [        
+    AllDeps = [
         {{<<"badindexchk">>,<<"1.0.0">>}, [[], ?bad_checksum, [<<"rebar3">>]]},
         {{<<"goodpkg">>,<<"1.0.0">>}, [[], ?good_checksum, [<<"rebar3">>]]},
         {{<<"goodpkg">>,<<"1.0.1">>}, [[], ?good_checksum, [<<"rebar3">>]]},
@@ -267,7 +267,7 @@ mock_config(Name, Config) ->
     ok = ets:tab2file(Tid, filename:join([CacheDir, "registry"])),
 
     catch ets:delete(?PACKAGE_TABLE),
-    rebar_packages:new_package_table(),    
+    rebar_packages:new_package_table(),
     lists:foreach(fun({{N, Vsn}, [Deps, Checksum, _]}) ->
                           case ets:member(?PACKAGE_TABLE, {ec_cnv:to_binary(N), Vsn, <<"hexpm">>}) of
                               false ->
@@ -279,18 +279,18 @@ mock_config(Name, Config) ->
                                   ok
                           end
                   end, AllDeps),
-    
+
 
     meck:new(hex_repo, [passthrough]),
-    meck:expect(hex_repo, get_package, 
+    meck:expect(hex_repo, get_package,
                 fun(_Config, PkgName) ->
                         Matches = ets:match_object(Tid, {{PkgName,'_'}, '_'}),
-                        Releases = 
+                        Releases =
                             [#{checksum => Checksum,
                                version => Vsn,
-                               dependencies => Deps} || 
+                               dependencies => Deps} ||
                                 {{_, Vsn}, [Deps, Checksum, _]} <- Matches],
-                        {ok, {200, #{}, #{releases => Releases}}}
+                        {ok, {200, #{}, Releases}}
                 end),
 
     %% The state returns us a fake registry
@@ -321,7 +321,7 @@ mock_config(Name, Config) ->
     %% Cache fetches are mocked -- we assume the server and clients are
     %% correctly used.
     GoodCache = ?config(good_cache, Config),
-    {Pkg,Vsn} = ?config(pkg, Config), 
+    {Pkg,Vsn} = ?config(pkg, Config),
     PkgFile = <<Pkg/binary, "-", Vsn/binary, ".tar">>,
     {ok, PkgContents} = file:read_file(filename:join(?config(data_dir, Config), PkgFile)),
 
@@ -329,8 +329,8 @@ mock_config(Name, Config) ->
                                                {ok, {304, #{<<"etag">> => ?good_etag}, <<>>}};
                                           (_, _, _) ->
                                                {ok, {200, #{<<"etag">> => ?good_etag}, PkgContents}}
-                                       end),   
- 
+                                       end),
+
     [{cache_root, CacheRoot},
      {cache_dir, CacheDir},
      {tmp_dir, TmpDir},
diff --git a/test/rebar_pkg_alias_SUITE.erl b/test/rebar_pkg_alias_SUITE.erl
index 079a3fd..1ef5a34 100644
--- a/test/rebar_pkg_alias_SUITE.erl
+++ b/test/rebar_pkg_alias_SUITE.erl
@@ -224,7 +224,7 @@ mock_config(Name, Config) ->
     meck:expect(rebar_prv_update, do, fun(State) -> {ok, State} end),
 
     catch ets:delete(?PACKAGE_TABLE),
-    rebar_packages:new_package_table(),    
+    rebar_packages:new_package_table(),
 
     lists:foreach(fun({{N, Vsn}, [Deps, Checksum, _]}) ->
                           case ets:member(?PACKAGE_TABLE, {ec_cnv:to_binary(N), Vsn, <<"hexpm">>}) of
@@ -238,11 +238,11 @@ mock_config(Name, Config) ->
                           end;
                      ({_N, _Vsns}) ->
                           ok
-                     
-                  end, AllDeps),    
+
+                  end, AllDeps),
 
     meck:new(hex_repo, [passthrough]),
-    meck:expect(hex_repo, get_package, 
+    meck:expect(hex_repo, get_package,
                 fun(_Config, PkgName) ->
                         Matches = ets:match_object(Tid, {{PkgName,'_'}, '_'}),
                         Releases =
@@ -251,13 +251,13 @@ mock_config(Name, Config) ->
                                dependencies => [{DAppName, {pkg, DN, DV, undefined}} ||
                                                    {DN, DV, _, DAppName} <- Deps]} ||
                                 {{_, Vsn}, [Deps, Checksum, _]} <- Matches],
-                        {ok, {200, #{}, #{releases => Releases}}}
+                        {ok, {200, #{}, Releases}}
                 end),
 
     meck:expect(hex_repo, get_tarball, fun(_, _, _) ->
                                                {ok, {304, #{<<"etag">> => EtagGood}, <<>>}}
-                                       end),   
-    
+                                       end),
+
     %% Move all packages to cache
     NewConf = [{cache_root, CacheRoot},
                {cache_dir, CacheDir},