support for hex v2, multiple repository fetching, private organizations (#1884)

* update to hex_core for hex-v2 repo support (#1865)

* update to hex_core for hex-v2 repo support

This patch adds only single repo hex-v2 support through hex_core.
Packages no longer filtered out by buildtool metadata and the
package index is updated per-package instead of fetched as one
large ets dump.

* tell travis to also build hex_core branch

* support list of repos for hex packages (#1866)

* support list of repos for hex packages

repos are defined under the hex key in rebar configs. They can be
defined at the top level of a project or globally, but not in
profiles and the repos configured in dependencies are also ignored.

Searching for packages involves first checking for a match in the
local repo index cache, in the order repos are defined. If not found
each repo is checked through the hex api for any known versions of
the package and the first repo with a version that fits the constraint
is used.

* add {repos, replace, []} for overriding the global & default repos

* add hex auth handling for repos (#1874)

auth token are kept in a hex.config file that is modified by the
rebar3 hex plugin.

Repo names that have a : separating a parent and child are considered
organizations. The parent repo's auth will be included with the child.
So an organization named hexpm:rebar3_test will include any hexpm
auth tokens found in the rebar3_test organization's configuration.

* move packages to top level of of hexpm cache dir (#1876)

* move packages to top level of of hexpm cache dir

* append organization name to parent's repo_url when parsing repos

* only eval config scripts and apply overrides once per app (#1879)

* only eval config scripts and apply overrides once per app

* move new resource behaviour to rebar_resource_v2 and keep v1

* cleanup use of rebar_resource module and unused functions

* cleanup error messages and unused code

* when discovering apps support mix packages as unbuilt apps (#1882)

* use hex_core tarball unpacking support in pkg resource (#1883)

* use hex_core tarball unpacking support in pkg resource

* ignore etag if package doesn't exist and delete if checksum fails

* add back tests for bad package checksums

* improve bad registry checksum error message

1 files changed, 340 insertions, 140 deletions

diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl
index 8cebeca..8a3ffea 100644
--- a/src/rebar_packages.erl
+++ b/src/rebar_packages.erl
@@ -1,18 +1,18 @@
 -module(rebar_packages).
 
--export([packages/1
-        ,close_packages/0
-        ,load_and_verify_version/1
-        ,deps/3
+-export([get/2
+        ,get_all_names/1
         ,registry_dir/1
-        ,package_dir/1
-        ,registry_checksum/2
-        ,find_highest_matching/6
-        ,find_highest_matching/4
-        ,find_highest_matching_/6
-        ,find_all/3
+        ,package_dir/2
+        ,find_highest_matching/5
         ,verify_table/1
-        ,format_error/1]).
+        ,format_error/1
+        ,update_package/3
+        ,resolve_version/5]).
+
+-ifdef(TEST).
+-export([new_package_table/0, find_highest_matching_/5, cmp_/4, cmpl_/4, valid_vsn/1]).
+-endif.
 
 -export_type([package/0]).
 
@@ -23,119 +23,133 @@
 -type vsn() :: binary().
 -type package() :: pkg_name() | {pkg_name(), vsn()}.
 
--spec packages(rebar_state:t()) -> ets:tid().
-packages(State) ->
-    catch ets:delete(?PACKAGE_TABLE),
-    case load_and_verify_version(State) of
-        true ->
-            ok;
-        false ->
-            ?DEBUG("Error loading package index.", []),
-            handle_bad_index(State)
+format_error({missing_package, Name, Vsn}) ->
+    io_lib:format("Package not found in any repo: ~ts-~ts.", [rebar_utils:to_binary(Name),
+                                                              rebar_utils:to_binary(Vsn)]);
+format_error({missing_package, Pkg}) ->
+    io_lib:format("Package not found in any repo: ~p.", [Pkg]).
+
+-spec get(rebar_hex_repos:repo(), binary()) -> {ok, map()} | {error, term()}.
+get(Config, Name) ->
+    try hex_api_package:get(Config, Name) of
+        {ok, {200, _Headers, PkgInfo}} ->
+            {ok, PkgInfo};
+        {ok, {404, _, _}} ->
+            {error, not_found};
+        Error ->
+            ?DEBUG("Hex api request failed: ~p", [Error]),
+            {error, unknown}
+    catch
+        error:{badmatch, {error, {failed_connect, _}}} ->
+            {error, failed_to_connect};
+        _:Exception ->
+            ?DEBUG("hex_api_package:get failed: ~p", [Exception]),
+            {error, unknown}
     end.
 
-handle_bad_index(State) ->
-    ?ERROR("Bad packages index. Trying to fix by updating the registry.", []),
-    {ok, State1} = rebar_prv_update:do(State),
-    case load_and_verify_version(State1) of
-        true ->
-            ok;
-        false ->
-            %% Still unable to load after an update, create an empty registry
-            ets:new(?PACKAGE_TABLE, [named_table, public])
+
+-spec get_all_names(rebar_state:t()) -> [binary()].
+get_all_names(State) ->    
+    verify_table(State),
+    lists:usort(ets:select(?PACKAGE_TABLE, [{#package{key={'$1', '_', '_'},
+                                                      _='_'}, 
+                                             [], ['$1']}])).
+
+-spec get_package_versions(unicode:unicode_binary(), unicode:unicode_binary(), ets:tid(), rebar_state:t())
+                          -> [vsn()].
+get_package_versions(Dep, Repo, Table, State) ->
+    ?MODULE:verify_table(State),
+    ets:select(Table, [{#package{key={Dep,'$1', Repo},
+                                 _='_'}, 
+                        [], ['$1']}]).
+
+
+get_package(Dep, Vsn, Hash, Repo, Table, State) ->
+    get_package(Dep, Vsn, Hash, false, [Repo], Table, State).
+
+-spec get_package(unicode:unicode_binary(), unicode:unicode_binary(),
+                  binary() | undefined | '_', boolean() | '_',
+                  unicode:unicode_binary() | '_' | list(), ets:tab(), rebar_state:t())
+                 -> {ok, #package{}} | not_found.
+get_package(Dep, Vsn, undefined, Retired, Repo, Table, State) ->
+    get_package(Dep, Vsn, '_', Retired, Repo, Table, State);
+get_package(Dep, Vsn, Hash, Retired, Repos, Table, State) ->
+    ?MODULE:verify_table(State),
+    case ets:select(Table, [{#package{key={Dep, Vsn, Repo},
+                                      checksum=Hash,
+                                      retired=Retired,
+                                      _='_'}, [], ['$_']} || Repo <- Repos]) of
+        %% have to allow multiple matches in the list for cases that Repo is `_`
+        [Package | _] ->
+            {ok, Package};
+        _ ->
+            not_found
     end.
 
-close_packages() ->
-    catch ets:delete(?PACKAGE_TABLE).
+new_package_table() ->    
+    ?PACKAGE_TABLE = ets:new(?PACKAGE_TABLE, [named_table, public, ordered_set, {keypos, 2}]),
+    ets:insert(?PACKAGE_TABLE, {?PACKAGE_INDEX_VERSION, package_index_version}).
 
 load_and_verify_version(State) ->
     {ok, RegistryDir} = registry_dir(State),
     case ets:file2tab(filename:join(RegistryDir, ?INDEX_FILE)) of
         {ok, _} ->
-            case ets:lookup_element(?PACKAGE_TABLE, package_index_version, 2) of
+            case ets:lookup_element(?PACKAGE_TABLE, package_index_version, 1) of
                 ?PACKAGE_INDEX_VERSION ->
                     true;
-                _ ->
+                V ->
+                    %% no reason to confuse the user since we just start fresh and they
+                    %% shouldn't notice, so log as a debug message only
+                    ?DEBUG("Package index version mismatch. Current version ~p, this rebar3 expecting ~p",
+                           [V, ?PACKAGE_INDEX_VERSION]),
                     (catch ets:delete(?PACKAGE_TABLE)),
-                    rebar_prv_update:hex_to_index(State)
+                    new_package_table()                    
             end;
-        _ ->
-            rebar_prv_update:hex_to_index(State)
+        _ ->            
+            new_package_table()
     end.
 
-deps(Name, Vsn, State) ->
-    try
-        deps_(Name, Vsn, State)
-    catch
-        _:_ ->
-            handle_missing_package({Name, Vsn}, State, fun(State1) -> deps_(Name, Vsn, State1) end)
-    end.
-
-deps_(Name, Vsn, State) ->
-    ?MODULE:verify_table(State),
-    ets:lookup_element(?PACKAGE_TABLE, {rebar_utils:to_binary(Name), rebar_utils:to_binary(Vsn)}, 2).
-
-handle_missing_package(Dep, State, Fun) ->
-    case Dep of
-        {Name, Vsn} ->
-            ?INFO("Package ~ts-~ts not found. Fetching registry updates and trying again...", [Name, Vsn]);
-        _ ->
-            ?INFO("Package ~p not found. Fetching registry updates and trying again...", [Dep])
-    end,
+handle_missing_package(PkgKey, Repo, State, Fun) ->
+    Name = 
+        case PkgKey of
+            {N, Vsn, _Repo} ->
+                ?DEBUG("Package ~ts-~ts not found. Fetching registry updates for "
+                       "package and trying again...", [N, Vsn]),
+                N;
+            _ ->
+                ?DEBUG("Package ~p not found. Fetching registry updates for "
+                       "package and trying again...", [PkgKey]),
+                PkgKey
+        end,
 
-    {ok, State1} = rebar_prv_update:do(State),
-    try
-        Fun(State1)
+    update_package(Name, Repo, State),
+    try 
+        Fun(State) 
     catch
         _:_ ->
             %% Even after an update the package is still missing, time to error out
-            throw(?PRV_ERROR({missing_package, Dep}))
+            throw(?PRV_ERROR({missing_package, PkgKey}))
     end.
 
 registry_dir(State) ->
     CacheDir = rebar_dir:global_cache_dir(rebar_state:opts(State)),
-    case rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN) of
-        ?DEFAULT_CDN ->
-            RegistryDir = filename:join([CacheDir, "hex", "default"]),
-            case filelib:ensure_dir(filename:join(RegistryDir, "placeholder")) of
-                ok -> ok;
-                {error, Posix} when Posix == eaccess; Posix == enoent ->
-                    ?ABORT("Could not write to ~p. Please ensure the path is writeable.",
-                           [RegistryDir])
-            end,
-            {ok, RegistryDir};
-        CDN ->
-            case rebar_utils:url_append_path(CDN, ?REMOTE_PACKAGE_DIR) of
-                {ok, Parsed} ->
-                    {ok, {_, _, Host, _, Path, _}} = http_uri:parse(Parsed),
-                    CDNHostPath = lists:reverse(rebar_string:lexemes(Host, ".")),
-                    CDNPath = tl(filename:split(Path)),
-                    RegistryDir = filename:join([CacheDir, "hex"] ++ CDNHostPath ++ CDNPath),
-                    ok = filelib:ensure_dir(filename:join(RegistryDir, "placeholder")),
-                    {ok, RegistryDir};
-                _ ->
-                    {uri_parse_error, CDN}
-            end
-    end.
+    RegistryDir = filename:join([CacheDir, "hex"]),
+    case filelib:ensure_dir(filename:join(RegistryDir, "placeholder")) of
+        ok -> ok;
+        {error, Posix} when Posix == eaccess; Posix == enoent ->
+            ?ABORT("Could not write to ~p. Please ensure the path is writeable.",
+                   [RegistryDir])
+    end,
+    {ok, RegistryDir}.
 
-package_dir(State) ->
-    case registry_dir(State) of
-        {ok, RegistryDir} ->
-            PackageDir = filename:join([RegistryDir, "packages"]),
-            ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")),
-            {ok, PackageDir};
-        Error ->
-            Error
-    end.
+-spec package_dir(rebar_hex_repos:repo(), rebar_state:t()) -> {ok, file:filename_all()}.
+package_dir(Repo, State) ->
+    {ok, RegistryDir} = registry_dir(State),
+    RepoName = maps:get(name, Repo),
+    PackageDir = filename:join([RegistryDir, rebar_utils:to_list(RepoName), "packages"]),
+    ok = filelib:ensure_dir(filename:join(PackageDir, "placeholder")),
+    {ok, PackageDir}.
 
-registry_checksum({pkg, Name, Vsn, _Hash}, State) ->
-    try
-        ?MODULE:verify_table(State),
-        ets:lookup_element(?PACKAGE_TABLE, {Name, Vsn}, 3)
-    catch
-        _:_ ->
-            throw(?PRV_ERROR({missing_package, rebar_utils:to_binary(Name), rebar_utils:to_binary(Vsn)}))
-    end.
 
 %% Hex supports use of ~> to specify the version required for a dependency.
 %% Since rebar3 requires exact versions to choose from we find the highest
@@ -152,31 +166,28 @@ registry_checksum({pkg, Name, Vsn, _Hash}, State) ->
 %% `~> 2.1.3-dev` | `>= 2.1.3-dev and < 2.2.0`
 %% `~> 2.0` | `>= 2.0.0 and < 3.0.0`
 %% `~> 2.1` | `>= 2.1.0 and < 3.0.0`
-find_highest_matching(Dep, Constraint, Table, State) ->
-    find_highest_matching(undefined, undefined, Dep, Constraint, Table, State).
-
-find_highest_matching(Pkg, PkgVsn, Dep, Constraint, Table, State) ->
-    try find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State) of
+find_highest_matching(Dep, Constraint, Repo, Table, State) ->
+    try find_highest_matching_(Dep, Constraint, Repo, Table, State) of
         none ->
-            handle_missing_package(Dep, State,
+            handle_missing_package(Dep, Repo, State,
                                    fun(State1) ->
-                                       find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State1)
+                                       find_highest_matching_(Dep, Constraint, Repo, Table, State1)
                                    end);
         Result ->
             Result
     catch
         _:_ ->
-            handle_missing_package(Dep, State,
+            handle_missing_package(Dep, Repo, State,
                                    fun(State1) ->
-                                       find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State1)
+                                       find_highest_matching_(Dep, Constraint, Repo, Table, State1)
                                    end)
     end.
 
-find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State) ->
-    try find_all(Dep, Table, State) of
-        {ok, [Vsn]} ->
-            handle_single_vsn(Pkg, PkgVsn, Dep, Vsn, Constraint);
-        {ok, Vsns} ->
+find_highest_matching_(Dep, Constraint, #{name := Repo}, Table, State) ->
+    try get_package_versions(Dep, Repo, Table, State) of
+        [Vsn] ->
+            handle_single_vsn(Vsn, Constraint);
+        Vsns ->
             case handle_vsns(Constraint, Vsns) of
                 none ->
                     none;
@@ -188,18 +199,6 @@ find_highest_matching_(Pkg, PkgVsn, Dep, Constraint, Table, State) ->
             none
     end.
 
-find_all(Dep, Table, State) ->
-    ?MODULE:verify_table(State),
-    try ets:lookup_element(Table, Dep, 2) of
-        [Vsns] when is_list(Vsns)->
-            {ok, Vsns};
-        Vsns ->
-            {ok, Vsns}
-    catch
-        error:badarg ->
-            none
-    end.
-
 handle_vsns(Constraint, Vsns) ->
     lists:foldl(fun(Version, Highest) ->
                         case ec_semver:pes(Version, Constraint) andalso
@@ -211,26 +210,227 @@ handle_vsns(Constraint, Vsns) ->
                         end
                 end, none, Vsns).
 
-handle_single_vsn(Pkg, PkgVsn, Dep, Vsn, Constraint) ->
+handle_single_vsn(Vsn, Constraint) ->
     case ec_semver:pes(Vsn, Constraint) of
         true ->
             {ok, Vsn};
         false ->
-            case {Pkg, PkgVsn} of
-                {undefined, undefined} ->
-                    ?DEBUG("Only existing version of ~ts is ~ts which does not match constraint ~~> ~ts. "
-                          "Using anyway, but it is not guaranteed to work.", [Dep, Vsn, Constraint]);
-                _ ->
-                    ?DEBUG("[~ts:~ts] Only existing version of ~ts is ~ts which does not match constraint ~~> ~ts. "
-                          "Using anyway, but it is not guaranteed to work.", [Pkg, PkgVsn, Dep, Vsn, Constraint])
-            end,
-            {ok, Vsn}
+            none
     end.
 
-format_error({missing_package, Name, Vsn}) ->
-    io_lib:format("Package not found in registry: ~ts-~ts.", [rebar_utils:to_binary(Name), rebar_utils:to_binary(Vsn)]);
-format_error({missing_package, Dep}) ->
-    io_lib:format("Package not found in registry: ~p.", [Dep]).
-
 verify_table(State) ->
     ets:info(?PACKAGE_TABLE, named_table) =:= true orelse load_and_verify_version(State).
+
+parse_deps(Deps) ->
+    [{maps:get(app, D, Name), {pkg, Name, Constraint, undefined}} 
+     || D=#{package := Name,
+            requirement := Constraint} <- Deps].
+
+parse_checksum(<<Checksum:256/big-unsigned>>) ->
+    list_to_binary(
+      rebar_string:uppercase(
+        lists:flatten(io_lib:format("~64.16.0b", [Checksum]))));
+parse_checksum(Checksum) ->
+    Checksum.
+
+update_package(Name, RepoConfig=#{name := Repo}, State) ->
+    ?MODULE:verify_table(State),
+    try hex_repo:get_package(RepoConfig#{repo_key => maps:get(read_key, RepoConfig, <<>>)}, Name) of
+        {ok, {200, _Headers, #{releases := Releases}}} ->
+            _ = insert_releases(Name, Releases, Repo, ?PACKAGE_TABLE),
+            {ok, RegistryDir} = rebar_packages:registry_dir(State),
+            PackageIndex = filename:join(RegistryDir, ?INDEX_FILE),
+            ok = ets:tab2file(?PACKAGE_TABLE, PackageIndex);
+        {ok, {403, _Headers, <<>>}} ->
+            not_found;
+        {ok, {404, _Headers, _}} ->
+            not_found;
+        Error ->
+            ?DEBUG("Hex get_package request failed: ~p", [Error]),
+            %% TODO: add better log message. hex_core should export a format_error
+            ?WARN("Failed to update package from repo ~ts", [Repo]),
+            fail
+    catch
+        _:Exception ->
+            ?DEBUG("hex_repo:get_package failed for package ~p: ~p", [Name, Exception]),
+            fail
+    end.
+
+insert_releases(Name, Releases, Repo, Table) ->
+    [true = ets:insert(Table,
+                       #package{key={Name, Version, Repo},
+                                checksum=parse_checksum(Checksum),
+                                retired=maps:get(retired, Release, false),
+                                dependencies=parse_deps(Dependencies)})
+     || Release=#{checksum := Checksum,
+                  version := Version,
+                  dependencies := Dependencies} <- Releases].
+
+-spec resolve_version(unicode:unicode_binary(), unicode:unicode_binary() | undefined,
+                      binary() | undefined,
+                      ets:tab(), rebar_state:t())
+                     -> {error, {invalid_vsn, unicode:unicode_binary()}} |
+                        not_found |
+                        {ok, #package{}, map()}.
+%% if checksum is defined search for any matching repo matching pkg-vsn and checksum
+resolve_version(Dep, DepVsn, Hash, HexRegistry, State) when is_binary(Hash) ->
+    Resources = rebar_state:resources(State),
+    #{repos := RepoConfigs} = rebar_resource_v2:find_resource_state(pkg, Resources),
+    RepoNames = [RepoName || #{name := RepoName} <- RepoConfigs],
+
+    %% allow retired packages when we have a checksum
+    case get_package(Dep, DepVsn, Hash, '_', RepoNames, HexRegistry, State) of
+        {ok, Package=#package{key={_, _, RepoName}}} ->
+            {ok, RepoConfig} = rebar_hex_repos:get_repo_config(RepoName, RepoConfigs),
+            {ok, Package, RepoConfig};
+        _ ->
+            Fun = fun(Repo) ->
+                      case resolve_version_(Dep, DepVsn, Repo, HexRegistry, State) of
+                          none ->
+                              not_found;
+                          {ok, Vsn} ->
+                              get_package(Dep, Vsn, Hash, Repo, HexRegistry, State)
+                      end
+                  end,
+            handle_missing_no_exception(Fun, Dep, State)
+    end;
+resolve_version(Dep, undefined, Hash, HexRegistry, State) ->
+    Fun = fun(Repo) ->
+              case highest_matching(Dep, "0", Repo, HexRegistry, State) of
+                  none ->
+                      not_found;
+                  {ok, Vsn} ->
+                      get_package(Dep, Vsn, Hash, Repo, HexRegistry, State)
+              end
+          end,
+    handle_missing_no_exception(Fun, Dep, State);
+resolve_version(Dep, DepVsn, Hash, HexRegistry, State) ->
+    case valid_vsn(DepVsn) of
+        false ->
+            {error, {invalid_vsn, DepVsn}};
+        _ ->
+            Fun = fun(Repo) ->
+                      case resolve_version_(Dep, DepVsn, Repo, HexRegistry, State) of
+                          none ->
+                              not_found;
+                          {ok, Vsn} ->
+                              get_package(Dep, Vsn, Hash, Repo, HexRegistry, State)
+                      end
+                  end,
+            handle_missing_no_exception(Fun, Dep, State)
+    end.
+
+check_all_repos(Fun, RepoConfigs) ->
+    ec_lists:search(fun(#{name := R}) ->
+                            Fun(R)
+                    end, RepoConfigs).
+
+handle_missing_no_exception(Fun, Dep, State) ->
+    Resources = rebar_state:resources(State),
+    #{repos := RepoConfigs} = rebar_resource_v2:find_resource_state(pkg, Resources),
+
+    %% first check all repos in order for a local match
+    %% if none is found then we step through checking after updating the repo registry
+    case check_all_repos(Fun, RepoConfigs) of
+        not_found ->
+            ec_lists:search(fun(Config=#{name := R}) ->
+                                    case ?MODULE:update_package(Dep, Config, State) of
+                                        ok ->
+                                            Fun(R);
+                                        _ ->
+                                            not_found
+                                    end
+                            end, RepoConfigs);
+        Result ->
+            Result
+    end.
+
+resolve_version_(Dep, DepVsn, Repo, HexRegistry, State) ->
+    case DepVsn of
+        <<"~>", Vsn/binary>> ->
+            highest_matching(Dep, rm_ws(Vsn), Repo, HexRegistry, State);
+        <<">=", Vsn/binary>> ->
+            cmp(Dep, rm_ws(Vsn), Repo, HexRegistry, State, fun ec_semver:gte/2);
+        <<">", Vsn/binary>> ->
+            cmp(Dep, rm_ws(Vsn), Repo, HexRegistry, State, fun ec_semver:gt/2);
+        <<"<=", Vsn/binary>> ->
+            cmpl(Dep, rm_ws(Vsn), Repo, HexRegistry, State, fun ec_semver:lte/2);
+        <<"<", Vsn/binary>> ->
+            cmpl(Dep, rm_ws(Vsn), Repo, HexRegistry, State, fun ec_semver:lt/2);
+        <<"==", Vsn/binary>> ->
+            {ok, Vsn};
+        Vsn ->
+            {ok, Vsn}
+    end.
+
+rm_ws(<<" ", R/binary>>) ->
+    rm_ws(R);
+rm_ws(R) ->
+    R.
+
+valid_vsn(Vsn) ->
+    %% Regepx from https://github.com/sindresorhus/semver-regex/blob/master/index.js
+    SemVerRegExp = "v?(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)(\\.(0|[1-9][0-9]*))?"
+        "(-[0-9a-z-]+(\\.[0-9a-z-]+)*)?(\\+[0-9a-z-]+(\\.[0-9a-z-]+)*)?",
+    SupportedVersions = "^(>=?|<=?|~>|==)?\\s*" ++ SemVerRegExp ++ "$",
+    re:run(Vsn, SupportedVersions, [unicode]) =/= nomatch.
+
+highest_matching(Dep, Vsn, Repo, HexRegistry, State) ->
+    find_highest_matching_(Dep, Vsn, #{name => Repo}, HexRegistry, State).
+
+cmp(Dep, Vsn, Repo, HexRegistry, State, CmpFun) ->
+    case get_package_versions(Dep, Repo, HexRegistry, State) of
+        [] ->
+            none;
+        Vsns ->
+            cmp_(undefined, Vsn, Vsns, CmpFun)
+    end.
+
+cmp_(undefined, MinVsn, [], _CmpFun) ->
+    {ok, MinVsn};
+cmp_(HighestDepVsn, _MinVsn, [], _CmpFun) ->
+    {ok, HighestDepVsn};
+
+cmp_(BestMatch, MinVsn, [Vsn | R], CmpFun) ->
+    case CmpFun(Vsn, MinVsn) of
+        true ->
+            cmp_(Vsn, Vsn, R, CmpFun);
+        false  ->
+            cmp_(BestMatch, MinVsn, R, CmpFun)
+    end.
+
+%% We need to treat this differently since we want a version that is LOWER but
+%% the higest possible one.
+cmpl(Dep, Vsn, Repo, HexRegistry, State, CmpFun) ->
+    case get_package_versions(Dep, Repo, HexRegistry, State) of
+        [] ->
+            none;
+        Vsns ->
+            cmpl_(undefined, Vsn, Vsns, CmpFun)
+    end.
+
+cmpl_(undefined, MaxVsn, [], _CmpFun) ->
+    {ok, MaxVsn};
+cmpl_(HighestDepVsn, _MaxVsn, [], _CmpFun) ->
+    {ok, HighestDepVsn};
+
+cmpl_(undefined, MaxVsn, [Vsn | R], CmpFun) ->
+    case CmpFun(Vsn, MaxVsn) of
+        true ->
+            cmpl_(Vsn, MaxVsn, R, CmpFun);
+        false  ->
+            cmpl_(undefined, MaxVsn, R, CmpFun)
+    end;
+
+cmpl_(BestMatch, MaxVsn, [Vsn | R], CmpFun) ->
+    case CmpFun(Vsn, MaxVsn) of
+        true ->
+            case ec_semver:gte(Vsn, BestMatch) of
+                true ->
+                    cmpl_(Vsn, MaxVsn, R, CmpFun);
+                false ->
+                    cmpl_(BestMatch, MaxVsn, R, CmpFun)
+            end;
+        false  ->
+            cmpl_(BestMatch, MaxVsn, R, CmpFun)
+    end.