From cb4599f828f0ec13a4ea18bcb1b0de4938d8dc7e Mon Sep 17 00:00:00 2001 From: Andrew Thompson Date: Thu, 19 Sep 2013 17:36:02 -0400 Subject: Make update-deps traverse deps breadth-first, top-down This ensures that deps of deps are updated AFTER the dep listing them is, so that a complicated project with many layers of deps will be updated correctly. Any new deps encountered along the way are also cloned, and THEIR deps are also evaluated. Also added was conflict detection, if a dep has differing versions or source information, inherited from different places, that will be logged at the end of update-deps, along with the origin of each conflicting dep. --- src/rebar_core.erl | 3 +- src/rebar_deps.erl | 98 ++++++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 90 insertions(+), 11 deletions(-) (limited to 'src') diff --git a/src/rebar_core.erl b/src/rebar_core.erl index fcfa62a..863d1d5 100644 --- a/src/rebar_core.erl +++ b/src/rebar_core.erl @@ -172,8 +172,9 @@ skip_or_process_dir1(AppFile, ModuleSet, Config, CurrentCodePath, CurrentCodePath, ModuleSet) end. -process_dir1(Dir, Command, DirSet, Config0, CurrentCodePath, +process_dir1(Dir, Command, DirSet, Config, CurrentCodePath, {DirModules, ModuleSetFile}) -> + Config0 = rebar_config:set(Config, command, Command), %% Get the list of modules for "any dir". This is a catch-all list %% of modules that are processed in addition to modules associated %% with this directory type. These any_dir modules are processed diff --git a/src/rebar_deps.erl b/src/rebar_deps.erl index be6283d..82c5061 100644 --- a/src/rebar_deps.erl +++ b/src/rebar_deps.erl @@ -97,8 +97,22 @@ preprocess(Config, _) -> %% deps-related can be executed on their directories. NonRawAvailableDeps = [D || D <- AvailableDeps, not D#dep.is_raw], - %% Return all the available dep directories for process - {ok, NewConfig, dep_dirs(NonRawAvailableDeps)}. + case rebar_config:get(Config, command, undefined) of + 'update-deps' -> + %% Skip ALL of the dep folders, we do this because we don't want + %% any other calls to preprocess() for update-deps beyond the + %% toplevel directory. They aren't actually harmful, but they slow + %% things down unnecessarily. + NewConfig2 = lists:foldl(fun(D, Acc) -> + rebar_config:set_skip_dir(Acc, D#dep.dir) + end, NewConfig, collect_deps(rebar_utils:get_cwd(),NewConfig)), + %% Return the empty list, as we don't want anything processed before + %% us. + {ok, NewConfig2, []}; + _ -> + %% Return all the available dep directories for process + {ok, NewConfig, dep_dirs(NonRawAvailableDeps)} + end. postprocess(Config, _) -> case rebar_config:get_xconf(Config, ?MODULE, undefined) of @@ -169,17 +183,23 @@ do_check_deps(Config) -> {ok, save_dep_dirs(Config2, lists:reverse(PulledDeps))}. 'update-deps'(Config, _) -> - %% Determine what deps are required - RawDeps = rebar_config:get_local(Config, deps, []), - {Config1, Deps} = find_deps(Config, read, RawDeps), - - %% Update each dep - UpdatedDeps = [update_source(Config1, D) - || D <- Deps, D#dep.source =/= undefined], + {Config2, UpdatedDeps} = update_deps_int(rebar_config:set(Config, depowner, dict:new()), []), + DepOwners = rebar_config:get(Config2, depowner, dict:new()), + + %% check for conflicting deps + [?ERROR("Conflicting dependencies for ~p: ~p~n", [K, + [{"From: " ++ string:join(dict:fetch(D, + DepOwners), + ", "), + {D#dep.vsn_regex, + D#dep.source}} || D <- V]]) || + {K, V} <- dict:to_list(lists:foldl(fun(Dep, Acc) -> + dict:append(Dep#dep.app, Dep, Acc) + end, dict:new(), UpdatedDeps)), length(V) > 1], %% Add each updated dep to our list of dirs for post-processing. This yields %% the necessary transitivity of the deps - {ok, save_dep_dirs(Config1, UpdatedDeps)}. + {ok, save_dep_dirs(Config, UpdatedDeps)}. 'delete-deps'(Config, _) -> %% Delete all the available deps in our deps/ directory, if any @@ -566,6 +586,64 @@ update_source1(AppDir, {fossil, _Url, Version}) -> rebar_utils:sh("fossil pull", [{cd, AppDir}]), rebar_utils:sh(?FMT("fossil update ~s", [Version]), []). +%% Recursively update deps, this is not done via rebar's usual dep traversal as +%% that is the wrong order (tips are updated before branches). Instead we do a +%% traverse the deps at each level completely before traversing *their* deps. +%% This allows updates to actually propogate down the tree, rather than fail to +%% flow up the tree, which was the previous behaviour. +update_deps_int(Config0, UDD) -> + %% Determine what deps are required + ConfDir = filename:basename(rebar_utils:get_cwd()), + RawDeps = rebar_config:get_local(Config0, deps, []), + {Config1, Deps} = find_deps(Config0, read, RawDeps), + + %% Update each dep + UpdatedDeps = [update_source(Config1, D) + || D <- Deps, D#dep.source =/= undefined, not lists:member(D, UDD)], + + lists:foldl(fun(Dep, {Config, Updated}) -> + {true, AppDir} = get_deps_dir(Config, Dep#dep.app), + Config2 = case has_vcs_dir(element(1, Dep#dep.source), AppDir) of + false -> + %% If the dep did not exist (maybe it was added) + %% clone it. We'll traverse ITS deps below. and + %% clone them if needed. + {C1, _D1} = use_source(Config, Dep), + C1; + true -> + Config + end, + ok = file:set_cwd(AppDir), + Config3 = rebar_config:new(Config2), + %% track where a dep comes from... + Config4 = rebar_config:set(Config3, depowner, + dict:append(Dep, ConfDir, + rebar_config:get(Config3, + depowner, + dict:new()))), + + {Config5, Res} = update_deps_int(Config4, Updated), + {Config5, lists:umerge(lists:sort(Res), + lists:sort(Updated))} + end, {Config1, lists:umerge(lists:sort(UpdatedDeps), + lists:sort(UDD))}, UpdatedDeps). + +%% Recursively walk the deps and build a list of them. +collect_deps(Dir, C) -> + case file:set_cwd(Dir) of + ok -> + Config = rebar_config:new(C), + RawDeps = rebar_config:get_local(Config, deps, []), + {Config1, Deps} = find_deps(Config, read, RawDeps), + + lists:flatten(Deps ++ [begin + {true, AppDir} = get_deps_dir(Config1, Dep#dep.app), + collect_deps(AppDir, C) + end || Dep <- Deps]); + _ -> + [] + end. + %% =================================================================== %% Source helper functions -- cgit v1.1 From 0977d58361d54c170cf45254d7b53e1cd03024e7 Mon Sep 17 00:00:00 2001 From: Andrew Thompson Date: Mon, 23 Sep 2013 15:11:11 -0400 Subject: Ignore skip_deps during update-deps as it has no meaning --- src/rebar_deps.erl | 56 +++++++++++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 28 deletions(-) (limited to 'src') diff --git a/src/rebar_deps.erl b/src/rebar_deps.erl index 82c5061..15e7594 100644 --- a/src/rebar_deps.erl +++ b/src/rebar_deps.erl @@ -68,31 +68,6 @@ preprocess(Config, _) -> %% Add available deps to code path Config3 = update_deps_code_path(Config2, AvailableDeps), - %% If skip_deps=true, mark each dep dir as a skip_dir w/ the core so that - %% the current command doesn't run on the dep dir. However, pre/postprocess - %% WILL run (and we want it to) for transitivity purposes. - %% - %% Also, if skip_deps=comma,separated,app,list, then only the given - %% dependencies are skipped. - NewConfig = case rebar_config:get_global(Config3, skip_deps, false) of - "true" -> - lists:foldl( - fun(#dep{dir = Dir}, C) -> - rebar_config:set_skip_dir(C, Dir) - end, Config3, AvailableDeps); - Apps when is_list(Apps) -> - SkipApps = [list_to_atom(App) || App <- string:tokens(Apps, ",")], - lists:foldl( - fun(#dep{dir = Dir, app = App}, C) -> - case lists:member(App, SkipApps) of - true -> rebar_config:set_skip_dir(C, Dir); - false -> C - end - end, Config3, AvailableDeps); - _ -> - Config3 - end, - %% Filtering out 'raw' dependencies so that no commands other than %% deps-related can be executed on their directories. NonRawAvailableDeps = [D || D <- AvailableDeps, not D#dep.is_raw], @@ -103,13 +78,38 @@ preprocess(Config, _) -> %% any other calls to preprocess() for update-deps beyond the %% toplevel directory. They aren't actually harmful, but they slow %% things down unnecessarily. - NewConfig2 = lists:foldl(fun(D, Acc) -> + NewConfig = lists:foldl(fun(D, Acc) -> rebar_config:set_skip_dir(Acc, D#dep.dir) - end, NewConfig, collect_deps(rebar_utils:get_cwd(),NewConfig)), + end, Config3, collect_deps(rebar_utils:get_cwd(), Config3)), %% Return the empty list, as we don't want anything processed before %% us. - {ok, NewConfig2, []}; + {ok, NewConfig, []}; _ -> + %% If skip_deps=true, mark each dep dir as a skip_dir w/ the core so that + %% the current command doesn't run on the dep dir. However, pre/postprocess + %% WILL run (and we want it to) for transitivity purposes. + %% + %% Also, if skip_deps=comma,separated,app,list, then only the given + %% dependencies are skipped. + NewConfig = case rebar_config:get_global(Config3, skip_deps, false) of + "true" -> + lists:foldl( + fun(#dep{dir = Dir}, C) -> + rebar_config:set_skip_dir(C, Dir) + end, Config3, AvailableDeps); + Apps when is_list(Apps) -> + SkipApps = [list_to_atom(App) || App <- string:tokens(Apps, ",")], + lists:foldl( + fun(#dep{dir = Dir, app = App}, C) -> + case lists:member(App, SkipApps) of + true -> rebar_config:set_skip_dir(C, Dir); + false -> C + end + end, Config3, AvailableDeps); + _ -> + Config3 + end, + %% Return all the available dep directories for process {ok, NewConfig, dep_dirs(NonRawAvailableDeps)} end. -- cgit v1.1 From f46e7b2e5c389ab8489121a3ab23159750dd1661 Mon Sep 17 00:00:00 2001 From: Andrew Thompson Date: Mon, 23 Sep 2013 15:13:26 -0400 Subject: Change how update-deps updates a git branch Previously, update-deps on a dep tagged as {branch, ...} would do the following: git fetch git checkout -q origin/ If you were already on that branch, the repo would end up in detached head state. This is kind of annoying if you're doing local development. This patch changes the behaviour to be git fetch git checkout -q git pull --ff-only --no-rebase -q The intent of this is to move the branch's HEAD forward to match upstream without destroying any local commits or changes, and without accidentally causing merges or rebases. It will fail if the operation can not be performed without losing history, merging or rebasing. The previous behaviour has been around a very long time: https://github.com/rebar/rebar/commit/064195dc5a90f5b0cc3ae92e8373671b0043033f#L0R308 It also exactly mirrors the download_source case, which is not really true. With git tags and SHAs, one can assume that they don't change, but branches move all the time. --- src/rebar_deps.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_deps.erl b/src/rebar_deps.erl index 15e7594..4a98ddf 100644 --- a/src/rebar_deps.erl +++ b/src/rebar_deps.erl @@ -560,7 +560,8 @@ update_source1(AppDir, {git, Url, ""}) -> update_source1(AppDir, {git, _Url, {branch, Branch}}) -> ShOpts = [{cd, AppDir}], rebar_utils:sh("git fetch origin", ShOpts), - rebar_utils:sh(?FMT("git checkout -q origin/~s", [Branch]), ShOpts); + rebar_utils:sh(?FMT("git checkout -q ~s", [Branch]), ShOpts), + rebar_utils:sh(?FMT("git pull --ff-only --no-rebase -q origin ~s", [Branch]), ShOpts); update_source1(AppDir, {git, _Url, {tag, Tag}}) -> ShOpts = [{cd, AppDir}], rebar_utils:sh("git fetch --tags origin", ShOpts), -- cgit v1.1 From d9aa65f118839f2ea76bbdace71166bad705423b Mon Sep 17 00:00:00 2001 From: Andrew Thompson Date: Mon, 23 Sep 2013 15:19:51 -0400 Subject: Make update-deps honor apps= and skip_apps= Because rebar_core handles skipping apps, we had to specialcase the handling in the case of update-deps because it has to do its own dep handling. The way this was done is not particularly clean, but there currently does not exist another way for a command to signal rebar_core that it doesn't want rebar_core to pay attention to skip_apps. With this change, however, you can update-deps even with local conflicting changes/commits by simply skipping the deps you don't wish to update, or whitelisting he ones you do wish to update. --- src/rebar_core.erl | 7 +++++++ src/rebar_deps.erl | 15 ++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rebar_core.erl b/src/rebar_core.erl index 863d1d5..42e106e 100644 --- a/src/rebar_core.erl +++ b/src/rebar_core.erl @@ -163,6 +163,13 @@ skip_or_process_dir({_, ModuleSetFile}=ModuleSet, Config, CurrentCodePath, skip_or_process_dir1(AppFile, ModuleSet, Config, CurrentCodePath, Dir, Command, DirSet) -> case rebar_app_utils:is_skipped_app(Config, AppFile) of + {Config1, {true, _SkippedApp}} when Command == 'update-deps' -> + %% update-deps does its own app skipping. Unfortunately there's no + %% way to signal this to rebar_core, so we have to explicitly do it + %% here... Otherwise if you use app=, it'll skip the toplevel + %% directory and nothing will be updated. + process_dir1(Dir, Command, DirSet, Config1, + CurrentCodePath, ModuleSet); {Config1, {true, SkippedApp}} -> ?DEBUG("Skipping app: ~p~n", [SkippedApp]), Config2 = increment_operations(Config1), diff --git a/src/rebar_deps.erl b/src/rebar_deps.erl index 4a98ddf..40000ad 100644 --- a/src/rebar_deps.erl +++ b/src/rebar_deps.erl @@ -600,7 +600,10 @@ update_deps_int(Config0, UDD) -> %% Update each dep UpdatedDeps = [update_source(Config1, D) - || D <- Deps, D#dep.source =/= undefined, not lists:member(D, UDD)], + || D <- Deps, D#dep.source =/= undefined, + not lists:member(D, UDD), + not should_skip_update_dep(Config1, D) + ], lists:foldl(fun(Dep, {Config, Updated}) -> {true, AppDir} = get_deps_dir(Config, Dep#dep.app), @@ -629,6 +632,16 @@ update_deps_int(Config0, UDD) -> end, {Config1, lists:umerge(lists:sort(UpdatedDeps), lists:sort(UDD))}, UpdatedDeps). +should_skip_update_dep(Config, Dep) -> + {true, AppDir} = get_deps_dir(Config, Dep#dep.app), + {true, AppFile} = rebar_app_utils:is_app_dir(AppDir), + case rebar_app_utils:is_skipped_app(Config, AppFile) of + {_Config, {true, _SkippedApp}} -> + true; + _ -> + false + end. + %% Recursively walk the deps and build a list of them. collect_deps(Dir, C) -> case file:set_cwd(Dir) of -- cgit v1.1 From eae26a5c4303f913b7d8d7177683cb4ae008d47c Mon Sep 17 00:00:00 2001 From: Andrew Thompson Date: Tue, 24 Sep 2013 12:07:41 -0400 Subject: Fix skip check for deps not present --- src/rebar_deps.erl | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/rebar_deps.erl b/src/rebar_deps.erl index 40000ad..6450715 100644 --- a/src/rebar_deps.erl +++ b/src/rebar_deps.erl @@ -634,12 +634,16 @@ update_deps_int(Config0, UDD) -> should_skip_update_dep(Config, Dep) -> {true, AppDir} = get_deps_dir(Config, Dep#dep.app), - {true, AppFile} = rebar_app_utils:is_app_dir(AppDir), - case rebar_app_utils:is_skipped_app(Config, AppFile) of - {_Config, {true, _SkippedApp}} -> - true; - _ -> - false + case rebar_app_utils:is_app_dir(AppDir) of + false -> + false; + {true, AppFile} -> + case rebar_app_utils:is_skipped_app(Config, AppFile) of + {_Config, {true, _SkippedApp}} -> + true; + _ -> + false + end end. %% Recursively walk the deps and build a list of them. -- cgit v1.1 From e74de95eebecae56b9278e9fda3941410678fedd Mon Sep 17 00:00:00 2001 From: Andrew Thompson Date: Mon, 30 Sep 2013 16:57:50 -0400 Subject: Address review comments and add inttest for update-deps --- src/rebar_core.erl | 4 ++-- src/rebar_deps.erl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rebar_core.erl b/src/rebar_core.erl index 42e106e..4d50f4f 100644 --- a/src/rebar_core.erl +++ b/src/rebar_core.erl @@ -168,7 +168,7 @@ skip_or_process_dir1(AppFile, ModuleSet, Config, CurrentCodePath, %% way to signal this to rebar_core, so we have to explicitly do it %% here... Otherwise if you use app=, it'll skip the toplevel %% directory and nothing will be updated. - process_dir1(Dir, Command, DirSet, Config1, + process_dir1(Dir, Command, DirSet, Config1, CurrentCodePath, ModuleSet); {Config1, {true, SkippedApp}} -> ?DEBUG("Skipping app: ~p~n", [SkippedApp]), @@ -181,7 +181,7 @@ skip_or_process_dir1(AppFile, ModuleSet, Config, CurrentCodePath, process_dir1(Dir, Command, DirSet, Config, CurrentCodePath, {DirModules, ModuleSetFile}) -> - Config0 = rebar_config:set(Config, command, Command), + Config0 = rebar_config:set(Config, current_command, Command), %% Get the list of modules for "any dir". This is a catch-all list %% of modules that are processed in addition to modules associated %% with this directory type. These any_dir modules are processed diff --git a/src/rebar_deps.erl b/src/rebar_deps.erl index 6450715..5e4f482 100644 --- a/src/rebar_deps.erl +++ b/src/rebar_deps.erl @@ -72,7 +72,7 @@ preprocess(Config, _) -> %% deps-related can be executed on their directories. NonRawAvailableDeps = [D || D <- AvailableDeps, not D#dep.is_raw], - case rebar_config:get(Config, command, undefined) of + case rebar_config:get(Config, current_command, undefined) of 'update-deps' -> %% Skip ALL of the dep folders, we do this because we don't want %% any other calls to preprocess() for update-deps beyond the -- cgit v1.1