From 2abb55170ea5408496c79b15a1c714263c66c0ed Mon Sep 17 00:00:00 2001 From: Kelly McLaughlin Date: Wed, 4 Feb 2015 10:05:19 -0700 Subject: Add xref provider for cross reference analysis * Add a provider for the xref tool for running cross reference analysis on a project. Most of the code has been ported directly from the rebar2 rebar_xref module with some modification and cleanup to support testing. * Port over the eunit test suite from rebar2, but convert it to common_test. The testing is the same, but now the erlang term output is examined to determine if the test run is successful instead of scanning the console output for the expected strings. --- README.md | 1 + THANKS | 3 +- src/rebar.app.src | 1 + src/rebar_prv_xref.erl | 296 ++++++++++++++++++++++++++++++++++++++++++++++ test/rebar_xref_SUITE.erl | 190 +++++++++++++++++++++++++++++ 5 files changed, 490 insertions(+), 1 deletion(-) create mode 100644 src/rebar_prv_xref.erl create mode 100644 test/rebar_xref_SUITE.erl diff --git a/README.md b/README.md index 85d95ea..dce1937 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ limit scope. | update | Update package index | | upgrade | Fetch latest version of dep | | version | Print current version of Erlang/OTP and rebar | +| xref | Run cross reference analysis on the project | ### Commands still to do diff --git a/THANKS b/THANKS index aef1e52..8aa2a8a 100644 --- a/THANKS +++ b/THANKS @@ -128,4 +128,5 @@ Alexander Verbitsky Andras Horvath Drew Varner Omar Yasin -Tristan Sloughter \ No newline at end of file +Tristan Sloughter +Kelly McLaughlin \ No newline at end of file diff --git a/src/rebar.app.src b/src/rebar.app.src index 73c1eff..1230436 100644 --- a/src/rebar.app.src +++ b/src/rebar.app.src @@ -43,6 +43,7 @@ rebar_prv_release, rebar_prv_version, rebar_prv_common_test, + rebar_prv_xref, rebar_prv_help]} ]} ]}. diff --git a/src/rebar_prv_xref.erl b/src/rebar_prv_xref.erl new file mode 100644 index 0000000..32ebf46 --- /dev/null +++ b/src/rebar_prv_xref.erl @@ -0,0 +1,296 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et + +-module(rebar_prv_xref). + +-behaviour(provider). + +-export([init/1, + do/1, + format_error/1]). + +-include("rebar.hrl"). + +-define(PROVIDER, xref). +-define(DEPS, [compile]). +-define(SUPPORTED_XREFS, [undefined_function_calls, undefined_functions, + locals_not_used, exports_not_used, + deprecated_function_calls, deprecated_functions]). + +%% =================================================================== +%% Public API +%% =================================================================== + +-spec init(rebar_state:t()) -> {ok, rebar_state:t()}. +init(State) -> + Provider = providers:create([{name, ?PROVIDER}, + {module, ?MODULE}, + {deps, ?DEPS}, + {bare, false}, + {example, "rebar3 xref"}, + {short_desc, short_desc()}, + {desc, desc()}]), + State1 = rebar_state:add_provider(State, Provider), + {ok, State1}. + +-spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. +do(State) -> + {OriginalPath, XrefChecks} = prepare(State), + + %% Run xref checks + ?INFO("Running cross reference analysis...", []), + XrefResults = xref_checks(XrefChecks), + + %% Run custom queries + QueryChecks = rebar_state:get(State, xref_queries, []), + QueryResults = lists:foldl(fun check_query/2, [], QueryChecks), + + ok = cleanup(OriginalPath), + + case XrefResults =:= [] andalso QueryResults =:= [] of + true -> + {ok, State}; + false -> + {error, {?MODULE, {xref_issues, XrefResults, QueryResults}}} + end. + +-spec format_error(any()) -> iolist(). +format_error({xref_issues, XrefResults, QueryResults}) -> + lists:flatten(display_results(XrefResults, QueryResults)); +format_error(Reason) -> + io_lib:format("~p", [Reason]). + +%% =================================================================== +%% Internal functions +%% =================================================================== + +short_desc() -> + "Run cross reference analysis". + +desc() -> + io_lib:format( + "~s~n" + "~n" + "Valid rebar.config options:~n" + " ~p~n" + " ~p~n" + " ~p~n" + " ~p~n", + [short_desc(), + {xref_warnings, false}, + {xref_extra_paths,[]}, + {xref_checks, [undefined_function_calls, undefined_functions, + locals_not_used, exports_not_used, + deprecated_function_calls, deprecated_functions]}, + {xref_queries, + [{"(xc - uc) || (xu - x - b" + " - (\"mod\":\".*foo\"/\"4\"))",[]}]} + ]). + +-spec prepare(rebar_state:t()) -> list(atom()). +prepare(State) -> + {ok, _} = xref:start(xref), + ok = xref:set_library_path(xref, code_path(State)), + + xref:set_default(xref, [{warnings, + rebar_state:get(State, xref_warnings, false)}, + {verbose, rebar_log:is_verbose(State)}]), + + {ok, _} = xref:add_directory(xref, "ebin"), + + %% Save the code path prior to doing any further code path + %% manipulation + OriginalPath = code:get_path(), + true = code:add_path(rebar_dir:ebin_dir()), + + %% Get list of xref checks we want to run + ConfXrefChecks = rebar_state:get(State, xref_checks, + [exports_not_used, + undefined_function_calls]), + + XrefChecks = sets:to_list(sets:intersection( + sets:from_list(?SUPPORTED_XREFS), + sets:from_list(ConfXrefChecks))), + {OriginalPath, XrefChecks}. + +cleanup(Path) -> + %% Restore the code path using the provided path + true = rebar_utils:cleanup_code_path(Path), + + %% Stop xref + stopped = xref:stop(xref), + ok. + +xref_checks(XrefChecks) -> + lists:foldl(fun run_xref_check/2, [], XrefChecks). + +run_xref_check(XrefCheck, Acc) -> + {ok, Results} = xref:analyze(xref, XrefCheck), + case filter_xref_results(XrefCheck, Results) of + [] -> + Acc; + FilterResult -> + [{XrefCheck, FilterResult} | Acc] + end. + +check_query({Query, Value}, Acc) -> + {ok, Answer} = xref:q(xref, Query), + case Answer =:= Value of + false -> + [{Query, Value, Answer} | Acc]; + _ -> + Acc + end. + +code_path(State) -> + [P || P <- code:get_path() ++ + rebar_state:get(State, xref_extra_paths, []), + filelib:is_dir(P)]. + +%% Ignore behaviour functions, and explicitly marked functions +%% +%% Functions can be ignored by using +%% -ignore_xref([{F, A}, {M, F, A}...]). +get_xref_ignorelist(Mod, XrefCheck) -> + %% Get ignore_xref attribute and combine them in one list + Attributes = + try + Mod:module_info(attributes) + catch + _Class:_Error -> [] + end, + + IgnoreXref = keyall(ignore_xref, Attributes), + + BehaviourCallbacks = get_behaviour_callbacks(XrefCheck, Attributes), + + %% And create a flat {M,F,A} list + lists:foldl( + fun({F, A}, Acc) -> [{Mod,F,A} | Acc]; + ({M, F, A}, Acc) -> [{M,F,A} | Acc] + end, [], lists:flatten([IgnoreXref, BehaviourCallbacks])). + +keyall(Key, List) -> + lists:flatmap(fun({K, L}) when Key =:= K -> L; (_) -> [] end, List). + +get_behaviour_callbacks(exports_not_used, Attributes) -> + [B:behaviour_info(callbacks) || B <- keyall(behaviour, Attributes)]; +get_behaviour_callbacks(_XrefCheck, _Attributes) -> + []. + +parse_xref_result({_, MFAt}) -> MFAt; +parse_xref_result(MFAt) -> MFAt. + +filter_xref_results(XrefCheck, XrefResults) -> + SearchModules = lists:usort( + lists:map( + fun({Mt,_Ft,_At}) -> Mt; + ({{Ms,_Fs,_As},{_Mt,_Ft,_At}}) -> Ms; + (_) -> undefined + end, XrefResults)), + + Ignores = lists:flatmap(fun(Module) -> + get_xref_ignorelist(Module, XrefCheck) + end, SearchModules), + + [Result || Result <- XrefResults, + not lists:member(parse_xref_result(Result), Ignores)]. + +display_results(XrefResults, QueryResults) -> + [lists:map(fun display_xref_results_for_type/1, XrefResults), + lists:map(fun display_query_result/1, QueryResults)]. + +display_query_result({Query, Answer, Value}) -> + io_lib:format("Query ~s~n answer ~p~n did not match ~p~n", + [Query, Answer, Value]). + +display_xref_results_for_type({Type, XrefResults}) -> + lists:map(display_xref_result_fun(Type), XrefResults). + +display_xref_result_fun(Type) -> + fun(XrefResult) -> + {Source, SMFA, TMFA} = + case XrefResult of + {MFASource, MFATarget} -> + {format_mfa_source(MFASource), + format_mfa(MFASource), + format_mfa(MFATarget)}; + MFATarget -> + {format_mfa_source(MFATarget), + format_mfa(MFATarget), + undefined} + end, + case Type of + undefined_function_calls -> + io_lib:format("~sWarning: ~s calls undefined function ~s (Xref)\n", + [Source, SMFA, TMFA]); + undefined_functions -> + io_lib:format("~sWarning: ~s is undefined function (Xref)\n", + [Source, SMFA]); + locals_not_used -> + io_lib:format("~sWarning: ~s is unused local function (Xref)\n", + [Source, SMFA]); + exports_not_used -> + io_lib:format("~sWarning: ~s is unused export (Xref)\n", + [Source, SMFA]); + deprecated_function_calls -> + io_lib:format("~sWarning: ~s calls deprecated function ~s (Xref)\n", + [Source, SMFA, TMFA]); + deprecated_functions -> + io_lib:format("~sWarning: ~s is deprecated function (Xref)\n", + [Source, SMFA]); + Other -> + io_lib:format("~sWarning: ~s - ~s xref check: ~s (Xref)\n", + [Source, SMFA, TMFA, Other]) + end + end. + +format_mfa({M, F, A}) -> + ?FMT("~s:~s/~w", [M, F, A]). + +format_mfa_source(MFA) -> + case find_mfa_source(MFA) of + {module_not_found, function_not_found} -> ""; + {Source, function_not_found} -> ?FMT("~s: ", [Source]); + {Source, Line} -> ?FMT("~s:~w: ", [Source, Line]) + end. + +%% +%% Extract an element from a tuple, or undefined if N > tuple size +%% +safe_element(N, Tuple) -> + case catch(element(N, Tuple)) of + {'EXIT', {badarg, _}} -> + undefined; + Value -> + Value + end. + +%% +%% Given a MFA, find the file and LOC where it's defined. Note that +%% xref doesn't work if there is no abstract_code, so we can avoid +%% being too paranoid here. +%% +find_mfa_source({M, F, A}) -> + case code:get_object_code(M) of + error -> {module_not_found, function_not_found}; + {M, Bin, _} -> find_function_source(M,F,A,Bin) + end. + +find_function_source(M, F, A, Bin) -> + AbstractCode = beam_lib:chunks(Bin, [abstract_code]), + {ok, {M, [{abstract_code, {raw_abstract_v1, Code}}]}} = AbstractCode, + %% Extract the original source filename from the abstract code + [{attribute, 1, file, {Source, _}} | _] = Code, + %% Extract the line number for a given function def + Fn = [E || E <- Code, + safe_element(1, E) == function, + safe_element(3, E) == F, + safe_element(4, E) == A], + case Fn of + [{function, Line, F, _, _}] -> {Source, Line}; + %% do not crash if functions are exported, even though they + %% are not in the source. + %% parameterized modules add new/1 and instance/1 for example. + [] -> {Source, function_not_found} + end. diff --git a/test/rebar_xref_SUITE.erl b/test/rebar_xref_SUITE.erl new file mode 100644 index 0000000..fde8c8f --- /dev/null +++ b/test/rebar_xref_SUITE.erl @@ -0,0 +1,190 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +-module(rebar_xref_SUITE). + +-export([suite/0, + init_per_suite/1, + end_per_suite/1, + init_per_testcase/2, + end_per_testcase/2, + all/0, + xref_test/1, + xref_ignore_test/1]). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("kernel/include/file.hrl"). + +%% =================================================================== +%% common_test callbacks +%% =================================================================== + +suite() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_testcase(Case, Config) -> + UpdConfig = rebar_test_utils:init_rebar_state(Config), + AppDir = ?config(apps, UpdConfig), + {ok, OrigDir} = file:get_cwd(), + file:set_cwd(AppDir), + Name = rebar_test_utils:create_random_name("xrefapp_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_empty_app(AppDir, Name, Vsn, [kernel, stdlib]), + AppModules = [behaviour1, behaviour2, mymod, othermod], + [write_src_file(AppDir, Name, Module, ignore_xref(Case)) || Module <- AppModules], + RebarConfig = [{erl_opts, [debug_info]}, + {xref_checks, [deprecated_function_calls,deprecated_functions, + undefined_function_calls,undefined_functions, + exports_not_used,locals_not_used]}], + [{app_name, Name}, + {rebar_config, RebarConfig}, + {orig_dir, OrigDir} | UpdConfig]. + +end_per_testcase(_, Config) -> + ?debugMsg("End test case cleanup"), + AppDir = ?config(apps, Config), + OrigDir = ?config(orig_dir, Config), + %% Code path cleanup because we set the CWD to the `AppDir' prior + %% to launching rebar and these paths make it into the code path + %% before the xref module executes so they don't get cleaned up + %% automatically after the xref run. Only have to do this because + %% we are about to remove the directory and there may be + %% subsequent test cases that error out when the code path tries + %% to include one of these soon-to-be nonexistent directories. + true = code:del_path(AppDir ++ "/."), + true = code:del_path(rebar_dir:ebin_dir()), + file:set_cwd(OrigDir), + ec_file:remove(AppDir, [recursive]), + ok. + +all() -> + [xref_test, xref_ignore_test]. + +%% =================================================================== +%% Test cases +%% =================================================================== + +xref_test(Config) -> + AppDir = ?config(apps, Config), + State = ?config(state, Config), + Name = ?config(app_name, Config), + RebarConfig = ?config(rebar_config, Config), + Result = rebar3:run(rebar_state:new(State, RebarConfig, AppDir), ["xref"]), + verify_results(xref_test, Name, Result). + +xref_ignore_test(Config) -> + AppDir = ?config(apps, Config), + State = ?config(state, Config), + Name = ?config(app_name, Config), + RebarConfig = ?config(rebar_config, Config), + Result = rebar3:run(rebar_state:new(State, RebarConfig, AppDir), ["xref"]), + verify_results(xref_ignore_test, Name, Result). + +%% =================================================================== +%% Helper functions +%% =================================================================== + +ignore_xref(xref_ignore_test) -> + true; +ignore_xref(_) -> + false. + +verify_results(TestCase, AppName, Results) -> + {error, {rebar_prv_xref, + {xref_issues, XrefResults, QueryResults}}} = Results, + verify_test_results(TestCase, AppName, XrefResults, QueryResults). + +verify_test_results(xref_test, AppName, XrefResults, _QueryResults) -> + AppModules = ["behaviour1", "behaviour2", "mymod", "othermod", "somemod"], + [Behaviour1Mod, Behaviour2Mod, MyMod, OtherMod, SomeMod] = + [list_to_atom(AppName ++ "_" ++ Mod) || Mod <- AppModules], + UndefFuns = proplists:get_value(undefined_functions, XrefResults), + UndefFunCalls = proplists:get_value(undefined_function_calls, XrefResults), + LocalsNotUsed = proplists:get_value(locals_not_used, XrefResults), + ExportsNotUsed = proplists:get_value(exports_not_used, XrefResults), + DeprecatedFuns = proplists:get_value(deprecated_functions, XrefResults), + DeprecatedFunCalls = proplists:get_value(deprecated_function_calls, XrefResults), + ?assert(lists:member({SomeMod, notavailable, 1}, UndefFuns)), + ?assert(lists:member({{OtherMod, somefunc, 0}, {SomeMod, notavailable, 1}}, + UndefFunCalls)), + ?assert(lists:member({MyMod, fdeprecated, 0}, DeprecatedFuns)), + ?assert(lists:member({{OtherMod, somefunc, 0}, {MyMod, fdeprecated, 0}}, + DeprecatedFunCalls)), + ?assert(lists:member({MyMod, localfunc2, 0}, LocalsNotUsed)), + ?assert(lists:member({Behaviour1Mod, behaviour_info, 1}, ExportsNotUsed)), + ?assert(lists:member({Behaviour2Mod, behaviour_info, 1}, ExportsNotUsed)), + ?assert(lists:member({MyMod, other2, 1}, ExportsNotUsed)), + ?assert(lists:member({OtherMod, somefunc, 0}, ExportsNotUsed)), + ?assertNot(lists:member({MyMod, bh1_a, 1}, ExportsNotUsed)), + ?assertNot(lists:member({MyMod, bh1_b, 1}, ExportsNotUsed)), + ?assertNot(lists:member({MyMod, bh2_a, 1}, ExportsNotUsed)), + ?assertNot(lists:member({MyMod, bh2_b, 1}, ExportsNotUsed)), + ok; +verify_test_results(xref_ignore_test, AppName, XrefResults, _QueryResults) -> + AppModules = ["behaviour1", "behaviour2", "mymod", "othermod", "somemod"], + [Behaviour1Mod, Behaviour2Mod, MyMod, OtherMod, SomeMod] = + [list_to_atom(AppName ++ "_" ++ Mod) || Mod <- AppModules], + UndefFuns = proplists:get_value(undefined_functions, XrefResults), + ?assertNot(lists:keymember(undefined_function_calls, 1, XrefResults)), + ?assertNot(lists:keymember(locals_not_used, 1, XrefResults)), + ?assertNot(lists:keymember(exports_not_used, 1, XrefResults)), + ?assertNot(lists:keymember(deprecated_functions, 1, XrefResults)), + ?assertNot(lists:keymember(deprecated_function_calls, 1, XrefResults)), + ?assert(lists:member({SomeMod, notavailable, 1}, UndefFuns)), + ok. + +write_src_file(Dir, AppName, Module, IgnoreXref) -> + Erl = filename:join([Dir, "src", module_name(AppName, Module)]), + ok = filelib:ensure_dir(Erl), + ok = ec_file:write(Erl, get_module_body(Module, AppName, IgnoreXref)). + +module_name(AppName, Module) -> + lists:flatten([AppName, "_", atom_to_list(Module), ".erl"]). + +get_module_body(behaviour1, AppName, IgnoreXref) -> + ["-module(", AppName, "_behaviour1).\n", + "-export([behaviour_info/1]).\n", + ["-ignore_xref({behaviour_info,1}).\n" || X <- [IgnoreXref], X =:= true], + "behaviour_info(callbacks) -> [{bh1_a,1},{bh1_b,1}];\n", + "behaviour_info(_Other) -> undefined.\n"]; +get_module_body(behaviour2, AppName, IgnoreXref) -> + ["-module(", AppName, "_behaviour2).\n", + "-export([behaviour_info/1]).\n", + ["-ignore_xref({behaviour_info,1}).\n" || X <- [IgnoreXref], X =:= true], + "behaviour_info(callbacks) -> [{bh2_a,1},{bh2_b,1}];\n", + "behaviour_info(_Other) -> undefined.\n"]; +get_module_body(mymod, AppName, IgnoreXref) -> + ["-module(", AppName, "_mymod).\n", + "-export([bh1_a/1,bh1_b/1,bh2_a/1,bh2_b/1," + "other1/1,other2/1,fdeprecated/0]).\n", + ["-ignore_xref([{other2,1},{localfunc2,0},{fdeprecated,0}]).\n" + || X <- [IgnoreXref], X =:= true], + "-behaviour(", AppName, "_behaviour1).\n", % 2 behaviours + "-behaviour(", AppName, "_behaviour2).\n", + "-deprecated({fdeprecated,0}).\n", % deprecated function + "bh1_a(A) -> localfunc1(bh1_a, A).\n", % behaviour functions + "bh1_b(A) -> localfunc1(bh1_b, A).\n", + "bh2_a(A) -> localfunc1(bh2_a, A).\n", + "bh2_b(A) -> localfunc1(bh2_b, A).\n", + "other1(A) -> localfunc1(other1, A).\n", % regular exported functions + "other2(A) -> localfunc1(other2, A).\n", + "localfunc1(A, B) -> {A, B}.\n", % used local + "localfunc2() -> ok.\n", % unused local + "fdeprecated() -> ok.\n" % deprecated function + ]; +get_module_body(othermod, AppName, IgnoreXref) -> + ["-module(", AppName, "_othermod).\n", + "-export([somefunc/0]).\n", + [["-ignore_xref([{", AppName, "_somemod,notavailable,1},{somefunc,0}]).\n", + "-ignore_xref({", AppName, "_mymod,fdeprecated,0}).\n"] + || X <- [IgnoreXref], X =:= true], + "somefunc() ->\n", + " ", AppName, "_mymod:other1(arg),\n", + " ", AppName, "_somemod:notavailable(arg),\n", + " ", AppName, "_mymod:fdeprecated().\n"]. -- cgit v1.1