implement --dump-unresolved-methods option

Author: phreppo

shim/shim.py

@@ -585,6 +585,11 @@ def _add_debug_arguments(parser: argparse.ArgumentParser) -> None:
         action="store_true",
         help="Dump the call graph in `call_graph.json`.",
     )
+    debug_arguments.add_argument(
+        "--dump-unresolved-methods",
+        action="store_true",
+        help="Dump the list of unresolved methods in `unresolved_methods.json`.",
+    )
     debug_arguments.add_argument(
         "--dump-dependencies",
         action="store_true",
@@ -737,6 +742,8 @@ def _get_command_options(
         options.append("--dump-overrides")
     if arguments.dump_call_graph:
         options.append("--dump-call-graph")
+    if arguments.dump_unresolved_methods:
+        options.append("--dump-unresolved-methods")
     if arguments.dump_dependencies:
         options.append("--dump-dependencies")
     if arguments.dump_methods:

source/BackwardTaintTransfer.cpp

@@ -733,7 +733,8 @@ bool BackwardTaintTransfer::analyze_invoke(
       aliasing.position(),
       get_source_register_types(context, instruction),
       source_constant_arguments,
-      get_is_this_call(aliasing.register_memory_locations_map(), instruction));
+      get_is_this_call(aliasing.register_memory_locations_map(), instruction),
+      context->statistics);
 
   TaintTree result_taint = TaintTree::bottom();
   if (callee.resolved_base_method &&

source/ForwardAliasTransfer.cpp

@@ -135,7 +135,8 @@ MemoryLocationsDomain invoke_result_memory_location(
       environment->last_position(),
       get_source_register_types(context, instruction),
       get_source_constant_arguments(register_memory_locations_map, instruction),
-      get_is_this_call(register_memory_locations_map, instruction));
+      get_is_this_call(register_memory_locations_map, instruction),
+      context->statistics);
 
   if (callee.resolved_base_method &&
       callee.resolved_base_method->returns_void()) {

source/ForwardTaintTransfer.cpp

@@ -910,7 +910,8 @@ bool ForwardTaintTransfer::analyze_invoke(
       aliasing.position(),
       get_source_register_types(context, instruction),
       source_constant_arguments,
-      get_is_this_call(aliasing.register_memory_locations_map(), instruction));
+      get_is_this_call(aliasing.register_memory_locations_map(), instruction),
+      context->statistics);
 
   const ForwardTaintEnvironment previous_environment = *environment;
 

source/MarianaTrench.cpp

@@ -484,6 +484,18 @@ void MarianaTrench::run(const program_options::variables_map& variables) {
         rule_coverage_timer.duration_in_seconds());
   }
 
+  if (options.dump_unresolved_methods()) {
+    Timer dump_unresolved_methods_timer;
+    auto unresolved_methods_output_path = options.unresolved_methods_output_path();
+    LOG(1, "Writing unresolved methods to `{}`.", unresolved_methods_output_path.native());
+    registry.dump_unresolved_methods(unresolved_methods_output_path);
+    context.statistics->log_time(
+        "dump_unresolved_methods", dump_unresolved_methods_timer);
+    LOG(1,
+        "Wrote unresolved methods list in {:.2f}s.",
+        dump_unresolved_methods_timer.duration_in_seconds());
+  }
+
   auto metadata_path = options.metadata_output_path();
   LOG(1, "Writing metadata to `{}`.", metadata_path.native());
   registry.dump_metadata(/* path */ metadata_path);

source/MethodContext.cpp

@@ -44,6 +44,7 @@ MethodContext::MethodContext(
       memory_factory(previous_model.method()),
       previous_model(previous_model),
       new_model(new_model),
+      statistics(*context.statistics),
       context_(context),
       dump_(previous_model.method()->should_be_logged(options)) {}
 

source/MethodContext.h

@@ -88,6 +88,7 @@ class MethodContext final {
   FulfilledPartialKindResults fulfilled_partial_sinks;
   const Model& previous_model;
   Model& new_model;
+  Statistics& statistics;
 
  private:
   struct CacheKey {

source/Options.cpp

@@ -140,6 +140,7 @@ Options::Options(
       dump_class_intervals_(false),
       dump_overrides_(false),
       dump_call_graph_(false),
+      dump_unresolved_methods_(false),
       dump_dependencies_(false),
       dump_methods_(false),
       dump_coverage_info_(false),
@@ -277,6 +278,7 @@ Options::Options(const boost::program_options::variables_map& variables) {
   dump_class_intervals_ = variables.count("dump-class-intervals") > 0;
   dump_overrides_ = variables.count("dump-overrides") > 0;
   dump_call_graph_ = variables.count("dump-call-graph") > 0;
+  dump_unresolved_methods_ = variables.count("dump-unresolved-methods") > 0;
   dump_dependencies_ = variables.count("dump-dependencies") > 0;
   dump_methods_ = variables.count("dump-methods") > 0;
   dump_coverage_info_ = variables.count("dump-coverage-info") > 0;
@@ -444,6 +446,8 @@ void Options::add_options(
       "dump-overrides", "Dump the override graph in `overrides.json`.");
   options.add_options()(
       "dump-call-graph", "Dump the call graph in `call_graph.json`.");
+  options.add_options()(
+      "dump-unresolved-methods", "Dump the list of unresolved methods in `unresolved_methods.json`.");
   options.add_options()(
       "dump-dependencies", "Dump the dependency graph in `dependencies.json`.");
   options.add_options()(
@@ -579,6 +583,10 @@ const std::filesystem::path Options::call_graph_output_path() const {
   return output_directory_;
 }
 
+const std::filesystem::path Options::unresolved_methods_output_path() const {
+  return output_directory_ / "unresolved_methods.json";
+}
+
 const std::filesystem::path Options::class_hierarchies_output_path() const {
   return output_directory_ / "class_hierarchies.json";
 }
@@ -688,6 +696,10 @@ bool Options::dump_call_graph() const {
   return dump_call_graph_;
 }
 
+bool Options::dump_unresolved_methods() const {
+  return dump_unresolved_methods_;
+}
+
 bool Options::dump_dependencies() const {
   return dump_dependencies_;
 }

source/Options.h

@@ -85,6 +85,7 @@ class Options final {
   const std::filesystem::path models_output_path() const;
   const std::filesystem::path methods_output_path() const;
   const std::filesystem::path call_graph_output_path() const;
+  const std::filesystem::path unresolved_methods_output_path() const;
   const std::filesystem::path class_hierarchies_output_path() const;
   const std::filesystem::path class_intervals_output_path() const;
   const std::filesystem::path overrides_output_path() const;
@@ -115,6 +116,7 @@ class Options final {
   bool dump_class_intervals() const;
   bool dump_overrides() const;
   bool dump_call_graph() const;
+  bool dump_unresolved_methods() const;
   bool dump_dependencies() const;
   bool dump_methods() const;
   bool dump_coverage_info() const;
@@ -176,6 +178,7 @@ class Options final {
   bool dump_class_intervals_;
   bool dump_overrides_;
   bool dump_call_graph_;
+  bool dump_unresolved_methods_;
   bool dump_dependencies_;
   bool dump_methods_;
   bool dump_coverage_info_;

source/Registry.cpp

@@ -475,4 +475,12 @@ void Registry::dump_rule_coverage_info(
   JsonWriter::write_json_file(output_path, rule_coverage.to_json());
 }
 
+void Registry::dump_unresolved_methods(const std::filesystem::path& output_path) const {
+  auto unresolved_methods_value = Json::Value(Json::arrayValue);
+  for (const auto& unresolved_method_name : context_.statistics->unresolved_methods()) {
+    unresolved_methods_value.append(Json::Value(show(unresolved_method_name)));
+  }
+  JsonWriter::write_json_file(output_path, unresolved_methods_value);
+}
+
 } // namespace marianatrench

source/Registry.h

@@ -104,6 +104,7 @@ class Registry final {
 
   void dump_file_coverage_info(const std::filesystem::path& path) const;
   void dump_rule_coverage_info(const std::filesystem::path& path) const;
+  void dump_unresolved_methods(const std::filesystem::path& path) const;
 
   std::string dump_models() const;
   Json::Value models_to_json() const;

source/Statistics.cpp

@@ -62,6 +62,15 @@ void Statistics::log_time(const Method* method, const Timer& timer) {
       record);
 }
 
+void Statistics::log_unable_to_resolve_call(const DexMethodRef* method) {
+  std::lock_guard<std::mutex> lock(mutex_);
+  unresolved_methods_.insert(method);
+}
+
+const std::unordered_set<const DexMethodRef*>& Statistics::unresolved_methods() const {
+  return unresolved_methods_;
+}
+
 namespace {
 
 double round(double x, int digits) {
@@ -77,6 +86,8 @@ Json::Value Statistics::to_json() const {
       Json::Value(static_cast<Json::UInt64>(number_iterations_));
   value["rss"] = Json::Value(round(max_resident_set_size_, 6));
   value["cores"] = Json::Value(sparta::parallel::default_num_threads());
+  value["unresolved_methods"] =
+    Json::Value(static_cast<Json::UInt64>(unresolved_methods_.size()));
 
   auto times_value = Json::Value(Json::objectValue);
   for (const auto& record : times_) {

source/Statistics.h

@@ -32,12 +32,15 @@ class Statistics final {
   void log_resident_set_size(double resident_set_size);
   void log_time(const std::string& name, const Timer& timer);
   void log_time(const Method* method, const Timer& timer);
+  void log_unable_to_resolve_call(const DexMethodRef* method);
 
   Json::Value to_json() const;
 
   /* Maximum number of slowest methods to record. */
   constexpr static std::size_t kRecordSlowestMethods = 20;
 
+  const std::unordered_set<const DexMethodRef*>& unresolved_methods() const;
+
  private:
   std::mutex mutex_;
 
@@ -52,6 +55,10 @@ class Statistics final {
 
   //  Sorted list of slowest methods to analyze (from slowest to fastest).
   std::vector<std::pair<const Method*, double>> slowest_methods_;
+
+  // Set of methods that could not be resolved.
+  std::unordered_set<const DexMethodRef*> unresolved_methods_ =
+      std::unordered_set<const DexMethodRef*>();
 };
 
 } // namespace marianatrench

source/TransferCall.cpp

@@ -12,6 +12,7 @@
 #include <mariana-trench/Log.h>
 #include <mariana-trench/Positions.h>
 #include <mariana-trench/TransferCall.h>
+#include <mariana-trench/Statistics.h>
 
 namespace marianatrench {
 
@@ -149,11 +150,13 @@ CalleeModel get_callee(
     const DexPosition* MT_NULLABLE dex_position,
     const std::vector<const DexType * MT_NULLABLE>& source_register_types,
     const std::vector<std::optional<std::string>>& source_constant_arguments,
-    bool is_this_call) {
+    bool is_this_call,
+    Statistics& statistics) {
   mt_assert(opcode::is_an_invoke(instruction->opcode()));
 
   auto call_target = context->call_graph.callee(context->method(), instruction);
   if (!call_target.resolved()) {
+    statistics.log_unable_to_resolve_call(instruction->get_method());
     WARNING_OR_DUMP(
         context,
         3,

source/TransferCall.h

@@ -55,7 +55,8 @@ CalleeModel get_callee(
     const DexPosition* MT_NULLABLE position,
     const std::vector<const DexType * MT_NULLABLE>& source_register_types,
     const std::vector<std::optional<std::string>>& source_constant_arguments,
-    bool is_this_call);
+    bool is_this_call,
+    Statistics& statistics);
 
 CalleeModel get_callee(
     const MethodContext* context,