Add support to avoid header expansion

Previously, clang-extract only properly supported flags to expand
headers. Now the new flag:
```
-DCE_NOT_EXPAND_INCLUDES=header1.h,...,headern.h
```
can be used to instruct it to not expand certain headers.  To enable it,
the expansion must go through the -DCE_KEEP_INCLUDES engine, which
means it is only activated if a certain policy is enabled.

Furthermore, this commit also adds a new program: `ce-includetree`
which shows in stdout the IncludeTree of the input source file.
it can be invoked using the compilation flags of the original file.

Signed-off-by: Giuliano Belinassi <gbelinassi@suse.de>

Author: giulianobelinassi

README.md

@@ -205,6 +205,7 @@ Clang-extract support many options which controls the output code:
 - `-DCE_KEEP_INCLUDES`            Keep all possible `#include<file>` directives.
 - `-DCE_KEEP_INCLUDES=<policy>`   Keep all possible `#include<file>` directives, but using the specified include expansion <policy>.  Valid values are `nothing`, `everything`, `kernel`, `system` and `compiler`.
 - `-DCE_EXPAND_INCLUDES=<args>`   Force expansion of the headers provided in <args>.
+- `-DCE_NOT_EXPAND_INCLUDES=<args>` Force the following headers to **not** be expanded.
 - `-DCE_RENAME_SYMBOLS`           Allow renaming of extracted symbols.
 - `-DCE_DEBUGINFO_PATH=<arg>`     Path to the compiled (ELF) object of the desired program to extract.  This is used to decide if externalization is necessary or not for given symbol.
 - `-DCE_IPACLONES_PATH=<arg>`     Path to gcc .ipa-clones files generated by gcc.  Used to decide if desired function to extract was inlined into other functions.

TreeDump.cpp

@@ -0,0 +1,77 @@
+//===- TreeDump.cpp - Dump IncludeTree to stdout               --*- C++ -*-===//
+//
+// This project is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// Entrypoint of the IncludeTree dump tool.
+//
+//===----------------------------------------------------------------------===//
+
+/* Author: Giuliano Belinassi  */
+
+#include "ArgvParser.hh"
+#include "Passes.hh"
+#include "Error.hh"
+
+#include <iostream>
+#include <stdio.h>
+
+using namespace llvm;
+using namespace clang;
+
+void print_usage_message(void)
+{
+  llvm::outs() <<
+"OVERVIEW: Tool to show the tree of includes of a C/C++ file.\n"
+"          It should be invoked as a C/C++ compiler.\n"
+"\n"
+"USAGE: ce-includetree [options] file...\n"
+"\n"
+"CLANG-EXTRACT OPTIONS:\n"
+"   <clang-switch>          A clang switch, as specified by calling clang --help.\n"
+"  -D__KERNEL__             Indicate that we are processing a Linux sourcefile.\n"
+"  -DCE_KEEP_INCLUDES       Keep all possible #include<file> directives.\n"
+"  -DCE_KEEP_INCLUDES=<policy>\n"
+"                           Keep all possible #include<file> directives, but using the\n"
+"                           specified include expansion <policy>.  Valid values are\n"
+"                           nothing, everything, kernel, system and compiler.\n"
+"  -DCE_EXPAND_INCLUDES=<args>\n"
+"                           Force expansion of the headers provided in <args>.\n"
+"  -DCE_NOT_EXPAND_INCLUDES=<args>\n"
+"                           Force the following headers to NOT be expanded.\n"
+"\n";
+}
+
+int main(int argc, char **argv)
+{
+  ArgvParser args(argc, argv);
+  PassManager::Context ctx(args);
+
+  if (argc <= 1) {
+    print_usage_message();
+    return 0;
+  }
+
+  /* Cleverly silence any error from clang.  */
+  fclose(stderr);
+  if (Build_ASTUnit(&ctx) == false){
+    llvm::outs() << "Unable to create ASTUnit of " << ctx.InputPath << '\n';
+    return 1;
+  }
+
+  const DiagnosticsEngine &de = ctx.AST->getDiagnostics();
+  if (de.hasErrorOccurred()) {
+    llvm::outs() << "ASTUnit of " << ctx.InputPath << " contain errors. Aborting.\n";
+    return 1;
+  }
+
+  IncludeTree IT(ctx.AST.get(), ctx.IncExpansionPolicy, ctx.HeadersToExpand,
+                 ctx.HeadersToNotExpand);
+  IT.Dump(llvm::outs());
+
+  return 0;
+}

libcextract/ArgvParser.cpp

@@ -61,6 +61,7 @@ ArgvParser::ArgvParser(int argc, char **argv)
     FunctionsToExtract(),
     SymbolsToExternalize(),
     HeadersToExpand(),
+    HeadersToNotExpand(),
     OutputFile(),
     IgnoreClangErrors(false),
     DisableExternalization(false),
@@ -156,6 +157,8 @@ void ArgvParser::Print_Usage_Message(void)
 "                           nothing, everything, kernel, system and compiler.\n"
 "  -DCE_EXPAND_INCLUDES=<args>\n"
 "                           Force expansion of the headers provided in <args>.\n"
+"  -DCE_NOT_EXPAND_INCLUDES=<args>\n"
+"                           Force the following headers to NOT be expanded.\n"
 "  -DCE_RENAME_SYMBOLS      Allow renaming of extracted symbols.\n"
 "  -DCE_DEBUGINFO_PATH=<arg>\n"
 "                           Path to the compiled (ELF) object of the desired program to\n"
@@ -255,6 +258,11 @@ bool ArgvParser::Handle_Clang_Extract_Arg(const char *str)
 
     return true;
   }
+  if (prefix("-DCE_NOT_EXPAND_INCLUDES=", str)) {
+    HeadersToNotExpand = Extract_Args(str);
+
+    return true;
+  }
   if (prefix("-DCE_DEBUGINFO_PATH=", str)) {
     Debuginfos = Extract_Args(str);
 

libcextract/ArgvParser.hh

@@ -59,6 +59,11 @@ class ArgvParser
     return HeadersToExpand;
   }
 
+  inline std::vector<std::string>& Get_Headers_To_Not_Expand(void)
+  {
+    return HeadersToNotExpand;
+  }
+
   inline std::string &Get_Output_File(void)
   {
     return OutputFile;
@@ -139,8 +144,6 @@ class ArgvParser
     return IgnoreClangErrors;
   }
 
-  const char *Get_Input_File(void);
-
   /** Print help usage message.  */
   void Print_Usage_Message(void);
 
@@ -153,6 +156,7 @@ class ArgvParser
   std::vector<std::string> FunctionsToExtract;
   std::vector<std::string> SymbolsToExternalize;
   std::vector<std::string> HeadersToExpand;
+  std::vector<std::string> HeadersToNotExpand;
   std::string OutputFile;
 
   bool IgnoreClangErrors;

libcextract/Error.cpp

@@ -28,7 +28,7 @@ DiagsClass DiagsClass::sDiag;
 DiagsClass::DiagsClass(void)
   : LangOpts(),
     DOpts(DiagnosticOptionsWithColor()),
-    DiagsEngine(llvm::outs(), LangOpts, DOpts.getDiagsOptsToEngine())
+    DiagsEngine(llvm::errs(), LangOpts, DOpts.getDiagsOptsToEngine())
 {}
 
 /* Print error giving a piece of source code that caused the error.  */
@@ -46,6 +46,6 @@ void DiagsClass::EmitMessage(const StringRef message, DiagnosticsEngine::Level l
 {
   bool colored = Is_Colored();
   const std::string ce_message = Append_CE(message);
-  TextDiagnostic::printDiagnosticLevel(llvm::outs(), level, colored);
-  TextDiagnostic::printDiagnosticMessage(llvm::outs(), false, ce_message, 0, 0, colored);
+  TextDiagnostic::printDiagnosticLevel(llvm::errs(), level, colored);
+  TextDiagnostic::printDiagnosticMessage(llvm::errs(), false, ce_message, 0, 0, colored);
 }

libcextract/ExpansionPolicy.cpp

@@ -21,12 +21,6 @@
 bool KernelExpansionPolicy::Must_Expand(const StringRef &absolute_path,
                                         const StringRef &relative_path)
 {
-  // absolute_path and relative_path are different when extracting code from a
-  // compiled kernel directory, so we need to check whether the include paths
-  // start from include paths.
-  if (absolute_path != relative_path)
-    return !relative_path.starts_with("./include");
-
   std::vector<std::string> include_paths = { "/include/", "/arch/" };
 
   for (auto &path : include_paths) {
@@ -75,6 +69,12 @@ bool CompilerExpansionPolicy::Must_Expand(const StringRef &absolute_path,
   return true;
 }
 
+bool CompilerExpansionPolicy::Must_Not_Expand(const StringRef &absolute_path,
+                                              const StringRef &relative_path)
+{
+  return !Must_Expand(absolute_path, relative_path);
+}
+
 std::unique_ptr<IncludeExpansionPolicy> IncludeExpansionPolicy::Get_Expansion_Policy_Unique(
                                                         IncludeExpansionPolicy::Policy p)
 {
@@ -134,3 +134,22 @@ IncludeExpansionPolicy::Policy IncludeExpansionPolicy::Get_From_String(const cha
 
   return IncludeExpansionPolicy::NOTHING;
 }
+
+
+/* Return true if headers passed through -include must be expanded.  */
+bool IncludeExpansionPolicy::Expand_Minus_Includes(IncludeExpansionPolicy::Policy policy)
+{
+  switch (policy) {
+    case INVALID:
+    case EVERYTHING:
+    case SYSTEM:
+    case COMPILER:
+      return true;
+      break;
+
+    case NOTHING:
+    case KERNEL:
+      return false;
+      break;
+  }
+}

libcextract/ExpansionPolicy.hh

@@ -25,6 +25,9 @@ class IncludeExpansionPolicy
   public:
   virtual bool Must_Expand(const StringRef &absolute_path, const StringRef &relative_path) = 0;
 
+  virtual bool Must_Not_Expand(const StringRef &absolute_path, const StringRef &relative_path) = 0;
+
+
   virtual ~IncludeExpansionPolicy() = default;
 
   enum Policy {
@@ -36,10 +39,12 @@ class IncludeExpansionPolicy
     COMPILER,
   };
 
-  static IncludeExpansionPolicy *Get_Expansion_Policy(Policy policy);
   static std::unique_ptr<IncludeExpansionPolicy>
                 Get_Expansion_Policy_Unique(Policy policy);
 
+  /* Return if headers passed through -include must be expanded.  */
+  static bool Expand_Minus_Includes(Policy policy);
+
   static Policy Get_From_String(const char *string);
   inline static Policy Get_Overriding(const char *string, bool is_kernel)
   {
@@ -66,6 +71,11 @@ class NoIncludeExpansionPolicy : public IncludeExpansionPolicy
     return false;
   }
 
+  virtual bool Must_Not_Expand(const StringRef &absolute_path, const StringRef &relative_path)
+  {
+    return false;
+  }
+
 };
 
 class ExpandEverythingExpansionPolicy : public IncludeExpansionPolicy
@@ -75,25 +85,41 @@ class ExpandEverythingExpansionPolicy : public IncludeExpansionPolicy
   {
     return true;
   }
+
+  virtual bool Must_Not_Expand(const StringRef &absolute_path, const StringRef &relative_path)
+  {
+    return false;
+  }
 };
 
 /** Expand any header according to kernel livepatching rules.  */
 class KernelExpansionPolicy : public IncludeExpansionPolicy
 {
   public:
   virtual bool Must_Expand(const StringRef &absolute_path, const StringRef &relative_path);
+
+  virtual bool Must_Not_Expand(const StringRef &absolute_path, const StringRef &relative_path)
+  {
+    return false;
+  }
 };
 
 /** Expand any header that is not installed in the system.  */
 class SystemExpansionPolicy : public IncludeExpansionPolicy
 {
   public:
   virtual bool Must_Expand(const StringRef &absolute_path, const StringRef &relative_path);
+
+  virtual bool Must_Not_Expand(const StringRef &absolute_path, const StringRef &relative_path)
+  {
+    return false;
+  }
 };
 
 /** Expand any header that is not compiler-specific.  */
 class CompilerExpansionPolicy : public IncludeExpansionPolicy
 {
   public:
   virtual bool Must_Expand(const StringRef &absolute_path, const StringRef &relative_path);
+  virtual bool Must_Not_Expand(const StringRef &absolute_path, const StringRef &relative_path);
 };

libcextract/FunctionDepsFinder.cpp

@@ -24,7 +24,7 @@
 /** FunctionDependencyFinder class methods implementation.  */
 FunctionDependencyFinder::FunctionDependencyFinder(PassManager::Context *ctx)
     : AST(ctx->AST.get()),
-      IT(AST, ctx->IncExpansionPolicy, ctx->HeadersToExpand),
+      IT(AST, ctx->IncExpansionPolicy, ctx->HeadersToExpand, ctx->HeadersToNotExpand),
       KeepIncludes(ctx->KeepIncludes),
       Visitor(AST)
 {