Add new expansion policies.

This commit add two new expansion policies:

- system: keep includes from system headers, such as stdio.h,
  stdlib.h, and other libraries that may be installed
  in the system.

- compiler: keep compiler-specific includes, such as stdatomic.h,
  stddef.h, and other includes that may expand into compiler-specific
  code.

Use `-DCE_KEEP_INCLUDES=<policy>` to use them.

Signed-off-by: Giuliano Belinassi <gbelinassi@suse.de>

Author: giulianobelinassi

README.md

@@ -93,6 +93,17 @@ Notice how *any reference to unused_function is removed* and *all headers has be
 $ clang out.c -O2 -o a
 ```
 If you desire to keep the includes, see `-DCE_KEEP_INCLUDES` options and the _Supported options_ chapter.
+The following expansion policies are supported:
+ - `nothing`: Do not expand any header.
+ - `everything`: Expand all headers.  Has the same semantic effect of not passing `-DCE_KEEP_INCLUDES`, but forces clang-extract to pass it through its header expansion logics (slow and very likely buggy!).
+ - `kernel`: Special policy used by the kernel livepatching developers.
+ - `system`: Keep all system headers installed in `/usr/include`, etc.
+ - `compiler`: Keep all compiler-specific headers, such as `stdatomic.h`. Useful if you want to expand everything but still want to ensure compatibility with other compilers.
+
+You may want to use `clang-tidy` to cleanup the generated file afterwards to remove duplicated includes:
+```
+$ clang-tidy -checks='-*,readability-duplicate-include,misc-include-cleaner' -fix <out.c>
+```
 
 ### Symbol Externalization
 
@@ -192,7 +203,7 @@ Clang-extract support many options which controls the output code:
 - `-DCE_NO_EXTERNALIZATION`       Disable symbol externalization.
 - `-DCE_DUMP_PASSES`              Dump the results of each transformation pass into files. Files will be dumped at the same path of the input files. Additional files are also generated on `/tmp/` folder.
 - `-DCE_KEEP_INCLUDES`            Keep all possible `#include<file>` directives.
-- `-DCE_KEEP_INCLUDES=<policy>`   Keep all possible `#include<file>` directives, but using the specified include expansion <policy>.  Valid values are nothing, everything and kernel.
+- `-DCE_KEEP_INCLUDES=<policy>`   Keep all possible `#include<file>` directives, but using the specified include expansion <policy>.  Valid values are `nothing`, `everything`, `kernel`, `system` and `compiler`.
 - `-DCE_EXPAND_INCLUDES=<args>`   Force expansion of the headers provided in <args>.
 - `-DCE_RENAME_SYMBOLS`           Allow renaming of extracted symbols.
 - `-DCE_DEBUGINFO_PATH=<arg>`     Path to the compiled (ELF) object of the desired program to extract.  This is used to decide if externalization is necessary or not for given symbol.

libcextract/ArgvParser.cpp

@@ -157,7 +157,7 @@ void ArgvParser::Print_Usage_Message(void)
 "  -DCE_KEEP_INCLUDES=<policy>\n"
 "                           Keep all possible #include<file> directives, but using the\n"
 "                           specified include expansion <policy>.  Valid values are\n"
-"                           nothing, everything and kernel.\n"
+"                           nothing, everything, kernel, system and compiler.\n"
 "  -DCE_EXPAND_INCLUDES=<args>\n"
 "                           Force expansion of the headers provided in <args>.\n"
 "  -DCE_RENAME_SYMBOLS      Allow renaming of extracted symbols.\n"

libcextract/ExpansionPolicy.cpp

@@ -39,25 +39,40 @@ bool KernelExpansionPolicy::Must_Expand(const StringRef &absolute_path,
   return true;
 }
 
-IncludeExpansionPolicy *IncludeExpansionPolicy::Get_Expansion_Policy(
-                                                IncludeExpansionPolicy::Policy p)
+bool SystemExpansionPolicy::Must_Expand(const StringRef &absolute_path,
+                                        const StringRef &relative_path)
 {
-  switch (p) {
-    case Policy::NOTHING:
-      return new NoIncludeExpansionPolicy();
-      break;
-
-    case Policy::EVERYTHING:
-      return new ExpandEverythingExpansionPolicy();
-      break;
+  const char *absolute_path_c = absolute_path.data();
+
+  /* Look for system headers by looking to specific prefixes.  */
+  const char *include_paths[] = { "/usr/include/", "/usr/lib64/",
+                                  "/usr/lib/", "/usr/local/include/", };
+  for (unsigned i = 0; i < ARRAY_LENGTH(include_paths); i++) {
+    if (prefix(include_paths[i], absolute_path_c)) {
+      return false; // Do not expand.
+    }
+  }
 
-    case Policy::KERNEL:
-      return new KernelExpansionPolicy();
-      break;
+  /* Expand anything that doesn't match this.  */
+  return true;
+}
 
-    default:
-      assert(false && "Invalid policy");
+bool CompilerExpansionPolicy::Must_Expand(const StringRef &absolute_path,
+                                          const StringRef &relative_path)
+{
+  const char *absolute_path_c = absolute_path.data();
+
+  /* Look for clang compiler headers by looking to specific prefixes.  */
+  const char *include_paths[] = { "/usr/lib64/clang/", "/usr/lib/clang/",
+                                  "/usr/local/lib64/clang/", "/usr/local/lib64/clang/", };
+  for (unsigned i = 0; i < ARRAY_LENGTH(include_paths); i++) {
+    if (prefix(include_paths[i], absolute_path_c)) {
+      return false; // Do not expand.
+    }
   }
+
+  /* Expand anything that doesn't match this.  */
+  return true;
 }
 
 std::unique_ptr<IncludeExpansionPolicy> IncludeExpansionPolicy::Get_Expansion_Policy_Unique(
@@ -79,6 +94,16 @@ std::unique_ptr<IncludeExpansionPolicy> IncludeExpansionPolicy::Get_Expansion_Po
                 KernelExpansionPolicy());
       break;
 
+    case Policy::SYSTEM:
+      return std::make_unique<SystemExpansionPolicy>(
+                SystemExpansionPolicy());
+      break;
+
+    case Policy::COMPILER:
+      return std::make_unique<CompilerExpansionPolicy>(
+                CompilerExpansionPolicy());
+      break;
+
     default:
       assert(false && "Invalid policy");
   }
@@ -96,7 +121,9 @@ IncludeExpansionPolicy::Policy IncludeExpansionPolicy::Get_From_String(const cha
   } policies[] = {
     { "nothing",    IncludeExpansionPolicy::NOTHING    },
     { "everything", IncludeExpansionPolicy::EVERYTHING },
-    { "kernel",     IncludeExpansionPolicy::KERNEL     }
+    { "kernel",     IncludeExpansionPolicy::KERNEL     },
+    { "system",     IncludeExpansionPolicy::SYSTEM     },
+    { "compiler",   IncludeExpansionPolicy::COMPILER   },
   };
 
   for (unsigned long i = 0; i < ARRAY_LENGTH(policies); i++) {

libcextract/ExpansionPolicy.hh

@@ -32,6 +32,8 @@ class IncludeExpansionPolicy
     NOTHING,
     EVERYTHING,
     KERNEL,
+    SYSTEM,
+    COMPILER,
   };
 
   static IncludeExpansionPolicy *Get_Expansion_Policy(Policy policy);
@@ -75,8 +77,23 @@ class ExpandEverythingExpansionPolicy : public IncludeExpansionPolicy
   }
 };
 
+/** Expand any header according to kernel livepatching rules.  */
 class KernelExpansionPolicy : public IncludeExpansionPolicy
 {
   public:
   virtual bool Must_Expand(const StringRef &absolute_path, const StringRef &relative_path);
 };
+
+/** Expand any header that is not installed in the system.  */
+class SystemExpansionPolicy : public IncludeExpansionPolicy
+{
+  public:
+  virtual bool Must_Expand(const StringRef &absolute_path, const StringRef &relative_path);
+};
+
+/** Expand any header that is not compiler-specific.  */
+class CompilerExpansionPolicy : public IncludeExpansionPolicy
+{
+  public:
+  virtual bool Must_Expand(const StringRef &absolute_path, const StringRef &relative_path);
+};

testsuite/includes/policy-compiler-1.c

@@ -0,0 +1,19 @@
+/* { dg-options "-DCE_EXTRACT_FUNCTIONS=fn_c -DCE_NO_EXTERNALIZATION -DCE_KEEP_INCLUDES=compiler" }*/
+
+#include <stdio.h>
+#include <stdatomic.h>
+#include <pthread.h>
+
+atomic_int cnt_atomic;
+
+void* fn_c(void *thr_data) {
+  (void)thr_data;
+  for (int i = 0; i < 40000; i++) {
+    atomic_fetch_add(&cnt_atomic, 1);
+  }
+  return NULL;
+}
+
+/* { dg-final { scan-tree-dump-not "#include <stdio.h>" } } */
+/* { dg-final { scan-tree-dump "#include <stdatomic.h>" } } */
+/* { dg-final { scan-tree-dump "#include <stddef.h>" } } */

testsuite/includes/policy-system-1.c

@@ -0,0 +1,21 @@
+/* { dg-options "-DCE_EXTRACT_FUNCTIONS=fn_c -DCE_NO_EXTERNALIZATION -DCE_KEEP_INCLUDES=system" }*/
+
+#include <stdio.h>
+#include <stdatomic.h>
+#include <pthread.h>
+#include "header-1.h"
+
+atomic_int cnt_atomic;
+
+void* fn_c(void *thr_data) {
+  (void)thr_data;
+  for (int i = 0; i < 40000; i++) {
+    atomic_fetch_add(&cnt_atomic, 1);
+  }
+  return NULL;
+}
+
+/* { dg-final { scan-tree-dump "#include <stdio.h>" } } */
+/* { dg-final { scan-tree-dump "#include <stdatomic.h>" } } */
+/* { dg-final { scan-tree-dump "#include <pthread.h>" } } */
+/* { dg-final { scan-tree-dump-not "#include \"header-1.h\"" } } */