Rename internalRuntimeForbidden and make heuristics reliable

build.xml

@@ -462,15 +462,15 @@
   </target>
 
   <target name="-check-myself" depends="compile,compile-tools,compile-test,-install-forbiddenapi-task">
-    <forbiddenapis internalRuntimeForbidden="true" failOnUnsupportedJava="false">
+    <forbiddenapis failOnUnsupportedJava="false">
       <classpath refid="path.all"/>
       <fileset dir="build/main"/>
       <fileset dir="build/tools"/>
       <fileset dir="build/test"/>
       <signatures>
         <bundled name="jdk-unsafe-${jdk.version}"/>
         <bundled name="jdk-deprecated-${jdk.version}"/>
-        <bundled name="jdk-internal-${jdk.version}"/>
+        <bundled name="jdk-non-portable"/>
         <bundled name="jdk-system-out"/>
         <bundled name="jdk-reflection"/>
       </signatures>

src/main/docs/ant-task.html

@@ -120,7 +120,8 @@ <h2>Parameters</h2>
   <td>internalRuntimeForbidden</td>
   <td><code>boolean</code></td>
   <td><code>false</code></td>
-  <td>Forbids calls to classes from the internal java runtime (like <code>sun.misc.Unsafe</code>).</td>
+  <td>Forbids calls to non-portable runtime APIs (like <code>sun.misc.Unsafe</code>). <em>Please note:</em> This enables <code>"jdk-non-portable"</code> bundled signatures for backwards compatibility.<br>
+  <strong>Deprecated.</strong> Use <a href="bundled-signatures.html">bundled signatures</a> <code>"jdk-non-portable"</code> or <code>"jdk-internal"</code> instead.</td>
 </tr>
 
 <tr>

src/main/docs/bundled-signatures.html

@@ -34,16 +34,24 @@ <h1>Bundled Signatures Documentation</h1>
 <li><strong><tt>jdk-deprecated-*</tt>:</strong> This disallows all deprecated
 methods from the JDK (for Java <tt>*</tt> = 1.6, 1.7, 1.8; Maven / Gradle automatically add the compile Java version).</li>
 
+<li><strong><tt>jdk-internal-*</tt>:</strong> Lists all internal packages of the JDK as of <code>Security.getProperty(&quot;package.access&quot;)</code>.
+Calling those methods will always trigger security manager and is completely forbidden from Java 9 on
+(for Java <tt>*</tt> = 1.6, 1.7, 1.8; Maven / Gradle automatically add the compile Java version, <em>since forbiddenapis v2.1</em>).</li>
+
+<li><strong><tt>jdk-non-portable</tt>:</strong> Signatures of all non-portable (like <code>com.sun.management.HotSpotDiagnosticMXBean</code>)
+or internal runtime APIs (like <code>sun.misc.Unsafe</code>). This is a superset of <tt>jdk-internal</tt>.<br>
+<em>Internally this is implemented using heuristics:</em> Any reference to an API that is part of the Java runtime (<tt>rt.jar</tt>, extensions,
+Java 9 <tt>java.*</tt> / <tt>jdk.*</tt> core modules) and is <strong>not</strong> part of the Java SE specification packages
+(mainly <tt>java</tt>, <tt>javax</tt>, but also <tt>org.ietf.jgss</tt>, <tt>org.omg</tt>, <tt>org.w3c.dom</tt>, and <tt>org.xml.sax</tt>) is forbidden
+(any java version, no specific JDK version, <em>since forbiddenapis v2.1 / replaces deprecated and wrong-named task
+setting <tt>internalRuntimeForbidden</tt></em>).</li>
+
 <li><strong><tt>jdk-system-out</tt>:</strong> On server-side applications or libraries used by other programs, printing to
 <tt>System.out</tt> or <tt>System.err</tt> is discouraged and should be avoided (any java version, no specific JDK version).</li>
 
 <li><strong><tt>jdk-reflection</tt>:</strong> Reflection usage to work around access flags fails with SecurityManagers
 and likely will not work anymore on runtime classes in Java 9 (any java version, no specific JDK version, <em>since forbiddenapis v2.1</em>).</li>
 
-<li><strong><tt>jdk-internal</tt>:</strong> Lists all internal packages of the JDK as of <code>Security.getProperty(&quot;package.access&quot;)</code>.
-Calling those methods will always trigger security manager and is completely forbidden from Java 9 on
-(for Java <tt>*</tt> = 1.6, 1.7, 1.8; Maven / Gradle automatically add the compile Java version, <em>since forbiddenapis v2.1</em>).</li>
-
 <li><strong><tt>commons-io-unsafe-*</tt>:</strong> If your application uses the famous <i>Apache Common-IO</i> library,
 this adds signatures of all methods that depend on default charset
 (for versions <tt>*</tt> = 1.0, 1.1, 1.2, 1.3, 1.4, 2.0, 2.1, 2.2, 2.3, 2.4).</li>

src/main/java/de/thetaphi/forbiddenapis/AsmUtils.java

@@ -16,7 +16,8 @@
  * limitations under the License.
  */
 
-import java.util.Arrays;
+import java.net.URISyntaxException;
+import java.net.URL;
 import java.util.Locale;
 import java.util.regex.Pattern;
 
@@ -26,24 +27,44 @@ public final class AsmUtils {
   private AsmUtils() {}
 
   private static final String REGEX_META_CHARS = ".^$+{}[]|()\\";
-  private static final Pattern INTERNAL_PACKAGE_PATTERN;
-  static {
+
+  /** Package prefixes of documented Java API (extracted from Javadocs of Java 8). */
+  private static final Pattern PORTABLE_RUNTIME_PACKAGE_PATTERN = makePkgPrefixPattern("java", "javax", "org.ietf.jgss", "org.omg", "org.w3c.dom", "org.xml.sax");
+
+  /** Pattern that matches all module names, which are shipped by default in Java.
+   * (see: {@code http://openjdk.java.net/projects/jigsaw/spec/sotms/}):
+   * The remaining platform modules will share the 'java.' name prefix and are likely to include,
+   * e.g., java.sql for database connectivity, java.xml for XML processing, and java.logging for
+   * logging. Modules that are not defined in the Java SE 9 Platform Specification but instead
+   * specific to the JDK will, by convention, share the 'jdk.' name prefix.
+   */
+  private static final Pattern RUNTIME_MODULES_PATTERN = makePkgPrefixPattern("java", "jdk");
+
+  private static Pattern makePkgPrefixPattern(String... prefixes) {
     final StringBuilder sb = new StringBuilder();
     boolean first = true;
-    for (final String pkg : Arrays.asList("sun.", "oracle.", "com.sun.", "com.oracle.", "jdk.", "sunw.")) {
-      sb.append(first ? '(' : '|').append(Pattern.quote(pkg));
+    for (final String p : prefixes) {
+      sb.append(first ? '(' : '|').append(Pattern.quote(p));
       first = false;
     }
-    INTERNAL_PACKAGE_PATTERN = Pattern.compile(sb.append(").*").toString());
+    sb.append(")").append(Pattern.quote(".")).append(".*");
+    return Pattern.compile(sb.toString());
   }
 
   private static boolean isRegexMeta(char c) {
     return REGEX_META_CHARS.indexOf(c) != -1;
   }
 
-  /** Returns true, if the given binary class name (dotted) is likely a internal class (like sun.misc.Unsafe) */
-  public static boolean isInternalClass(String className) {
-    return INTERNAL_PACKAGE_PATTERN.matcher(className).matches();
+  /** Returns true, if the given binary class name (dotted) is part of the documented and portable Java APIs. */
+  public static boolean isPortableRuntimeClass(String className) {
+    return PORTABLE_RUNTIME_PACKAGE_PATTERN.matcher(className).matches();
+  }
+
+  /** Returns true, if the given Java 9 module name is part of the runtime (no custom 3rd party module).
+   * @param module the module name or {@code null}, if in unnamed module
+   */
+  public static boolean isRuntimeModule(String module) {
+    return module != null && RUNTIME_MODULES_PATTERN.matcher(module).matches();
   }
 
   /** Converts a binary class name (dotted) to the JVM internal one (slashed). Only accepts valid class names, no arrays. */
@@ -103,5 +124,27 @@ public static Pattern glob2Pattern(String... globs) {
     }
     return Pattern.compile(regex.toString(), 0);
   }
+
+  /** Returns the module name from a {@code jrt:/} URL; returns null if no module given or wrong URL type. */
+  public static String getModuleName(URL jrtUrl) {
+    if (!"jrt".equalsIgnoreCase(jrtUrl.getProtocol())) {
+      return null;
+    }
+    try {
+      // use URI class to also decode path and remove escapes:
+      String mod = jrtUrl.toURI().getPath();
+      if (mod != null && mod.length() >= 1) {
+        mod = mod.substring(1);
+        int p = mod.indexOf('/');
+        if (p >= 0) {
+          mod = mod.substring(0, p);
+        }
+        return mod.isEmpty() ? null : mod;
+      }
+      return null;
+    } catch (URISyntaxException use) {
+      return null;
+    }
+  }
 
 }

src/main/java/de/thetaphi/forbiddenapis/Checker.java

@@ -54,10 +54,9 @@
 /**
  * Forbidden APIs checker class.
  */
-public final class Checker implements RelatedClassLookup {
+public final class Checker implements RelatedClassLookup, Constants {
 
   public static enum Option {
-    INTERNAL_RUNTIME_FORBIDDEN,
     FAIL_ON_MISSING_CLASSES,
     FAIL_ON_VIOLATION,
     FAIL_ON_UNRESOLVABLE_SIGNATURES
@@ -66,27 +65,28 @@ public static enum Option {
   public final boolean isSupportedJDK;
 
   private final long start;
-
+  private final NavigableSet<String> runtimePaths;
+
   final Logger logger;
 
-  final NavigableSet<String> runtimePaths;
-
   final ClassLoader loader;
-  final java.lang.reflect.Method method_Class_getModule, method_Module_getResourceAsStream;
+  final java.lang.reflect.Method method_Class_getModule, method_Module_getResourceAsStream, method_Module_getName;
   final EnumSet<Option> options;
 
   // key is the binary name (dotted):
   final Map<String,ClassSignature> classesToCheck = new HashMap<String,ClassSignature>();
   // key is the binary name (dotted):
   final Map<String,ClassSignature> classpathClassCache = new HashMap<String,ClassSignature>();
 
+  // if enabled, the bundled signature to enable heuristics for detection of non-portable runtime calls is used:
+  private boolean forbidNonPortableRuntime = false;  
   // key is the internal name (slashed), followed by \000 and the field name:
   final Map<String,String> forbiddenFields = new HashMap<String,String>();
   // key is the internal name (slashed), followed by \000 and the method signature:
   final Map<String,String> forbiddenMethods = new HashMap<String,String>();
   // key is the internal name (slashed):
   final Map<String,String> forbiddenClasses = new HashMap<String,String>();
-  // key is pattern to binary class name:
+  // set of patterns of forbidden classes:
   final Set<ClassPatternRule> forbiddenClassPatterns = new LinkedHashSet<ClassPatternRule>();
   // descriptors (not internal names) of all annotations that suppress:
   final Set<String> suppressAnnotations = new LinkedHashSet<String>();
@@ -129,19 +129,22 @@ public Checker(Logger logger, ClassLoader loader, EnumSet<Option> options) {
 
     boolean isSupportedJDK = false;
 
-    // first try Java 9 mdoule system (Jigsaw)
+    // first try Java 9 module system (Jigsaw)
     // Please note: This code is not guaranteed to work with final Java 9 version. This is just for testing!
-    java.lang.reflect.Method method_Class_getModule, method_Module_getResourceAsStream;
+    java.lang.reflect.Method method_Class_getModule, method_Module_getResourceAsStream, method_Module_getName;
     try {
       method_Class_getModule = Class.class.getMethod("getModule");
       method_Module_getResourceAsStream = method_Class_getModule
           .getReturnType().getMethod("getResourceAsStream", String.class);
+      method_Module_getName = method_Class_getModule
+          .getReturnType().getMethod("getName");
       isSupportedJDK = true;
     } catch (NoSuchMethodException e) {
-      method_Class_getModule = method_Module_getResourceAsStream = null;
+      method_Class_getModule = method_Module_getResourceAsStream = method_Module_getName = null;
     }
     this.method_Class_getModule = method_Class_getModule;
     this.method_Module_getResourceAsStream = method_Module_getResourceAsStream;
+    this.method_Module_getName = method_Module_getName;
 
     final NavigableSet<String> runtimePaths = new TreeSet<String>();
 
@@ -222,17 +225,30 @@ public Checker(Logger logger, ClassLoader loader, EnumSet<Option> options) {
    * This code is not guaranteed to work with final Java 9 version.
    * This is just for testing!
    **/
-  private InputStream getBytecodeFromJigsaw(String classname) {
-    if (method_Class_getModule == null || method_Module_getResourceAsStream == null) {
+  private ClassSignature loadClassFromJigsaw(String classname) throws IOException {
+    if (method_Class_getModule == null || method_Module_getResourceAsStream == null || method_Module_getName == null) {
       return null; // not Java 9 JIGSAW
     }
+
+    final InputStream in;
+    final String moduleName;
     try {
       final Class<?> clazz = Class.forName(classname, false, loader);
       final Object module = method_Class_getModule.invoke(clazz);
-      return (InputStream) method_Module_getResourceAsStream.invoke(module, AsmUtils.getClassResourceName(classname));
+      moduleName = (String) method_Module_getName.invoke(module);
+      in = (InputStream) method_Module_getResourceAsStream.invoke(module, AsmUtils.getClassResourceName(classname));
+      if (in == null) {
+        return null;
+      }
     } catch (Exception e) {
       return null; // not found
     }
+
+    try {
+      return new ClassSignature(new ClassReader(in), AsmUtils.isRuntimeModule(moduleName), false);
+    } finally {
+      in.close();
+    }
   }
 
   private boolean isRuntimePath(URL url) throws IOException {
@@ -260,7 +276,7 @@ private boolean isRuntimeClass(URLConnection conn) throws IOException {
       // all 'jrt:' URLs refer to a module in the Java 9+ runtime (see http://openjdk.java.net/jeps/220)
       // This may still be different with module system. We support both variants for now.
       // Please note: This code is not guaranteed to work with final Java 9 version. This is just for testing!
-      return true;
+      return AsmUtils.isRuntimeModule(AsmUtils.getModuleName(url));
     }
     return false;
   }
@@ -287,14 +303,9 @@ private ClassSignature getClassFromClassLoader(final String clazz) throws ClassN
         }
         return c;
       } else {
-        final InputStream in = getBytecodeFromJigsaw(clazz);
-        if (in != null) {
-          try {
-            // we mark it as runtime class because it was derived from the module system:
-            classpathClassCache.put(clazz, c = new ClassSignature(new ClassReader(in), true, false));
-          } finally {
-            in.close();
-          }
+        final ClassSignature jigsawCl = loadClassFromJigsaw(clazz);
+        if (jigsawCl != null) {
+          classpathClassCache.put(clazz, c = jigsawCl);
           return c;
         }
       }
@@ -336,7 +347,8 @@ public ClassSignature lookupRelatedClass(String internalName) {
 
   /** Adds the method signature to the list of disallowed methods. The Signature is checked against the given ClassLoader. */
   private void addSignature(final String line, final String defaultMessage, final UnresolvableReporting report) throws ParseException,IOException {
-    final String clazz, field, signature, message;
+    final String clazz, field, signature;
+    String message = null;
     final Method method;
     int p = line.indexOf('@');
     if (p >= 0) {
@@ -371,15 +383,17 @@ private void addSignature(final String line, final String defaultMessage, final
       method = null;
       field = null;
     }
+    if (message != null && message.isEmpty()) {
+      message = null;
+    }
     // create printout message:
-    final String printout = (message != null && message.length() > 0) ?
-      (signature + " [" + message + "]") : signature;
+    final String printout = (message != null) ? (signature + " [" + message + "]") : signature;
     // check class & method/field signature, if it is really existent (in classpath), but we don't really load the class into JVM:
     if (AsmUtils.isGlob(clazz)) {
       if (method != null || field != null) {
         throw new ParseException(String.format(Locale.ENGLISH, "Class level glob pattern cannot be combined with methods/fields: %s", signature));
       }
-      forbiddenClassPatterns.add(new ClassPatternRule(clazz, printout));
+      forbiddenClassPatterns.add(new ClassPatternRule(clazz, message));
     } else {
       final ClassSignature c;
       try {
@@ -419,14 +433,19 @@ private void addSignature(final String line, final String defaultMessage, final
   }
 
   /** Reads a list of bundled API signatures from classpath. */
-  public void parseBundledSignatures(String name, String jdkTargetVersion) throws IOException,ParseException {
-    parseBundledSignatures(name, jdkTargetVersion, true);
+  public void addBundledSignatures(String name, String jdkTargetVersion) throws IOException,ParseException {
+    addBundledSignatures(name, jdkTargetVersion, true);
   }
 
-  private void parseBundledSignatures(String name, String jdkTargetVersion, boolean logging) throws IOException,ParseException {
+  private void addBundledSignatures(String name, String jdkTargetVersion, boolean logging) throws IOException,ParseException {
     if (!name.matches("[A-Za-z0-9\\-\\.]+")) {
       throw new ParseException("Invalid bundled signature reference: " + name);
     }
+    if (BS_JDK_NONPORTABLE.equals(name)) {
+      if (logging) logger.info("Reading bundled API signatures: " + name);
+      forbidNonPortableRuntime = true;
+      return;
+    }
     // use Checker.class hardcoded (not getClass) so we have a fixed package name:
     InputStream in = Checker.class.getResourceAsStream("signatures/" + name + ".txt");
     // automatically expand the compiler version in here (for jdk-* signatures without version):
@@ -487,7 +506,7 @@ private void parseSignaturesFile(Reader reader, boolean isBundled) throws IOExce
         if (line.startsWith("@")) {
           if (isBundled && line.startsWith(BUNDLED_PREFIX)) {
             final String name = line.substring(BUNDLED_PREFIX.length()).trim();
-            parseBundledSignatures(name, null, false);
+            addBundledSignatures(name, null, false);
           } else if (line.startsWith(DEFAULT_MESSAGE_PREFIX)) {
             defaultMessage = line.substring(DEFAULT_MESSAGE_PREFIX.length()).trim();
             if (defaultMessage.length() == 0) defaultMessage = null;
@@ -549,7 +568,11 @@ public void addClassesToCheck(File basedir, String... relativeNames) throws IOEx
   }
 
   public boolean hasNoSignatures() {
-    return forbiddenMethods.isEmpty() && forbiddenFields.isEmpty() && forbiddenClasses.isEmpty() && forbiddenClassPatterns.isEmpty() && (!options.contains(Option.INTERNAL_RUNTIME_FORBIDDEN));
+    return 0 == forbiddenMethods.size() + 
+        forbiddenFields.size() + 
+        forbiddenClasses.size() + 
+        forbiddenClassPatterns.size() +
+        (forbidNonPortableRuntime ? 1 : 0);
   }
 
   /** Adds the given annotation class for suppressing errors. */
@@ -565,7 +588,7 @@ public void addSuppressAnnotation(String annoName) {
   /** Parses a class and checks for valid method invocations */
   private int checkClass(final ClassReader reader, Pattern suppressAnnotationsPattern) {
     final String className = Type.getObjectType(reader.getClassName()).getClassName();
-    final ClassScanner scanner = new ClassScanner(this, forbiddenClasses, forbiddenClassPatterns, forbiddenMethods, forbiddenFields, suppressAnnotationsPattern, options.contains(Option.INTERNAL_RUNTIME_FORBIDDEN)); 
+    final ClassScanner scanner = new ClassScanner(this, forbiddenClasses, forbiddenClassPatterns, forbiddenMethods, forbiddenFields, suppressAnnotationsPattern, forbidNonPortableRuntime); 
     reader.accept(scanner, ClassReader.SKIP_FRAMES);
     final List<ForbiddenViolation> violations = scanner.getSortedViolations();
     final Pattern splitter = Pattern.compile(Pattern.quote(ForbiddenViolation.SEPARATOR));