Add --experimental_check_non_output_external_files option

This option disables the code path that checks so-called "non-output
external" files.  These are files outside of roots that Bazel tracks;
they occur when outputs/sources/repos symlink from the host.

Addresses #25954.

RELNOTES: File change checks for non-output external files (host files
and embedded tools) can now be disabled with the
`--experimental_check_non_output_external_files` flag.

Author: hanwen-flow

src/main/java/com/google/devtools/build/lib/pkgcache/PackageOptions.java

@@ -188,6 +188,15 @@ public ParallelismConverter() throws OptionsParsingException {
               + "previous run's cache.")
   public boolean checkOutputFiles;
 
+  @Option(
+          name = "experimental_check_non_output_external_files",
+          defaultValue = "true",
+          documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+          effectTags = {OptionEffectTag.UNKNOWN},
+          help =
+                  "Check for modifications made to the non-output external files (install root, host files)")
+  public boolean checkNonOutputExternalFiles;
+
   /** A converter from strings containing comma-separated names of packages to lists of strings. */
   public static class CommaSeparatedPackageNameListConverter
       extends Converter.Contextless<List<PackageIdentifier>> {

src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java

@@ -3445,17 +3445,21 @@ public boolean hasDiffAwareness() {
     return diffAwarenessManager != null;
   }
 
+  @VisibleForTesting
+  public void handleDiffsForTesting(ExtendedEventHandler eventHandler) throws InterruptedException, AbruptExitException {
+    handleDiffsForTesting(eventHandler, Options.getDefaults(PackageOptions.class));
+  }
+
   /** Uses diff awareness on all the package paths to invalidate changed files. */
   @VisibleForTesting
-  public void handleDiffsForTesting(ExtendedEventHandler eventHandler)
+  public void handleDiffsForTesting(ExtendedEventHandler eventHandler, PackageOptions packageOptions)
       throws InterruptedException, AbruptExitException {
     if (lastAnalysisDiscarded) {
       // Values were cleared last build, but they couldn't be deleted because they were needed for
       // the execution phase. We can delete them now.
       dropConfiguredTargetsNow(eventHandler);
       lastAnalysisDiscarded = false;
     }
-    PackageOptions packageOptions = Options.getDefaults(PackageOptions.class);
     packageOptions.checkOutputFiles = false;
     ClassToInstanceMap<OptionsBase> options =
         ImmutableClassToInstanceMap.of(PackageOptions.class, packageOptions);
@@ -3558,12 +3562,14 @@ protected WorkspaceInfoFromDiff handleDiffs(
     }
     RepositoryOptions repoOptions = options.getOptions(RepositoryOptions.class);
     try (SilentCloseable c = Profiler.instance().profile("handleDiffsWithMissingDiffInformation")) {
+      PackageOptions packageOptions = options.getOptions(PackageOptions.class);
       handleDiffsWithMissingDiffInformation(
           eventHandler,
           tsgm,
           pathEntriesWithoutDiffInformation,
-          options.getOptions(PackageOptions.class).checkOutputFiles,
-          repoOptions != null && repoOptions.checkExternalRepositoryFiles,
+          packageOptions.checkOutputFiles,
+	  repoOptions != null && repoOptions.checkExternalRepositoryFiles,
+          packageOptions.checkNonOutputExternalFiles,
           fsvcThreads);
     }
     handleClientEnvironmentChanges();
@@ -3640,6 +3646,7 @@ protected void handleDiffsWithMissingDiffInformation(
       Set<Pair<Root, ProcessableModifiedFileSet>> pathEntriesWithoutDiffInformation,
       boolean checkOutputFiles,
       boolean checkExternalRepositoryFiles,
+      boolean checkNonoutputExternalFiles,
       int fsvcThreads)
       throws InterruptedException, AbruptExitException {
 
@@ -3692,8 +3699,8 @@ protected void handleDiffsWithMissingDiffInformation(
       if (checkExternalRepositoryFiles) {
         fileTypesToCheck = EnumSet.of(FileType.EXTERNAL_REPO);
       }
-      if (externalFilesKnowledge.tooManyNonOutputExternalFilesSeen
-          || !externalFilesKnowledge.nonOutputExternalFilesSeen.isEmpty()) {
+      if (checkNonoutputExternalFiles && (externalFilesKnowledge.tooManyNonOutputExternalFilesSeen
+          || !externalFilesKnowledge.nonOutputExternalFilesSeen.isEmpty())) {
         fileTypesToCheck.add(FileType.EXTERNAL);
       }
       // See the comment for FileType.OUTPUT for why we need to consider output files here.
@@ -3727,8 +3734,8 @@ protected void handleDiffsWithMissingDiffInformation(
       // once an external file gets into the Skyframe graph, we'll overly-conservatively always
       // think the graph needs to be scanned.
       externalFilesHelper.setExternalFilesKnowledge(
-          tmpExternalFilesHelper.getExternalFilesKnowledge());
-    } else if (!externalFilesKnowledge.nonOutputExternalFilesSeen.isEmpty()) {
+          tmpExternalFilesHelper.getExternalFilesKnowledge());      
+    } else if (checkNonoutputExternalFiles && !externalFilesKnowledge.nonOutputExternalFilesSeen.isEmpty()) {
       logger.atInfo().log(
           "About to scan %d external files",
           externalFilesKnowledge.nonOutputExternalFilesSeen.size());

src/test/java/com/google/devtools/build/lib/pkgcache/IncrementalLoadingTest.java

@@ -406,8 +406,13 @@ def b():
     tester.addSymlink("a/b.bzl", "/b.bzl");
     tester.sync();
     tester.getTarget("//a:BUILD");
+    PackageOptions packageOptions = Options.getDefaults(PackageOptions.class);
+    packageOptions.checkNonOutputExternalFiles = false;
     tester.modifyFile("/b.bzl", "ERROR ERROR");
-    tester.sync();
+    tester.syncWithOptions(packageOptions);
+    tester.getTarget("//a:BUILD");
+    packageOptions.checkNonOutputExternalFiles = true;
+    tester.syncWithOptions(packageOptions);
 
     assertThrows(NoSuchThingException.class, () -> tester.getTarget("//a:BUILD"));
   }
@@ -618,10 +623,13 @@ private ModifiedFileSet getModifiedFileSet() {
     }
 
     void sync() throws InterruptedException, AbruptExitException {
+      syncWithOptions(Options.getDefaults(PackageOptions.class));
+    }
+
+    void syncWithOptions(PackageOptions packageOptions) throws InterruptedException, AbruptExitException {
       clock.advanceMillis(1);
 
       modifiedFileSet = getModifiedFileSet();
-      PackageOptions packageOptions = Options.getDefaults(PackageOptions.class);
       packageOptions.defaultVisibility = RuleVisibility.PUBLIC;
       packageOptions.showLoadingProgress = true;
       packageOptions.globbingThreads = 7;
@@ -642,7 +650,7 @@ void sync() throws InterruptedException, AbruptExitException {
       skyframeExecutor.invalidateFilesUnderPathForTesting(
           new Reporter(new EventBus()), modifiedFileSet, Root.fromPath(workspace));
       ((SequencedSkyframeExecutor) skyframeExecutor)
-          .handleDiffsForTesting(new Reporter(new EventBus()));
+          .handleDiffsForTesting(new Reporter(new EventBus()), packageOptions);
 
       changes.clear();
     }