Safeprefetch: document and in sync with HADOOP-18577.

Author: steveloughran

README.md

@@ -620,7 +620,8 @@ You can also explore what directory tree structure is most efficient here.
 
 ## Command `mkcsv`
 
-Creates a CSV file.
+Creates a large CSV file designed to trigger/validate the ABFS prefetching bug which
+came in HADOOP-17156/
 
 See [mkcsv](src/main/site/mkcsv.md)
 
@@ -720,6 +721,14 @@ a failure to connect to the instance metadata server.
 
 This is to be expected, given that the service isn't there.
 
+## Command `safeprefetch`
+
+Probes an abfs store for being vulnerable to
+prefetch data corruption, providing the configuration
+information to disable it if so.
+
+See [safeprefetch](src/main/site/safeprefetch.md)
+
 ## Command `tlsinfo`
 
 Print out tls information. The `storediag` command prints the same information;

src/main/extra/org/apache/hadoop/fs/gs/package-info.java

@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;

src/main/extra/org/apache/hadoop/fs/s3a/extra/package-info.java

@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.extra;

src/main/extra/org/apache/hadoop/fs/store/commands/CommitterInfo.java

@@ -26,6 +26,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StreamCapabilities;
 import org.apache.hadoop.fs.store.StoreDurationInfo;
 import org.apache.hadoop.fs.store.StoreEntryPoint;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
@@ -47,6 +48,7 @@
  *
  * Prints some performance numbers at the end.
  */
+@SuppressWarnings("InstanceofIncompatibleInterface")
 public class CommitterInfo extends StoreEntryPoint {
 
   private static final Logger LOG = LoggerFactory.getLogger(CommitterInfo.class);
@@ -94,6 +96,11 @@ public int run(String[] args) throws Exception {
               " %s:\n" +
               " %s",
           committer.getClass().getCanonicalName(), committer);
+      if (committer instanceof StreamCapabilities
+        && ((StreamCapabilities) committer).hasCapability(
+                    "mapreduce.job.committer.dynamic.partitioning")) {
+          println("Committer declares support for spark dynamic partitioning");
+      }
     }
     return 0;
   }

src/main/extra/org/apache/hadoop/fs/store/commands/package-info.java

@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.store.commands;

src/main/java/org/apache/hadoop/fs/store/abfs/SafePrefetch.java

@@ -75,7 +75,7 @@ public int run(String[] args) throws Exception {
 
     // path on the CLI
     Path path = new Path(argList.get(0));
-    println("Probing %s for prefetch safety %s", path);
+    println("\nProbing %s for prefetch safety", path);
 
     FileSystem fs = path.getFileSystem(conf);
     if (!(fs instanceof AzureBlobFileSystem)) {
@@ -84,7 +84,7 @@ public int run(String[] args) throws Exception {
       return 0;
     }
     String etag_capability = "fs.capability.etags.available";
-    String hadoop_18546 = "hadoop-18546";
+    String readahead_safe = "fs.azure.capability.readahead.safe";
 
     println("Using filesystem %s", fs.getUri());
     Path abfsPath = path.makeQualified(fs.getUri(), fs.getWorkingDirectory());
@@ -98,13 +98,13 @@ public int run(String[] args) throws Exception {
           etag_capability);
       return 0;
     }
-    if (checker.hasPathCapability(abfsPath, hadoop_18546)) {
+    if (checker.hasPathCapability(abfsPath, readahead_safe)) {
 
-      println("Filesystem has prefetch issue fixed (has path capability %s)",
-          abfsPath, hadoop_18546);
+      println("Filesystem %s has prefetch issue fixed (has path capability %s)",
+          abfsPath, readahead_safe);
       return 0;
     }
-    println("Store is vulnerable to inconsistent prefetching. This MUST be disabled");
+    println("Store is vulnerable to inconsistent prefetching. This MUST be disabled\n");
     final Configuration fsConf = fs.getConf();
     List<EnvEntry> entries = new ArrayList<>();
     entries.add(new EnvEntry(FS_AZURE_READAHEADQUEUE_DEPTH, "", "0"));
@@ -130,16 +130,12 @@ public int run(String[] args) throws Exception {
       // no disable option; don't confuse the user by mentioning it.
 
     }
-    EnvEntry property = new EnvEntry(FS_AZURE_READAHEADQUEUE_DEPTH, "", "0");
-    println("To disable prefetching, set %s to %s", property.getName(), property.getValue());
-    println("\n%s\n", property.xml());
-    println("\n%s\n", property.spark());
 
     warn("Filesystem is vulnerable until prefetching is disabled");
     StringBuilder xml = new StringBuilder();
-    xml.append("<configuration>\n\n");
+    xml.append("<configuration>\n");
     entries.forEach(e -> xml.append(e.xml()));
-    xml.append("\n</configuration>\n");
+    xml.append("</configuration>\n");
 
     println("hadoop XML: %n%s%n", xml);
 

src/main/site/safeprefetch.md

@@ -0,0 +1,71 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+# Command `safeprefetch`: validate prefetch safety of abfs client
+
+Command to probe an abfs store for prefetch safety.
+
+If safe: returns a status code of 0
+
+If unsafe, prints configuration options to disable prefetching
+and then returns the exit code -1.
+
+The safety probe considers an abfs store safe if *any* of the conditions are met
+
+* It is from a release *before* `HADOOP-17156. Clear abfs readahead requests on stream close`,
+* It has the fix `HADOOP-18546. ABFS. disable purging list of in progress reads in abfs stream close()`
+* The `fs.azure.readaheadqueue.depth` is 0
+* Cloudera releases: readahead has been completely disabled
+  (`HADOOP-18517. ABFS: Add fs.azure.enable.readahead option to disable readahead` is in all CDH releases with the bug)
+
+The probe for the fix relies on `HADOOP-18577. ABFS: Add probes of readahead fix`; a pathcapabilities probe
+`fs.azure.capability.readahead.safe` has been added to all abfs releases with the fix.
+
+## Example, probe of a (safe) hadoop 3.3.5
+
+
+```
+bin/hadoop jar $CLOUDSTORE safeprefetch abfs://stevel-testing@stevelukwest.dfs.core.windows.net/user
+
+Probing abfs://stevel-testing@stevelukwest.dfs.core.windows.net/user for prefetch safety
+Using filesystem abfs://stevel-testing@stevelukwest.dfs.core.windows.net
+Filesystem abfs://stevel-testing@stevelukwest.dfs.core.windows.net/user has prefetch issue fixed (has path capability fs.azure.capability.readahead.safe)
+```
+
+## Example, probe of hadoop 3.3.4 -unsafe
+
+```
+bin/hadoop jar $CLOUDSTORE safeprefetch abfs://stevel-testing@stevelukwest.dfs.core.windows.net/user
+Probing abfs://stevel-testing@stevelukwest.dfs.core.windows.net/user for prefetch safety
+Using filesystem abfs://stevel-testing@stevelukwest.dfs.core.windows.net
+Store is vulnerable to inconsistent prefetching. This MUST be disabled
+
+WARNING: Filesystem is vulnerable until prefetching is disabled
+hadoop XML: 
+<configuration>
+<fs.azure.readaheadqueue.depth>
+  0
+</fs.azure.readaheadqueue.depth>
+</configuration>
+
+
+
+spark: 
+spark.hadoop.fs.azure.readaheadqueue.depth 0 
+
+
+2022-12-19 12:32:06,003 [main] INFO  util.ExitUtil (ExitUtil.java:terminate(241)) - Exiting with status -1: 
+
+```
+