Cloudup and bandwidth commands print iostats (through reflection)

Author: steveloughran

src/main/java/org/apache/hadoop/fs/store/commands/Bandwidth.java

@@ -44,6 +44,7 @@
 import org.apache.hadoop.fs.store.StoreDurationInfo;
 import org.apache.hadoop.fs.store.StoreEntryPoint;
 import org.apache.hadoop.fs.store.StoreUtils;
+import org.apache.hadoop.fs.store.logging.IOStatisticsIntegration;
 import org.apache.hadoop.fs.tools.csv.CsvWriterWithCRC;
 import org.apache.hadoop.util.Progressable;
 import org.apache.hadoop.util.ShutdownHookManager;
@@ -375,6 +376,15 @@ public int run(String[] args) throws Exception {
       }
     }
 
+    final IOStatisticsIntegration ios = new IOStatisticsIntegration();
+    if (ios.available()) {
+      final String prettyString = ios.ioStatisticsToPrettyString(fs);
+      if (!prettyString.isEmpty()) {
+        heading("Destination Filesystem IO Statistics");
+        println(prettyString);
+      }
+    }
+
     // now print summaries
     summarize("Upload", uploadDurationTracker, fileSizeBytes,
         "Blocks uploaded (ignoring close() overhead):", blockUploads);

src/main/java/org/apache/hadoop/fs/store/logging/IOStatisticsIntegration.java

@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.store.logging;
+
+import org.apache.hadoop.fs.statistics.IOStatistics;
+import org.apache.hadoop.fs.store.shim.impl.Invocation;
+import org.apache.hadoop.fs.store.shim.impl.ShimReflectionSupport;
+
+import static org.apache.hadoop.fs.store.shim.impl.Invocation.unavailable;
+import static org.apache.hadoop.fs.store.shim.impl.ShimReflectionSupport.loadClass;
+import static org.apache.hadoop.fs.store.shim.impl.ShimReflectionSupport.loadInvocation;
+
+/**
+ * Support for IO statistics (initially through reflection).
+ */
+public class IOStatisticsIntegration {
+
+  public static final String CLASSNAME_IOSTATISTICS = "org.apache.hadoop.fs.statistics.IOStatistics";
+
+  public static final String CLASSNAME_IOSTATISTICS_LOGGING = "org.apache.hadoop.fs.statistics.IOStatisticsLogging";
+  public static final String CLASSNAME_IOSTATISTICS_SUPPORT = "org.apache.hadoop.fs.statistics.IOStatisticsSupport";
+
+  private final Class<?> ioStatisticsClass;
+
+  private final Class<?> ioStatisticsLogging;
+  private final Class<?> ioStatisticsSupport;
+
+  private final Invocation<String> _ioStatisticsToPrettyString;
+
+  private final Invocation<?> _retrieveIOStatistics;
+
+  public IOStatisticsIntegration() {
+    // try to load the class
+    ioStatisticsClass = loadClass(CLASSNAME_IOSTATISTICS);
+    if (ioStatisticsClass == null) {
+      // if that class is missing, so is the rest.
+      ioStatisticsSupport = null;
+      ioStatisticsLogging = null;
+      _ioStatisticsToPrettyString = unavailable("ioStatisticsToPrettyString");
+      _retrieveIOStatistics = unavailable("retrieveIOStatistics");
+    } else {
+      ioStatisticsSupport = loadClass(CLASSNAME_IOSTATISTICS_SUPPORT);
+      _retrieveIOStatistics = loadInvocation(ioStatisticsSupport,
+          ioStatisticsClass, "retrieveIOStatistics", Object.class);
+
+      ioStatisticsLogging = loadClass(CLASSNAME_IOSTATISTICS_LOGGING);
+
+      _ioStatisticsToPrettyString = loadInvocation(ioStatisticsLogging,
+          String.class, "ioStatisticsToPrettyString", ioStatisticsClass);
+    }
+  }
+
+  public boolean available() {
+    return ioStatisticsClass != null;
+  }
+
+  public String ioStatisticsToPrettyString(Object source) {
+    if (!available()) {
+      return "";
+    }
+    return _ioStatisticsToPrettyString.invokeUnchecked(null,
+        _retrieveIOStatistics.invokeUnchecked(null, source));
+  }
+}

src/main/java/org/apache/hadoop/fs/tools/cloudup/Cloudup.java

@@ -59,6 +59,7 @@
 import org.apache.hadoop.fs.store.StoreDurationInfo;
 import org.apache.hadoop.fs.store.StoreEntryPoint;
 import org.apache.hadoop.fs.store.StoreUtils;
+import org.apache.hadoop.fs.store.logging.IOStatisticsIntegration;
 import org.apache.hadoop.util.Progressable;
 import org.apache.hadoop.util.ToolRunner;
 
@@ -782,7 +783,16 @@ private void dumpStats(FileSystem fs, String header) {
     // Not supported on Hadoop 2.7
     println();
     println("%s: %s", header, fs.getUri());
+    final IOStatisticsIntegration ios = new IOStatisticsIntegration();
+    if (ios.available()) {
+      final String prettyString = ios.ioStatisticsToPrettyString(fs);
+      if (!prettyString.isEmpty()) {
+        println(prettyString);
+        return;
+      }
+    }
 
+    // only get here if there are no iostats
     if (verbose) {
       // hope to see FS IOStats
       println("Filesystem %s", fs);