Operation timeouts can create a pathological feedback loop with multigets

Author: Hernan Gelaf-Romer

hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java

@@ -19,7 +19,6 @@
 
 import java.io.IOException;
 import java.io.InterruptedIOException;
-import java.net.SocketTimeoutException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -208,12 +207,13 @@ public void run() {
             // Cancelled
             return;
           }
-        } catch (OperationTimeoutExceededException e) {
+        } catch (OperationTimeoutExceededException
+          | InterruptedOperationTimeoutExceededException e) {
           // The operation has timed out before executing the actual callable. This may be due to
           // slow/hotspotted meta or the operation timeout set too low for the number of requests.
           // Circumventing the usual failure flow ensure the meta cache is not cleared and will not
           // result in a doomed feedback loop in which the meta continues to be hotspotted.
-          // See HBASE-27487
+          // See HBASE-27487 + HBASE-29265
           failAll(multiAction, server, numAttempt, e);
           return;
         } catch (IOException e) {
@@ -1254,7 +1254,8 @@ public void waitUntilDone() throws InterruptedIOException {
         // stuck here forever
         long cutoff = (EnvironmentEdgeManager.currentTime() + this.operationTimeout) * 1000L;
         if (!waitUntilDone(cutoff)) {
-          throw new SocketTimeoutException("time out before the actionsInProgress changed to zero");
+          throw new InterruptedOperationTimeoutExceededException(
+            "time out before the actionsInProgress changed to zero");
         }
       } else {
         waitUntilDone(Long.MAX_VALUE);

hbase-client/src/main/java/org/apache/hadoop/hbase/client/InterruptedOperationTimeoutExceededException.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import java.net.SocketTimeoutException;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * Thrown when a batch operation exceeds the operation timeout, but requests are still in-flight and
+ * need to be interrupted
+ */
+@InterfaceAudience.Public
+public class InterruptedOperationTimeoutExceededException extends SocketTimeoutException {
+
+  public InterruptedOperationTimeoutExceededException(String msg) {
+    super(msg);
+  }
+}