Fix a bug in setting retry expiration while getting history (#528)

Author: meiliang86

src/main/java/com/uber/cadence/internal/replay/ReplayDecider.java

@@ -598,7 +598,7 @@ private class DecisionTaskWithHistoryIteratorImpl implements DecisionTaskWithHis
     private final Duration paginationStart = Duration.ofMillis(System.currentTimeMillis());
     private Duration decisionTaskStartToCloseTimeout;
 
-    private final Duration retryServiceOperationExpirationInterval() {
+    private final Duration decisionTaskRemainingTime() {
       Duration passed = Duration.ofMillis(System.currentTimeMillis()).minus(paginationStart);
       return decisionTaskStartToCloseTimeout.minus(passed);
     }
@@ -642,11 +642,18 @@ public HistoryEvent next() {
             return current.next();
           }
 
+          Duration decisionTaskRemainingTime = decisionTaskRemainingTime();
+          if (decisionTaskRemainingTime.isNegative() || decisionTaskRemainingTime.isZero()) {
+            throw new Error(
+                "Decision task timed out while querying history. If this happens consistently please consider " +
+                    "increase decision task timeout or reduce history size.");
+          }
+
           metricsScope.counter(MetricsType.WORKFLOW_GET_HISTORY_COUNTER).inc(1);
           Stopwatch sw = metricsScope.timer(MetricsType.WORKFLOW_GET_HISTORY_LATENCY).start();
           RetryOptions retryOptions =
               new RetryOptions.Builder()
-                  .setExpiration(retryServiceOperationExpirationInterval())
+                  .setExpiration(decisionTaskRemainingTime)
                   .setInitialInterval(retryServiceOperationInitialInterval)
                   .setMaximumInterval(retryServiceOperationMaxInterval)
                   .build();