From c1ef66746ddf31c37cf001abfc8e83c3ac49f875 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Tue, 27 May 2025 17:34:07 +0200 Subject: [PATCH 01/36] [WIP] feat: add hedging ability to RetryingRpcClient with first tests --- .../client/retry/AbstractRetryingClient.java | 127 +++- .../armeria/client/retry/RetryConfig.java | 26 +- .../client/retry/RetryConfigBuilder.java | 17 +- .../client/retry/RetryingRpcClient.java | 175 ++++- .../common/logging/DefaultRequestLog.java | 21 +- .../common/logging/RequestLogBuilder.java | 6 +- .../armeria/internal/client/ClientUtil.java | 12 +- .../RetryingRpcClientWithHedgingTest.java | 612 ++++++++++++++++++ 8 files changed, 936 insertions(+), 60 deletions(-) create mode 100644 thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java index e8e5cc277e3..a46d12b914e 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java @@ -1,7 +1,7 @@ /* - * Copyright 2017 LINE Corporation + * Copyright 2025 LY Corporation * - * LINE Corporation licenses this file to you under the Apache License, + * LY Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * @@ -15,9 +15,12 @@ */ package com.linecorp.armeria.client.retry; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static java.util.Objects.requireNonNull; +import java.util.LinkedList; +import java.util.List; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -51,7 +54,7 @@ public abstract class AbstractRetryingClient extends SimpleDecoratingClient { - private static final Logger logger = LoggerFactory.getLogger(AbstractRetryingClient.class); + protected static final Logger logger = LoggerFactory.getLogger(AbstractRetryingClient.class); /** * The header which indicates the retry count of a {@link Request}. @@ -100,11 +103,18 @@ protected final RetryConfigMapping mapping() { */ protected abstract O doExecute(ClientRequestContext ctx, I req) throws Exception; + protected static void onRetryingComplete(ClientRequestContext ctx) { + ctx.logBuilder().endResponseWithLastChild(); + } + /** * This should be called when retrying is finished. */ - protected static void onRetryingComplete(ClientRequestContext ctx) { - ctx.logBuilder().endResponseWithLastChild(); + protected static void onRetryingComplete(ClientRequestContext ctx, + ClientRequestContext derivedCtx) { + // Cancel every in-flight attempt. + // The following is not long true. + ctx.logBuilder().endResponseWithChild(derivedCtx.log()); } /** @@ -154,7 +164,9 @@ protected static void scheduleNextRetry(ClientRequestContext ctx, @SuppressWarnings("unchecked") final ScheduledFuture scheduledFuture = (ScheduledFuture) ctx .eventLoop().schedule(retryTask, nextDelayMillis, TimeUnit.MILLISECONDS); + scheduledFuture.addListener(future -> { + state(ctx).removeRetryTask(scheduledFuture); if (future.isCancelled()) { // future is cancelled when the client factory is closed. actionOnException.accept(new IllegalStateException( @@ -164,12 +176,59 @@ protected static void scheduleNextRetry(ClientRequestContext ctx, actionOnException.accept(future.cause()); } }); + + + state(ctx).addRetryTask(scheduledFuture); } } catch (Throwable t) { actionOnException.accept(t); } } + protected static void scheduleNextAttemptTimeout(ClientRequestContext ctx, + Runnable attemptTimeoutTask, + Consumer actionOnException, + long nextAttemptTimeoutMillis) { + checkArgument(nextAttemptTimeoutMillis > 0, "nextAttemptTimeoutMillis must be > 0"); + + try { + final ScheduledFuture nextAttemptTimeoutFuture = (ScheduledFuture) ctx + .eventLoop() + .schedule(attemptTimeoutTask, nextAttemptTimeoutMillis, TimeUnit.MILLISECONDS); + + nextAttemptTimeoutFuture.addListener(thisAttemptTimeoutFuture -> { + if (thisAttemptTimeoutFuture.isCancelled()) { + if (state(ctx).getCurrentAttemptTimeoutFuture() == thisAttemptTimeoutFuture) { + actionOnException.accept( + new IllegalStateException(ClientFactory.class.getSimpleName() + " has been " + + "closed.")); + } else { + // It is fine that attempt timeout tasks are cancelled to be replaced with the next + // attempt timeout task. + } + } else if (thisAttemptTimeoutFuture.cause() != null) { + actionOnException.accept(thisAttemptTimeoutFuture.cause()); + } + }); + + state(ctx).setCurrentAttemptTimeoutFuture(nextAttemptTimeoutFuture); + } catch (Throwable t) { + actionOnException.accept(t); + } + } + + protected static void cancelRetryTasks(ClientRequestContext ctx) { + state(ctx).cancelAllRetryTasks(); + } + + protected static void cancelAttemptTimeout(ClientRequestContext ctx) { + final ScheduledFuture currentAttemptTimeoutFuture = state(ctx).getCurrentAttemptTimeoutFuture(); + if (currentAttemptTimeoutFuture != null) { + currentAttemptTimeoutFuture.cancel(false); + state(ctx).setCurrentAttemptTimeoutFuture(null); + } + } + /** * Resets the {@link ClientRequestContext#responseTimeoutMillis()}. * @@ -252,6 +311,15 @@ protected static int getTotalAttempts(ClientRequestContext ctx) { return state.totalAttemptNo; } + protected static boolean areAttemptsExhausted(ClientRequestContext ctx) { + final State state = ctx.attr(STATE); + if (state == null) { + // todo(szymon): when does this happen? + return true; // No retrying is in progress. + } + return state.areAttemptsExhausted(); + } + /** * Creates a new derived {@link ClientRequestContext}, replacing the requests. * If {@link ClientRequestContext#endpointGroup()} exists, a new {@link Endpoint} will be selected. @@ -278,7 +346,11 @@ private static final class State { @Nullable private Backoff lastBackoff; private int currentAttemptNoWithLastBackoff; + // Starting with 1 private int totalAttemptNo; + @Nullable + private ScheduledFuture currentAttemptTimeoutFuture; + private final List> pendingRetryTasks; State(RetryConfig config, long responseTimeoutMillis) { this.config = config; @@ -291,6 +363,42 @@ private static final class State { isTimeoutEnabled = true; } totalAttemptNo = 1; + + // todo(szymon) can initialize with null first + pendingRetryTasks = new LinkedList<>(); + } + + @Nullable + ScheduledFuture getCurrentAttemptTimeoutFuture() { + return currentAttemptTimeoutFuture; + } + + @Nullable + void setCurrentAttemptTimeoutFuture(@Nullable ScheduledFuture nextAttemptTimeoutFuture) { + if (currentAttemptTimeoutFuture != null) { + currentAttemptTimeoutFuture.cancel(false); + } + + currentAttemptTimeoutFuture = nextAttemptTimeoutFuture; + } + + + void addRetryTask(ScheduledFuture retryTask) { + pendingRetryTasks.add(retryTask); + } + + + void removeRetryTask(ScheduledFuture retryTask) { + final boolean retryTaskFound = pendingRetryTasks.remove(retryTask); + assert retryTaskFound; + } + + + void cancelAllRetryTasks() { + for (ScheduledFuture retryTask : pendingRetryTasks) { + // They will all call removeRetryTask() when they are done. + retryTask.cancel(false); + } } /** @@ -327,10 +435,17 @@ long actualResponseTimeoutMillis() { return TimeUnit.NANOSECONDS.toMillis(deadlineNanos - System.nanoTime()); } + boolean areAttemptsExhausted() { + return totalAttemptNo >= config.maxTotalAttempts(); + } + int currentAttemptNoWith(Backoff backoff) { - if (totalAttemptNo++ >= config.maxTotalAttempts()) { + // todo(szymon): is it okay to not increment totalAttemptNo for this check? + if (areAttemptsExhausted()) { return -1; } + + totalAttemptNo++; if (lastBackoff != backoff) { lastBackoff = backoff; currentAttemptNoWithLastBackoff = 1; diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfig.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfig.java index 31f7892815d..5f4d3c33900 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfig.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfig.java @@ -1,7 +1,7 @@ /* - * Copyright 2020 LINE Corporation + * Copyright 2025 LY Corporation * - * LINE Corporation licenses this file to you under the Apache License, + * LY Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * @@ -77,6 +77,7 @@ static RetryConfigBuilder builder0( private final int maxTotalAttempts; private final long responseTimeoutMillisForEachAttempt; + private final boolean abortAttemptOnPerAttemptResponseTimeout; private final int maxContentLength; @Nullable @@ -88,9 +89,11 @@ static RetryConfigBuilder builder0( @Nullable private RetryRuleWithContent fromRetryRule; - RetryConfig(RetryRule retryRule, int maxTotalAttempts, long responseTimeoutMillisForEachAttempt) { + RetryConfig(RetryRule retryRule, int maxTotalAttempts, long responseTimeoutMillisForEachAttempt, + boolean abortAttemptOnPerAttemptResponseTimeout) { this(requireNonNull(retryRule, "retryRule"), null, - maxTotalAttempts, responseTimeoutMillisForEachAttempt, 0); + maxTotalAttempts, responseTimeoutMillisForEachAttempt, + abortAttemptOnPerAttemptResponseTimeout, 0); checkArguments(maxTotalAttempts, responseTimeoutMillisForEachAttempt); } @@ -98,9 +101,11 @@ static RetryConfigBuilder builder0( RetryRuleWithContent retryRuleWithContent, int maxContentLength, int maxTotalAttempts, - long responseTimeoutMillisForEachAttempt) { + long responseTimeoutMillisForEachAttempt, + boolean abortAttemptOnPerAttemptResponseTimeout) { this(null, requireNonNull(retryRuleWithContent, "retryRuleWithContent"), - maxTotalAttempts, responseTimeoutMillisForEachAttempt, maxContentLength); + maxTotalAttempts, responseTimeoutMillisForEachAttempt, + abortAttemptOnPerAttemptResponseTimeout, maxContentLength); } private RetryConfig( @@ -108,12 +113,14 @@ private RetryConfig( @Nullable RetryRuleWithContent retryRuleWithContent, int maxTotalAttempts, long responseTimeoutMillisForEachAttempt, + boolean abortAttemptOnPerAttemptResponseTimeout, int maxContentLength) { checkArguments(maxTotalAttempts, responseTimeoutMillisForEachAttempt); this.retryRule = retryRule; this.retryRuleWithContent = retryRuleWithContent; this.maxTotalAttempts = maxTotalAttempts; this.responseTimeoutMillisForEachAttempt = responseTimeoutMillisForEachAttempt; + this.abortAttemptOnPerAttemptResponseTimeout = abortAttemptOnPerAttemptResponseTimeout; this.maxContentLength = maxContentLength; if (retryRuleWithContent == null) { fromRetryRuleWithContent = null; @@ -147,7 +154,8 @@ public RetryConfigBuilder toBuilder() { } return builder .maxTotalAttempts(maxTotalAttempts) - .responseTimeoutMillisForEachAttempt(responseTimeoutMillisForEachAttempt); + .responseTimeoutMillisForEachAttempt(responseTimeoutMillisForEachAttempt) + .abortAttemptOnPerAttemptResponseTimeout(abortAttemptOnPerAttemptResponseTimeout); } /** @@ -166,6 +174,10 @@ public long responseTimeoutMillisForEachAttempt() { return responseTimeoutMillisForEachAttempt; } + public boolean abortAttemptOnPerAttemptResponseTimeout() { + return abortAttemptOnPerAttemptResponseTimeout; + } + /** * Returns the {@link RetryRule} which was specified with {@link RetryConfig#builder(RetryRule)}. */ diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfigBuilder.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfigBuilder.java index cbda418e44f..662cabe0963 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfigBuilder.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfigBuilder.java @@ -1,7 +1,7 @@ /* - * Copyright 2020 LINE Corporation + * Copyright 2025 LY Corporation * - * LINE Corporation licenses this file to you under the Apache License, + * LY Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * @@ -36,6 +36,7 @@ public final class RetryConfigBuilder { private int maxTotalAttempts = Flags.defaultMaxTotalAttempts(); private long responseTimeoutMillisForEachAttempt = Flags.defaultResponseTimeoutMillis(); + private boolean abortAttemptOnPerAttemptResponseTimeout = true; private int maxContentLength; @Nullable @@ -96,6 +97,12 @@ public RetryConfigBuilder responseTimeoutMillisForEachAttempt(long responseTi return this; } + + public RetryConfigBuilder abortAttemptOnPerAttemptResponseTimeout(boolean abortAttemptOnPerAttemptResponseTimeout) { + this.abortAttemptOnPerAttemptResponseTimeout = abortAttemptOnPerAttemptResponseTimeout; + return this; + } + /** * Sets the specified {@link Duration} by converting responseTimeoutForEachAttempt to millis. */ @@ -116,14 +123,15 @@ public RetryConfigBuilder responseTimeoutForEachAttempt(Duration responseTime */ public RetryConfig build() { if (retryRule != null) { - return new RetryConfig<>(retryRule, maxTotalAttempts, responseTimeoutMillisForEachAttempt); + return new RetryConfig<>(retryRule, maxTotalAttempts, responseTimeoutMillisForEachAttempt, abortAttemptOnPerAttemptResponseTimeout); } assert retryRuleWithContent != null; return new RetryConfig<>( retryRuleWithContent, maxContentLength, maxTotalAttempts, - responseTimeoutMillisForEachAttempt); + responseTimeoutMillisForEachAttempt, + abortAttemptOnPerAttemptResponseTimeout); } @Override @@ -139,6 +147,7 @@ ToStringHelper toStringHelper() { .add("retryRuleWithContent", retryRuleWithContent) .add("maxTotalAttempts", maxTotalAttempts) .add("responseTimeoutMillisForEachAttempt", responseTimeoutMillisForEachAttempt) + .add("abortAttemptOnPerAttemptResponseTimeout", abortAttemptOnPerAttemptResponseTimeout) .add("maxContentLength", maxContentLength); } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java index 32146db4c68..ac0096eea52 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java @@ -1,7 +1,7 @@ /* - * Copyright 2017 LINE Corporation + * Copyright 2025 LY Corporation * - * LINE Corporation licenses this file to you under the Apache License, + * LY Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * @@ -18,6 +18,8 @@ import static com.linecorp.armeria.internal.client.ClientUtil.executeWithFallback; import static com.linecorp.armeria.internal.client.ClientUtil.initContextAndExecuteWithFallback; +import java.util.LinkedList; +import java.util.List; import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; import java.util.function.Function; @@ -30,6 +32,7 @@ import com.linecorp.armeria.common.Request; import com.linecorp.armeria.common.RpcRequest; import com.linecorp.armeria.common.RpcResponse; +import com.linecorp.armeria.common.annotation.Nullable; import com.linecorp.armeria.internal.client.ClientPendingThrowableUtil; import com.linecorp.armeria.internal.client.ClientRequestContextExtension; import com.linecorp.armeria.internal.common.util.StringUtil; @@ -73,7 +76,7 @@ public final class RetryingRpcClient extends AbstractRetryingClient newDecorator(RetryConfigMapping mapping) { @@ -144,26 +147,50 @@ public static RetryingRpcClientBuilder builder(RetryConfigMapping m protected RpcResponse doExecute(ClientRequestContext ctx, RpcRequest req) throws Exception { final CompletableFuture future = new CompletableFuture<>(); final RpcResponse res = RpcResponse.from(future); - doExecute0(ctx, req, res, future); + + final List attemptCtxs = new LinkedList<>(); + + doExecute0(ctx, req, res, future, attemptCtxs); return res; } private void doExecute0(ClientRequestContext ctx, RpcRequest req, - RpcResponse returnedRes, CompletableFuture future) { + RpcResponse returnedRes, CompletableFuture future, + List attemptCtxs) { final int totalAttempts = getTotalAttempts(ctx); final boolean initialAttempt = totalAttempts <= 1; if (returnedRes.isDone()) { // The response has been cancelled by the client before it receives a response, so stop retrying. - handleException(ctx, future, new CancellationException( - "the response returned to the client has been cancelled"), initialAttempt); + handleException(ctx, mappedRetryConfig(ctx), attemptCtxs, future, + new CancellationException( + "the response returned to the client has been cancelled"), initialAttempt); return; } - if (!setResponseTimeout(ctx)) { - handleException(ctx, future, ResponseTimeoutException.get(), initialAttempt); - return; + + final RetryConfig retryConfig = mappedRetryConfig(ctx); + final RetryRuleWithContent retryRule = + retryConfig.needsContentInRule() ? + retryConfig.retryRuleWithContent() : retryConfig.fromRetryRule(); + assert retryRule != null; + + if (retryConfig.abortAttemptOnPerAttemptResponseTimeout()) { + if (!setResponseTimeout(ctx)) { + handleException(ctx, mappedRetryConfig(ctx), attemptCtxs, future, + ResponseTimeoutException.get(), + initialAttempt); + return; + } + } else { + // As the retry config stays constant for the entire retry process, + // retryConfig.abortAttemptOnPerAttemptResponseTimeout() stays + // constant too. This means that if abortAttemptOnPerAttemptResponseTimeout() + // returns false, we will never change the response timeout. + // This means we do not have to reset it here to the actual + // response timeout. } final ClientRequestContext derivedCtx = newDerivedContext(ctx, null, req, initialAttempt); + attemptCtxs.add(derivedCtx); if (!initialAttempt) { derivedCtx.mutateAdditionalRequestHeaders( @@ -188,52 +215,134 @@ private void doExecute0(ClientRequestContext ctx, RpcRequest req, req, true); } - final RetryConfig retryConfig = mappedRetryConfig(ctx); - final RetryRuleWithContent retryRule = - retryConfig.needsContentInRule() ? - retryConfig.retryRuleWithContent() : retryConfig.fromRetryRule(); + if (!retryConfig.abortAttemptOnPerAttemptResponseTimeout()) { + scheduleNextAttemptTimeout(ctx, () -> { + // todo(szymon): Can this happen? + if (returnedRes.isDone()) { + return; + } + + try { + retryRule.shouldRetry(derivedCtx, res, + ResponseTimeoutException.get()).handle( + (decision, unused2) -> { + final Backoff backoff = decision != null ? decision.backoff() : null; + if (backoff != null && areAttemptsExhausted(ctx)) { + // Rule gave us allowance to continue but we + // do not have any attempts left. + // In that case we do not want to complete + // retrying immediately but let the pending + // requests complete. + return null; + } + + handleRetryDecision(ctx, derivedCtx, req, + returnedRes, + res, future, + attemptCtxs, + decision); + return null; + } + ); + } catch (Throwable cause) { + handleException(ctx, mappedRetryConfig(ctx), + attemptCtxs, future, cause, + false); + } + }, cause0 -> handleException(ctx, mappedRetryConfig(ctx), + attemptCtxs, future, + cause0, false), // todo + // (szymon): false or initialAttempt? + retryConfig.responseTimeoutMillisForEachAttempt()); + } + res.handle((unused1, cause) -> { + if (returnedRes.isDone()) { + // With hedging it could be that another attempt has already provided a response. + // If this is the case every cleanup has already been done. + return null; + } + try { - assert retryRule != null; retryRule.shouldRetry(derivedCtx, res, cause).handle((decision, unused3) -> { - final Backoff backoff = decision != null ? decision.backoff() : null; - if (backoff != null) { - final long nextDelay = getNextDelay(derivedCtx, backoff); - if (nextDelay < 0) { - onRetryComplete(ctx, derivedCtx, res, future); - return null; - } - - scheduleNextRetry(ctx, cause0 -> handleException(ctx, future, cause0, false), - () -> doExecute0(ctx, req, returnedRes, future), nextDelay); - } else { - onRetryComplete(ctx, derivedCtx, res, future); - } + handleRetryDecision(ctx, derivedCtx, req, returnedRes, res, future, + attemptCtxs, decision); return null; }); } catch (Throwable t) { - handleException(ctx, future, t, false); + handleException(ctx, mappedRetryConfig(ctx), attemptCtxs, future, t, false); } return null; }); } + private void handleRetryDecision(ClientRequestContext ctx, ClientRequestContext derivedCtx, + RpcRequest req, RpcResponse returnedRes, RpcResponse res, + CompletableFuture future, + List attemptCtxs, + @Nullable RetryDecision decision) { + final Backoff backoff = decision != null ? decision.backoff() : null; + if (backoff != null) { + final long nextDelay = getNextDelay(derivedCtx, backoff); + if (nextDelay < 0) { + onRetryComplete(ctx, derivedCtx, mappedRetryConfig(ctx), res, future, attemptCtxs); + } + + scheduleNextRetry(ctx, cause0 -> handleException(ctx, mappedRetryConfig(ctx), + attemptCtxs, future, cause0, + false), + () -> doExecute0(ctx, req, returnedRes, future, attemptCtxs), + nextDelay); + } else { + onRetryComplete(ctx, derivedCtx, mappedRetryConfig(ctx), res, future, attemptCtxs); + } + } + private static void onRetryComplete(ClientRequestContext ctx, ClientRequestContext derivedCtx, - RpcResponse res, CompletableFuture future) { - onRetryingComplete(ctx); + RetryConfig retryConfig, + RpcResponse res, CompletableFuture future, + List attemptCtxs) { + if (future.isDone()) { + return; + } + + onRetryingComplete(ctx, derivedCtx); + final HttpRequest actualHttpReq = derivedCtx.request(); if (actualHttpReq != null) { ctx.updateRequest(actualHttpReq); } + future.complete(res); + + cancelPendingAttempts(ctx, retryConfig, attemptCtxs); } - private static void handleException(ClientRequestContext ctx, CompletableFuture future, + private static void handleException(ClientRequestContext ctx, + RetryConfig retryConfig, + List attemptCtxs, + CompletableFuture future, Throwable cause, boolean endRequestLog) { future.completeExceptionally(cause); if (endRequestLog) { ctx.logBuilder().endRequest(cause); } ctx.logBuilder().endResponse(cause); + + cancelPendingAttempts(ctx, retryConfig, attemptCtxs); + } + + private static void cancelPendingAttempts(ClientRequestContext ctx, RetryConfig retryConfig, + List attemptCtxs) { + if (!retryConfig.abortAttemptOnPerAttemptResponseTimeout()) { + for (ClientRequestContext attemptCtx : attemptCtxs) { + if (!attemptCtx.isCancelled()) { + attemptCtx.cancel(); + } + } + + // Cancel in-flight retry tasks. + cancelRetryTasks(ctx); + } } } diff --git a/core/src/main/java/com/linecorp/armeria/common/logging/DefaultRequestLog.java b/core/src/main/java/com/linecorp/armeria/common/logging/DefaultRequestLog.java index 250eed2fae3..f34c13b9804 100644 --- a/core/src/main/java/com/linecorp/armeria/common/logging/DefaultRequestLog.java +++ b/core/src/main/java/com/linecorp/armeria/common/logging/DefaultRequestLog.java @@ -1,7 +1,7 @@ /* - * Copyright 2016 LINE Corporation + * Copyright 2025 LY Corporation * - * LINE Corporation licenses this file to you under the Apache License, + * LY Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * @@ -595,6 +595,15 @@ private void propagateRequestSideLog(RequestLogAccess child) { }); } + @Override + public void endResponseWithChild(RequestLogAccess child) { + checkState(!hasLastChild, "last child is already added"); + checkState(children != null && !children.isEmpty(), "at least one child should be already added"); + checkState(children.stream().anyMatch(c -> c == child), "child is not a child of this log: %s", child); + hasLastChild = true; + propagateResponseSideLog(child.partial()); + } + @Override public void endResponseWithLastChild() { checkState(!hasLastChild, "last child is already added"); @@ -1068,8 +1077,8 @@ private void endRequest0(@Nullable Throwable requestCause, long requestEndTimeNa final int deferredFlags; if (requestCause != null) { // Will auto-fill request content and its preview if request has failed. - deferredFlags = this.deferredFlags & (~(RequestLogProperty.REQUEST_CONTENT.flag() | - RequestLogProperty.REQUEST_CONTENT_PREVIEW.flag())); + deferredFlags = this.deferredFlags & ~(RequestLogProperty.REQUEST_CONTENT.flag() | + RequestLogProperty.REQUEST_CONTENT_PREVIEW.flag()); } else { deferredFlags = this.deferredFlags; } @@ -1419,8 +1428,8 @@ private void endResponse0(@Nullable Throwable responseCause, long responseEndTim final int deferredFlags; if (responseCause != null) { // Will auto-fill response content and its preview if response has failed. - deferredFlags = this.deferredFlags & (~(RequestLogProperty.RESPONSE_CONTENT.flag() | - RequestLogProperty.RESPONSE_CONTENT_PREVIEW.flag())); + deferredFlags = this.deferredFlags & ~(RequestLogProperty.RESPONSE_CONTENT.flag() | + RequestLogProperty.RESPONSE_CONTENT_PREVIEW.flag()); } else { deferredFlags = this.deferredFlags; } diff --git a/core/src/main/java/com/linecorp/armeria/common/logging/RequestLogBuilder.java b/core/src/main/java/com/linecorp/armeria/common/logging/RequestLogBuilder.java index 61915cfaaa6..c450bbca547 100644 --- a/core/src/main/java/com/linecorp/armeria/common/logging/RequestLogBuilder.java +++ b/core/src/main/java/com/linecorp/armeria/common/logging/RequestLogBuilder.java @@ -1,7 +1,7 @@ /* - * Copyright 2016 LINE Corporation + * Copyright 2025 LY Corporation * - * LINE Corporation licenses this file to you under the Apache License, + * LY Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * @@ -437,6 +437,8 @@ void session(@Nullable Channel channel, SessionProtocol sessionProtocol, @Nullab */ void addChild(RequestLogAccess child); + void endResponseWithChild(RequestLogAccess child); + /** * Fills the response-side logs from the last added child. Note that already collected properties * in the child log will be propagated immediately. diff --git a/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java b/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java index a4f65d598fc..bfd7ed3ec08 100644 --- a/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java +++ b/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java @@ -1,7 +1,7 @@ /* - * Copyright 2018 LINE Corporation + * Copyright 2025 LY Corporation * - * LINE Corporation licenses this file to you under the Apache License, + * LY Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * @@ -235,6 +235,11 @@ public static ClientRequestContext newDerivedContext(ClientRequestContext ctx, derived = ctx.newDerivedContext(id, req, rpcReq, ctx.endpoint()); } + // We want to add the request log of the derived context as a child to the request log of the context + // we are deriving from. + // For that we copy over all log properties from the parent log to the derived log + // and add future actions to copy over content (previews). + // We are doing this because final RequestLogAccess parentLog = ctx.log(); final RequestLog partial = parentLog.partial(); final RequestLogBuilder logBuilder = derived.logBuilder(); @@ -277,7 +282,10 @@ public static ClientRequestContext newDerivedContext(ClientRequestContext ctx, .thenAccept(requestLog -> logBuilder.responseContentPreview( requestLog.responseContentPreview())); } + + // We finally add the derived log as a child of the parent log. ctx.logBuilder().addChild(derived.log()); + return derived; } diff --git a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java new file mode 100644 index 00000000000..454e13542e2 --- /dev/null +++ b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java @@ -0,0 +1,612 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.armeria.it.client.retry; + +import static com.linecorp.armeria.client.retry.AbstractRetryingClient.ARMERIA_RETRY_COUNT; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.assertj.core.api.Assertions.fail; +import static org.awaitility.Awaitility.await; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; + +import org.apache.thrift.TApplicationException; +import org.apache.thrift.TException; +import org.apache.thrift.async.AsyncMethodCallback; +import org.apache.thrift.transport.TTransportException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import com.linecorp.armeria.client.ClientRequestContext; +import com.linecorp.armeria.client.ClientRequestContextCaptor; +import com.linecorp.armeria.client.Clients; +import com.linecorp.armeria.client.ResponseCancellationException; +import com.linecorp.armeria.client.ResponseTimeoutException; +import com.linecorp.armeria.client.endpoint.EndpointGroup; +import com.linecorp.armeria.client.endpoint.EndpointSelectionStrategy; +import com.linecorp.armeria.client.retry.Backoff; +import com.linecorp.armeria.client.retry.RetryConfig; +import com.linecorp.armeria.client.retry.RetryRule; +import com.linecorp.armeria.client.retry.RetryingRpcClient; +import com.linecorp.armeria.client.thrift.ThriftClients; +import com.linecorp.armeria.common.RpcRequest; +import com.linecorp.armeria.common.RpcResponse; +import com.linecorp.armeria.common.SessionProtocol; +import com.linecorp.armeria.common.annotation.Nullable; +import com.linecorp.armeria.common.logging.RequestLog; +import com.linecorp.armeria.common.logging.RequestLogAccess; +import com.linecorp.armeria.common.logging.RequestLogProperty; +import com.linecorp.armeria.server.ServerBuilder; +import com.linecorp.armeria.server.ServiceRequestContext; +import com.linecorp.armeria.server.logging.LoggingService; +import com.linecorp.armeria.server.thrift.THttpService; +import com.linecorp.armeria.testing.junit5.server.ServerExtension; + +import testing.thrift.main.HelloService; +import testing.thrift.main.HelloService.Iface; + +class RetryingRpcClientWithHedgingTest { + private static final long LOOSING_SERVER_RESPONSE_DELAY_MILLIS = 50; + + private static class TestServer extends ServerExtension { + private CountDownLatch responseLatch = new CountDownLatch(1); + private CountDownLatch requestLatch = new CountDownLatch(1); + private volatile HelloService.Iface serviceHandler; + + TestServer() { + super(true); + } + + @Override + protected void configure(ServerBuilder sb) throws Exception { + sb.service("/thrift", THttpService.of((Iface) name -> getServiceHandler().hello(name)) + .decorate( + (delegate, ctx, req) -> { + getRequestLatch().countDown(); + getResponseLatch().await(); + return delegate.serve(ctx, req); + } + ) + .decorate(LoggingService.newDecorator()) + ); + } + + private void reset() { + serviceHandler = mock(HelloService.Iface.class); + responseLatch = new CountDownLatch(1); + requestLatch = new CountDownLatch(1); + } + + public HelloService.Iface getServiceHandler() { + return serviceHandler; + } + + public CountDownLatch getResponseLatch() { + return responseLatch; + } + + public CountDownLatch getRequestLatch() { + return requestLatch; + } + + public void unlatchResponse() { + responseLatch.countDown(); + } + + public void waitForFirstRequest() { + try { + requestLatch.await(); + } catch (InterruptedException e) { + fail(e); + } + } + } + + @RegisterExtension + private static final TestServer server1 = new TestServer(); + @RegisterExtension + private static final TestServer server2 = new TestServer(); + @RegisterExtension + private static final TestServer server3 = new TestServer(); + + @BeforeEach + void beforeEach() { + server1.reset(); + server2.reset(); + server3.reset(); + } + + @AfterEach + void afterEach() { + // Unblock all servers. + server1.unlatchResponse(); + server2.unlatchResponse(); + server3.unlatchResponse(); + } + + /* + todo(szymon): Tests for hedging. + todo(szymon): Sometimes returnErrorWhenSecondErrors blocks in a second iteration. + Each test: + - are connections closed on the client and server side? + - are decorators before invoked with the right request context? + - are decorators after invoked for every attempt? Are they invoked when we abort pending attempts? + - the order and timing of the request send out (via client request contexts). Do we respect the + per-attempt timeouts? + - does a server receive a cancellation signal when it is lost? + Test cases: + Positive outcome: + - First server wins, before the per attempt timeout + - First server wins, after the per attempt timeout + - Third server wins, before the per attempt timeout + - Third server wins, after the per attempt timeout + - First, second and third requests are issued in a row (Backoff.fixed(0)). + - First, second and third requests are issued with backoff 0, 100, 0ms (non-monotonic). + - First, second and third request each arrive earlier than the timeout. + Negative outcome: + - Request times out even before first server answers + - Request times out shortly before third server, who should win, answers + - First, second and third requests are issued; second request errors out; should abort every + other request + - Interaction with CircuitBreaker? + */ + @Test + void execute_hedging_lastWins() throws Exception { + when(server1.getServiceHandler().hello(anyString())).thenReturn("server1"); + when(server2.getServiceHandler().hello(anyString())).thenReturn("server2"); + when(server3.getServiceHandler().hello(anyString())).thenReturn("server3"); + + final HelloService.AsyncIface client = helloClientThreeEndpoints( + RetryConfig. + builderForRpc( + RetryRule + .builder() + .onTimeoutException() + .thenBackoff(Backoff.withoutDelay()) + ) + .maxTotalAttempts(3) + .responseTimeoutMillisForEachAttempt(50) + .abortAttemptOnPerAttemptResponseTimeout(false) + .build() + ); + + final CompletableFuture result; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + result = asyncHelloWith(client); + ctx = captor.get(); + } + + server1.waitForFirstRequest(); + server2.waitForFirstRequest(); + server3.waitForFirstRequest(); + + // Let server 3 win. + server3.unlatchResponse(); + Thread.sleep(LOOSING_SERVER_RESPONSE_DELAY_MILLIS); + server1.unlatchResponse(); + server2.unlatchResponse(); + + await() + .untilAsserted(() -> { + assertValidServerRequestContext(server1, 1); + assertValidServerRequestContext(server2, 2); + assertValidServerRequestContext(server3, 3); + + assertThat(result.get()).isEqualTo("server3"); + assertValidClientRequestContext( + ctx,GET_VERIFY_RESPONSE_HAS_CONTENT.apply("server3"), VERIFY_REQUEST_CANCELLED , + VERIFY_REQUEST_CANCELLED, + GET_VERIFY_RESPONSE_HAS_CONTENT.apply("server3") + ); + }); + } + + @Test + void execute_hedging_thirdWinsEventAfterPerAttemptTimeout() throws Exception { + when(server1.getServiceHandler().hello(anyString())).thenReturn("server1"); + when(server2.getServiceHandler().hello(anyString())).thenReturn("server2"); + when(server3.getServiceHandler().hello(anyString())).thenReturn("server3"); + + final HelloService.AsyncIface client = helloClientThreeEndpoints( + RetryConfig. + builderForRpc( + RetryRule + .builder() + .onTimeoutException() + .thenBackoff(Backoff.withoutDelay()) + ) + .maxTotalAttempts(3) + .responseTimeoutMillisForEachAttempt(50) + .abortAttemptOnPerAttemptResponseTimeout(false) + .build() + ); + + + final CompletableFuture result; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + result = asyncHelloWith(client); + ctx = captor.get(); + } + + // After 3 * , we should have called the + // per-attempt timeout handler a third time. It should not cancel + // any request but should just wait for some request to finish (or for the request timeout). + Thread.sleep(3 * 50 + 100); + + server1.waitForFirstRequest(); + server2.waitForFirstRequest(); + server3.waitForFirstRequest(); + + // Let the third server win. + server3.unlatchResponse(); + Thread.sleep(LOOSING_SERVER_RESPONSE_DELAY_MILLIS); + server1.unlatchResponse(); + server2.unlatchResponse(); + + await() + .untilAsserted(() -> { + assertValidServerRequestContext(server1, 1); + assertValidServerRequestContext(server2, 2); + assertValidServerRequestContext(server3, 3); + + assertThat(result.get()).isEqualTo("server3"); + assertValidClientRequestContext(ctx,GET_VERIFY_RESPONSE_HAS_CONTENT.apply("server3"), + VERIFY_REQUEST_CANCELLED, VERIFY_REQUEST_CANCELLED, + GET_VERIFY_RESPONSE_HAS_CONTENT.apply("server3")); + }); + } + + @Test + void execute_hedging_thirdWinsEvenWhenFirstErrors() throws Exception { + final String errorMessage = "it's a me! non-retried error!"; + when(server1.getServiceHandler().hello(anyString())).thenThrow( + new TApplicationException(TApplicationException.INTERNAL_ERROR, errorMessage)); + + when(server2.getServiceHandler().hello(anyString())).thenReturn("server2"); + when(server3.getServiceHandler().hello(anyString())).thenReturn("server3"); + + final HelloService.AsyncIface client = helloClientThreeEndpoints( + RetryConfig. + builderForRpc( + RetryRule + .builder() + .onTimeoutException() + .thenBackoff(Backoff.withoutDelay()) + ) + .maxTotalAttempts(3) + .responseTimeoutMillisForEachAttempt(1) + .abortAttemptOnPerAttemptResponseTimeout(false) + .build() + ); + + final CompletableFuture result; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + result = asyncHelloWith(client); + ctx = captor.get(); + } + + server1.waitForFirstRequest(); + server2.waitForFirstRequest(); + server3.waitForFirstRequest(); + + server3.unlatchResponse(); + Thread.sleep(LOOSING_SERVER_RESPONSE_DELAY_MILLIS); + server1.unlatchResponse(); + server2.unlatchResponse(); + + await().untilAsserted(() -> { + assertValidServerRequestContext(server1, 1); + assertValidServerRequestContext(server2, 2); + assertValidServerRequestContext(server3, 3); + + assertThat(result.get()).isEqualTo("server3"); + assertValidClientRequestContext( + ctx,GET_VERIFY_RESPONSE_HAS_CONTENT.apply("server3"), VERIFY_REQUEST_CANCELLED, + VERIFY_REQUEST_CANCELLED, GET_VERIFY_RESPONSE_HAS_CONTENT.apply("server3") + ); + }); + } + + @Test + void execute_hedging_returnErrorWhenSecondErrors() throws Exception { + when(server1.getServiceHandler().hello(anyString())).thenReturn("server1"); + final String errorMessage = "it's a me! non-retried error!"; + when(server2.getServiceHandler().hello(anyString())).thenThrow( + new TApplicationException(TApplicationException.INTERNAL_ERROR, errorMessage)); + when(server3.getServiceHandler().hello(anyString())).thenReturn("server3"); + + final HelloService.AsyncIface client = helloClientThreeEndpoints( + RetryConfig. + builderForRpc( + RetryRule + .builder() + .onTimeoutException() + .thenBackoff(Backoff.withoutDelay()) + ) + .maxTotalAttempts(3) + .responseTimeoutMillisForEachAttempt(1) + .abortAttemptOnPerAttemptResponseTimeout(false) + .build() + ); + + final CompletableFuture result; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + result = asyncHelloWith(client); + ctx = captor.get(); + } + + server1.waitForFirstRequest(); + server2.waitForFirstRequest(); + server3.waitForFirstRequest(); + + + // Let the second server win. + server2.unlatchResponse(); + Thread.sleep(LOOSING_SERVER_RESPONSE_DELAY_MILLIS); + server1.unlatchResponse(); + server3.unlatchResponse(); + + await().untilAsserted(() -> { + assertValidServerRequestContext(server1, 1); + assertValidServerRequestContext(server2, 2); + assertValidServerRequestContext(server3, 3); + + assertThat(result) + .isCompletedExceptionally(); + + assertThatExceptionOfType(ExecutionException.class) + .isThrownBy(result::get) + .satisfies(e -> + assertThat(e.getCause()) + .isInstanceOf(TApplicationException.class) + .hasMessageContaining(errorMessage) + .satisfies(cause -> assertThat( + ((TApplicationException) cause).getType()) + .isEqualTo(TApplicationException.INTERNAL_ERROR))); + + + assertValidClientRequestContext( + ctx, GET_VERIFY_RESPONSE_HAS_APPLICATION_EXCEPTION.apply(TApplicationException.INTERNAL_ERROR, + errorMessage), + VERIFY_REQUEST_CANCELLED, + GET_VERIFY_RESPONSE_HAS_APPLICATION_EXCEPTION.apply(TApplicationException.INTERNAL_ERROR, + errorMessage), + VERIFY_REQUEST_CANCELLED); + }); + } + + + @Test + void execute_hedging_honorResponseTimeout() throws TException { + when(server1.getServiceHandler().hello(anyString())).thenReturn("server1"); + when(server2.getServiceHandler().hello(anyString())).thenReturn("server2"); + + final HelloService.AsyncIface client = helloClientThreeEndpoints( + RetryConfig. + builderForRpc( + RetryRule + .builder() + .onTimeoutException() + .thenBackoff(Backoff.withoutDelay()) + ) + .maxTotalAttempts(3) + .responseTimeoutMillisForEachAttempt(300) + .abortAttemptOnPerAttemptResponseTimeout(false) + .build(), 300 + 100 // Lets give the client 100ms to schedule the second attempt. + ); + + final CompletableFuture result; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + result = asyncHelloWith(client); + ctx = captor.get(); + } + // The first request is issued "immediately". + // The second request issued after the per-attempt request timeout which is 300ms. + // The third request would be issued after the per-attempt request timeout which is 300ms. + // However, the request timeout is set to 400ms, so we should never call this. + await().untilAsserted(() -> { + assertThat(result) + .isCompletedExceptionally(); + + assertThatExceptionOfType(ExecutionException.class) + .isThrownBy(result::get) + .satisfies(cause -> + { + + assertThat(cause.getCause()).isInstanceOf(TTransportException.class); + assertThat(cause.getCause().getCause()).isInstanceOf(ResponseTimeoutException.class); + } + ); + + // The first request timed out and the second was cancelled when it was detected. + assertValidClientRequestContext( + ctx, VERIFY_RESPONSE_TIMEOUT, VERIFY_RESPONSE_TIMEOUT, VERIFY_REQUEST_CANCELLED, null); + }); + + server1.unlatchResponse(); + server2.unlatchResponse(); + server3.unlatchResponse(); + + await().untilAsserted(() -> { + assertValidServerRequestContext(server1, 1); + assertValidServerRequestContext(server2, 2); + assertNoServerRequestContext(server3); + }); + + await().pollDelay(1000, TimeUnit.MILLISECONDS).untilAsserted(() -> { + assertNoServerRequestContext(server3); + }); + } + + private CompletableFuture asyncHelloWith(HelloService.AsyncIface client) throws TException { + final CompletableFuture future = new CompletableFuture<>(); + try { + client.hello("hello", new AsyncMethodCallback() { + @Override + public void onComplete(String response) { + future.complete(response); + } + + @Override + public void onError(Exception exception) { + future.completeExceptionally(exception); + } + }); + } catch (TException e) { + future.completeExceptionally(e); + } + + return future; + } + + private static HelloService.AsyncIface helloClientThreeEndpoints(RetryConfig config) { + return helloClientThreeEndpoints(config, 10000); + } + + private static HelloService.AsyncIface helloClientThreeEndpoints(RetryConfig config, + long responseTimeoutMillis) { + return ThriftClients.builder( + SessionProtocol.HTTP, + EndpointGroup.of( + EndpointSelectionStrategy.roundRobin(), + server1.endpoint(SessionProtocol.HTTP), + server2.endpoint(SessionProtocol.HTTP), + server3.endpoint(SessionProtocol.HTTP) + ) + ) + .responseTimeoutMillis(responseTimeoutMillis) + .path("/thrift") + .rpcDecorator(RetryingRpcClient.builder(config).newDecorator()) + .build(HelloService.AsyncIface.class); + } + + private interface RequestLogVerifier extends Consumer {} + private static final RequestLogVerifier VERIFY_REQUEST_CANCELLED = + log -> { + assertThat(log.responseCause()).isInstanceOf(TTransportException.class); + assertThat(log.responseCause().getCause()).isInstanceOf(ResponseCancellationException.class); + }; + + private static final RequestLogVerifier VERIFY_RESPONSE_TIMEOUT = + log -> { + assertThat(log.responseCause()).isInstanceOf(TTransportException.class); + assertThat(log.responseCause().getCause()).isInstanceOf(ResponseTimeoutException.class); + }; + + private static final Function GET_VERIFY_RESPONSE_HAS_CONTENT = + expectedResponseContent -> log -> { + assertThat(log.responseContent()).isInstanceOf(RpcResponse.class); + assertThat((CompletionStage) log.responseContent()) + .isCompletedWithValue(expectedResponseContent); + }; + + private static final BiFunction + GET_VERIFY_RESPONSE_HAS_APPLICATION_EXCEPTION = + (expectedType, expectedMessage) -> log -> { + assertThat(log.responseCause()).isInstanceOf(TApplicationException.class); + final TApplicationException cause = (TApplicationException) log.responseCause(); + assertThat(cause.getType()).isEqualTo(expectedType); + assertThat(cause.getMessage()).contains(expectedMessage); + }; + + private void assertValidClientRequestContext(ClientRequestContext ctx, + RequestLogVerifier logVerifierCtx, + RequestLogVerifier logVerifierServer1, + RequestLogVerifier logVerifierServer2, + @Nullable RequestLogVerifier logVerifierServer3 + ) { + assertThat(ctx.log().isComplete()).isTrue(); + assertThat(ctx.log().children()).hasSize(logVerifierServer3 == null ? 2 : 3); + final RequestLog log = ctx.log().getIfAvailable(RequestLogProperty.RESPONSE_CONTENT, + RequestLogProperty.RESPONSE_CAUSE, + RequestLogProperty.REQUEST_HEADERS); + assertThat(log).isNotNull(); + logVerifierCtx.accept(log); + assertValidChildLog(ctx.log().children().get(0), 1, logVerifierServer1); + assertValidChildLog(ctx.log().children().get(1), 2, logVerifierServer2); + if (logVerifierServer3 != null) { + assertValidChildLog(ctx.log().children().get(2), 3, logVerifierServer3); + } + } + + void assertValidChildLog(RequestLogAccess logAccess, int attemptNumber, + RequestLogVerifier requestLogVerifier) { + assertThat(logAccess.isComplete()).isTrue(); + // After the check right above, all properties of the RequestLog should be available. + final @Nullable RequestLog log = logAccess.getIfAvailable(RequestLogProperty.RESPONSE_CONTENT, + RequestLogProperty.RESPONSE_CAUSE, + RequestLogProperty.REQUEST_HEADERS); + assertThat(log).isNotNull(); + + if (attemptNumber > 1) { + assertThat(log.requestHeaders().getInt(ARMERIA_RETRY_COUNT)).isEqualTo(attemptNumber - 1); + } else { + assertThat(log.requestHeaders().contains(ARMERIA_RETRY_COUNT)).isFalse(); + } + + requestLogVerifier.accept(log); + } + + private void assertValidServerRequestContext(ServerExtension server, int attemptNumber) { + assertThat(server.requestContextCaptor().size()).isEqualTo(1); + + final ServiceRequestContext sctx; + + try { + sctx = server.requestContextCaptor().take(); + } catch (InterruptedException e) { + fail(e); + return; + } + + assertThat(sctx.log().isComplete()).isTrue(); + + final RequestLog slog = sctx.log().getIfAvailable(RequestLogProperty.REQUEST_HEADERS, + RequestLogProperty.REQUEST_CONTENT); + + assertThat(slog).isNotNull(); + if (attemptNumber > 1) { + assertThat(slog.requestHeaders().getInt(ARMERIA_RETRY_COUNT)).isEqualTo(attemptNumber - 1); + } else { + assertThat(slog.requestHeaders().contains(ARMERIA_RETRY_COUNT)).isFalse(); + } + + assertThat(slog.requestContent()).isInstanceOf(RpcRequest.class); + assertThat(((RpcRequest) slog.requestContent()).params().get(0)).isEqualTo("hello"); + } + + private void assertNoServerRequestContext(ServerExtension server) { + assertThat(server.requestContextCaptor().size()).isEqualTo(0); + } + +} From 27e22d71c379de54aa96610f6e0bc3640bf20383 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Mon, 2 Jun 2025 12:27:25 +0200 Subject: [PATCH 02/36] refactor: improve variable naming and reduce method parameter count in RetryingClient --- .../client/retry/AbstractRetryingClient.java | 8 +- .../armeria/client/retry/RetryingClient.java | 322 ++++++++++-------- .../client/retry/RetryingRpcClient.java | 2 +- 3 files changed, 193 insertions(+), 139 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java index a46d12b914e..ea63f74ee38 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java @@ -111,10 +111,10 @@ protected static void onRetryingComplete(ClientRequestContext ctx) { * This should be called when retrying is finished. */ protected static void onRetryingComplete(ClientRequestContext ctx, - ClientRequestContext derivedCtx) { + ClientRequestContext attemptCtx) { // Cancel every in-flight attempt. // The following is not long true. - ctx.logBuilder().endResponseWithChild(derivedCtx.log()); + ctx.logBuilder().endResponseWithChild(attemptCtx.log()); } /** @@ -321,10 +321,10 @@ protected static boolean areAttemptsExhausted(ClientRequestContext ctx) { } /** - * Creates a new derived {@link ClientRequestContext}, replacing the requests. + * Creates a new derived {@link ClientRequestContext} for a retrying attempt, replacing the requests. * If {@link ClientRequestContext#endpointGroup()} exists, a new {@link Endpoint} will be selected. */ - protected static ClientRequestContext newDerivedContext(ClientRequestContext ctx, + protected static ClientRequestContext newAttemptContext(ClientRequestContext ctx, @Nullable HttpRequest req, @Nullable RpcRequest rpcReq, boolean initialAttempt) { diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index 5694a0716d7..b92c8303041 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -1,7 +1,7 @@ /* - * Copyright 2017 LINE Corporation + * Copyright 2025 LY Corporation * - * LINE Corporation licenses this file to you under the Apache License, + * LY Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * @@ -246,7 +246,8 @@ protected HttpResponse doExecute(ClientRequestContext ctx, HttpRequest req) thro final HttpResponse res = HttpResponse.of(responseFuture, ctx.eventLoop()); if (ctx.exchangeType().isRequestStreaming()) { final HttpRequestDuplicator reqDuplicator = req.toDuplicator(ctx.eventLoop().withoutContext(), 0); - doExecute0(ctx, reqDuplicator, req, res, responseFuture); + doExecute0(new RetryingContext(mappedRetryConfig(ctx), ctx, reqDuplicator, req, res, + responseFuture)); } else { req.aggregate(AggregationOptions.usePooledObjects(ctx.alloc(), ctx.eventLoop())) .handle((agg, cause) -> { @@ -254,7 +255,8 @@ protected HttpResponse doExecute(ClientRequestContext ctx, HttpRequest req) thro handleException(ctx, null, responseFuture, cause, true); } else { final HttpRequestDuplicator reqDuplicator = new AggregatedHttpRequestDuplicator(agg); - doExecute0(ctx, reqDuplicator, req, res, responseFuture); + doExecute0(new RetryingContext(mappedRetryConfig(ctx), ctx, reqDuplicator, req, res, + responseFuture)); } return null; }); @@ -262,16 +264,21 @@ protected HttpResponse doExecute(ClientRequestContext ctx, HttpRequest req) thro return res; } - private void doExecute0(ClientRequestContext ctx, HttpRequestDuplicator rootReqDuplicator, - HttpRequest originalReq, HttpResponse returnedRes, - CompletableFuture future) { + private void doExecute0(RetryingContext retryingContext) { + final RetryConfig config = retryingContext.config(); + final ClientRequestContext ctx = retryingContext.ctx(); + final HttpRequestDuplicator rootReqDuplicator = retryingContext.reqDuplicator(); + final HttpRequest originalReq = retryingContext.req(); + final HttpResponse returnedRes = retryingContext.res(); + final CompletableFuture returnedResFuture = retryingContext.resFuture(); + final int totalAttempts = getTotalAttempts(ctx); final boolean initialAttempt = totalAttempts <= 1; - // The request or response has been aborted by the client before it receives a response, + // The request or attemptRes has been aborted by the client before it receives a attemptRes, // so stop retrying. if (originalReq.whenComplete().isCompletedExceptionally()) { originalReq.whenComplete().handle((unused, cause) -> { - handleException(ctx, rootReqDuplicator, future, cause, initialAttempt); + handleException(retryingContext, cause, initialAttempt); return null; }); return; @@ -284,212 +291,201 @@ private void doExecute0(ClientRequestContext ctx, HttpRequestDuplicator rootReqD } else { abortCause = AbortedStreamException.get(); } - handleException(ctx, rootReqDuplicator, future, abortCause, initialAttempt); + handleException(retryingContext, abortCause, initialAttempt); return null; }); return; } if (!setResponseTimeout(ctx)) { - handleException(ctx, rootReqDuplicator, future, ResponseTimeoutException.get(), initialAttempt); + handleException(retryingContext, ResponseTimeoutException.get(), initialAttempt); return; } - final HttpRequest duplicateReq; + final HttpRequest attemptReq; if (initialAttempt) { - duplicateReq = rootReqDuplicator.duplicate(); + attemptReq = rootReqDuplicator.duplicate(); } else { final RequestHeadersBuilder newHeaders = originalReq.headers().toBuilder(); newHeaders.setInt(ARMERIA_RETRY_COUNT, totalAttempts - 1); - duplicateReq = rootReqDuplicator.duplicate(newHeaders.build()); + attemptReq = rootReqDuplicator.duplicate(newHeaders.build()); } - final ClientRequestContext derivedCtx; + final ClientRequestContext attemptCtx; try { - derivedCtx = newDerivedContext(ctx, duplicateReq, ctx.rpcRequest(), initialAttempt); + attemptCtx = newAttemptContext(ctx, attemptReq, ctx.rpcRequest(), initialAttempt); } catch (Throwable t) { - handleException(ctx, rootReqDuplicator, future, t, initialAttempt); + handleException(retryingContext, t, initialAttempt); return; } - final HttpRequest ctxReq = derivedCtx.request(); - assert ctxReq != null; - final HttpResponse response; - final ClientRequestContextExtension ctxExtension = derivedCtx.as(ClientRequestContextExtension.class); - if (!initialAttempt && ctxExtension != null && derivedCtx.endpoint() == null) { + final HttpRequest attemptCtxReq = attemptCtx.request(); + assert attemptCtxReq != null; + final HttpResponse attemptRes; + final ClientRequestContextExtension ctxExtension = attemptCtx.as(ClientRequestContextExtension.class); + if (!initialAttempt && ctxExtension != null && attemptCtx.endpoint() == null) { // clear the pending throwable to retry endpoint selection - ClientPendingThrowableUtil.removePendingThrowable(derivedCtx); + ClientPendingThrowableUtil.removePendingThrowable(attemptCtx); // if the endpoint hasn't been selected, try to initialize the ctx with a new endpoint/event loop - response = initContextAndExecuteWithFallback( + attemptRes = initContextAndExecuteWithFallback( unwrap(), ctxExtension, HttpResponse::of, - (context, cause) -> HttpResponse.ofFailure(cause), ctxReq, false); + (context, cause) -> HttpResponse.ofFailure(cause), attemptCtxReq, false); } else { - response = executeWithFallback(unwrap(), derivedCtx, - (context, cause) -> HttpResponse.ofFailure(cause), ctxReq, false); + attemptRes = executeWithFallback(unwrap(), attemptCtx, + (context, cause) -> HttpResponse.ofFailure(cause), attemptCtxReq, false); } - final RetryConfig config = mappedRetryConfig(ctx); if (!ctx.exchangeType().isResponseStreaming() || config.requiresResponseTrailers()) { - response.aggregate().handle((aggregated, cause) -> { + attemptRes.aggregate().handle((attemptAggResponse, cause) -> { if (cause != null) { - derivedCtx.logBuilder().endRequest(cause); - derivedCtx.logBuilder().endResponse(cause); - handleResponseWithoutContent(config, ctx, rootReqDuplicator, originalReq, returnedRes, - future, derivedCtx, HttpResponse.ofFailure(cause), cause); + attemptCtx.logBuilder().endRequest(cause); + attemptCtx.logBuilder().endResponse(cause); + handleResponseWithoutContent(retryingContext, attemptCtx, + HttpResponse.ofFailure(cause), + cause); } else { - completeLogIfBytesNotTransferred(aggregated, derivedCtx); - derivedCtx.log().whenAvailable(RequestLogProperty.RESPONSE_END_TIME).thenRun(() -> { - handleAggregatedResponse(config, ctx, rootReqDuplicator, originalReq, returnedRes, - future, derivedCtx, aggregated); + completeAttemptLogIfBytesNotTransferred(attemptCtx, attemptAggResponse); + attemptCtx.log().whenAvailable(RequestLogProperty.RESPONSE_END_TIME).thenRun(() -> { + handleAggregatedResponse(retryingContext, attemptCtx, attemptAggResponse); }); } return null; }); } else { - handleStreamingResponse(config, ctx, rootReqDuplicator, originalReq, returnedRes, - future, derivedCtx, response); + handleStreamingResponse(retryingContext, attemptCtx, attemptRes); } } // TODO(ikhoon): Add a request-scope class such as RetryRequestContext to avoid passing too many parameters. - private void handleResponseWithoutContent(RetryConfig config, ClientRequestContext ctx, - HttpRequestDuplicator rootReqDuplicator, HttpRequest originalReq, - HttpResponse returnedRes, CompletableFuture future, - ClientRequestContext derivedCtx, HttpResponse response, - @Nullable Throwable responseCause) { - if (responseCause != null) { - responseCause = Exceptions.peel(responseCause); + private void handleResponseWithoutContent(RetryingContext retryingContext, + ClientRequestContext attemptCtx, HttpResponse attemptRes, + @Nullable Throwable attemptResCause) { + if (attemptResCause != null) { + attemptResCause = Exceptions.peel(attemptResCause); } try { - final RetryRule retryRule = retryRule(config); - final CompletionStage f = retryRule.shouldRetry(derivedCtx, responseCause); + final RetryRule retryRule = retryRule(retryingContext.config()); + final CompletionStage f = retryRule.shouldRetry(attemptCtx, attemptResCause); f.handle((decision, shouldRetryCause) -> { warnIfExceptionIsRaised(retryRule, shouldRetryCause); - handleRetryDecision(decision, ctx, derivedCtx, rootReqDuplicator, - originalReq, returnedRes, future, response); + handleRetryDecision(retryingContext, decision, attemptCtx, attemptRes); return null; }); } catch (Throwable cause) { - response.abort(); - handleException(ctx, rootReqDuplicator, future, cause, false); + attemptRes.abort(); + handleException(retryingContext, cause, false); } } - private void handleStreamingResponse(RetryConfig retryConfig, ClientRequestContext ctx, - HttpRequestDuplicator rootReqDuplicator, - HttpRequest originalReq, HttpResponse returnedRes, - CompletableFuture future, - ClientRequestContext derivedCtx, - HttpResponse response) { - final SplitHttpResponse splitResponse = response.split(); - splitResponse.headers().handle((headers, headersCause) -> { + private void handleStreamingResponse(RetryingContext retryingContext, + ClientRequestContext attemptCtx, + HttpResponse attemptRes) { + final SplitHttpResponse attemptSplitRes = attemptRes.split(); + attemptSplitRes.headers().handle((headers, headersCause) -> { final Throwable responseCause; if (headersCause == null) { - final RequestLog log = derivedCtx.log().getIfAvailable(RequestLogProperty.RESPONSE_CAUSE); + final RequestLog log = attemptCtx.log().getIfAvailable(RequestLogProperty.RESPONSE_CAUSE); responseCause = log != null ? log.responseCause() : null; } else { responseCause = Exceptions.peel(headersCause); } - completeLogIfBytesNotTransferred(response, headers, derivedCtx, responseCause); - - derivedCtx.log().whenAvailable(RequestLogProperty.RESPONSE_HEADERS).thenRun(() -> { - if (retryConfig.needsContentInRule() && responseCause == null) { - final HttpResponse response0 = HttpResponse.of(headers, splitResponse.body()); - final HttpResponseDuplicator duplicator = - response0.toDuplicator(derivedCtx.eventLoop().withoutContext(), - derivedCtx.maxResponseLength()); + + completeAttemptLogIfBytesNotTransferred(attemptCtx, attemptRes, headers, responseCause); + + attemptCtx.log().whenAvailable(RequestLogProperty.RESPONSE_HEADERS).thenRun(() -> { + if (retryingContext.config().needsContentInRule() && responseCause == null) { + final HttpResponse attemptUnsplitRes = HttpResponse.of(headers, attemptSplitRes.body()); + final HttpResponseDuplicator attemptResDuplicator = + attemptUnsplitRes.toDuplicator(attemptCtx.eventLoop().withoutContext(), + attemptCtx.maxResponseLength()); try { - final TruncatingHttpResponse truncatingHttpResponse = - new TruncatingHttpResponse(duplicator.duplicate(), - retryConfig.maxContentLength()); - final HttpResponse duplicated = duplicator.duplicate(); - duplicator.close(); + final TruncatingHttpResponse attemptTruncatedRes = + new TruncatingHttpResponse(attemptResDuplicator.duplicate(), + retryingContext.config().maxContentLength()); + final HttpResponse attemptDuplicatedRes = attemptResDuplicator.duplicate(); + attemptResDuplicator.close(); final RetryRuleWithContent ruleWithContent = - retryConfig.retryRuleWithContent(); + retryingContext.config().retryRuleWithContent(); assert ruleWithContent != null; - ruleWithContent.shouldRetry(derivedCtx, truncatingHttpResponse, null) + ruleWithContent.shouldRetry(attemptCtx, attemptTruncatedRes, null) .handle((decision, cause) -> { warnIfExceptionIsRaised(ruleWithContent, cause); - truncatingHttpResponse.abort(); - handleRetryDecision(decision, ctx, derivedCtx, rootReqDuplicator, - originalReq, returnedRes, future, duplicated); + attemptTruncatedRes.abort(); + handleRetryDecision(retryingContext, decision, attemptCtx, + attemptDuplicatedRes); return null; }); } catch (Throwable cause) { - duplicator.abort(cause); - handleException(ctx, rootReqDuplicator, future, cause, false); + attemptResDuplicator.abort(cause); + handleException(retryingContext, cause, false); } } else { - final HttpResponse response0; + final HttpResponse attemptUnsplitRes; if (responseCause != null) { - splitResponse.body().abort(responseCause); - response0 = HttpResponse.ofFailure(responseCause); + attemptSplitRes.body().abort(responseCause); + attemptUnsplitRes = HttpResponse.ofFailure(responseCause); } else { - response0 = HttpResponse.of(headers, splitResponse.body()); + attemptUnsplitRes = HttpResponse.of(headers, attemptSplitRes.body()); } - handleResponseWithoutContent(retryConfig, ctx, rootReqDuplicator, originalReq, returnedRes, - future, derivedCtx, response0, responseCause); + handleResponseWithoutContent(retryingContext, attemptCtx, attemptUnsplitRes, responseCause); } }); return null; }); } - private void handleAggregatedResponse(RetryConfig retryConfig, ClientRequestContext ctx, - HttpRequestDuplicator rootReqDuplicator, - HttpRequest originalReq, HttpResponse returnedRes, - CompletableFuture future, - ClientRequestContext derivedCtx, - AggregatedHttpResponse aggregatedRes) { - if (retryConfig.needsContentInRule()) { - final RetryRuleWithContent ruleWithContent = retryConfig.retryRuleWithContent(); + private void handleAggregatedResponse(RetryingContext retryingContext, + ClientRequestContext attemptCtx, + AggregatedHttpResponse attemptAggRes) { + if (retryingContext.config().needsContentInRule()) { + final RetryRuleWithContent ruleWithContent = retryingContext.config().retryRuleWithContent(); assert ruleWithContent != null; try { - ruleWithContent.shouldRetry(derivedCtx, aggregatedRes.toHttpResponse(), null) + ruleWithContent.shouldRetry(attemptCtx, attemptAggRes.toHttpResponse(), null) .handle((decision, cause) -> { warnIfExceptionIsRaised(ruleWithContent, cause); - handleRetryDecision( - decision, ctx, derivedCtx, rootReqDuplicator, originalReq, - returnedRes, future, aggregatedRes.toHttpResponse()); + handleRetryDecision(retryingContext, + decision, attemptCtx, attemptAggRes.toHttpResponse()); return null; }); } catch (Throwable cause) { - handleException(ctx, rootReqDuplicator, future, cause, false); + handleException(retryingContext, cause, false); } return; } - handleResponseWithoutContent(retryConfig, ctx, rootReqDuplicator, originalReq, returnedRes, - future, derivedCtx, aggregatedRes.toHttpResponse(), null); + + handleResponseWithoutContent(retryingContext, attemptCtx, attemptAggRes.toHttpResponse(), + null); } - private static void completeLogIfBytesNotTransferred(AggregatedHttpResponse response, - ClientRequestContext ctx) { - if (!ctx.log().isAvailable(RequestLogProperty.REQUEST_FIRST_BYTES_TRANSFERRED_TIME)) { - final RequestLogBuilder logBuilder = ctx.logBuilder(); + private static void completeAttemptLogIfBytesNotTransferred(ClientRequestContext attemptCtx, + AggregatedHttpResponse attemptAggRes) { + if (!attemptCtx.log().isAvailable(RequestLogProperty.REQUEST_FIRST_BYTES_TRANSFERRED_TIME)) { + final RequestLogBuilder logBuilder = attemptCtx.logBuilder(); logBuilder.endRequest(); - logBuilder.responseHeaders(response.headers()); - if (!response.trailers().isEmpty()) { - logBuilder.responseTrailers(response.trailers()); + logBuilder.responseHeaders(attemptAggRes.headers()); + if (!attemptAggRes.trailers().isEmpty()) { + logBuilder.responseTrailers(attemptAggRes.trailers()); } logBuilder.endResponse(); } } - private static void completeLogIfBytesNotTransferred( - HttpResponse response, @Nullable ResponseHeaders headers, ClientRequestContext ctx, - @Nullable Throwable responseCause) { - if (!ctx.log().isAvailable(RequestLogProperty.REQUEST_FIRST_BYTES_TRANSFERRED_TIME)) { - final RequestLogBuilder logBuilder = ctx.logBuilder(); - if (responseCause != null) { - logBuilder.endRequest(responseCause); - logBuilder.endResponse(responseCause); + private static void completeAttemptLogIfBytesNotTransferred( + ClientRequestContext attemptCtx, HttpResponse attemptRes, @Nullable ResponseHeaders attemptResHeaders, + @Nullable Throwable attemptResCause) { + if (!attemptCtx.log().isAvailable(RequestLogProperty.REQUEST_FIRST_BYTES_TRANSFERRED_TIME)) { + final RequestLogBuilder logBuilder = attemptCtx.logBuilder(); + if (attemptResCause != null) { + logBuilder.endRequest(attemptResCause); + logBuilder.endResponse(attemptResCause); } else { logBuilder.endRequest(); - if (headers != null) { - logBuilder.responseHeaders(headers); + if (attemptResHeaders != null) { + logBuilder.responseHeaders(attemptResHeaders); } - response.whenComplete().handle((unused, cause) -> { + attemptRes.whenComplete().handle((unused, cause) -> { if (cause != null) { logBuilder.endResponse(cause); } else { @@ -507,6 +503,13 @@ private static void warnIfExceptionIsRaised(Object retryRule, @Nullable Throwabl } } + private static void handleException(RetryingContext retryingContext, Throwable cause, + boolean endRequestLog) { + handleException( + retryingContext.ctx(), retryingContext.reqDuplicator(), retryingContext.resFuture(), + cause, endRequestLog); + } + private static void handleException(ClientRequestContext ctx, @Nullable HttpRequestDuplicator rootReqDuplicator, CompletableFuture future, Throwable cause, @@ -521,31 +524,28 @@ private static void handleException(ClientRequestContext ctx, ctx.logBuilder().endResponse(cause); } - private void handleRetryDecision(@Nullable RetryDecision decision, ClientRequestContext ctx, - ClientRequestContext derivedCtx, HttpRequestDuplicator rootReqDuplicator, - HttpRequest originalReq, HttpResponse returnedRes, - CompletableFuture future, HttpResponse originalRes) { + private void handleRetryDecision(RetryingContext retryingContext, @Nullable RetryDecision decision, + ClientRequestContext attemptCtx, HttpResponse attemptRes) { final Backoff backoff = decision != null ? decision.backoff() : null; if (backoff != null) { - final long millisAfter = useRetryAfter ? getRetryAfterMillis(derivedCtx) : -1; - final long nextDelay = getNextDelay(ctx, backoff, millisAfter); + final long millisAfter = useRetryAfter ? getRetryAfterMillis(attemptCtx) : -1; + final long nextDelay = getNextDelay(retryingContext.ctx(), backoff, millisAfter); if (nextDelay >= 0) { - abortResponse(originalRes, derivedCtx); + abortAttempt(attemptCtx, attemptRes); scheduleNextRetry( - ctx, cause -> handleException(ctx, rootReqDuplicator, future, cause, false), - () -> doExecute0(ctx, rootReqDuplicator, originalReq, returnedRes, future), + retryingContext.ctx(), cause -> handleException(retryingContext, cause, false), + () -> doExecute0(retryingContext), nextDelay); return; } } - onRetryingComplete(ctx); - future.complete(originalRes); - rootReqDuplicator.close(); + + onRetryingComplete(retryingContext, attemptCtx, attemptRes); } - private static void abortResponse(HttpResponse originalRes, ClientRequestContext derivedCtx) { + private static void abortAttempt(ClientRequestContext attemptCtx, HttpResponse originalRes) { // Set response content with null to make sure that the log is complete. - final RequestLogBuilder logBuilder = derivedCtx.logBuilder(); + final RequestLogBuilder logBuilder = attemptCtx.logBuilder(); logBuilder.responseContent(null, null); logBuilder.responseContentPreview(null); originalRes.abort(); @@ -591,4 +591,58 @@ private static RetryRule retryRule(RetryConfig retryConfig) { return rule; } } + + + void onRetryingComplete(RetryingContext retryingContext, + ClientRequestContext attemptCtx, + HttpResponse attemptRes) { + onRetryingComplete(retryingContext.ctx(), attemptCtx); + retryingContext.resFuture().complete(attemptRes); + retryingContext.reqDuplicator().close(); + } + + private static class RetryingContext { + private final ClientRequestContext ctx; + private final HttpRequestDuplicator reqDuplicator; + private final HttpRequest req; + private final HttpResponse res; + private final CompletableFuture resFuture; + private final RetryConfig retryConfig; + + RetryingContext(RetryConfig retryConfig, ClientRequestContext ctx, + HttpRequestDuplicator reqDuplicator, + HttpRequest req, HttpResponse res, + CompletableFuture resFuture) { + this.retryConfig = retryConfig; + this.ctx = ctx; + this.reqDuplicator = reqDuplicator; + this.req = req; + this.res = res; + this.resFuture = resFuture; + } + + RetryConfig config() { + return retryConfig; + } + + ClientRequestContext ctx() { + return ctx; + } + + HttpRequestDuplicator reqDuplicator() { + return reqDuplicator; + } + + HttpRequest req() { + return req; + } + + HttpResponse res() { + return res; + } + + CompletableFuture resFuture() { + return resFuture; + } + } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java index ac0096eea52..0fbeb930363 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java @@ -189,7 +189,7 @@ private void doExecute0(ClientRequestContext ctx, RpcRequest req, // response timeout. } - final ClientRequestContext derivedCtx = newDerivedContext(ctx, null, req, initialAttempt); + final ClientRequestContext derivedCtx = newAttemptContext(ctx, null, req, initialAttempt); attemptCtxs.add(derivedCtx); if (!initialAttempt) { From 057e441ac2a8404965b4d56f9b20fa1066360d0f Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Fri, 6 Jun 2025 18:22:45 +0200 Subject: [PATCH 03/36] [WIP] feat: add hedging ability to retrying (rpc) client --- .../client/retry/AbstractRetryingClient.java | 512 ++++++++++++------ .../armeria/client/retry/RetryConfig.java | 49 +- .../client/retry/RetryConfigBuilder.java | 29 +- .../armeria/client/retry/RetryScheduler.java | 235 ++++++++ .../armeria/client/retry/RetryingClient.java | 175 ++++-- .../client/retry/RetryingRpcClient.java | 238 +++----- .../client/retry/RetryingClientTest.java | 28 +- .../RetryingClientWithContextAwareTest.java | 16 +- .../retry/RetryingClientWithHedgingTest.java | 453 ++++++++++++++++ .../server/ServiceRequestContextCaptor.java | 9 +- .../client/retry/RetryingRpcClientTest.java | 15 +- .../RetryingRpcClientWithHedgingTest.java | 10 +- 12 files changed, 1355 insertions(+), 414 deletions(-) create mode 100644 core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java create mode 100644 core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java index ea63f74ee38..7f9ac7fc85e 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java @@ -19,8 +19,7 @@ import static com.google.common.base.Preconditions.checkState; import static java.util.Objects.requireNonNull; -import java.util.LinkedList; -import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -28,10 +27,10 @@ import org.slf4j.LoggerFactory; import com.linecorp.armeria.client.Client; -import com.linecorp.armeria.client.ClientFactory; import com.linecorp.armeria.client.ClientRequestContext; import com.linecorp.armeria.client.Endpoint; import com.linecorp.armeria.client.SimpleDecoratingClient; +import com.linecorp.armeria.client.retry.AbstractRetryingClient.RetrySchedulabilityDecision.Rationale; import com.linecorp.armeria.common.HttpHeaderNames; import com.linecorp.armeria.common.HttpRequest; import com.linecorp.armeria.common.Request; @@ -43,7 +42,6 @@ import io.netty.util.AsciiString; import io.netty.util.AttributeKey; -import io.netty.util.concurrent.ScheduledFuture; /** * A {@link Client} decorator that handles failures of remote invocation and retries requests. @@ -82,12 +80,67 @@ public abstract class AbstractRetryingClient rfp = getResponseFuturePair(ctx); + + if (!ctx.eventLoop().inEventLoop()) { + ctx.eventLoop().execute(() -> { + try { + doFirstExecute(ctx, req, rfp.response(), rfp.responseFuture()); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } else { + doFirstExecute(ctx, req, rfp.response(), rfp.responseFuture()); + } + + return rfp.response(); + } + + private void doFirstExecute(ClientRequestContext ctx, I req, O res, CompletableFuture resFuture) + throws Exception { final RetryConfig config = mapping.get(ctx, req); requireNonNull(config, "mapping.get() returned null"); - final State state = new State(config, ctx.responseTimeoutMillis()); + final State state; + if (ctx.responseTimeoutMillis() <= 0 || ctx.responseTimeoutMillis() == Long.MAX_VALUE) { + final RetryScheduler scheduler = new RetryScheduler(ctx.eventLoop()); + state = new State(config, scheduler); + } else { + final long responseTimeoutTimeNanos = + System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(ctx.responseTimeoutMillis()); + final RetryScheduler scheduler = new RetryScheduler(ctx.eventLoop(), responseTimeoutTimeNanos); + state = new State(config, scheduler, responseTimeoutTimeNanos); + } + ctx.setAttr(STATE, state); - return doExecute(ctx, req); + + state.whenRetryingComplete().handle((f, t) -> { + state.scheduler().close(); + return null; + }); + + doExecute(ctx, req, res, resFuture); + } + + abstract ResponseFuturePair getResponseFuturePair(ClientRequestContext ctx); + + protected static class ResponseFuturePair { + private final O response; + private final CompletableFuture responseFuture; + + ResponseFuturePair(O response, CompletableFuture responseFuture) { + this.response = requireNonNull(response, "response"); + this.responseFuture = requireNonNull(responseFuture, "responseFuture"); + } + + public O response() { + return response; + } + + public CompletableFuture responseFuture() { + return responseFuture; + } } /** @@ -101,20 +154,23 @@ protected final RetryConfigMapping mapping() { * Invoked by {@link #execute(ClientRequestContext, Request)} * after the deadline for response timeout is set. */ - protected abstract O doExecute(ClientRequestContext ctx, I req) throws Exception; - - protected static void onRetryingComplete(ClientRequestContext ctx) { - ctx.logBuilder().endResponseWithLastChild(); - } + protected abstract void doExecute(ClientRequestContext ctx, I req, O res, CompletableFuture resFuture) + throws Exception; /** * This should be called when retrying is finished. */ protected static void onRetryingComplete(ClientRequestContext ctx, ClientRequestContext attemptCtx) { - // Cancel every in-flight attempt. - // The following is not long true. ctx.logBuilder().endResponseWithChild(attemptCtx.log()); + + state(ctx).complete(attemptCtx); + } + + protected static void onRetryingCompleteExceptionally(ClientRequestContext ctx, + Throwable cause) { + ctx.logBuilder().endResponse(cause); + state(ctx).completeExceptionally(cause); } /** @@ -151,82 +207,76 @@ protected final RetryRuleWithContent retryRuleWithContent() { return retryRuleWithContent; } - /** - * Schedules next retry. - */ - protected static void scheduleNextRetry(ClientRequestContext ctx, - Consumer actionOnException, - Runnable retryTask, long nextDelayMillis) { - try { - if (nextDelayMillis == 0) { - ctx.eventLoop().execute(retryTask); - } else { - @SuppressWarnings("unchecked") - final ScheduledFuture scheduledFuture = (ScheduledFuture) ctx - .eventLoop().schedule(retryTask, nextDelayMillis, TimeUnit.MILLISECONDS); - - scheduledFuture.addListener(future -> { - state(ctx).removeRetryTask(scheduledFuture); - if (future.isCancelled()) { - // future is cancelled when the client factory is closed. - actionOnException.accept(new IllegalStateException( - ClientFactory.class.getSimpleName() + " has been closed.")); - } else if (future.cause() != null) { - // Other unexpected exceptions. - actionOnException.accept(future.cause()); - } - }); - - - state(ctx).addRetryTask(scheduledFuture); + protected static void onAttemptStarted(ClientRequestContext ctx, + ClientRequestContext attemptCtx, + Consumer<@Nullable Throwable> onAttemptAbortedHandler + ) { + requireNonNull(ctx, "ctx"); + requireNonNull(attemptCtx, "attemptCtx"); + // todo(szymon) [Q]: should we track the attemptCtxs and check if we have multiple attempts started? + state(ctx).incrementPendingAttemptCount(); + state(ctx).whenRetryingComplete().handle((winningAttemptCtx, cause) -> { + if (attemptCtx == winningAttemptCtx) { + return null; } - } catch (Throwable t) { - actionOnException.accept(t); - } + + onAttemptAbortedHandler.accept(cause); + return null; + }); + logger.debug("onAttemptStarted: {}. numRemainingPendingAttempts = {}, hasScheduledRetryTask={}", + ctx, state(ctx).numPendingAttempts, state(ctx).scheduler().hasScheduledRetryTask()); + } - protected static void scheduleNextAttemptTimeout(ClientRequestContext ctx, - Runnable attemptTimeoutTask, - Consumer actionOnException, - long nextAttemptTimeoutMillis) { - checkArgument(nextAttemptTimeoutMillis > 0, "nextAttemptTimeoutMillis must be > 0"); - - try { - final ScheduledFuture nextAttemptTimeoutFuture = (ScheduledFuture) ctx - .eventLoop() - .schedule(attemptTimeoutTask, nextAttemptTimeoutMillis, TimeUnit.MILLISECONDS); - - nextAttemptTimeoutFuture.addListener(thisAttemptTimeoutFuture -> { - if (thisAttemptTimeoutFuture.isCancelled()) { - if (state(ctx).getCurrentAttemptTimeoutFuture() == thisAttemptTimeoutFuture) { - actionOnException.accept( - new IllegalStateException(ClientFactory.class.getSimpleName() + " has been " - + "closed.")); - } else { - // It is fine that attempt timeout tasks are cancelled to be replaced with the next - // attempt timeout task. - } - } else if (thisAttemptTimeoutFuture.cause() != null) { - actionOnException.accept(thisAttemptTimeoutFuture.cause()); - } - }); + protected boolean onAttemptEnded(ClientRequestContext ctx) { + final int numRemainingPendingAttempts = state(ctx).decrementPendingAttemptCount(); + logger.debug("onAttemptEnded: {}. numRemainingPendingAttempts = {}, hasScheduledRetryTask={}", + ctx, numRemainingPendingAttempts, state(ctx).scheduler().hasScheduledRetryTask()); + return numRemainingPendingAttempts > 0 || state(ctx).scheduler().hasScheduledRetryTask(); + } - state(ctx).setCurrentAttemptTimeoutFuture(nextAttemptTimeoutFuture); - } catch (Throwable t) { - actionOnException.accept(t); - } + // an attempt did not trigger a retry. when does the attempt end? + + protected static boolean isRetryingComplete(ClientRequestContext ctx) { + requireNonNull(ctx, "ctx"); + return state(ctx).whenRetryingComplete().isDone(); } - protected static void cancelRetryTasks(ClientRequestContext ctx) { - state(ctx).cancelAllRetryTasks(); + protected static void scheduleNextRetry(ClientRequestContext ctx, + Runnable retryTask, + long retryTimeNanos, + Consumer actionOnException) { + requireNonNull(ctx, "ctx"); + requireNonNull(actionOnException, "actionOnException"); + requireNonNull(retryTask, "retryTask"); + + final RetryScheduler scheduler = state(ctx).scheduler(); + + scheduleNextRetry(ctx, retryTask, retryTimeNanos, + scheduler.getEarliestNextRetryTimeNanos(), actionOnException); } - protected static void cancelAttemptTimeout(ClientRequestContext ctx) { - final ScheduledFuture currentAttemptTimeoutFuture = state(ctx).getCurrentAttemptTimeoutFuture(); - if (currentAttemptTimeoutFuture != null) { - currentAttemptTimeoutFuture.cancel(false); - state(ctx).setCurrentAttemptTimeoutFuture(null); - } + protected static void scheduleNextRetry(ClientRequestContext ctx, + Runnable retryTask, + long retryTimeNanos, + long earliestNextRetryTimeFromServerNanos, + Consumer actionOnException) { + requireNonNull(ctx, "ctx"); + requireNonNull(actionOnException, "actionOnException"); + requireNonNull(retryTask, "retryTask"); + + final RetryScheduler scheduler = state(ctx).scheduler(); + + scheduler.addEarliestNextRetryTimeNanos(earliestNextRetryTimeFromServerNanos); + scheduler.schedule(retryTask, retryTimeNanos, actionOnException); + } + + protected static void addEarliestNextRetryTimeNanos(ClientRequestContext ctx, + long earliestNextRetryTimeNanos) { + requireNonNull(ctx, "ctx"); + final RetryScheduler scheduler = state(ctx).scheduler(); + scheduler.addEarliestNextRetryTimeNanos(earliestNextRetryTimeNanos); + scheduler.rescheduleCurrentRetryTaskIfTooEarly(); } /** @@ -235,9 +285,9 @@ protected static void cancelAttemptTimeout(ClientRequestContext ctx) { * @return {@code true} if the response timeout is set, {@code false} if it can't be set due to the timeout */ @SuppressWarnings("MethodMayBeStatic") // Intentionally left non-static for better user experience. - protected final boolean setResponseTimeout(ClientRequestContext ctx) { + protected final boolean updateResponseTimeout(ClientRequestContext ctx) { requireNonNull(ctx, "ctx"); - final long responseTimeoutMillis = state(ctx).responseTimeoutMillis(); + final long responseTimeoutMillis = state(ctx).responseTimeoutMillisForAttempt(); if (responseTimeoutMillis < 0) { return false; } else if (responseTimeoutMillis == 0) { @@ -249,14 +299,81 @@ protected final boolean setResponseTimeout(ClientRequestContext ctx) { } } + protected final RetrySchedulabilityDecision canScheduleWith(ClientRequestContext ctx, Backoff backoff) { + return canScheduleWith(ctx, backoff, -1); + } + + protected final RetrySchedulabilityDecision canScheduleWith(ClientRequestContext ctx, Backoff backoff, + long millisFromServer) { + requireNonNull(ctx, "ctx"); + requireNonNull(backoff, "backoff"); + final State state = state(ctx); + final RetryScheduler scheduler = state.scheduler(); + final long nowTimeNanos = System.nanoTime(); + + final long earliestNextRetryTimeNanos = Math.max(scheduler.getEarliestNextRetryTimeNanos(), + millisFromServer < 0 ? + scheduler.getEarliestNextRetryTimeNanos() : + nowTimeNanos + TimeUnit.MILLISECONDS.toNanos( + millisFromServer)); + + if (state.timeoutForWholeRetryEnabled()) { + if (earliestNextRetryTimeNanos > state.responseTimeoutTimeNanos()) { + logger.debug("The earliest next retry time {} is after the response timeout time {}. " + + "Not scheduling a retry.", + earliestNextRetryTimeNanos, state.responseTimeoutTimeNanos()); + return new RetrySchedulabilityDecision(Rationale.EXCEEDS_RESPONSE_TIMEOUT, Long.MAX_VALUE, + scheduler.getEarliestNextRetryTimeNanos()); + } + } + + final int nextAttemptNo = state.nextAttemptNoWithBackoff(backoff); + if (nextAttemptNo < 0) { + logger.debug("Exceeded the default number of max attempt: {}", state.config.maxTotalAttempts()); + return new RetrySchedulabilityDecision(Rationale.NO_MORE_ATTEMPTS, Long.MAX_VALUE, + earliestNextRetryTimeNanos); + } + + final long nextDelay = backoff.nextDelayMillis(nextAttemptNo); + if (nextDelay < 0) { + logger.debug("Exceeded the number of max attempts in the backoff: {}", backoff); + return new RetrySchedulabilityDecision(Rationale.NO_MORE_ATTEMPTS_IN_BACKOFF, Long.MAX_VALUE, + earliestNextRetryTimeNanos); + } + + final long nextRetryTimeNanos = Math.max(nowTimeNanos + TimeUnit.MILLISECONDS.toNanos(nextDelay), + earliestNextRetryTimeNanos); + + if (state(ctx).timeoutForWholeRetryEnabled()) { + if (nextRetryTimeNanos > state(ctx).responseTimeoutTimeNanos()) { + logger.debug("The next retry time {} is after the response timeout time {}. " + + "Not scheduling a retry.", + nextRetryTimeNanos, state(ctx).responseTimeoutTimeNanos()); + return new RetrySchedulabilityDecision(Rationale.EXCEEDS_RESPONSE_TIMEOUT, Long.MAX_VALUE, + earliestNextRetryTimeNanos); + } + } + + if (scheduler.hasAlreadyRetryScheduledBefore(nextRetryTimeNanos, earliestNextRetryTimeNanos)) { + return new RetrySchedulabilityDecision(Rationale.HAS_EARLIER_RETRY, + nextRetryTimeNanos, earliestNextRetryTimeNanos); + } + + // todo(szymon): do we want to wait for acquisition when we schedule it? + state(ctx).acquireAttemptNoWithCurrentBackoff(backoff); + + return new RetrySchedulabilityDecision(Rationale.SCHEDULABLE, nextRetryTimeNanos, + earliestNextRetryTimeNanos); + } + /** * Returns the next delay which retry will be made after. The delay will be: * *

{@code Math.min(responseTimeoutMillis, Backoff.nextDelayMillis(int))} * * @return the number of milliseconds to wait for before attempting a retry. -1 if the - * {@code currentAttemptNo} exceeds the {@code maxAttempts} or the {@code nextDelay} is after - * the moment which timeout happens. + * {@code currentAttemptNo} exceeds the {@code maxAttempts} or the {@code nextDelay} is after + * the moment which timeout happens. */ protected final long getNextDelay(ClientRequestContext ctx, Backoff backoff) { return getNextDelay(ctx, backoff, -1); @@ -266,37 +383,20 @@ protected final long getNextDelay(ClientRequestContext ctx, Backoff backoff) { * Returns the next delay which retry will be made after. The delay will be: * *

{@code Math.min(responseTimeoutMillis, Math.max(Backoff.nextDelayMillis(int), - * millisAfterFromServer))} + * millisFromServer))} + *

+ * If delay is non-negative, we expect a retry to be issued and so a retry attempt is consumed. * - * @return the number of milliseconds to wait for before attempting a retry. -1 if the - * {@code currentAttemptNo} exceeds the {@code maxAttempts} or the {@code nextDelay} is after - * the moment which timeout happens. + * @return the number of milliseconds to wait for before attempting a retry. -1 if either + * - {@code currentAttemptNo} exceeds the {@code maxAttempts} or + * - the {@code nextDelay} is after the moment which timeout happens or + * - there is a pending retry task that is shorter than the next delay. */ @SuppressWarnings("MethodMayBeStatic") // Intentionally left non-static for better user experience. - protected final long getNextDelay(ClientRequestContext ctx, Backoff backoff, long millisAfterFromServer) { - requireNonNull(ctx, "ctx"); - requireNonNull(backoff, "backoff"); - final State state = state(ctx); - final int currentAttemptNo = state.currentAttemptNoWith(backoff); - - if (currentAttemptNo < 0) { - logger.debug("Exceeded the default number of max attempt: {}", state.config.maxTotalAttempts()); - return -1; - } - - long nextDelay = backoff.nextDelayMillis(currentAttemptNo); - if (nextDelay < 0) { - logger.debug("Exceeded the number of max attempts in the backoff: {}", backoff); - return -1; - } - - nextDelay = Math.max(nextDelay, millisAfterFromServer); - if (state.timeoutForWholeRetryEnabled() && nextDelay > state.actualResponseTimeoutMillis()) { - // The nextDelay will be after the moment which timeout will happen. So return just -1. - return -1; - } - - return nextDelay; + protected final long getNextDelay(ClientRequestContext ctx, Backoff backoff, + long millisFromServer) { + // todo(szymon): map to canschedulewith. + return -1; } /** @@ -311,15 +411,6 @@ protected static int getTotalAttempts(ClientRequestContext ctx) { return state.totalAttemptNo; } - protected static boolean areAttemptsExhausted(ClientRequestContext ctx) { - final State state = ctx.attr(STATE); - if (state == null) { - // todo(szymon): when does this happen? - return true; // No retrying is in progress. - } - return state.areAttemptsExhausted(); - } - /** * Creates a new derived {@link ClientRequestContext} for a retrying attempt, replacing the requests. * If {@link ClientRequestContext#endpointGroup()} exists, a new {@link Endpoint} will be selected. @@ -337,83 +428,112 @@ private static State state(ClientRequestContext ctx) { return state; } - private static final class State { + protected static final class RetrySchedulabilityDecision { + enum Rationale { + SCHEDULABLE(true), + NO_MORE_ATTEMPTS(false), + NO_MORE_ATTEMPTS_IN_BACKOFF(false), + EXCEEDS_RESPONSE_TIMEOUT(false), + HAS_EARLIER_RETRY(false); - private final RetryConfig config; - private final long deadlineNanos; - private final boolean isTimeoutEnabled; - - @Nullable - private Backoff lastBackoff; - private int currentAttemptNoWithLastBackoff; - // Starting with 1 - private int totalAttemptNo; - @Nullable - private ScheduledFuture currentAttemptTimeoutFuture; - private final List> pendingRetryTasks; - - State(RetryConfig config, long responseTimeoutMillis) { - this.config = config; + private final boolean canSchedule; - if (responseTimeoutMillis <= 0 || responseTimeoutMillis == Long.MAX_VALUE) { - deadlineNanos = 0; - isTimeoutEnabled = false; - } else { - deadlineNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(responseTimeoutMillis); - isTimeoutEnabled = true; + Rationale(boolean canSchedule) { + this.canSchedule = canSchedule; } - totalAttemptNo = 1; - // todo(szymon) can initialize with null first - pendingRetryTasks = new LinkedList<>(); + public boolean canSchedule() { + return canSchedule; + } } - @Nullable - ScheduledFuture getCurrentAttemptTimeoutFuture() { - return currentAttemptTimeoutFuture; + private final Rationale outcome; + private final long nextRetryTimeNanos; + private final long earliestNextRetryTimeNanos; + + RetrySchedulabilityDecision(Rationale outcome, long nextRetryTimeNanos, + long earliestNextRetryTimeNanos) { + requireNonNull(outcome, "outcome"); + checkArgument(earliestNextRetryTimeNanos <= nextRetryTimeNanos); + + this.outcome = outcome; + this.nextRetryTimeNanos = nextRetryTimeNanos; + this.earliestNextRetryTimeNanos = earliestNextRetryTimeNanos; } - @Nullable - void setCurrentAttemptTimeoutFuture(@Nullable ScheduledFuture nextAttemptTimeoutFuture) { - if (currentAttemptTimeoutFuture != null) { - currentAttemptTimeoutFuture.cancel(false); - } + long nextRetryTimeNanos() { + return nextRetryTimeNanos; + } - currentAttemptTimeoutFuture = nextAttemptTimeoutFuture; + long earliestNextRetryTimeNanos() { + return earliestNextRetryTimeNanos; } + boolean canSchedule() { + return outcome.canSchedule(); + } - void addRetryTask(ScheduledFuture retryTask) { - pendingRetryTasks.add(retryTask); + @Override + public String toString() { + return "RetrySchedulabilityDecision{" + + "outcome=" + outcome + + ", nextRetryTimeNanos=" + nextRetryTimeNanos + + ", earliestNextRetryTimeNanos=" + earliestNextRetryTimeNanos + + '}'; } + } + + private static final class State { + private final RetryConfig config; + private final long deadlineNanos; + private final boolean isTimeoutEnabled; + + private final RetryScheduler retryScheduler; + private int numPendingAttempts; + private final CompletableFuture retryingCompleteFuture; - void removeRetryTask(ScheduledFuture retryTask) { - final boolean retryTaskFound = pendingRetryTasks.remove(retryTask); - assert retryTaskFound; + @Nullable + private Backoff lastBackoff; + private int currentAttemptNoWithLastBackoff; + // Starting with 1 + private int totalAttemptNo; + + State(RetryConfig config, RetryScheduler retryScheduler) { + this.config = config; + this.retryScheduler = retryScheduler; + totalAttemptNo = 1; + retryingCompleteFuture = new CompletableFuture<>(); + deadlineNanos = 0; + isTimeoutEnabled = false; } + State(RetryConfig config, RetryScheduler retryScheduler, long responseTimeoutTimeNanos) { + this.config = config; + this.retryScheduler = retryScheduler; + totalAttemptNo = 1; + retryingCompleteFuture = new CompletableFuture<>(); + deadlineNanos = responseTimeoutTimeNanos; + isTimeoutEnabled = true; + } - void cancelAllRetryTasks() { - for (ScheduledFuture retryTask : pendingRetryTasks) { - // They will all call removeRetryTask() when they are done. - retryTask.cancel(false); - } + RetryScheduler scheduler() { + return retryScheduler; } /** * Returns the smaller value between {@link RetryConfig#responseTimeoutMillisForEachAttempt()} and - * remaining {@link #responseTimeoutMillis}. + * remaining {@link #responseTimeoutMillisForAttempt}. * * @return 0 if the response timeout for both of each request and whole retry is disabled or - * -1 if the elapsed time from the first request has passed {@code responseTimeoutMillis} + * -1 if the elapsed time from the first request has passed {@code responseTimeoutMillis} */ - long responseTimeoutMillis() { + long responseTimeoutMillisForAttempt() { if (!timeoutForWholeRetryEnabled()) { return config.responseTimeoutMillisForEachAttempt(); } - final long actualResponseTimeoutMillis = actualResponseTimeoutMillis(); + final long actualResponseTimeoutMillis = responseTimeoutMillis(); // Consider 0 or less than 0 of actualResponseTimeoutMillis as timed out. if (actualResponseTimeoutMillis <= 0) { @@ -431,26 +551,62 @@ boolean timeoutForWholeRetryEnabled() { return isTimeoutEnabled; } - long actualResponseTimeoutMillis() { - return TimeUnit.NANOSECONDS.toMillis(deadlineNanos - System.nanoTime()); + long responseTimeoutMillis() { + assert isTimeoutEnabled; + return Math.max(TimeUnit.NANOSECONDS.toMillis(deadlineNanos - System.nanoTime()), -1); } - boolean areAttemptsExhausted() { - return totalAttemptNo >= config.maxTotalAttempts(); + long responseTimeoutTimeNanos() { + assert isTimeoutEnabled; + return deadlineNanos; } - int currentAttemptNoWith(Backoff backoff) { - // todo(szymon): is it okay to not increment totalAttemptNo for this check? - if (areAttemptsExhausted()) { + int nextAttemptNoWithBackoff(Backoff backoff) { + if (totalAttemptNo >= config.maxTotalAttempts()) { return -1; } + if (lastBackoff != backoff) { + return 1; + } + return currentAttemptNoWithLastBackoff + 1; + } + + void acquireAttemptNoWithCurrentBackoff(Backoff backoff) { + checkState((totalAttemptNo + 1) <= config.maxTotalAttempts(), + "Exceeded the maximum number of attempts: %s", config.maxTotalAttempts()); + totalAttemptNo++; + if (lastBackoff != backoff) { lastBackoff = backoff; currentAttemptNoWithLastBackoff = 1; + return; } - return currentAttemptNoWithLastBackoff++; + + currentAttemptNoWithLastBackoff++; + } + + void incrementPendingAttemptCount() { + numPendingAttempts++; + } + + int decrementPendingAttemptCount() { + checkArgument(numPendingAttempts > 0, "numPendingAttempts must be greater than 0. did you call " + + "incrementPendingAttemptCount() before?"); + return --numPendingAttempts; + } + + void complete(ClientRequestContext winningAttemptCtx) { + retryingCompleteFuture.complete(winningAttemptCtx); + } + + void completeExceptionally(Throwable cause) { + retryingCompleteFuture.completeExceptionally(cause); + } + + CompletableFuture whenRetryingComplete() { + return retryingCompleteFuture; } } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfig.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfig.java index 5f4d3c33900..18e2f557c12 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfig.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfig.java @@ -77,7 +77,7 @@ static RetryConfigBuilder builder0( private final int maxTotalAttempts; private final long responseTimeoutMillisForEachAttempt; - private final boolean abortAttemptOnPerAttemptResponseTimeout; + private final @Nullable Backoff hedgingBackoff; private final int maxContentLength; @Nullable @@ -89,23 +89,40 @@ static RetryConfigBuilder builder0( @Nullable private RetryRuleWithContent fromRetryRule; + RetryConfig(RetryRule retryRule, int maxTotalAttempts, long responseTimeoutMillisForEachAttempt) { + this(requireNonNull(retryRule, "retryRule"), null, + maxTotalAttempts, responseTimeoutMillisForEachAttempt, + 0, null); + checkArguments(maxTotalAttempts, responseTimeoutMillisForEachAttempt); + } + RetryConfig(RetryRule retryRule, int maxTotalAttempts, long responseTimeoutMillisForEachAttempt, - boolean abortAttemptOnPerAttemptResponseTimeout) { + Backoff hedgingBackoff) { this(requireNonNull(retryRule, "retryRule"), null, maxTotalAttempts, responseTimeoutMillisForEachAttempt, - abortAttemptOnPerAttemptResponseTimeout, 0); + 0, requireNonNull(hedgingBackoff, "hedgingBackoff")); checkArguments(maxTotalAttempts, responseTimeoutMillisForEachAttempt); } + RetryConfig( + RetryRuleWithContent retryRuleWithContent, + int maxContentLength, + int maxTotalAttempts, + long responseTimeoutMillisForEachAttempt) { + this(null, requireNonNull(retryRuleWithContent, "retryRuleWithContent"), + maxTotalAttempts, responseTimeoutMillisForEachAttempt, + maxContentLength, null); + } + RetryConfig( RetryRuleWithContent retryRuleWithContent, int maxContentLength, int maxTotalAttempts, long responseTimeoutMillisForEachAttempt, - boolean abortAttemptOnPerAttemptResponseTimeout) { + Backoff hedgingBackoff) { this(null, requireNonNull(retryRuleWithContent, "retryRuleWithContent"), maxTotalAttempts, responseTimeoutMillisForEachAttempt, - abortAttemptOnPerAttemptResponseTimeout, maxContentLength); + maxContentLength, requireNonNull(hedgingBackoff, "hedgingBackoff")); } private RetryConfig( @@ -113,15 +130,16 @@ private RetryConfig( @Nullable RetryRuleWithContent retryRuleWithContent, int maxTotalAttempts, long responseTimeoutMillisForEachAttempt, - boolean abortAttemptOnPerAttemptResponseTimeout, - int maxContentLength) { + int maxContentLength, + @Nullable Backoff hedgingBackoff + ) { checkArguments(maxTotalAttempts, responseTimeoutMillisForEachAttempt); this.retryRule = retryRule; this.retryRuleWithContent = retryRuleWithContent; this.maxTotalAttempts = maxTotalAttempts; this.responseTimeoutMillisForEachAttempt = responseTimeoutMillisForEachAttempt; - this.abortAttemptOnPerAttemptResponseTimeout = abortAttemptOnPerAttemptResponseTimeout; this.maxContentLength = maxContentLength; + this.hedgingBackoff = hedgingBackoff; if (retryRuleWithContent == null) { fromRetryRuleWithContent = null; } else { @@ -152,10 +170,15 @@ public RetryConfigBuilder toBuilder() { assert retryRule != null; builder = builder0(retryRule); } - return builder + builder .maxTotalAttempts(maxTotalAttempts) - .responseTimeoutMillisForEachAttempt(responseTimeoutMillisForEachAttempt) - .abortAttemptOnPerAttemptResponseTimeout(abortAttemptOnPerAttemptResponseTimeout); + .responseTimeoutMillisForEachAttempt(responseTimeoutMillisForEachAttempt); + + if (hedgingBackoff != null) { + builder.hedgingBackoff(hedgingBackoff); + } + + return builder; } /** @@ -174,8 +197,8 @@ public long responseTimeoutMillisForEachAttempt() { return responseTimeoutMillisForEachAttempt; } - public boolean abortAttemptOnPerAttemptResponseTimeout() { - return abortAttemptOnPerAttemptResponseTimeout; + public @Nullable Backoff hedgingBackoff() { + return hedgingBackoff; } /** diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfigBuilder.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfigBuilder.java index 662cabe0963..c1f453198a0 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfigBuilder.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfigBuilder.java @@ -36,7 +36,7 @@ public final class RetryConfigBuilder { private int maxTotalAttempts = Flags.defaultMaxTotalAttempts(); private long responseTimeoutMillisForEachAttempt = Flags.defaultResponseTimeoutMillis(); - private boolean abortAttemptOnPerAttemptResponseTimeout = true; + private @Nullable Backoff hedgingBackoff; private int maxContentLength; @Nullable @@ -98,8 +98,8 @@ public RetryConfigBuilder responseTimeoutMillisForEachAttempt(long responseTi } - public RetryConfigBuilder abortAttemptOnPerAttemptResponseTimeout(boolean abortAttemptOnPerAttemptResponseTimeout) { - this.abortAttemptOnPerAttemptResponseTimeout = abortAttemptOnPerAttemptResponseTimeout; + public RetryConfigBuilder hedgingBackoff(Backoff hedgingBackoff) { + this.hedgingBackoff = requireNonNull(hedgingBackoff); return this; } @@ -123,15 +123,22 @@ public RetryConfigBuilder responseTimeoutForEachAttempt(Duration responseTime */ public RetryConfig build() { if (retryRule != null) { - return new RetryConfig<>(retryRule, maxTotalAttempts, responseTimeoutMillisForEachAttempt, abortAttemptOnPerAttemptResponseTimeout); + if (hedgingBackoff != null) { + return new RetryConfig<>(retryRule, maxTotalAttempts, responseTimeoutMillisForEachAttempt, + hedgingBackoff); + } else { + return new RetryConfig<>(retryRule, maxTotalAttempts, responseTimeoutMillisForEachAttempt); + } } assert retryRuleWithContent != null; - return new RetryConfig<>( - retryRuleWithContent, - maxContentLength, - maxTotalAttempts, - responseTimeoutMillisForEachAttempt, - abortAttemptOnPerAttemptResponseTimeout); + + if (hedgingBackoff != null) { + return new RetryConfig<>(retryRuleWithContent, maxContentLength, maxTotalAttempts, + responseTimeoutMillisForEachAttempt, hedgingBackoff); + } else { + return new RetryConfig<>(retryRuleWithContent, maxContentLength, maxTotalAttempts, + responseTimeoutMillisForEachAttempt); + } } @Override @@ -147,7 +154,7 @@ ToStringHelper toStringHelper() { .add("retryRuleWithContent", retryRuleWithContent) .add("maxTotalAttempts", maxTotalAttempts) .add("responseTimeoutMillisForEachAttempt", responseTimeoutMillisForEachAttempt) - .add("abortAttemptOnPerAttemptResponseTimeout", abortAttemptOnPerAttemptResponseTimeout) + .add("hedgingBackoff", hedgingBackoff) .add("maxContentLength", maxContentLength); } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java new file mode 100644 index 00000000000..553bef85390 --- /dev/null +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java @@ -0,0 +1,235 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.client.retry; + +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; + +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.linecorp.armeria.client.ClientFactory; +import com.linecorp.armeria.common.annotation.Nullable; + +import io.netty.channel.EventLoop; +import io.netty.util.concurrent.ScheduledFuture; + +class RetryScheduler { + private static class ScheduledRetryTask { + private final Runnable retryTask; + private final ScheduledFuture scheduledFuture; + + private final Consumer onRetryTaskFailedHandler; + + private final long retryTimeNanos; + private boolean ignoreCancellation; + + ScheduledRetryTask(ScheduledFuture scheduledFuture, Runnable retryTask, + long retryTimeNanos, + Consumer onRetryTaskFailedHandler) { + this.retryTask = retryTask; + this.scheduledFuture = scheduledFuture; + + this.onRetryTaskFailedHandler = onRetryTaskFailedHandler; + + this.retryTimeNanos = retryTimeNanos; + + scheduledFuture.addListener(future -> { + if (future.isCancelled()) { + if (!isIgnoringCancellation()) { + onRetryTaskFailedHandler.accept(new IllegalStateException( + ClientFactory.class.getSimpleName() + " has been closed.")); + } else { + // We are cancelled because we are getting rescheduled. + } + } else if (!future.isSuccess()) { + onRetryTaskFailedHandler.accept(future.cause()); + } + }); + } + + private boolean isIgnoringCancellation() { + return ignoreCancellation; + } + + public Runnable getRetryTaskRunnable() { + return retryTask; + } + + public long retryTimeNanos() { + return retryTimeNanos; + } + + public Consumer getOnRetryTaskFailedHandler() { + return onRetryTaskFailedHandler; + } + + public boolean cancel() { + ignoreCancellation = true; + final boolean couldCancel = scheduledFuture.cancel(false); + ignoreCancellation = false; + return couldCancel; + } + } + + private static final Logger logger = LoggerFactory.getLogger(RetryScheduler.class); + + private final EventLoop eventLoop; + private long earliestNextRetryTimeNanos; + private final long latestNextRetryTimeNanos; + private @Nullable ScheduledRetryTask currentRetryTask; + + RetryScheduler(EventLoop eventLoop) { + this(eventLoop, Long.MAX_VALUE); + } + + RetryScheduler(EventLoop eventLoop, long latestNextRetryTimeNanos) { + this.eventLoop = requireNonNull(eventLoop, "eventLoop"); + currentRetryTask = null; + earliestNextRetryTimeNanos = Long.MIN_VALUE; + this.latestNextRetryTimeNanos = latestNextRetryTimeNanos; + } + + public void schedule(Runnable retryTask, long nextRetryTimeNanos, + Consumer onRetryTaskFailedHandler) { + requireNonNull(retryTask, "retryTask"); + requireNonNull(onRetryTaskFailedHandler, "onRetryTaskFailedHandler"); + + if (nextRetryTimeNanos < earliestNextRetryTimeNanos) { + // The next retry time is before the earliestNextRetryTimeNanos. + // We need to update the earliestNextRetryTimeNanos. + onRetryTaskFailedHandler.accept( + new IllegalStateException( + "nextRetryTimeNanos is before the earliestNextRetryTimeNanos: " + + nextRetryTimeNanos + " < " + earliestNextRetryTimeNanos)); + } + + if (nextRetryTimeNanos > latestNextRetryTimeNanos) { + // The next retry time is after the latestNextRetryTimeNanos. + onRetryTaskFailedHandler.accept( + new IllegalStateException("nextRetryTimeNanos is after the latestNextRetryTimeNanos: " + + nextRetryTimeNanos + " > " + latestNextRetryTimeNanos)); + return; + } + + // "fast-path" + if (currentRetryTask == null || nextRetryTimeNanos >= currentRetryTask.retryTimeNanos()) { + // No retry task scheduled. We can schedule a new one directly. + scheduleNextRetryTask(retryTask, nextRetryTimeNanos, + onRetryTaskFailedHandler); + return; + } + + onRetryTaskFailedHandler.accept( + new IllegalStateException("A retry task is already scheduled at " + + currentRetryTask.retryTimeNanos() + ". " + + "nextRetryTimeNanos: " + nextRetryTimeNanos)); + } + + private void scheduleNextRetryTask(Runnable retryRunnable, long retryTimeNanos, + Consumer onRetryTaskFailedHandler) { + assert earliestNextRetryTimeNanos <= retryTimeNanos; + assert retryTimeNanos <= latestNextRetryTimeNanos; + + if (currentRetryTask != null) { + if (!currentRetryTask.cancel()) { + onRetryTaskFailedHandler.accept( + new IllegalStateException("Could not cancel the current retry task.")); + return; + } + } + + assert currentRetryTask == null; + + try { + final long delayNanos = Math.max(retryTimeNanos - System.nanoTime(), 0); + final Runnable wrappedRetryRunnable = () -> { + logger.debug("Retry task starting. Resetting..."); + currentRetryTask = null; + earliestNextRetryTimeNanos = Long.MIN_VALUE; + retryRunnable.run(); + }; + + logger.debug("Scheduling the retry task. delayNanos = {}, " + + "retryTimeNanos = {}, earliestNextRetryTimeNanos = {}", + delayNanos, retryTimeNanos, earliestNextRetryTimeNanos); + + //noinspection unchecked + final ScheduledFuture nextRetryTaskFuture = + (ScheduledFuture) eventLoop.schedule(wrappedRetryRunnable, delayNanos, + TimeUnit.NANOSECONDS); + + // We are passing in the original to avoid multiple wrapping in case of the retry task being + // rescheduled multiple times. + currentRetryTask = new ScheduledRetryTask(nextRetryTaskFuture, retryRunnable, + retryTimeNanos, + onRetryTaskFailedHandler); + } catch (Throwable t) { + onRetryTaskFailedHandler.accept(t); + } + } + + public void close() { + if (currentRetryTask != null) { + currentRetryTask.cancel(); + } + } + + // todo(szymon): Remove dependency on nextEarliestNextRetryTimeNanos. Users should simply set it before + // calling this method. + public boolean hasAlreadyRetryScheduledBefore(long nextRetryTimeNanos, + long nextEarliestNextRetryTimeNanos) { + checkState(nextEarliestNextRetryTimeNanos <= latestNextRetryTimeNanos); + earliestNextRetryTimeNanos = Math.max(earliestNextRetryTimeNanos, nextEarliestNextRetryTimeNanos); + + if (currentRetryTask == null) { + return false; + } + + return Math.max(currentRetryTask.retryTimeNanos(), earliestNextRetryTimeNanos) + <= nextRetryTimeNanos; + } + + public void addEarliestNextRetryTimeNanos(long earliestNextRetryTimeNanos) { + checkState(earliestNextRetryTimeNanos <= latestNextRetryTimeNanos); + this.earliestNextRetryTimeNanos = Math.max(this.earliestNextRetryTimeNanos, + earliestNextRetryTimeNanos); + } + + public long getEarliestNextRetryTimeNanos() { + return earliestNextRetryTimeNanos; + } + + public void rescheduleCurrentRetryTaskIfTooEarly() { + if (currentRetryTask != null) { + if (currentRetryTask.retryTimeNanos() < earliestNextRetryTimeNanos) { + // Current retry task is going to be executed before the earliestNextRetryTimeNanos so + // we need to reschedule it. + scheduleNextRetryTask(currentRetryTask.getRetryTaskRunnable(), + earliestNextRetryTimeNanos, + currentRetryTask.getOnRetryTaskFailedHandler()); + } + } + } + + public boolean hasScheduledRetryTask() { + return currentRetryTask != null; + } +} diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index b92c8303041..93b91474a65 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -96,7 +96,7 @@ public static RetryingClientBuilder builder(RetryRuleWithContent r * it will hand over the stream to the client. * * @throws IllegalArgumentException if the specified {@code maxContentLength} is equal to or - * less than {@code 0} + * less than {@code 0} */ public static RetryingClientBuilder builder(RetryRuleWithContent retryRuleWithContent, int maxContentLength) { @@ -170,7 +170,7 @@ public static Function newDecorator(RetryRul * @param retryRule the retry rule * @param maxTotalAttempts the maximum number of total attempts * @param responseTimeoutMillisForEachAttempt response timeout for each attempt. {@code 0} disables - * the timeout + * the timeout * * @deprecated Use {@link #newDecorator(RetryConfig)} instead. */ @@ -189,7 +189,7 @@ public static Function newDecorator(RetryRul * @param retryRuleWithContent the retry rule * @param maxTotalAttempts the maximum number of total attempts * @param responseTimeoutMillisForEachAttempt response timeout for each attempt. {@code 0} disables - * the timeout + * the timeout * * @deprecated Use {@link #newDecorator(RetryConfig)} instead. */ @@ -219,7 +219,7 @@ public static Function newDecorator(RetryRul * requests. * * @param mapping the mapping that returns a {@link RetryConfig} for a given {@link ClientRequestContext} - * and {@link Request}. + * and {@link Request}. */ public static Function newDecoratorWithMapping(RetryConfigMapping mapping) { @@ -241,9 +241,16 @@ public static Function newDecorator(RetryRul } @Override - protected HttpResponse doExecute(ClientRequestContext ctx, HttpRequest req) throws Exception { + protected ResponseFuturePair getResponseFuturePair( + ClientRequestContext ctx) { final CompletableFuture responseFuture = new CompletableFuture<>(); final HttpResponse res = HttpResponse.of(responseFuture, ctx.eventLoop()); + return new ResponseFuturePair<>(res, responseFuture); + } + + @Override + protected void doExecute(ClientRequestContext ctx, HttpRequest req, HttpResponse res, + CompletableFuture responseFuture) throws Exception { if (ctx.exchangeType().isRequestStreaming()) { final HttpRequestDuplicator reqDuplicator = req.toDuplicator(ctx.eventLoop().withoutContext(), 0); doExecute0(new RetryingContext(mappedRetryConfig(ctx), ctx, reqDuplicator, req, res, @@ -261,7 +268,6 @@ protected HttpResponse doExecute(ClientRequestContext ctx, HttpRequest req) thro return null; }); } - return res; } private void doExecute0(RetryingContext retryingContext) { @@ -270,9 +276,9 @@ private void doExecute0(RetryingContext retryingContext) { final HttpRequestDuplicator rootReqDuplicator = retryingContext.reqDuplicator(); final HttpRequest originalReq = retryingContext.req(); final HttpResponse returnedRes = retryingContext.res(); - final CompletableFuture returnedResFuture = retryingContext.resFuture(); final int totalAttempts = getTotalAttempts(ctx); + logger.trace("doExecute0: {}", totalAttempts); final boolean initialAttempt = totalAttempts <= 1; // The request or attemptRes has been aborted by the client before it receives a attemptRes, // so stop retrying. @@ -297,7 +303,7 @@ private void doExecute0(RetryingContext retryingContext) { return; } - if (!setResponseTimeout(ctx)) { + if (!updateResponseTimeout(ctx)) { handleException(retryingContext, ResponseTimeoutException.get(), initialAttempt); return; } @@ -332,19 +338,38 @@ private void doExecute0(RetryingContext retryingContext) { (context, cause) -> HttpResponse.ofFailure(cause), attemptCtxReq, false); } else { attemptRes = executeWithFallback(unwrap(), attemptCtx, - (context, cause) -> HttpResponse.ofFailure(cause), attemptCtxReq, false); + (context, cause) -> HttpResponse.ofFailure(cause), attemptCtxReq, + false); } + onAttemptStarted(ctx, attemptCtx, (@Nullable Throwable cause) -> abortAttempt(attemptCtx, attemptRes, + cause)); + if (!ctx.exchangeType().isResponseStreaming() || config.requiresResponseTrailers()) { attemptRes.aggregate().handle((attemptAggResponse, cause) -> { if (cause != null) { attemptCtx.logBuilder().endRequest(cause); attemptCtx.logBuilder().endResponse(cause); + + // At that point we completed the request log for the attempt. + // What comes next investigates the attempt's response to decide + // on retry. We do not want to continue if we already completed + // the whole retrying process/ if we have a winning attempt. + if (isRetryingComplete(ctx)) { + return null; + } + handleResponseWithoutContent(retryingContext, attemptCtx, HttpResponse.ofFailure(cause), cause); } else { completeAttemptLogIfBytesNotTransferred(attemptCtx, attemptAggResponse); + + // see above + if (isRetryingComplete(ctx)) { + return null; + } + attemptCtx.log().whenAvailable(RequestLogProperty.RESPONSE_END_TIME).thenRun(() -> { handleAggregatedResponse(retryingContext, attemptCtx, attemptAggResponse); }); @@ -354,25 +379,41 @@ private void doExecute0(RetryingContext retryingContext) { } else { handleStreamingResponse(retryingContext, attemptCtx, attemptRes); } + + @Nullable + final Backoff hedgingBackoff = config.hedgingBackoff(); + + if (hedgingBackoff != null) { + final RetrySchedulabilityDecision retrySchedulabilityDecision = canScheduleWith(ctx, hedgingBackoff, + -1); + if (retrySchedulabilityDecision.canSchedule()) { + scheduleNextRetry(ctx, () -> doExecute0(retryingContext), + retrySchedulabilityDecision.nextRetryTimeNanos(), + cause -> handleException(retryingContext, cause, false)); + } + } } - // TODO(ikhoon): Add a request-scope class such as RetryRequestContext to avoid passing too many parameters. private void handleResponseWithoutContent(RetryingContext retryingContext, ClientRequestContext attemptCtx, HttpResponse attemptRes, @Nullable Throwable attemptResCause) { if (attemptResCause != null) { attemptResCause = Exceptions.peel(attemptResCause); } + try { final RetryRule retryRule = retryRule(retryingContext.config()); final CompletionStage f = retryRule.shouldRetry(attemptCtx, attemptResCause); f.handle((decision, shouldRetryCause) -> { + if (isRetryingComplete(retryingContext.ctx())) { + return null; + } + warnIfExceptionIsRaised(retryRule, shouldRetryCause); handleRetryDecision(retryingContext, decision, attemptCtx, attemptRes); return null; }); } catch (Throwable cause) { - attemptRes.abort(); handleException(retryingContext, cause, false); } } @@ -392,12 +433,28 @@ private void handleStreamingResponse(RetryingContext retryingContext, completeAttemptLogIfBytesNotTransferred(attemptCtx, attemptRes, headers, responseCause); + // see above + if (isRetryingComplete(retryingContext.ctx())) { + attemptSplitRes.body().abort(); + return null; + } + attemptCtx.log().whenAvailable(RequestLogProperty.RESPONSE_HEADERS).thenRun(() -> { + // see above + if (isRetryingComplete(retryingContext.ctx())) { + if (responseCause != null) { + attemptSplitRes.body().abort(responseCause); + } else { + attemptSplitRes.body().abort(); + } + return; + } + if (retryingContext.config().needsContentInRule() && responseCause == null) { final HttpResponse attemptUnsplitRes = HttpResponse.of(headers, attemptSplitRes.body()); final HttpResponseDuplicator attemptResDuplicator = attemptUnsplitRes.toDuplicator(attemptCtx.eventLoop().withoutContext(), - attemptCtx.maxResponseLength()); + attemptCtx.maxResponseLength()); try { final TruncatingHttpResponse attemptTruncatedRes = new TruncatingHttpResponse(attemptResDuplicator.duplicate(), @@ -412,6 +469,12 @@ private void handleStreamingResponse(RetryingContext retryingContext, .handle((decision, cause) -> { warnIfExceptionIsRaised(ruleWithContent, cause); attemptTruncatedRes.abort(); + + if (isRetryingComplete(retryingContext.ctx())) { + attemptResDuplicator.abort(); + return null; + } + handleRetryDecision(retryingContext, decision, attemptCtx, attemptDuplicatedRes); return null; @@ -439,14 +502,20 @@ private void handleAggregatedResponse(RetryingContext retryingContext, ClientRequestContext attemptCtx, AggregatedHttpResponse attemptAggRes) { if (retryingContext.config().needsContentInRule()) { - final RetryRuleWithContent ruleWithContent = retryingContext.config().retryRuleWithContent(); + final RetryRuleWithContent ruleWithContent = + retryingContext.config().retryRuleWithContent(); assert ruleWithContent != null; try { ruleWithContent.shouldRetry(attemptCtx, attemptAggRes.toHttpResponse(), null) .handle((decision, cause) -> { warnIfExceptionIsRaised(ruleWithContent, cause); + + if (isRetryingComplete(retryingContext.ctx())) { + return null; + } + handleRetryDecision(retryingContext, - decision, attemptCtx, attemptAggRes.toHttpResponse()); + decision, attemptCtx, attemptAggRes.toHttpResponse()); return null; }); } catch (Throwable cause) { @@ -473,7 +542,8 @@ private static void completeAttemptLogIfBytesNotTransferred(ClientRequestContext } private static void completeAttemptLogIfBytesNotTransferred( - ClientRequestContext attemptCtx, HttpResponse attemptRes, @Nullable ResponseHeaders attemptResHeaders, + ClientRequestContext attemptCtx, HttpResponse attemptRes, + @Nullable ResponseHeaders attemptResHeaders, @Nullable Throwable attemptResCause) { if (!attemptCtx.log().isAvailable(RequestLogProperty.REQUEST_FIRST_BYTES_TRANSFERRED_TIME)) { final RequestLogBuilder logBuilder = attemptCtx.logBuilder(); @@ -512,43 +582,79 @@ private static void handleException(RetryingContext retryingContext, Throwable c private static void handleException(ClientRequestContext ctx, @Nullable HttpRequestDuplicator rootReqDuplicator, - CompletableFuture future, Throwable cause, + CompletableFuture returnedResFuture, Throwable cause, boolean endRequestLog) { - future.completeExceptionally(cause); + if (isRetryingComplete(ctx)) { + return; + } + if (rootReqDuplicator != null) { rootReqDuplicator.abort(cause); } + if (endRequestLog) { ctx.logBuilder().endRequest(cause); } - ctx.logBuilder().endResponse(cause); + + returnedResFuture.completeExceptionally(cause); + + onRetryingCompleteExceptionally(ctx, cause); } private void handleRetryDecision(RetryingContext retryingContext, @Nullable RetryDecision decision, ClientRequestContext attemptCtx, HttpResponse attemptRes) { final Backoff backoff = decision != null ? decision.backoff() : null; + final boolean shouldContinueRetry; + if (backoff != null) { + shouldContinueRetry = true; final long millisAfter = useRetryAfter ? getRetryAfterMillis(attemptCtx) : -1; - final long nextDelay = getNextDelay(retryingContext.ctx(), backoff, millisAfter); - if (nextDelay >= 0) { - abortAttempt(attemptCtx, attemptRes); - scheduleNextRetry( - retryingContext.ctx(), cause -> handleException(retryingContext, cause, false), - () -> doExecute0(retryingContext), - nextDelay); - return; + final RetrySchedulabilityDecision schedulabilityDecision = canScheduleWith(retryingContext.ctx(), + backoff, + millisAfter); + + if (schedulabilityDecision.canSchedule()) { + logger.debug("Scheduling next retry for {} with backoff: {}, " + + "schedulabilityDecision: {}", + retryingContext.ctx(), backoff, schedulabilityDecision); + scheduleNextRetry(retryingContext.ctx(), + () -> doExecute0(retryingContext), + schedulabilityDecision.nextRetryTimeNanos(), + schedulabilityDecision.earliestNextRetryTimeNanos(), + cause -> handleException(retryingContext, cause, false)); + } else { + logger.debug("Not scheduling next retry for {} with backoff: {}, " + + "schedulabilityDecision: {}", + retryingContext.ctx(), backoff, schedulabilityDecision); + addEarliestNextRetryTimeNanos(retryingContext.ctx(), + schedulabilityDecision.earliestNextRetryTimeNanos()); } + } else { + shouldContinueRetry = false; } - onRetryingComplete(retryingContext, attemptCtx, attemptRes); + final boolean isOtherAttemptInProgress = onAttemptEnded(retryingContext.ctx()); + + if (!shouldContinueRetry || !isOtherAttemptInProgress) { + onRetryingComplete(retryingContext, attemptCtx, attemptRes); + } else { + logger.debug("Retrying is not complete for {} with decision: {}", + retryingContext.ctx(), decision); + } } - private static void abortAttempt(ClientRequestContext attemptCtx, HttpResponse originalRes) { + private static void abortAttempt(ClientRequestContext attemptCtx, HttpResponse attemptRes, + @Nullable Throwable cause) { // Set response content with null to make sure that the log is complete. final RequestLogBuilder logBuilder = attemptCtx.logBuilder(); logBuilder.responseContent(null, null); logBuilder.responseContentPreview(null); - originalRes.abort(); + attemptCtx.cancel(); + if (cause != null) { + attemptRes.abort(cause); + } else { + attemptRes.abort(); + } } private static long getRetryAfterMillis(ClientRequestContext ctx) { @@ -592,13 +698,18 @@ private static RetryRule retryRule(RetryConfig retryConfig) { } } - void onRetryingComplete(RetryingContext retryingContext, ClientRequestContext attemptCtx, HttpResponse attemptRes) { - onRetryingComplete(retryingContext.ctx(), attemptCtx); - retryingContext.resFuture().complete(attemptRes); + if (isRetryingComplete(retryingContext.ctx())) { + return; + } + + logger.debug("Completing retrying"); + retryingContext.reqDuplicator().close(); + retryingContext.resFuture().complete(attemptRes); + onRetryingComplete(retryingContext.ctx(), attemptCtx); } private static class RetryingContext { diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java index 0fbeb930363..7cfd5b90ac3 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java @@ -18,8 +18,6 @@ import static com.linecorp.armeria.internal.client.ClientUtil.executeWithFallback; import static com.linecorp.armeria.internal.client.ClientUtil.initContextAndExecuteWithFallback; -import java.util.LinkedList; -import java.util.List; import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; import java.util.function.Function; @@ -76,7 +74,7 @@ public final class RetryingRpcClient extends AbstractRetryingClient newDecorator(RetryConfigMapping mapping) { @@ -144,205 +142,139 @@ public static RetryingRpcClientBuilder builder(RetryConfigMapping m } @Override - protected RpcResponse doExecute(ClientRequestContext ctx, RpcRequest req) throws Exception { - final CompletableFuture future = new CompletableFuture<>(); - final RpcResponse res = RpcResponse.from(future); - - final List attemptCtxs = new LinkedList<>(); + protected ResponseFuturePair getResponseFuturePair( + ClientRequestContext ctx) { + final CompletableFuture responseFuture = new CompletableFuture<>(); + final RpcResponse res = RpcResponse.from(responseFuture); + return new ResponseFuturePair<>(res, responseFuture); + } - doExecute0(ctx, req, res, future, attemptCtxs); - return res; + @Override + protected void doExecute(ClientRequestContext ctx, RpcRequest req, RpcResponse res, + CompletableFuture returnedResFuture) throws Exception { + doExecute0(ctx, req, res, returnedResFuture); } private void doExecute0(ClientRequestContext ctx, RpcRequest req, - RpcResponse returnedRes, CompletableFuture future, - List attemptCtxs) { + RpcResponse returnedRes, CompletableFuture returnedResFuture) { final int totalAttempts = getTotalAttempts(ctx); final boolean initialAttempt = totalAttempts <= 1; if (returnedRes.isDone()) { // The response has been cancelled by the client before it receives a response, so stop retrying. - handleException(ctx, mappedRetryConfig(ctx), attemptCtxs, future, - new CancellationException( - "the response returned to the client has been cancelled"), initialAttempt); + handleException(ctx, returnedResFuture, new CancellationException( + "the response returned to the client has been cancelled"), initialAttempt); return; } - - final RetryConfig retryConfig = mappedRetryConfig(ctx); - final RetryRuleWithContent retryRule = - retryConfig.needsContentInRule() ? - retryConfig.retryRuleWithContent() : retryConfig.fromRetryRule(); - assert retryRule != null; - - if (retryConfig.abortAttemptOnPerAttemptResponseTimeout()) { - if (!setResponseTimeout(ctx)) { - handleException(ctx, mappedRetryConfig(ctx), attemptCtxs, future, - ResponseTimeoutException.get(), - initialAttempt); - return; - } - } else { - // As the retry config stays constant for the entire retry process, - // retryConfig.abortAttemptOnPerAttemptResponseTimeout() stays - // constant too. This means that if abortAttemptOnPerAttemptResponseTimeout() - // returns false, we will never change the response timeout. - // This means we do not have to reset it here to the actual - // response timeout. + if (!updateResponseTimeout(ctx)) { + handleException(ctx, returnedResFuture, ResponseTimeoutException.get(), initialAttempt); + return; } - final ClientRequestContext derivedCtx = newAttemptContext(ctx, null, req, initialAttempt); - attemptCtxs.add(derivedCtx); + final ClientRequestContext attemptCtx = newAttemptContext(ctx, null, req, initialAttempt); if (!initialAttempt) { - derivedCtx.mutateAdditionalRequestHeaders( + attemptCtx.mutateAdditionalRequestHeaders( mutator -> mutator.add(ARMERIA_RETRY_COUNT, StringUtil.toString(totalAttempts - 1))); } - final RpcResponse res; + final RpcResponse attemptRes; - final ClientRequestContextExtension ctxExtension = derivedCtx.as(ClientRequestContextExtension.class); - final EndpointGroup endpointGroup = derivedCtx.endpointGroup(); + final ClientRequestContextExtension ctxExtension = attemptCtx.as(ClientRequestContextExtension.class); + final EndpointGroup endpointGroup = attemptCtx.endpointGroup(); if (!initialAttempt && ctxExtension != null && - endpointGroup != null && derivedCtx.endpoint() == null) { + endpointGroup != null && attemptCtx.endpoint() == null) { // clear the pending throwable to retry endpoint selection - ClientPendingThrowableUtil.removePendingThrowable(derivedCtx); + ClientPendingThrowableUtil.removePendingThrowable(attemptCtx); // if the endpoint hasn't been selected, try to initialize the ctx with a new endpoint/event loop - res = initContextAndExecuteWithFallback(unwrap(), ctxExtension, RpcResponse::from, + attemptRes = initContextAndExecuteWithFallback(unwrap(), ctxExtension, RpcResponse::from, (context, cause) -> RpcResponse.ofFailure(cause), req, true); } else { - res = executeWithFallback(unwrap(), derivedCtx, + attemptRes = executeWithFallback(unwrap(), attemptCtx, (context, cause) -> RpcResponse.ofFailure(cause), req, true); } - if (!retryConfig.abortAttemptOnPerAttemptResponseTimeout()) { - scheduleNextAttemptTimeout(ctx, () -> { - // todo(szymon): Can this happen? - if (returnedRes.isDone()) { - return; - } - - try { - retryRule.shouldRetry(derivedCtx, res, - ResponseTimeoutException.get()).handle( - (decision, unused2) -> { - final Backoff backoff = decision != null ? decision.backoff() : null; - if (backoff != null && areAttemptsExhausted(ctx)) { - // Rule gave us allowance to continue but we - // do not have any attempts left. - // In that case we do not want to complete - // retrying immediately but let the pending - // requests complete. - return null; - } - - handleRetryDecision(ctx, derivedCtx, req, - returnedRes, - res, future, - attemptCtxs, - decision); - return null; - } - ); - } catch (Throwable cause) { - handleException(ctx, mappedRetryConfig(ctx), - attemptCtxs, future, cause, - false); - } - }, cause0 -> handleException(ctx, mappedRetryConfig(ctx), - attemptCtxs, future, - cause0, false), // todo - // (szymon): false or initialAttempt? - retryConfig.responseTimeoutMillisForEachAttempt()); - } - - res.handle((unused1, cause) -> { - if (returnedRes.isDone()) { - // With hedging it could be that another attempt has already provided a response. - // If this is the case every cleanup has already been done. - return null; - } + onAttemptStarted(ctx, attemptCtx, (@Nullable Throwable cause) -> { + attemptCtx.cancel(); + }); + final RetryConfig retryConfig = mappedRetryConfig(ctx); + final RetryRuleWithContent retryRule = + retryConfig.needsContentInRule() ? + retryConfig.retryRuleWithContent() : retryConfig.fromRetryRule(); + attemptRes.handle((unused1, cause) -> { try { - retryRule.shouldRetry(derivedCtx, res, cause).handle((decision, unused3) -> { - handleRetryDecision(ctx, derivedCtx, req, returnedRes, res, future, - attemptCtxs, decision); + assert retryRule != null; + retryRule.shouldRetry(attemptCtx, attemptRes, cause).handle((decision, unused3) -> { + final Backoff backoff = decision != null ? decision.backoff() : null; + + if (backoff != null) { + final RetrySchedulabilityDecision schedulabilityDecision = canScheduleWith(ctx, backoff); + + if (schedulabilityDecision.canSchedule()) { + scheduleNextRetry(ctx, + () -> doExecute0(ctx, req, returnedRes, returnedResFuture), + schedulabilityDecision.nextRetryTimeNanos(), + cause0 -> handleException(ctx, returnedResFuture, cause0, false)); + } + } + + final boolean isOtherAttemptInProgress = onAttemptEnded(ctx); + + if (!isOtherAttemptInProgress) { + onRetryingComplete(ctx, returnedResFuture, attemptCtx, attemptRes); + } + return null; }); } catch (Throwable t) { - handleException(ctx, mappedRetryConfig(ctx), attemptCtxs, future, t, false); + handleException(ctx, returnedResFuture, t, false); } return null; }); - } - private void handleRetryDecision(ClientRequestContext ctx, ClientRequestContext derivedCtx, - RpcRequest req, RpcResponse returnedRes, RpcResponse res, - CompletableFuture future, - List attemptCtxs, - @Nullable RetryDecision decision) { - final Backoff backoff = decision != null ? decision.backoff() : null; - if (backoff != null) { - final long nextDelay = getNextDelay(derivedCtx, backoff); - if (nextDelay < 0) { - onRetryComplete(ctx, derivedCtx, mappedRetryConfig(ctx), res, future, attemptCtxs); - } - scheduleNextRetry(ctx, cause0 -> handleException(ctx, mappedRetryConfig(ctx), - attemptCtxs, future, cause0, - false), - () -> doExecute0(ctx, req, returnedRes, future, attemptCtxs), - nextDelay); - } else { - onRetryComplete(ctx, derivedCtx, mappedRetryConfig(ctx), res, future, attemptCtxs); + final @Nullable Backoff hedgingBackoff = retryConfig.hedgingBackoff(); + + if (hedgingBackoff != null) { + final RetrySchedulabilityDecision schedulabilityDecision = + canScheduleWith(ctx, hedgingBackoff); + if (schedulabilityDecision.canSchedule()) { + scheduleNextRetry(ctx, () -> doExecute0(ctx, req, returnedRes, returnedResFuture), + schedulabilityDecision.nextRetryTimeNanos(), + cause -> handleException(ctx, returnedResFuture, cause, false)); + } } } - private static void onRetryComplete(ClientRequestContext ctx, ClientRequestContext derivedCtx, - RetryConfig retryConfig, - RpcResponse res, CompletableFuture future, - List attemptCtxs) { - if (future.isDone()) { + private static void handleException(ClientRequestContext ctx, CompletableFuture returnedResFuture, + Throwable cause, boolean endRequestLog) { + if (isRetryingComplete(ctx)) { return; } - onRetryingComplete(ctx, derivedCtx); - - final HttpRequest actualHttpReq = derivedCtx.request(); - if (actualHttpReq != null) { - ctx.updateRequest(actualHttpReq); - } - - future.complete(res); - - cancelPendingAttempts(ctx, retryConfig, attemptCtxs); - } - - private static void handleException(ClientRequestContext ctx, - RetryConfig retryConfig, - List attemptCtxs, - CompletableFuture future, - Throwable cause, boolean endRequestLog) { - future.completeExceptionally(cause); + returnedResFuture.completeExceptionally(cause); if (endRequestLog) { ctx.logBuilder().endRequest(cause); } - ctx.logBuilder().endResponse(cause); - - cancelPendingAttempts(ctx, retryConfig, attemptCtxs); + onRetryingCompleteExceptionally(ctx, cause); } - private static void cancelPendingAttempts(ClientRequestContext ctx, RetryConfig retryConfig, - List attemptCtxs) { - if (!retryConfig.abortAttemptOnPerAttemptResponseTimeout()) { - for (ClientRequestContext attemptCtx : attemptCtxs) { - if (!attemptCtx.isCancelled()) { - attemptCtx.cancel(); - } - } + void onRetryingComplete(ClientRequestContext ctx, + CompletableFuture returnedResFuture, + ClientRequestContext attemptCtx, + RpcResponse attemptRes) { + if (isRetryingComplete(ctx)) { + return; + } - // Cancel in-flight retry tasks. - cancelRetryTasks(ctx); + final HttpRequest actualHttpReq = attemptCtx.request(); + if (actualHttpReq != null) { + ctx.updateRequest(actualHttpReq); } + + returnedResFuture.complete(attemptRes); + onRetryingComplete(ctx, attemptCtx); } } diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java index c68446ad7e3..c632ed74de0 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java @@ -1,7 +1,7 @@ /* - * Copyright 2017 LINE Corporation + * Copyright 2025 LY Corporation * - * LINE Corporation licenses this file to you under the Apache License, + * LY Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * @@ -80,6 +80,7 @@ import com.linecorp.armeria.server.AbstractHttpService; import com.linecorp.armeria.server.ServerBuilder; import com.linecorp.armeria.server.ServiceRequestContext; +import com.linecorp.armeria.server.logging.LoggingService; import com.linecorp.armeria.testing.junit5.server.ServerExtension; import io.netty.channel.EventLoop; @@ -119,6 +120,8 @@ protected boolean runForEachTest() { @Override protected void configure(ServerBuilder sb) throws Exception { + sb.decorator(LoggingService.newDecorator()); + sb.service("/retry-content", new AbstractHttpService() { @Override protected HttpResponse doGet(ServiceRequestContext ctx, HttpRequest req) @@ -526,15 +529,15 @@ void honorRetryMapping() { void evaluatesMappingOnce() { final AtomicInteger evaluations = new AtomicInteger(0); final RetryConfigMapping mapping = - (ctx, req) -> { - evaluations.incrementAndGet(); - return RetryConfig - .builder0(RetryRule.builder() - .onStatus(HttpStatus.valueOf(500)) - .thenBackoff()) - .maxTotalAttempts(2) - .build(); - }; + (ctx, req) -> { + evaluations.incrementAndGet(); + return RetryConfig + .builder0(RetryRule.builder() + .onStatus(HttpStatus.valueOf(500)) + .thenBackoff()) + .maxTotalAttempts(2) + .build(); + }; final WebClient client = client(mapping); @@ -660,7 +663,8 @@ void shouldGetExceptionWhenFactoryIsClosed() { } assertThat(t).isInstanceOf(IllegalStateException.class) .satisfies(cause -> assertThat(cause.getMessage()).matches( - "(?i).*(factory has been closed|not accepting a task).*")); + "(?i).*(factory has been closed|not accepting a task|factory is closing or " + + "closed).*")); } @Test diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithContextAwareTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithContextAwareTest.java index a33f0382322..0e7654bd8ac 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithContextAwareTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithContextAwareTest.java @@ -1,11 +1,11 @@ /* - * Copyright 2019 LINE Corporation + * Copyright 2025 LY Corporation * - * LINE Corporation licenses this file to you under the Apache License, + * LY Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT @@ -57,7 +57,15 @@ void contextAwareDoesNotThrowException() { final ServiceRequestContext dummyCtx = ServiceRequestContext.of(HttpRequest.of(HttpMethod.GET, "/")); try (SafeCloseable ignored = dummyCtx.push()) { final CompletableFuture future = client.get("/").aggregate(); - assertThatThrownBy(() -> dummyCtx.makeContextAware(future).join()).hasCauseInstanceOf( + assertThatThrownBy(() -> { + try { + dummyCtx.makeContextAware(future).join(); + } catch (Exception e) { + throw e; + } + + System.out.println(future); + }).hasCauseInstanceOf( ResponseTimeoutException.class); } } diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java new file mode 100644 index 00000000000..09869095592 --- /dev/null +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java @@ -0,0 +1,453 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.client.retry; + +import static com.linecorp.armeria.client.retry.AbstractRetryingClient.ARMERIA_RETRY_COUNT; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; +import static org.awaitility.Awaitility.await; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.nio.charset.Charset; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.function.Function; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.linecorp.armeria.client.ClientFactory; +import com.linecorp.armeria.client.ClientRequestContext; +import com.linecorp.armeria.client.ClientRequestContextCaptor; +import com.linecorp.armeria.client.Clients; +import com.linecorp.armeria.client.ResponseCancellationException; +import com.linecorp.armeria.client.ResponseTimeoutException; +import com.linecorp.armeria.client.WebClient; +import com.linecorp.armeria.client.endpoint.EndpointGroup; +import com.linecorp.armeria.client.endpoint.EndpointSelectionStrategy; +import com.linecorp.armeria.common.AggregatedHttpResponse; +import com.linecorp.armeria.common.ExchangeType; +import com.linecorp.armeria.common.HttpRequest; +import com.linecorp.armeria.common.HttpResponse; +import com.linecorp.armeria.common.HttpStatus; +import com.linecorp.armeria.common.SessionProtocol; +import com.linecorp.armeria.common.annotation.Nullable; +import com.linecorp.armeria.common.logging.RequestLog; +import com.linecorp.armeria.common.logging.RequestLogAccess; +import com.linecorp.armeria.common.logging.RequestLogProperty; +import com.linecorp.armeria.server.HttpService; +import com.linecorp.armeria.server.RoutingContext; +import com.linecorp.armeria.server.ServerBuilder; +import com.linecorp.armeria.server.ServiceRequestContext; +import com.linecorp.armeria.server.logging.LoggingService; +import com.linecorp.armeria.testing.junit5.server.ServerExtension; + +class RetryingClientWithHedgingTest { + private static final long LOOSING_SERVER_RESPONSE_DELAY_MILLIS = 300; + + private static final String SERVER1_RESPONSE = "s1"; + private static final String SERVER2_RESPONSE = "s2#"; + private static final String SERVER3_RESPONSE = "s3##"; + + private static final Logger log = LoggerFactory.getLogger(RetryingClientWithHedgingTest.class); + + private static class TestServer extends ServerExtension { + private CountDownLatch responseLatch = new CountDownLatch(1); + private final AtomicInteger numRequests = new AtomicInteger(); + private HttpService helloService = mock(HttpService.class); + + TestServer() { + super(true); + } + + @Override + protected void configure(ServerBuilder sb) throws Exception { + sb.decorator(LoggingService.newDecorator()); + sb.service("/hello", + new HttpService() { + @Override + public HttpResponse serve(ServiceRequestContext ctx, HttpRequest req) + throws Exception { + numRequests.incrementAndGet(); + try { + responseLatch.await(); + } catch (InterruptedException e) { + fail(e); + } + + req.whenComplete().handle((t, tt) -> { + System.out.println("Request completed: " + req); + return null; + }); + + final HttpResponse res = helloService.serve(ctx, req); + + res.whenComplete().handle((t, tt) -> { + System.out.println("Response completed: " + res); + return null; + }); + + return res; + } + + @Override + public ExchangeType exchangeType(RoutingContext routingContext) { + return ExchangeType.UNARY; + } + }); + } + + private void reset() { + helloService = mock(HttpService.class); + responseLatch = new CountDownLatch(1); + numRequests.set(0); + } + + public HttpService getHelloService() { + return helloService; + } + + public int getNumRequests() { + return numRequests.get(); + } + + public void unlatchResponse() { + responseLatch.countDown(); + } + } + + @RegisterExtension + private static final TestServer server1 = new TestServer(); + @RegisterExtension + private static final TestServer server2 = new TestServer(); + @RegisterExtension + private static final TestServer server3 = new TestServer(); + + private static ClientFactory clientFactory; + private final RetryRule NO_RETRY_RULE = RetryRule.builder().thenNoRetry(); + + @BeforeAll + static void beforeAll() { + // use different eventLoop from server's so that clients don't hang when the eventLoop in server hangs + clientFactory = ClientFactory.builder() + .workerGroup(5).build(); + } + + @AfterAll + static void afterAll() { + clientFactory.closeAsync(); + } + + @BeforeEach + void beforeEach() { + server1.reset(); + server2.reset(); + server3.reset(); + } + + @AfterEach + void afterEach() { + server1.unlatchResponse(); + server2.unlatchResponse(); + server3.unlatchResponse(); + } + + @Test + void letSecondServerWins() throws Exception { + when(server1.getHelloService().serve(any(), any())).thenReturn(HttpResponse.of(SERVER1_RESPONSE)); + when(server2.getHelloService().serve(any(), any())).thenReturn(HttpResponse.of(SERVER2_RESPONSE)); + when(server3.getHelloService().serve(any(), any())).thenReturn(HttpResponse.of(SERVER3_RESPONSE)); + + final RetryConfig hedgingNoRetryConfig = RetryConfig + .builder(NO_RETRY_RULE) + .maxTotalAttempts(3) + .hedgingBackoff(Backoff.fixed(100)) + .build(); + + final WebClient client = WebClient.builder(SessionProtocol.H2C, + EndpointGroup.of(EndpointSelectionStrategy.roundRobin(), + server1.httpEndpoint(), + server2.httpEndpoint(), + server3.httpEndpoint())) + .factory(clientFactory) + .decorator( + RetryingClient.newDecorator(hedgingNoRetryConfig) + ) + .build(); + + final CompletableFuture responseFuture; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + responseFuture = client.get("/hello").aggregate(); + ctx = captor.get(); + } + + await().untilAsserted(() -> { + assertThat(server1.getNumRequests()).isOne(); + assertThat(server2.getNumRequests()).isOne(); + assertThat(server3.getNumRequests()).isOne(); + }); + + server2.unlatchResponse(); + Thread.sleep(LOOSING_SERVER_RESPONSE_DELAY_MILLIS); + server1.unlatchResponse(); + server3.unlatchResponse(); + + await() + .untilAsserted(() -> { + assertValidServerRequestContext(server1, 1); + assertValidServerRequestContext(server2, 2); + assertValidServerRequestContext(server3, 3); + + assertValidAggregatedResponse(responseFuture, SERVER2_RESPONSE); + assertValidClientRequestContext( + ctx, GET_VERIFY_RESPONSE_HAS_CONTENT.apply(SERVER2_RESPONSE), + VERIFY_REQUEST_CANCELLED, + GET_VERIFY_RESPONSE_HAS_CONTENT.apply(SERVER2_RESPONSE), + VERIFY_REQUEST_CANCELLED + ); + }); + } + + @Test + void letThirdServerWin() throws Exception { + when(server1.getHelloService().serve(any(), any())).thenReturn(HttpResponse.of(SERVER1_RESPONSE)); + when(server2.getHelloService().serve(any(), any())).thenReturn(HttpResponse.of(SERVER2_RESPONSE)); + when(server3.getHelloService().serve(any(), any())).thenReturn(HttpResponse.of(SERVER3_RESPONSE)); + + final RetryConfig hedgingNoRetryConfig = RetryConfig + .builder(NO_RETRY_RULE) + .maxTotalAttempts(3) + .hedgingBackoff(Backoff.fixed(100)) + .build(); + + final WebClient client = WebClient.builder(SessionProtocol.H2C, + EndpointGroup.of(EndpointSelectionStrategy.roundRobin(), + server1.httpEndpoint(), + server2.httpEndpoint(), + server3.httpEndpoint())) + .factory(clientFactory) + .decorator( + RetryingClient.newDecorator(hedgingNoRetryConfig) + ) + .build(); + + final CompletableFuture responseFuture; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + responseFuture = client.get("/hello").aggregate(); + ctx = captor.get(); + } + + await().untilAsserted(() -> { + assertThat(server1.getNumRequests()).isOne(); + assertThat(server2.getNumRequests()).isOne(); + assertThat(server3.getNumRequests()).isOne(); + }); + + server3.unlatchResponse(); + Thread.sleep(LOOSING_SERVER_RESPONSE_DELAY_MILLIS); + server1.unlatchResponse(); + server2.unlatchResponse(); + + await() + .untilAsserted(() -> { + assertValidServerRequestContext(server1, 1); + assertValidServerRequestContext(server2, 2); + assertValidServerRequestContext(server3, 3); + + assertValidAggregatedResponse(responseFuture, SERVER3_RESPONSE); + assertValidClientRequestContext( + ctx, GET_VERIFY_RESPONSE_HAS_CONTENT.apply(SERVER3_RESPONSE), + VERIFY_REQUEST_CANCELLED, + VERIFY_REQUEST_CANCELLED, + GET_VERIFY_RESPONSE_HAS_CONTENT.apply(SERVER3_RESPONSE) + ); + }); + } + + @Test + void thirdWinsEvenAfterPerAttemptTimeout() throws Exception { + when(server1.getHelloService().serve(any(), any())).thenReturn(HttpResponse.of(SERVER1_RESPONSE)); + when(server2.getHelloService().serve(any(), any())).thenReturn(HttpResponse.of(SERVER2_RESPONSE)); + when(server3.getHelloService().serve(any(), any())).thenReturn(HttpResponse.of(SERVER3_RESPONSE)); + + final RetryConfig hedgingNoRetryConfig = RetryConfig + .builder(RetryRule.builder().onTimeoutException().thenBackoff(Backoff.fixed(10_000))) // should + // be always overtaken by hedging task + .maxTotalAttempts(3) + .responseTimeoutMillisForEachAttempt(200) + .hedgingBackoff(Backoff.fixed(500)) + .build(); + + final WebClient client = WebClient.builder(SessionProtocol.H2C, + EndpointGroup.of(EndpointSelectionStrategy.roundRobin(), + server1.httpEndpoint(), + server2.httpEndpoint(), + server3.httpEndpoint())) + .factory(clientFactory) + .decorator( + RetryingClient.newDecorator(hedgingNoRetryConfig) + ) + .build(); + + final CompletableFuture responseFuture; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + responseFuture = client.get("/hello").aggregate(); + ctx = captor.get(); + } + + await().untilAsserted(() -> { + assertThat(server1.getNumRequests()).isOne(); + assertThat(server2.getNumRequests()).isOne(); + assertThat(server3.getNumRequests()).isOne(); + }); + + // Let the third server win + server1.unlatchResponse(); + server2.unlatchResponse(); + server3.unlatchResponse(); + + await() + .untilAsserted(() -> { + assertValidServerRequestContext(server1, 1); + assertValidServerRequestContext(server2, 2); + assertValidServerRequestContext(server3, 3); + + assertValidAggregatedResponse(responseFuture, SERVER3_RESPONSE); + assertValidClientRequestContext( + ctx, GET_VERIFY_RESPONSE_HAS_CONTENT.apply(SERVER3_RESPONSE), + VERIFY_REQUEST_TIMED_OUT, + VERIFY_REQUEST_TIMED_OUT, GET_VERIFY_RESPONSE_HAS_CONTENT.apply(SERVER3_RESPONSE) + ); + }); + } + + private static String getResponseContent(AggregatedHttpResponse response) { + return response.content().toString(Charset.defaultCharset()); + } + + private void assertValidAggregatedResponse(CompletableFuture resFuture, + String expectedContent) { + assertThat(resFuture.isDone()).isTrue(); + final AggregatedHttpResponse res = resFuture.getNow(null); + assertThat(res.status()).isEqualTo(HttpStatus.OK); + assertThat(getResponseContent(res)).isEqualTo(expectedContent); + } + + private void assertValidClientRequestContext(ClientRequestContext ctx, + RequestLogVerifier logVerifierCtx, + RequestLogVerifier logVerifierServer1, + RequestLogVerifier logVerifierServer2, + @Nullable RequestLogVerifier logVerifierServer3 + ) { + assertThat(ctx.log().isComplete()).isTrue(); + assertThat(ctx.log().children()).hasSize(logVerifierServer3 == null ? 2 : 3); + final RequestLog log = ctx.log().getIfAvailable(RequestLogProperty.RESPONSE_CONTENT, + RequestLogProperty.RESPONSE_CAUSE, + RequestLogProperty.REQUEST_HEADERS); + assertThat(log).isNotNull(); + logVerifierCtx.accept(log); + assertValidChildLog(ctx.log().children().get(0), 1, logVerifierServer1); + assertValidChildLog(ctx.log().children().get(1), 2, logVerifierServer2); + if (logVerifierServer3 != null) { + assertValidChildLog(ctx.log().children().get(2), 3, logVerifierServer3); + } + } + + void assertValidChildLog(RequestLogAccess logAccess, int attemptNumber, + RequestLogVerifier requestLogVerifier) { + assertThat(logAccess.isComplete()).isTrue(); + // After the check right above, all properties of the RequestLog should be available. + final @Nullable RequestLog log = logAccess.getIfAvailable(RequestLogProperty.RESPONSE_CONTENT, + RequestLogProperty.RESPONSE_CAUSE, + RequestLogProperty.REQUEST_HEADERS); + assertThat(log).isNotNull(); + + if (attemptNumber > 1) { + assertThat(log.requestHeaders().getInt(ARMERIA_RETRY_COUNT)).isEqualTo(attemptNumber - 1); + } else { + assertThat(log.requestHeaders().contains(ARMERIA_RETRY_COUNT)).isFalse(); + } + + requestLogVerifier.accept(log); + } + + private void assertValidServerRequestContext(ServerExtension server, int attemptNumber) { + assertThat(server.requestContextCaptor().size()).isEqualTo(1); + + final ServiceRequestContext sctx = server.requestContextCaptor().peek(); + assertThat(sctx).isNotNull(); + assertThat(sctx.log().isComplete()).isTrue(); + + final RequestLog slog = sctx.log().getIfAvailable(RequestLogProperty.REQUEST_HEADERS, + RequestLogProperty.REQUEST_CONTENT); + + assertThat(slog).isNotNull(); + if (attemptNumber > 1) { + assertThat(slog.requestHeaders().getInt(ARMERIA_RETRY_COUNT)).isEqualTo(attemptNumber - 1); + } else { + assertThat(slog.requestHeaders().contains(ARMERIA_RETRY_COUNT)).isFalse(); + } + + assertThat(slog.requestHeaders().path()).contains("hello"); + } + + @FunctionalInterface + private interface RequestLogVerifier extends Consumer {} + + private static final RequestLogVerifier VERIFY_REQUEST_CANCELLED = + log -> { + assertThat(log.responseCause()).isInstanceOf(ResponseCancellationException.class); + }; + + private static final RequestLogVerifier VERIFY_REQUEST_TIMED_OUT = + log -> { + assertThat(log.responseCause()).isInstanceOf(ResponseTimeoutException.class); + }; + // +// private static final RequestLogVerifier VERIFY_RESPONSE_TIMEOUT = +// log -> { +// assertThat(log.responseCause()).isInstanceOf(TTransportException.class); +// assertThat(log.responseCause().getCause()).isInstanceOf(ResponseTimeoutException.class); +// }; +// + private static final Function GET_VERIFY_RESPONSE_HAS_CONTENT = + expectedResponseContent -> log -> { + assertThat(log.responseLength()).isEqualTo(expectedResponseContent.length()); + }; +// +// private static final BiFunction +// GET_VERIFY_RESPONSE_HAS_APPLICATION_EXCEPTION = +// (expectedType, expectedMessage) -> log -> { +// assertThat(log.responseCause()).isInstanceOf(TApplicationException.class); +// final TApplicationException cause = (TApplicationException) log.responseCause(); +// assertThat(cause.getType()).isEqualTo(expectedType); +// assertThat(cause.getMessage()).contains(expectedMessage); +// }; +} diff --git a/junit5/src/main/java/com/linecorp/armeria/testing/server/ServiceRequestContextCaptor.java b/junit5/src/main/java/com/linecorp/armeria/testing/server/ServiceRequestContextCaptor.java index de83161c4b7..7cc44b45ce5 100644 --- a/junit5/src/main/java/com/linecorp/armeria/testing/server/ServiceRequestContextCaptor.java +++ b/junit5/src/main/java/com/linecorp/armeria/testing/server/ServiceRequestContextCaptor.java @@ -1,7 +1,7 @@ /* - * Copyright 2021 LINE Corporation + * Copyright 2025 LY Corporation * - * LINE Corporation licenses this file to you under the Apache License, + * LY Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * @@ -151,6 +151,11 @@ public ServiceRequestContext take() throws InterruptedException { return serviceContexts.take(); } + @Nullable + public ServiceRequestContext peek() { + return serviceContexts.peek(); + } + /** * Retrieves and removes the first captured {@link ServiceRequestContext}, waiting up to * {@value DEFAULT_TIMEOUT_IN_SECONDS} seconds if necessary until an element becomes available. diff --git a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java index 1e56c8b0472..41cab0ed0b8 100644 --- a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java +++ b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java @@ -1,7 +1,7 @@ /* - * Copyright 2017 LINE Corporation + * Copyright 2025 LY Corporation * - * LINE Corporation licenses this file to you under the Apache License, + * LY Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * @@ -334,9 +334,16 @@ void doNotRetryWhenResponseIsCancelled() throws Exception { .path("/thrift") .rpcDecorator(RetryingRpcClient.builder(retryAlways).newDecorator()) .rpcDecorator((delegate, ctx, req) -> { - context.set(ctx); final RpcResponse res = delegate.execute(ctx, req); - res.cancel(true); + + // We are not guarenteed that the retrying client will immediately retry but + // execute the first attempt in the event loop. It should be enough to just + // enqueue in the loop and only then cancel the response. + ctx.eventLoop().execute(() -> { + context.set(ctx); + res.cancel(true); + }); + return res; }) .build(HelloService.Iface.class); diff --git a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java index 454e13542e2..9811f9b4a13 100644 --- a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java +++ b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java @@ -191,7 +191,7 @@ void execute_hedging_lastWins() throws Exception { ) .maxTotalAttempts(3) .responseTimeoutMillisForEachAttempt(50) - .abortAttemptOnPerAttemptResponseTimeout(false) + .hedgingBackoff(Backoff.fixed(20)) .build() ); @@ -243,7 +243,7 @@ void execute_hedging_thirdWinsEventAfterPerAttemptTimeout() throws Exception { ) .maxTotalAttempts(3) .responseTimeoutMillisForEachAttempt(50) - .abortAttemptOnPerAttemptResponseTimeout(false) + .hedgingBackoff(Backoff.fixed(20)) .build() ); @@ -302,7 +302,7 @@ void execute_hedging_thirdWinsEvenWhenFirstErrors() throws Exception { ) .maxTotalAttempts(3) .responseTimeoutMillisForEachAttempt(1) - .abortAttemptOnPerAttemptResponseTimeout(false) + .hedgingBackoff(Backoff.fixed(20)) .build() ); @@ -353,7 +353,7 @@ void execute_hedging_returnErrorWhenSecondErrors() throws Exception { ) .maxTotalAttempts(3) .responseTimeoutMillisForEachAttempt(1) - .abortAttemptOnPerAttemptResponseTimeout(false) + .hedgingBackoff(Backoff.fixed(20)) .build() ); @@ -420,7 +420,7 @@ void execute_hedging_honorResponseTimeout() throws TException { ) .maxTotalAttempts(3) .responseTimeoutMillisForEachAttempt(300) - .abortAttemptOnPerAttemptResponseTimeout(false) + .hedgingBackoff(Backoff.fixed(20)) .build(), 300 + 100 // Lets give the client 100ms to schedule the second attempt. ); From b74afcad3003a529cfd26fb65ea48437a797871b Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Sat, 7 Jun 2025 16:25:51 +0200 Subject: [PATCH 04/36] [WIP] feat: make scheduler thread-safe and add test when hedging task is cancelled --- .../client/retry/AbstractRetryingClient.java | 31 +++- .../armeria/client/retry/RetryScheduler.java | 165 +++++++++++------- .../armeria/client/retry/RetryingClient.java | 9 +- .../client/retry/RetryingRpcClient.java | 21 ++- .../retry/RetryingClientWithHedgingTest.java | 151 ++++++++++++---- 5 files changed, 254 insertions(+), 123 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java index 7f9ac7fc85e..c1196dfc9d9 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java @@ -162,13 +162,14 @@ protected abstract void doExecute(ClientRequestContext ctx, I req, O res, Comple */ protected static void onRetryingComplete(ClientRequestContext ctx, ClientRequestContext attemptCtx) { + logger.debug("onRetryingComplete: {}", attemptCtx); ctx.logBuilder().endResponseWithChild(attemptCtx.log()); - state(ctx).complete(attemptCtx); } protected static void onRetryingCompleteExceptionally(ClientRequestContext ctx, Throwable cause) { + logger.debug("onRetryingCompleteExceptionally: {}", ctx, cause); ctx.logBuilder().endResponse(cause); state(ctx).completeExceptionally(cause); } @@ -214,6 +215,7 @@ protected static void onAttemptStarted(ClientRequestContext ctx, requireNonNull(ctx, "ctx"); requireNonNull(attemptCtx, "attemptCtx"); // todo(szymon) [Q]: should we track the attemptCtxs and check if we have multiple attempts started? + // todo(szymon): do we want to wait for acquisition when we schedule it? state(ctx).incrementPendingAttemptCount(); state(ctx).whenRetryingComplete().handle((winningAttemptCtx, cause) -> { if (attemptCtx == winningAttemptCtx) { @@ -232,7 +234,11 @@ protected boolean onAttemptEnded(ClientRequestContext ctx) { final int numRemainingPendingAttempts = state(ctx).decrementPendingAttemptCount(); logger.debug("onAttemptEnded: {}. numRemainingPendingAttempts = {}, hasScheduledRetryTask={}", ctx, numRemainingPendingAttempts, state(ctx).scheduler().hasScheduledRetryTask()); - return numRemainingPendingAttempts > 0 || state(ctx).scheduler().hasScheduledRetryTask(); + return hasPendingOrScheduledAttempts(ctx); + } + + protected static boolean hasPendingOrScheduledAttempts(ClientRequestContext ctx) { + return state(ctx).numPendingAttempts() > 0 || state(ctx).scheduler().hasScheduledRetryTask(); } // an attempt did not trigger a retry. when does the attempt end? @@ -245,6 +251,7 @@ protected static boolean isRetryingComplete(ClientRequestContext ctx) { protected static void scheduleNextRetry(ClientRequestContext ctx, Runnable retryTask, long retryTimeNanos, + Backoff responsibleBackoff, Consumer actionOnException) { requireNonNull(ctx, "ctx"); requireNonNull(actionOnException, "actionOnException"); @@ -252,23 +259,30 @@ protected static void scheduleNextRetry(ClientRequestContext ctx, final RetryScheduler scheduler = state(ctx).scheduler(); - scheduleNextRetry(ctx, retryTask, retryTimeNanos, + scheduleNextRetry(ctx, retryTask, retryTimeNanos, responsibleBackoff, scheduler.getEarliestNextRetryTimeNanos(), actionOnException); } protected static void scheduleNextRetry(ClientRequestContext ctx, Runnable retryTask, long retryTimeNanos, + Backoff backoff, long earliestNextRetryTimeFromServerNanos, Consumer actionOnException) { requireNonNull(ctx, "ctx"); requireNonNull(actionOnException, "actionOnException"); requireNonNull(retryTask, "retryTask"); - final RetryScheduler scheduler = state(ctx).scheduler(); + final State state = state(ctx); + final RetryScheduler scheduler = state.scheduler(); + // todo(szymon): remove scheduler.addEarliestNextRetryTimeNanos(earliestNextRetryTimeFromServerNanos); - scheduler.schedule(retryTask, retryTimeNanos, actionOnException); + scheduler.schedule(() -> { + checkState(!isRetryingComplete(ctx)); + state.acquireAttemptNoWithCurrentBackoff(backoff); + retryTask.run(); + }, retryTimeNanos, actionOnException); } protected static void addEarliestNextRetryTimeNanos(ClientRequestContext ctx, @@ -359,9 +373,6 @@ protected final RetrySchedulabilityDecision canScheduleWith(ClientRequestContext nextRetryTimeNanos, earliestNextRetryTimeNanos); } - // todo(szymon): do we want to wait for acquisition when we schedule it? - state(ctx).acquireAttemptNoWithCurrentBackoff(backoff); - return new RetrySchedulabilityDecision(Rationale.SCHEDULABLE, nextRetryTimeNanos, earliestNextRetryTimeNanos); } @@ -597,6 +608,10 @@ int decrementPendingAttemptCount() { return --numPendingAttempts; } + int numPendingAttempts() { + return numPendingAttempts; + } + void complete(ClientRequestContext winningAttemptCtx) { retryingCompleteFuture.complete(winningAttemptCtx); } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java index 553bef85390..e813fbb6b0e 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java @@ -25,48 +25,28 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.linecorp.armeria.client.ClientFactory; import com.linecorp.armeria.common.annotation.Nullable; import io.netty.channel.EventLoop; import io.netty.util.concurrent.ScheduledFuture; class RetryScheduler { - private static class ScheduledRetryTask { + private static class RetryTaskHandle { private final Runnable retryTask; private final ScheduledFuture scheduledFuture; + private boolean isCancellationMuted; - private final Consumer onRetryTaskFailedHandler; + private final Consumer<@Nullable ? super Throwable> onExceptionHandler; private final long retryTimeNanos; - private boolean ignoreCancellation; - ScheduledRetryTask(ScheduledFuture scheduledFuture, Runnable retryTask, - long retryTimeNanos, - Consumer onRetryTaskFailedHandler) { + RetryTaskHandle(ScheduledFuture scheduledFuture, Runnable retryTask, + long retryTimeNanos, + Consumer<@Nullable ? super Throwable> onExceptionHandler) { this.retryTask = retryTask; this.scheduledFuture = scheduledFuture; - - this.onRetryTaskFailedHandler = onRetryTaskFailedHandler; - + this.onExceptionHandler = onExceptionHandler; this.retryTimeNanos = retryTimeNanos; - - scheduledFuture.addListener(future -> { - if (future.isCancelled()) { - if (!isIgnoringCancellation()) { - onRetryTaskFailedHandler.accept(new IllegalStateException( - ClientFactory.class.getSimpleName() + " has been closed.")); - } else { - // We are cancelled because we are getting rescheduled. - } - } else if (!future.isSuccess()) { - onRetryTaskFailedHandler.accept(future.cause()); - } - }); - } - - private boolean isIgnoringCancellation() { - return ignoreCancellation; } public Runnable getRetryTaskRunnable() { @@ -77,24 +57,34 @@ public long retryTimeNanos() { return retryTimeNanos; } - public Consumer getOnRetryTaskFailedHandler() { - return onRetryTaskFailedHandler; + public boolean isCancellationMuted() { + return isCancellationMuted; + } + + public void muteCancellation() { + isCancellationMuted = true; + } + + public void unmuteCancellation() { + isCancellationMuted = false; + } + + public Consumer getOnExceptionHandler() { + return onExceptionHandler; } - public boolean cancel() { - ignoreCancellation = true; - final boolean couldCancel = scheduledFuture.cancel(false); - ignoreCancellation = false; - return couldCancel; + public ScheduledFuture getFuture() { + return scheduledFuture; } } private static final Logger logger = LoggerFactory.getLogger(RetryScheduler.class); private final EventLoop eventLoop; - private long earliestNextRetryTimeNanos; private final long latestNextRetryTimeNanos; - private @Nullable ScheduledRetryTask currentRetryTask; + + private long earliestNextRetryTimeNanos; + private @Nullable RetryTaskHandle currentRetryTask; RetryScheduler(EventLoop eventLoop) { this(eventLoop, Long.MAX_VALUE); @@ -107,8 +97,8 @@ public boolean cancel() { this.latestNextRetryTimeNanos = latestNextRetryTimeNanos; } - public void schedule(Runnable retryTask, long nextRetryTimeNanos, - Consumer onRetryTaskFailedHandler) { + public synchronized void schedule(Runnable retryTask, long nextRetryTimeNanos, + Consumer onRetryTaskFailedHandler) { requireNonNull(retryTask, "retryTask"); requireNonNull(onRetryTaskFailedHandler, "onRetryTaskFailedHandler"); @@ -119,6 +109,7 @@ public void schedule(Runnable retryTask, long nextRetryTimeNanos, new IllegalStateException( "nextRetryTimeNanos is before the earliestNextRetryTimeNanos: " + nextRetryTimeNanos + " < " + earliestNextRetryTimeNanos)); + return; } if (nextRetryTimeNanos > latestNextRetryTimeNanos) { @@ -129,11 +120,8 @@ public void schedule(Runnable retryTask, long nextRetryTimeNanos, return; } - // "fast-path" - if (currentRetryTask == null || nextRetryTimeNanos >= currentRetryTask.retryTimeNanos()) { - // No retry task scheduled. We can schedule a new one directly. - scheduleNextRetryTask(retryTask, nextRetryTimeNanos, - onRetryTaskFailedHandler); + if (currentRetryTask == null || nextRetryTimeNanos < currentRetryTask.retryTimeNanos()) { + scheduleNextRetryTask(retryTask, nextRetryTimeNanos, onRetryTaskFailedHandler); return; } @@ -143,17 +131,59 @@ public void schedule(Runnable retryTask, long nextRetryTimeNanos, "nextRetryTimeNanos: " + nextRetryTimeNanos)); } - private void scheduleNextRetryTask(Runnable retryRunnable, long retryTimeNanos, - Consumer onRetryTaskFailedHandler) { + private synchronized boolean cancelCurrentRetryTask() { + if (currentRetryTask != null) { + final ScheduledFuture retryTaskFuture = currentRetryTask.getFuture(); + + currentRetryTask.muteCancellation(); + if (!retryTaskFuture.cancel(false)) { + currentRetryTask.unmuteCancellation(); + return false; + } else { + clearCurrentRetryTask(); + return true; + } + } + + return true; + } + + private synchronized void handleRetryTaskCompletion(RetryTaskHandle retryTaskHandle) { + final ScheduledFuture retryTaskFuture = retryTaskHandle.getFuture(); + assert retryTaskFuture.isDone(); + + if (currentRetryTask == retryTaskHandle) { + clearCurrentRetryTask(); + } + + if (retryTaskFuture.isCancelled()) { + if (!retryTaskHandle.isCancellationMuted()) { + // The retry task was cancelled by the user, not by the scheduler. + retryTaskHandle.getOnExceptionHandler().accept( + new IllegalStateException("Retry task was cancelled by the user.")); + } + return; + } + + if (!retryTaskFuture.isSuccess()) { + retryTaskHandle.getOnExceptionHandler().accept(retryTaskFuture.cause()); + } + } + + private synchronized void clearCurrentRetryTask() { + currentRetryTask = null; + earliestNextRetryTimeNanos = Long.MIN_VALUE; + } + + private synchronized void scheduleNextRetryTask(Runnable retryRunnable, long retryTimeNanos, + Consumer onExceptionHandler) { assert earliestNextRetryTimeNanos <= retryTimeNanos; assert retryTimeNanos <= latestNextRetryTimeNanos; - if (currentRetryTask != null) { - if (!currentRetryTask.cancel()) { - onRetryTaskFailedHandler.accept( - new IllegalStateException("Could not cancel the current retry task.")); - return; - } + if (!cancelCurrentRetryTask()) { + onExceptionHandler.accept( + new IllegalStateException("Could not cancel the current retry task.")); + return; } assert currentRetryTask == null; @@ -162,8 +192,8 @@ private void scheduleNextRetryTask(Runnable retryRunnable, long retryTimeNanos, final long delayNanos = Math.max(retryTimeNanos - System.nanoTime(), 0); final Runnable wrappedRetryRunnable = () -> { logger.debug("Retry task starting. Resetting..."); - currentRetryTask = null; - earliestNextRetryTimeNanos = Long.MIN_VALUE; + // todo(szymon): do sanity check that we are clearing this task (very bad otherwise). + clearCurrentRetryTask(); retryRunnable.run(); }; @@ -178,24 +208,25 @@ private void scheduleNextRetryTask(Runnable retryRunnable, long retryTimeNanos, // We are passing in the original to avoid multiple wrapping in case of the retry task being // rescheduled multiple times. - currentRetryTask = new ScheduledRetryTask(nextRetryTaskFuture, retryRunnable, - retryTimeNanos, - onRetryTaskFailedHandler); + final RetryTaskHandle nextRetryTask = new RetryTaskHandle(nextRetryTaskFuture, retryRunnable, + retryTimeNanos, + onExceptionHandler); + + nextRetryTaskFuture.addListener(f -> handleRetryTaskCompletion(nextRetryTask)); + currentRetryTask = nextRetryTask; } catch (Throwable t) { - onRetryTaskFailedHandler.accept(t); + onExceptionHandler.accept(t); } } public void close() { - if (currentRetryTask != null) { - currentRetryTask.cancel(); - } + cancelCurrentRetryTask(); } // todo(szymon): Remove dependency on nextEarliestNextRetryTimeNanos. Users should simply set it before // calling this method. - public boolean hasAlreadyRetryScheduledBefore(long nextRetryTimeNanos, - long nextEarliestNextRetryTimeNanos) { + public synchronized boolean hasAlreadyRetryScheduledBefore(long nextRetryTimeNanos, + long nextEarliestNextRetryTimeNanos) { checkState(nextEarliestNextRetryTimeNanos <= latestNextRetryTimeNanos); earliestNextRetryTimeNanos = Math.max(earliestNextRetryTimeNanos, nextEarliestNextRetryTimeNanos); @@ -207,29 +238,29 @@ public boolean hasAlreadyRetryScheduledBefore(long nextRetryTimeNanos, <= nextRetryTimeNanos; } - public void addEarliestNextRetryTimeNanos(long earliestNextRetryTimeNanos) { + public synchronized void addEarliestNextRetryTimeNanos(long earliestNextRetryTimeNanos) { checkState(earliestNextRetryTimeNanos <= latestNextRetryTimeNanos); this.earliestNextRetryTimeNanos = Math.max(this.earliestNextRetryTimeNanos, earliestNextRetryTimeNanos); } - public long getEarliestNextRetryTimeNanos() { + public synchronized long getEarliestNextRetryTimeNanos() { return earliestNextRetryTimeNanos; } - public void rescheduleCurrentRetryTaskIfTooEarly() { + public synchronized void rescheduleCurrentRetryTaskIfTooEarly() { if (currentRetryTask != null) { if (currentRetryTask.retryTimeNanos() < earliestNextRetryTimeNanos) { // Current retry task is going to be executed before the earliestNextRetryTimeNanos so // we need to reschedule it. scheduleNextRetryTask(currentRetryTask.getRetryTaskRunnable(), earliestNextRetryTimeNanos, - currentRetryTask.getOnRetryTaskFailedHandler()); + currentRetryTask.getOnExceptionHandler()); } } } - public boolean hasScheduledRetryTask() { + public synchronized boolean hasScheduledRetryTask() { return currentRetryTask != null; } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index 93b91474a65..f02d0d6e608 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -387,8 +387,10 @@ private void doExecute0(RetryingContext retryingContext) { final RetrySchedulabilityDecision retrySchedulabilityDecision = canScheduleWith(ctx, hedgingBackoff, -1); if (retrySchedulabilityDecision.canSchedule()) { + logger.debug("Scheduling hedging with backoff: " + hedgingBackoff); scheduleNextRetry(ctx, () -> doExecute0(retryingContext), retrySchedulabilityDecision.nextRetryTimeNanos(), + hedgingBackoff, cause -> handleException(retryingContext, cause, false)); } } @@ -620,6 +622,7 @@ private void handleRetryDecision(RetryingContext retryingContext, @Nullable Retr scheduleNextRetry(retryingContext.ctx(), () -> doExecute0(retryingContext), schedulabilityDecision.nextRetryTimeNanos(), + backoff, schedulabilityDecision.earliestNextRetryTimeNanos(), cause -> handleException(retryingContext, cause, false)); } else { @@ -649,11 +652,11 @@ private static void abortAttempt(ClientRequestContext attemptCtx, HttpResponse a final RequestLogBuilder logBuilder = attemptCtx.logBuilder(); logBuilder.responseContent(null, null); logBuilder.responseContentPreview(null); - attemptCtx.cancel(); + if (cause != null) { - attemptRes.abort(cause); + attemptCtx.cancel(cause); } else { - attemptRes.abort(); + attemptCtx.cancel(); } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java index 7cfd5b90ac3..a02e148b1c4 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java @@ -187,12 +187,12 @@ private void doExecute0(ClientRequestContext ctx, RpcRequest req, ClientPendingThrowableUtil.removePendingThrowable(attemptCtx); // if the endpoint hasn't been selected, try to initialize the ctx with a new endpoint/event loop attemptRes = initContextAndExecuteWithFallback(unwrap(), ctxExtension, RpcResponse::from, - (context, cause) -> RpcResponse.ofFailure(cause), - req, true); + (context, cause) -> RpcResponse.ofFailure(cause), + req, true); } else { attemptRes = executeWithFallback(unwrap(), attemptCtx, - (context, cause) -> RpcResponse.ofFailure(cause), - req, true); + (context, cause) -> RpcResponse.ofFailure(cause), + req, true); } onAttemptStarted(ctx, attemptCtx, (@Nullable Throwable cause) -> { @@ -210,12 +210,14 @@ private void doExecute0(ClientRequestContext ctx, RpcRequest req, final Backoff backoff = decision != null ? decision.backoff() : null; if (backoff != null) { - final RetrySchedulabilityDecision schedulabilityDecision = canScheduleWith(ctx, backoff); + final RetrySchedulabilityDecision schedulabilityDecision = canScheduleWith(ctx, + backoff); if (schedulabilityDecision.canSchedule()) { scheduleNextRetry(ctx, () -> doExecute0(ctx, req, returnedRes, returnedResFuture), schedulabilityDecision.nextRetryTimeNanos(), + backoff, cause0 -> handleException(ctx, returnedResFuture, cause0, false)); } } @@ -234,7 +236,6 @@ private void doExecute0(ClientRequestContext ctx, RpcRequest req, return null; }); - final @Nullable Backoff hedgingBackoff = retryConfig.hedgingBackoff(); if (hedgingBackoff != null) { @@ -242,13 +243,15 @@ private void doExecute0(ClientRequestContext ctx, RpcRequest req, canScheduleWith(ctx, hedgingBackoff); if (schedulabilityDecision.canSchedule()) { scheduleNextRetry(ctx, () -> doExecute0(ctx, req, returnedRes, returnedResFuture), - schedulabilityDecision.nextRetryTimeNanos(), - cause -> handleException(ctx, returnedResFuture, cause, false)); + schedulabilityDecision.nextRetryTimeNanos(), + hedgingBackoff, + cause -> handleException(ctx, returnedResFuture, cause, false)); } } } - private static void handleException(ClientRequestContext ctx, CompletableFuture returnedResFuture, + private static void handleException(ClientRequestContext ctx, + CompletableFuture returnedResFuture, Throwable cause, boolean endRequestLog) { if (isRetryingComplete(ctx)) { return; diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java index 09869095592..eb13183b6dd 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java @@ -28,6 +28,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; @@ -37,8 +38,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.linecorp.armeria.client.ClientFactory; import com.linecorp.armeria.client.ClientRequestContext; @@ -47,6 +46,7 @@ import com.linecorp.armeria.client.ResponseCancellationException; import com.linecorp.armeria.client.ResponseTimeoutException; import com.linecorp.armeria.client.WebClient; +import com.linecorp.armeria.client.WebClientBuilder; import com.linecorp.armeria.client.endpoint.EndpointGroup; import com.linecorp.armeria.client.endpoint.EndpointSelectionStrategy; import com.linecorp.armeria.common.AggregatedHttpResponse; @@ -54,6 +54,7 @@ import com.linecorp.armeria.common.HttpRequest; import com.linecorp.armeria.common.HttpResponse; import com.linecorp.armeria.common.HttpStatus; +import com.linecorp.armeria.common.MediaType; import com.linecorp.armeria.common.SessionProtocol; import com.linecorp.armeria.common.annotation.Nullable; import com.linecorp.armeria.common.logging.RequestLog; @@ -73,8 +74,6 @@ class RetryingClientWithHedgingTest { private static final String SERVER2_RESPONSE = "s2#"; private static final String SERVER3_RESPONSE = "s3##"; - private static final Logger log = LoggerFactory.getLogger(RetryingClientWithHedgingTest.class); - private static class TestServer extends ServerExtension { private CountDownLatch responseLatch = new CountDownLatch(1); private final AtomicInteger numRequests = new AtomicInteger(); @@ -188,16 +187,11 @@ void letSecondServerWins() throws Exception { .hedgingBackoff(Backoff.fixed(100)) .build(); - final WebClient client = WebClient.builder(SessionProtocol.H2C, - EndpointGroup.of(EndpointSelectionStrategy.roundRobin(), - server1.httpEndpoint(), - server2.httpEndpoint(), - server3.httpEndpoint())) - .factory(clientFactory) - .decorator( - RetryingClient.newDecorator(hedgingNoRetryConfig) - ) - .build(); + final WebClient client = clientBuilder() + .decorator( + RetryingClient.newDecorator(hedgingNoRetryConfig) + ) + .build(); final CompletableFuture responseFuture; final ClientRequestContext ctx; @@ -245,16 +239,11 @@ void letThirdServerWin() throws Exception { .hedgingBackoff(Backoff.fixed(100)) .build(); - final WebClient client = WebClient.builder(SessionProtocol.H2C, - EndpointGroup.of(EndpointSelectionStrategy.roundRobin(), - server1.httpEndpoint(), - server2.httpEndpoint(), - server3.httpEndpoint())) - .factory(clientFactory) - .decorator( - RetryingClient.newDecorator(hedgingNoRetryConfig) - ) - .build(); + final WebClient client = clientBuilder() + .decorator( + RetryingClient.newDecorator(hedgingNoRetryConfig) + ) + .build(); final CompletableFuture responseFuture; final ClientRequestContext ctx; @@ -304,16 +293,11 @@ void thirdWinsEvenAfterPerAttemptTimeout() throws Exception { .hedgingBackoff(Backoff.fixed(500)) .build(); - final WebClient client = WebClient.builder(SessionProtocol.H2C, - EndpointGroup.of(EndpointSelectionStrategy.roundRobin(), - server1.httpEndpoint(), - server2.httpEndpoint(), - server3.httpEndpoint())) - .factory(clientFactory) - .decorator( - RetryingClient.newDecorator(hedgingNoRetryConfig) - ) - .build(); + final WebClient client = clientBuilder() + .decorator( + RetryingClient.newDecorator(hedgingNoRetryConfig) + ) + .build(); final CompletableFuture responseFuture; final ClientRequestContext ctx; @@ -348,6 +332,94 @@ void thirdWinsEvenAfterPerAttemptTimeout() throws Exception { }); } + @Test + void thirdWinsEvenAfterRetriableError() throws Exception { + when(server1.getHelloService().serve(any(), any())) + .thenReturn(HttpResponse.of(HttpStatus.TOO_MANY_REQUESTS, MediaType.PLAIN_TEXT, + SERVER1_RESPONSE)); + when(server2.getHelloService().serve(any(), any())) + .thenReturn(HttpResponse.of(HttpStatus.TOO_MANY_REQUESTS, MediaType.PLAIN_TEXT, + SERVER2_RESPONSE)); + when(server3.getHelloService().serve(any(), any())) + .thenReturn(HttpResponse.of(SERVER3_RESPONSE)); + + final RetryConfig config = RetryConfig + .builder(RetryRule.of( + RetryRule + .builder() + .onStatus(HttpStatus.TOO_MANY_REQUESTS) + .thenBackoff(Backoff.withoutDelay()), + NO_RETRY_RULE + )) + .maxTotalAttempts(3) + .hedgingBackoff(Backoff.fixed(200)) + .build(); + + final WebClient client = client(config); + + /* + We expect the following to happen: + 1. The first request will be sent to server 1 while a hedged request is scheduled after 100ms after + that. + 2. The first request will fail with TOO_MANY_REQUESTS and order an immediate retry. This will + cancel the hedged request. + 3. The retry will be sent to server 2 and with that again schedule a hedged request after 100ms. + 4. The second request will fail with TOO_MANY_REQUESTS and order an immediate retry. This will again + cancel the hedged request of the second request. + 5. The immediate retry will be sent to server 3, which will succeed. + */ + + final CompletableFuture responseFuture; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + responseFuture = client.get("/hello").aggregate(); + ctx = captor.get(); + } + + server1.unlatchResponse(); + server2.unlatchResponse(); + server3.unlatchResponse(); + + await().untilAsserted(() -> { + assertThat(server1.getNumRequests()).isOne(); + assertThat(server2.getNumRequests()).isOne(); + assertThat(server3.getNumRequests()).isOne(); + }); + + await().untilAsserted(() -> { + assertValidServerRequestContext(server1, 1); + assertValidServerRequestContext(server2, 2); + assertValidServerRequestContext(server3, 3); + + assertValidAggregatedResponse(responseFuture, SERVER3_RESPONSE); + assertValidClientRequestContext( + ctx, GET_VERIFY_RESPONSE_HAS_CONTENT.apply(SERVER3_RESPONSE), + GET_VERIFY_RESPONSE_HAS_CONTENT_AND_STATUS.apply(SERVER1_RESPONSE, + HttpStatus.TOO_MANY_REQUESTS), + GET_VERIFY_RESPONSE_HAS_CONTENT_AND_STATUS.apply(SERVER2_RESPONSE, + HttpStatus.TOO_MANY_REQUESTS), + GET_VERIFY_RESPONSE_HAS_CONTENT.apply(SERVER3_RESPONSE) + ); + }); + } + + @Test + void loosesAfterNonRetriableError() {} + + private static WebClientBuilder clientBuilder() { + return WebClient.builder(SessionProtocol.H2C, + EndpointGroup.of(EndpointSelectionStrategy.roundRobin(), + server1.httpEndpoint(), + server2.httpEndpoint(), + server3.httpEndpoint())) + .factory(clientFactory); + } + + private static WebClient client(RetryConfig config) { + return clientBuilder() + .decorator(RetryingClient.newDecorator(config)).build(); + } + private static String getResponseContent(AggregatedHttpResponse response) { return response.content().toString(Charset.defaultCharset()); } @@ -437,10 +509,17 @@ private interface RequestLogVerifier extends Consumer {} // assertThat(log.responseCause().getCause()).isInstanceOf(ResponseTimeoutException.class); // }; // - private static final Function GET_VERIFY_RESPONSE_HAS_CONTENT = - expectedResponseContent -> log -> { + + private static final BiFunction + GET_VERIFY_RESPONSE_HAS_CONTENT_AND_STATUS = + (expectedResponseContent, expectedStatus) -> log -> { assertThat(log.responseLength()).isEqualTo(expectedResponseContent.length()); + assertThat(log.responseStatus()).isEqualTo(expectedStatus); }; + + private static final Function GET_VERIFY_RESPONSE_HAS_CONTENT = + expectedResponseContent -> GET_VERIFY_RESPONSE_HAS_CONTENT_AND_STATUS.apply(expectedResponseContent, + HttpStatus.OK); // // private static final BiFunction // GET_VERIFY_RESPONSE_HAS_APPLICATION_EXCEPTION = From fe9f063ca82eb18a01f1e475740a4d73e203c8f2 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Sat, 7 Jun 2025 21:30:25 +0200 Subject: [PATCH 05/36] [WIP] feat: add test verifying that RetryingClient stops hedging after non-retriable response --- .../retry/RetryingClientWithHedgingTest.java | 110 +++++++++++------- 1 file changed, 70 insertions(+), 40 deletions(-) diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java index eb13183b6dd..be2c1b29f3f 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java @@ -27,6 +27,7 @@ import java.nio.charset.Charset; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; import java.util.function.Consumer; @@ -187,11 +188,7 @@ void letSecondServerWins() throws Exception { .hedgingBackoff(Backoff.fixed(100)) .build(); - final WebClient client = clientBuilder() - .decorator( - RetryingClient.newDecorator(hedgingNoRetryConfig) - ) - .build(); + final WebClient client = client(hedgingNoRetryConfig); final CompletableFuture responseFuture; final ClientRequestContext ctx; @@ -236,14 +233,10 @@ void letThirdServerWin() throws Exception { final RetryConfig hedgingNoRetryConfig = RetryConfig .builder(NO_RETRY_RULE) .maxTotalAttempts(3) - .hedgingBackoff(Backoff.fixed(100)) + .hedgingBackoff(Backoff.fixed(10)) .build(); - final WebClient client = clientBuilder() - .decorator( - RetryingClient.newDecorator(hedgingNoRetryConfig) - ) - .build(); + final WebClient client = client(hedgingNoRetryConfig); final CompletableFuture responseFuture; final ClientRequestContext ctx; @@ -289,8 +282,8 @@ void thirdWinsEvenAfterPerAttemptTimeout() throws Exception { .builder(RetryRule.builder().onTimeoutException().thenBackoff(Backoff.fixed(10_000))) // should // be always overtaken by hedging task .maxTotalAttempts(3) - .responseTimeoutMillisForEachAttempt(200) - .hedgingBackoff(Backoff.fixed(500)) + .responseTimeoutMillisForEachAttempt(100) + .hedgingBackoff(Backoff.fixed(200)) .build(); final WebClient client = clientBuilder() @@ -306,16 +299,16 @@ void thirdWinsEvenAfterPerAttemptTimeout() throws Exception { ctx = captor.get(); } - await().untilAsserted(() -> { + await().pollInterval(25, TimeUnit.MILLISECONDS).untilAsserted(() -> { assertThat(server1.getNumRequests()).isOne(); assertThat(server2.getNumRequests()).isOne(); assertThat(server3.getNumRequests()).isOne(); }); // Let the third server win - server1.unlatchResponse(); - server2.unlatchResponse(); - server3.unlatchResponse(); + server1.unlatchResponse(); // issued at T + server2.unlatchResponse(); // issued at T + 200 (request 1 timed out) + server3.unlatchResponse(); // issued at T + 400 (request 2 timed out) await() .untilAsserted(() -> { @@ -333,7 +326,7 @@ void thirdWinsEvenAfterPerAttemptTimeout() throws Exception { } @Test - void thirdWinsEvenAfterRetriableError() throws Exception { + void thirdServerWinsEvenAfterRetriableResponse() throws Exception { when(server1.getHelloService().serve(any(), any())) .thenReturn(HttpResponse.of(HttpStatus.TOO_MANY_REQUESTS, MediaType.PLAIN_TEXT, SERVER1_RESPONSE)); @@ -357,18 +350,6 @@ void thirdWinsEvenAfterRetriableError() throws Exception { final WebClient client = client(config); - /* - We expect the following to happen: - 1. The first request will be sent to server 1 while a hedged request is scheduled after 100ms after - that. - 2. The first request will fail with TOO_MANY_REQUESTS and order an immediate retry. This will - cancel the hedged request. - 3. The retry will be sent to server 2 and with that again schedule a hedged request after 100ms. - 4. The second request will fail with TOO_MANY_REQUESTS and order an immediate retry. This will again - cancel the hedged request of the second request. - 5. The immediate retry will be sent to server 3, which will succeed. - */ - final CompletableFuture responseFuture; final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { @@ -404,7 +385,53 @@ void thirdWinsEvenAfterRetriableError() throws Exception { } @Test - void loosesAfterNonRetriableError() {} + void loosesAfterNonRetriableResponse() throws Exception { + when(server1.getHelloService().serve(any(), any())) + .thenReturn(HttpResponse.of(SERVER1_RESPONSE)); + when(server2.getHelloService().serve(any(), any())) + .thenReturn(HttpResponse.of(HttpStatus.INTERNAL_SERVER_ERROR, MediaType.PLAIN_TEXT, + SERVER2_RESPONSE)); + + final RetryConfig config = RetryConfig + .builder(NO_RETRY_RULE) + .maxTotalAttempts(3) + .hedgingBackoff(Backoff.fixed(100)) + .build(); + + final WebClient client = client(config); + + final CompletableFuture responseFuture; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + responseFuture = client.get("/hello").aggregate(); + ctx = captor.get(); + } + + server2.unlatchResponse(); + + await().untilAsserted(() -> { + assertThat(server2.getNumRequests()).isOne(); + }); + + Thread.sleep(10); + server1.unlatchResponse(); + server2.unlatchResponse(); + + await().untilAsserted(() -> { + assertValidServerRequestContext(server1, 1); + assertValidServerRequestContext(server2, 2); + assertNoServerRequestContext(server3); + + assertValidAggregatedResponse(responseFuture, HttpStatus.INTERNAL_SERVER_ERROR, SERVER2_RESPONSE); + assertValidClientRequestContext( + ctx, GET_VERIFY_RESPONSE_HAS_CONTENT_AND_STATUS.apply(SERVER2_RESPONSE, + HttpStatus.INTERNAL_SERVER_ERROR), + VERIFY_REQUEST_CANCELLED, + GET_VERIFY_RESPONSE_HAS_CONTENT_AND_STATUS.apply(SERVER2_RESPONSE, + HttpStatus.INTERNAL_SERVER_ERROR), null + ); + }); + } private static WebClientBuilder clientBuilder() { return WebClient.builder(SessionProtocol.H2C, @@ -432,6 +459,14 @@ private void assertValidAggregatedResponse(CompletableFuture resFuture, + HttpStatus expectedStatus, String expectedContent) { + assertThat(resFuture.isDone()).isTrue(); + final AggregatedHttpResponse res = resFuture.getNow(null); + assertThat(res.status()).isEqualTo(expectedStatus); + assertThat(getResponseContent(res)).isEqualTo(expectedContent); + } + private void assertValidClientRequestContext(ClientRequestContext ctx, RequestLogVerifier logVerifierCtx, RequestLogVerifier logVerifierServer1, @@ -490,6 +525,10 @@ private void assertValidServerRequestContext(ServerExtension server, int attempt assertThat(slog.requestHeaders().path()).contains("hello"); } + private void assertNoServerRequestContext(ServerExtension server) { + assertThat(server.requestContextCaptor().size()).isEqualTo(0); + } + @FunctionalInterface private interface RequestLogVerifier extends Consumer {} @@ -520,13 +559,4 @@ private interface RequestLogVerifier extends Consumer {} private static final Function GET_VERIFY_RESPONSE_HAS_CONTENT = expectedResponseContent -> GET_VERIFY_RESPONSE_HAS_CONTENT_AND_STATUS.apply(expectedResponseContent, HttpStatus.OK); -// -// private static final BiFunction -// GET_VERIFY_RESPONSE_HAS_APPLICATION_EXCEPTION = -// (expectedType, expectedMessage) -> log -> { -// assertThat(log.responseCause()).isInstanceOf(TApplicationException.class); -// final TApplicationException cause = (TApplicationException) log.responseCause(); -// assertThat(cause.getType()).isEqualTo(expectedType); -// assertThat(cause.getMessage()).contains(expectedMessage); -// }; } From fbc4455315413a68ac26aad36337128ecfcd3dda Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Sun, 8 Jun 2025 12:51:47 +0200 Subject: [PATCH 06/36] [WIP] feat: add test verifying that RetryingClient stops correctly after response timeout --- .../armeria/client/retry/RetryingClient.java | 4 +- .../retry/RetryingClientWithHedgingTest.java | 180 +++++++++++++----- 2 files changed, 134 insertions(+), 50 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index f02d0d6e608..355e43cff71 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -342,7 +342,7 @@ private void doExecute0(RetryingContext retryingContext) { false); } - onAttemptStarted(ctx, attemptCtx, (@Nullable Throwable cause) -> abortAttempt(attemptCtx, attemptRes, + onAttemptStarted(ctx, attemptCtx, (@Nullable Throwable cause) -> abortAttempt(attemptCtx, cause)); if (!ctx.exchangeType().isResponseStreaming() || config.requiresResponseTrailers()) { @@ -646,7 +646,7 @@ private void handleRetryDecision(RetryingContext retryingContext, @Nullable Retr } } - private static void abortAttempt(ClientRequestContext attemptCtx, HttpResponse attemptRes, + private static void abortAttempt(ClientRequestContext attemptCtx, @Nullable Throwable cause) { // Set response content with null to make sure that the log is complete. final RequestLogBuilder logBuilder = attemptCtx.logBuilder(); diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java index be2c1b29f3f..91d03965fe1 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java @@ -18,6 +18,7 @@ import static com.linecorp.armeria.client.retry.AbstractRetryingClient.ARMERIA_RETRY_COUNT; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.fail; import static org.awaitility.Awaitility.await; import static org.mockito.ArgumentMatchers.any; @@ -25,6 +26,8 @@ import static org.mockito.Mockito.when; import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -37,6 +40,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.RepeatedTest; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -61,6 +65,7 @@ import com.linecorp.armeria.common.logging.RequestLog; import com.linecorp.armeria.common.logging.RequestLogAccess; import com.linecorp.armeria.common.logging.RequestLogProperty; +import com.linecorp.armeria.common.util.Exceptions; import com.linecorp.armeria.server.HttpService; import com.linecorp.armeria.server.RoutingContext; import com.linecorp.armeria.server.ServerBuilder; @@ -68,6 +73,8 @@ import com.linecorp.armeria.server.logging.LoggingService; import com.linecorp.armeria.testing.junit5.server.ServerExtension; +// todo(szymon): change tests that we wait for the response and demand that immediately after we see all the +// logs in the appropriate state. class RetryingClientWithHedgingTest { private static final long LOOSING_SERVER_RESPONSE_DELAY_MILLIS = 300; @@ -87,28 +94,32 @@ private static class TestServer extends ServerExtension { @Override protected void configure(ServerBuilder sb) throws Exception { sb.decorator(LoggingService.newDecorator()); + sb.blockingTaskExecutor(1); sb.service("/hello", new HttpService() { @Override public HttpResponse serve(ServiceRequestContext ctx, HttpRequest req) throws Exception { - numRequests.incrementAndGet(); - try { - responseLatch.await(); - } catch (InterruptedException e) { - fail(e); - } - - req.whenComplete().handle((t, tt) -> { - System.out.println("Request completed: " + req); - return null; - }); - - final HttpResponse res = helloService.serve(ctx, req); - res.whenComplete().handle((t, tt) -> { - System.out.println("Response completed: " + res); - return null; + final CompletableFuture responseFuture = new CompletableFuture<>(); + final HttpResponse res = HttpResponse.of(responseFuture); + + // We are using a blocking task executor to not block the event loop so we are + // able to receive request cancellations. + ctx.blockingTaskExecutor().execute(() -> { + numRequests.incrementAndGet(); + try { + responseLatch.await(); + } catch (InterruptedException e) { + responseFuture.completeExceptionally(e); + fail(e); + } + + try { + responseFuture.complete(helloService.serve(ctx, req)); + } catch (Exception e) { + responseFuture.completeExceptionally(e); + } }); return res; @@ -210,9 +221,9 @@ void letSecondServerWins() throws Exception { await() .untilAsserted(() -> { - assertValidServerRequestContext(server1, 1); - assertValidServerRequestContext(server2, 2); - assertValidServerRequestContext(server3, 3); + assertValidServerRequestContext(server1, 1, true); + assertValidServerRequestContext(server2, 2, false); + assertValidServerRequestContext(server3, 3, true); assertValidAggregatedResponse(responseFuture, SERVER2_RESPONSE); assertValidClientRequestContext( @@ -258,9 +269,9 @@ void letThirdServerWin() throws Exception { await() .untilAsserted(() -> { - assertValidServerRequestContext(server1, 1); - assertValidServerRequestContext(server2, 2); - assertValidServerRequestContext(server3, 3); + assertValidServerRequestContext(server1, 1, true); + assertValidServerRequestContext(server2, 2, true); + assertValidServerRequestContext(server3, 3, false); assertValidAggregatedResponse(responseFuture, SERVER3_RESPONSE); assertValidClientRequestContext( @@ -312,9 +323,9 @@ void thirdWinsEvenAfterPerAttemptTimeout() throws Exception { await() .untilAsserted(() -> { - assertValidServerRequestContext(server1, 1); - assertValidServerRequestContext(server2, 2); - assertValidServerRequestContext(server3, 3); + assertValidServerRequestContext(server1, 1, true); + assertValidServerRequestContext(server2, 2, true); + assertValidServerRequestContext(server3, 3, false); assertValidAggregatedResponse(responseFuture, SERVER3_RESPONSE); assertValidClientRequestContext( @@ -418,8 +429,8 @@ void loosesAfterNonRetriableResponse() throws Exception { server2.unlatchResponse(); await().untilAsserted(() -> { - assertValidServerRequestContext(server1, 1); - assertValidServerRequestContext(server2, 2); + assertValidServerRequestContext(server1, 1, true); + assertValidServerRequestContext(server2, 2, false); assertNoServerRequestContext(server3); assertValidAggregatedResponse(responseFuture, HttpStatus.INTERNAL_SERVER_ERROR, SERVER2_RESPONSE); @@ -433,6 +444,67 @@ void loosesAfterNonRetriableResponse() throws Exception { }); } + @RepeatedTest(5) + void loosesAfterResponseTimeout() throws Exception { + final RetryConfig config = RetryConfig + .builder(NO_RETRY_RULE) + .maxTotalAttempts(3) + .hedgingBackoff(Backoff.withoutDelay()) + .build(); + + final WebClient client = clientBuilder() + .responseTimeoutMillis(500) + .decorator( + RetryingClient.newDecorator(config) + ).build(); + + final CompletableFuture responseFuture; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + responseFuture = client.get("/hello").aggregate(); + ctx = captor.get(); + } + + await().atLeast(400, TimeUnit.MILLISECONDS).atMost(600, TimeUnit.MILLISECONDS).untilAsserted(() -> { + assertThat(responseFuture).isCompletedExceptionally(); + assertThatThrownBy(responseFuture::get).satisfies(throwable -> { + final Throwable rootCause = Exceptions.peel(throwable); + assertThat(rootCause).isInstanceOf(ResponseTimeoutException.class); + }); + }); + + final List childLogExceptions = new ArrayList<>(); + + final RequestLogVerifier catchException = log -> { + assertThat(log.responseCause()).isNotNull(); + childLogExceptions.add(log.responseCause()); + }; + + assertValidClientRequestContext(ctx, VERIFY_RESPONSE_TIMEOUT, catchException, catchException, + catchException); + + int numTimeouts = 0; + int numCancelled = 0; + + for (final @Nullable Throwable childException : childLogExceptions) { + if (childException instanceof ResponseTimeoutException) { + numTimeouts++; + } else if (childException instanceof ResponseCancellationException) { + numCancelled++; + } else { + fail("Unexpected exception: " + childException); + } + } + + assertThat(numTimeouts + numCancelled).isEqualTo(3); + // At least one attempt needs to time out. + assertThat(numTimeouts).isPositive(); + + assertValidServerRequestContext(server1, 1, true); + assertValidServerRequestContext(server2, 2, true); + assertValidServerRequestContext(server3, 3, true); + } + private static WebClientBuilder clientBuilder() { return WebClient.builder(SessionProtocol.H2C, EndpointGroup.of(EndpointSelectionStrategy.roundRobin(), @@ -451,35 +523,41 @@ private static String getResponseContent(AggregatedHttpResponse response) { return response.content().toString(Charset.defaultCharset()); } - private void assertValidAggregatedResponse(CompletableFuture resFuture, - String expectedContent) { + private static void assertValidAggregatedResponse(CompletableFuture resFuture, + String expectedContent) { assertThat(resFuture.isDone()).isTrue(); final AggregatedHttpResponse res = resFuture.getNow(null); assertThat(res.status()).isEqualTo(HttpStatus.OK); assertThat(getResponseContent(res)).isEqualTo(expectedContent); } - private void assertValidAggregatedResponse(CompletableFuture resFuture, - HttpStatus expectedStatus, String expectedContent) { + private static void assertValidAggregatedResponse(CompletableFuture resFuture, + HttpStatus expectedStatus, String expectedContent) { assertThat(resFuture.isDone()).isTrue(); final AggregatedHttpResponse res = resFuture.getNow(null); assertThat(res.status()).isEqualTo(expectedStatus); assertThat(getResponseContent(res)).isEqualTo(expectedContent); } - private void assertValidClientRequestContext(ClientRequestContext ctx, - RequestLogVerifier logVerifierCtx, - RequestLogVerifier logVerifierServer1, - RequestLogVerifier logVerifierServer2, - @Nullable RequestLogVerifier logVerifierServer3 - ) { + private static void assertValidRootClientRequestContext(ClientRequestContext ctx, + RequestLogVerifier logVerifierCtx, + int expectedNumChildren) { assertThat(ctx.log().isComplete()).isTrue(); - assertThat(ctx.log().children()).hasSize(logVerifierServer3 == null ? 2 : 3); + assertThat(ctx.log().children()).hasSize(expectedNumChildren); final RequestLog log = ctx.log().getIfAvailable(RequestLogProperty.RESPONSE_CONTENT, RequestLogProperty.RESPONSE_CAUSE, RequestLogProperty.REQUEST_HEADERS); assertThat(log).isNotNull(); logVerifierCtx.accept(log); + } + + private static void assertValidClientRequestContext(ClientRequestContext ctx, + RequestLogVerifier logVerifierCtx, + RequestLogVerifier logVerifierServer1, + RequestLogVerifier logVerifierServer2, + @Nullable RequestLogVerifier logVerifierServer3 + ) { + assertValidRootClientRequestContext(ctx, logVerifierCtx, logVerifierServer3 == null ? 2 : 3); assertValidChildLog(ctx.log().children().get(0), 1, logVerifierServer1); assertValidChildLog(ctx.log().children().get(1), 2, logVerifierServer2); if (logVerifierServer3 != null) { @@ -487,8 +565,8 @@ private void assertValidClientRequestContext(ClientRequestContext ctx, } } - void assertValidChildLog(RequestLogAccess logAccess, int attemptNumber, - RequestLogVerifier requestLogVerifier) { + private static void assertValidChildLog(RequestLogAccess logAccess, int attemptNumber, + RequestLogVerifier requestLogVerifier) { assertThat(logAccess.isComplete()).isTrue(); // After the check right above, all properties of the RequestLog should be available. final @Nullable RequestLog log = logAccess.getIfAvailable(RequestLogProperty.RESPONSE_CONTENT, @@ -505,13 +583,20 @@ void assertValidChildLog(RequestLogAccess logAccess, int attemptNumber, requestLogVerifier.accept(log); } - private void assertValidServerRequestContext(ServerExtension server, int attemptNumber) { + private static void assertValidServerRequestContext(ServerExtension server, int attemptNumber) { + assertValidServerRequestContext(server, attemptNumber, false); + } + + private static void assertValidServerRequestContext(ServerExtension server, int attemptNumber, + boolean expectCancelled) { assertThat(server.requestContextCaptor().size()).isEqualTo(1); final ServiceRequestContext sctx = server.requestContextCaptor().peek(); assertThat(sctx).isNotNull(); assertThat(sctx.log().isComplete()).isTrue(); + assertThat(sctx.isCancelled()).isEqualTo(expectCancelled); + final RequestLog slog = sctx.log().getIfAvailable(RequestLogProperty.REQUEST_HEADERS, RequestLogProperty.REQUEST_CONTENT); @@ -523,9 +608,10 @@ private void assertValidServerRequestContext(ServerExtension server, int attempt } assertThat(slog.requestHeaders().path()).contains("hello"); + } - private void assertNoServerRequestContext(ServerExtension server) { + private static void assertNoServerRequestContext(ServerExtension server) { assertThat(server.requestContextCaptor().size()).isEqualTo(0); } @@ -542,12 +628,10 @@ private interface RequestLogVerifier extends Consumer {} assertThat(log.responseCause()).isInstanceOf(ResponseTimeoutException.class); }; // -// private static final RequestLogVerifier VERIFY_RESPONSE_TIMEOUT = -// log -> { -// assertThat(log.responseCause()).isInstanceOf(TTransportException.class); -// assertThat(log.responseCause().getCause()).isInstanceOf(ResponseTimeoutException.class); -// }; -// + private static final RequestLogVerifier VERIFY_RESPONSE_TIMEOUT = + log -> { + assertThat(log.responseCause()).isInstanceOf(ResponseTimeoutException.class); + }; private static final BiFunction GET_VERIFY_RESPONSE_HAS_CONTENT_AND_STATUS = From 60ea9a87ecd5c383c8fa85cf9481b4a1d3c2ee9d Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Sun, 8 Jun 2025 14:35:03 +0200 Subject: [PATCH 07/36] [WIP] feat: change hedging API to accept long or Duration instead of Backoff --- .../armeria/client/retry/RetryConfig.java | 34 ++++++++-------- .../client/retry/RetryConfigBuilder.java | 39 +++++++++++++------ .../armeria/client/retry/RetryingClient.java | 12 +++--- .../client/retry/RetryingRpcClient.java | 5 ++- .../retry/RetryingClientWithHedgingTest.java | 12 +++--- .../RetryingRpcClientWithHedgingTest.java | 37 +++++++++--------- 6 files changed, 78 insertions(+), 61 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfig.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfig.java index 18e2f557c12..151b6bd1099 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfig.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfig.java @@ -77,7 +77,7 @@ static RetryConfigBuilder builder0( private final int maxTotalAttempts; private final long responseTimeoutMillisForEachAttempt; - private final @Nullable Backoff hedgingBackoff; + private final long hedgingDelayMillis; private final int maxContentLength; @Nullable @@ -92,15 +92,15 @@ static RetryConfigBuilder builder0( RetryConfig(RetryRule retryRule, int maxTotalAttempts, long responseTimeoutMillisForEachAttempt) { this(requireNonNull(retryRule, "retryRule"), null, maxTotalAttempts, responseTimeoutMillisForEachAttempt, - 0, null); + 0, -1); checkArguments(maxTotalAttempts, responseTimeoutMillisForEachAttempt); } RetryConfig(RetryRule retryRule, int maxTotalAttempts, long responseTimeoutMillisForEachAttempt, - Backoff hedgingBackoff) { + long hedgingDelayMillis) { this(requireNonNull(retryRule, "retryRule"), null, - maxTotalAttempts, responseTimeoutMillisForEachAttempt, - 0, requireNonNull(hedgingBackoff, "hedgingBackoff")); + maxTotalAttempts, responseTimeoutMillisForEachAttempt, + 0, hedgingDelayMillis); checkArguments(maxTotalAttempts, responseTimeoutMillisForEachAttempt); } @@ -111,7 +111,7 @@ static RetryConfigBuilder builder0( long responseTimeoutMillisForEachAttempt) { this(null, requireNonNull(retryRuleWithContent, "retryRuleWithContent"), maxTotalAttempts, responseTimeoutMillisForEachAttempt, - maxContentLength, null); + maxContentLength, -1); } RetryConfig( @@ -119,10 +119,10 @@ static RetryConfigBuilder builder0( int maxContentLength, int maxTotalAttempts, long responseTimeoutMillisForEachAttempt, - Backoff hedgingBackoff) { + long hedgingDelayMillis) { this(null, requireNonNull(retryRuleWithContent, "retryRuleWithContent"), - maxTotalAttempts, responseTimeoutMillisForEachAttempt, - maxContentLength, requireNonNull(hedgingBackoff, "hedgingBackoff")); + maxTotalAttempts, responseTimeoutMillisForEachAttempt, + maxContentLength, hedgingDelayMillis); } private RetryConfig( @@ -131,7 +131,7 @@ private RetryConfig( int maxTotalAttempts, long responseTimeoutMillisForEachAttempt, int maxContentLength, - @Nullable Backoff hedgingBackoff + long hedgingDelayMillis ) { checkArguments(maxTotalAttempts, responseTimeoutMillisForEachAttempt); this.retryRule = retryRule; @@ -139,7 +139,7 @@ private RetryConfig( this.maxTotalAttempts = maxTotalAttempts; this.responseTimeoutMillisForEachAttempt = responseTimeoutMillisForEachAttempt; this.maxContentLength = maxContentLength; - this.hedgingBackoff = hedgingBackoff; + this.hedgingDelayMillis = hedgingDelayMillis; if (retryRuleWithContent == null) { fromRetryRuleWithContent = null; } else { @@ -174,11 +174,11 @@ public RetryConfigBuilder toBuilder() { .maxTotalAttempts(maxTotalAttempts) .responseTimeoutMillisForEachAttempt(responseTimeoutMillisForEachAttempt); - if (hedgingBackoff != null) { - builder.hedgingBackoff(hedgingBackoff); - } + if (hedgingDelayMillis >= 0) { + builder.hedgingDelayMillis(hedgingDelayMillis); + } - return builder; + return builder; } /** @@ -197,8 +197,8 @@ public long responseTimeoutMillisForEachAttempt() { return responseTimeoutMillisForEachAttempt; } - public @Nullable Backoff hedgingBackoff() { - return hedgingBackoff; + public long hedgingDelayMillis() { + return hedgingDelayMillis; } /** diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfigBuilder.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfigBuilder.java index c1f453198a0..326de6d3ed5 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfigBuilder.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfigBuilder.java @@ -36,7 +36,7 @@ public final class RetryConfigBuilder { private int maxTotalAttempts = Flags.defaultMaxTotalAttempts(); private long responseTimeoutMillisForEachAttempt = Flags.defaultResponseTimeoutMillis(); - private @Nullable Backoff hedgingBackoff; + private long hedgingDelayMillis = -1; private int maxContentLength; @Nullable @@ -85,6 +85,27 @@ public RetryConfigBuilder maxTotalAttempts(int maxTotalAttempts) { return this; } + public RetryConfigBuilder hedgingDelay(Duration hedgingDelay) { + final long millis = + requireNonNull(hedgingDelay, "hedgingDelay") + .toMillis(); + checkArgument( + millis >= 0, + "responseTimeoutForEachAttempt.toMillis(): %s (expected: >= 0)", + millis); + hedgingDelayMillis = millis; + return this; + } + + public RetryConfigBuilder hedgingDelayMillis(long hedgingDelayMillis) { + checkArgument( + hedgingDelayMillis >= 0, + "hedgingDelayMillis: %s (expected: >= 0)", + hedgingDelayMillis); + this.hedgingDelayMillis = hedgingDelayMillis; + return this; + } + /** * Sets the specified {@code responseTimeoutMillisForEachAttempt}. */ @@ -97,12 +118,6 @@ public RetryConfigBuilder responseTimeoutMillisForEachAttempt(long responseTi return this; } - - public RetryConfigBuilder hedgingBackoff(Backoff hedgingBackoff) { - this.hedgingBackoff = requireNonNull(hedgingBackoff); - return this; - } - /** * Sets the specified {@link Duration} by converting responseTimeoutForEachAttempt to millis. */ @@ -123,18 +138,18 @@ public RetryConfigBuilder responseTimeoutForEachAttempt(Duration responseTime */ public RetryConfig build() { if (retryRule != null) { - if (hedgingBackoff != null) { + if (hedgingDelayMillis >= 0) { return new RetryConfig<>(retryRule, maxTotalAttempts, responseTimeoutMillisForEachAttempt, - hedgingBackoff); + hedgingDelayMillis); } else { return new RetryConfig<>(retryRule, maxTotalAttempts, responseTimeoutMillisForEachAttempt); } } assert retryRuleWithContent != null; - if (hedgingBackoff != null) { + if (hedgingDelayMillis >= 0) { return new RetryConfig<>(retryRuleWithContent, maxContentLength, maxTotalAttempts, - responseTimeoutMillisForEachAttempt, hedgingBackoff); + responseTimeoutMillisForEachAttempt, hedgingDelayMillis); } else { return new RetryConfig<>(retryRuleWithContent, maxContentLength, maxTotalAttempts, responseTimeoutMillisForEachAttempt); @@ -154,7 +169,7 @@ ToStringHelper toStringHelper() { .add("retryRuleWithContent", retryRuleWithContent) .add("maxTotalAttempts", maxTotalAttempts) .add("responseTimeoutMillisForEachAttempt", responseTimeoutMillisForEachAttempt) - .add("hedgingBackoff", hedgingBackoff) + .add("hedgingDelayMillis", hedgingDelayMillis) .add("maxContentLength", maxContentLength); } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index 355e43cff71..c37605e1157 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -381,16 +381,18 @@ private void doExecute0(RetryingContext retryingContext) { } @Nullable - final Backoff hedgingBackoff = config.hedgingBackoff(); + final long hedgingDelayMillis = config.hedgingDelayMillis(); - if (hedgingBackoff != null) { - final RetrySchedulabilityDecision retrySchedulabilityDecision = canScheduleWith(ctx, hedgingBackoff, + if (hedgingDelayMillis >= 0) { + final Backoff hedgingDelayBackoff = Backoff.fixed(hedgingDelayMillis); + final RetrySchedulabilityDecision retrySchedulabilityDecision = canScheduleWith(ctx, + hedgingDelayBackoff, -1); if (retrySchedulabilityDecision.canSchedule()) { - logger.debug("Scheduling hedging with backoff: " + hedgingBackoff); + logger.debug("Scheduling hedging with backoff: {}", hedgingDelayBackoff); scheduleNextRetry(ctx, () -> doExecute0(retryingContext), retrySchedulabilityDecision.nextRetryTimeNanos(), - hedgingBackoff, + hedgingDelayBackoff, cause -> handleException(retryingContext, cause, false)); } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java index a02e148b1c4..ff46f09e52e 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java @@ -236,9 +236,10 @@ private void doExecute0(ClientRequestContext ctx, RpcRequest req, return null; }); - final @Nullable Backoff hedgingBackoff = retryConfig.hedgingBackoff(); + final long hedgingDelayMillis = retryConfig.hedgingDelayMillis(); - if (hedgingBackoff != null) { + if (hedgingDelayMillis >= 0) { + final Backoff hedgingBackoff = Backoff.fixed(hedgingDelayMillis); final RetrySchedulabilityDecision schedulabilityDecision = canScheduleWith(ctx, hedgingBackoff); if (schedulabilityDecision.canSchedule()) { diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java index 91d03965fe1..a3addfb8b6a 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java @@ -196,7 +196,7 @@ void letSecondServerWins() throws Exception { final RetryConfig hedgingNoRetryConfig = RetryConfig .builder(NO_RETRY_RULE) .maxTotalAttempts(3) - .hedgingBackoff(Backoff.fixed(100)) + .hedgingDelayMillis(100) .build(); final WebClient client = client(hedgingNoRetryConfig); @@ -244,7 +244,7 @@ void letThirdServerWin() throws Exception { final RetryConfig hedgingNoRetryConfig = RetryConfig .builder(NO_RETRY_RULE) .maxTotalAttempts(3) - .hedgingBackoff(Backoff.fixed(10)) + .hedgingDelayMillis(10) .build(); final WebClient client = client(hedgingNoRetryConfig); @@ -294,7 +294,7 @@ void thirdWinsEvenAfterPerAttemptTimeout() throws Exception { // be always overtaken by hedging task .maxTotalAttempts(3) .responseTimeoutMillisForEachAttempt(100) - .hedgingBackoff(Backoff.fixed(200)) + .hedgingDelayMillis(200) .build(); final WebClient client = clientBuilder() @@ -356,7 +356,7 @@ void thirdServerWinsEvenAfterRetriableResponse() throws Exception { NO_RETRY_RULE )) .maxTotalAttempts(3) - .hedgingBackoff(Backoff.fixed(200)) + .hedgingDelayMillis(200) .build(); final WebClient client = client(config); @@ -406,7 +406,7 @@ void loosesAfterNonRetriableResponse() throws Exception { final RetryConfig config = RetryConfig .builder(NO_RETRY_RULE) .maxTotalAttempts(3) - .hedgingBackoff(Backoff.fixed(100)) + .hedgingDelayMillis(100) .build(); final WebClient client = client(config); @@ -449,7 +449,7 @@ void loosesAfterResponseTimeout() throws Exception { final RetryConfig config = RetryConfig .builder(NO_RETRY_RULE) .maxTotalAttempts(3) - .hedgingBackoff(Backoff.withoutDelay()) + .hedgingDelayMillis(0) .build(); final WebClient client = clientBuilder() diff --git a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java index 9811f9b4a13..291240f33b2 100644 --- a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java +++ b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java @@ -74,8 +74,8 @@ class RetryingRpcClientWithHedgingTest { private static final long LOOSING_SERVER_RESPONSE_DELAY_MILLIS = 50; private static class TestServer extends ServerExtension { - private CountDownLatch responseLatch = new CountDownLatch(1); - private CountDownLatch requestLatch = new CountDownLatch(1); + private CountDownLatch responseLatch = new CountDownLatch(1); + private CountDownLatch requestLatch = new CountDownLatch(1); private volatile HelloService.Iface serviceHandler; TestServer() { @@ -118,7 +118,7 @@ public void unlatchResponse() { responseLatch.countDown(); } - public void waitForFirstRequest() { + public void waitForFirstRequest() { try { requestLatch.await(); } catch (InterruptedException e) { @@ -191,7 +191,7 @@ void execute_hedging_lastWins() throws Exception { ) .maxTotalAttempts(3) .responseTimeoutMillisForEachAttempt(50) - .hedgingBackoff(Backoff.fixed(20)) + .hedgingDelayMillis(20) .build() ); @@ -220,7 +220,7 @@ void execute_hedging_lastWins() throws Exception { assertThat(result.get()).isEqualTo("server3"); assertValidClientRequestContext( - ctx,GET_VERIFY_RESPONSE_HAS_CONTENT.apply("server3"), VERIFY_REQUEST_CANCELLED , + ctx, GET_VERIFY_RESPONSE_HAS_CONTENT.apply("server3"), VERIFY_REQUEST_CANCELLED, VERIFY_REQUEST_CANCELLED, GET_VERIFY_RESPONSE_HAS_CONTENT.apply("server3") ); @@ -243,11 +243,10 @@ void execute_hedging_thirdWinsEventAfterPerAttemptTimeout() throws Exception { ) .maxTotalAttempts(3) .responseTimeoutMillisForEachAttempt(50) - .hedgingBackoff(Backoff.fixed(20)) + .hedgingDelayMillis(20) .build() ); - final CompletableFuture result; final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { @@ -277,7 +276,7 @@ void execute_hedging_thirdWinsEventAfterPerAttemptTimeout() throws Exception { assertValidServerRequestContext(server3, 3); assertThat(result.get()).isEqualTo("server3"); - assertValidClientRequestContext(ctx,GET_VERIFY_RESPONSE_HAS_CONTENT.apply("server3"), + assertValidClientRequestContext(ctx, GET_VERIFY_RESPONSE_HAS_CONTENT.apply("server3"), VERIFY_REQUEST_CANCELLED, VERIFY_REQUEST_CANCELLED, GET_VERIFY_RESPONSE_HAS_CONTENT.apply("server3")); }); @@ -302,7 +301,7 @@ void execute_hedging_thirdWinsEvenWhenFirstErrors() throws Exception { ) .maxTotalAttempts(3) .responseTimeoutMillisForEachAttempt(1) - .hedgingBackoff(Backoff.fixed(20)) + .hedgingDelayMillis(20) .build() ); @@ -329,7 +328,7 @@ void execute_hedging_thirdWinsEvenWhenFirstErrors() throws Exception { assertThat(result.get()).isEqualTo("server3"); assertValidClientRequestContext( - ctx,GET_VERIFY_RESPONSE_HAS_CONTENT.apply("server3"), VERIFY_REQUEST_CANCELLED, + ctx, GET_VERIFY_RESPONSE_HAS_CONTENT.apply("server3"), VERIFY_REQUEST_CANCELLED, VERIFY_REQUEST_CANCELLED, GET_VERIFY_RESPONSE_HAS_CONTENT.apply("server3") ); }); @@ -353,7 +352,7 @@ void execute_hedging_returnErrorWhenSecondErrors() throws Exception { ) .maxTotalAttempts(3) .responseTimeoutMillisForEachAttempt(1) - .hedgingBackoff(Backoff.fixed(20)) + .hedgingDelayMillis(20) .build() ); @@ -368,7 +367,6 @@ void execute_hedging_returnErrorWhenSecondErrors() throws Exception { server2.waitForFirstRequest(); server3.waitForFirstRequest(); - // Let the second server win. server2.unlatchResponse(); Thread.sleep(LOOSING_SERVER_RESPONSE_DELAY_MILLIS); @@ -393,10 +391,10 @@ void execute_hedging_returnErrorWhenSecondErrors() throws Exception { ((TApplicationException) cause).getType()) .isEqualTo(TApplicationException.INTERNAL_ERROR))); - assertValidClientRequestContext( - ctx, GET_VERIFY_RESPONSE_HAS_APPLICATION_EXCEPTION.apply(TApplicationException.INTERNAL_ERROR, - errorMessage), + ctx, + GET_VERIFY_RESPONSE_HAS_APPLICATION_EXCEPTION.apply(TApplicationException.INTERNAL_ERROR, + errorMessage), VERIFY_REQUEST_CANCELLED, GET_VERIFY_RESPONSE_HAS_APPLICATION_EXCEPTION.apply(TApplicationException.INTERNAL_ERROR, errorMessage), @@ -404,7 +402,6 @@ void execute_hedging_returnErrorWhenSecondErrors() throws Exception { }); } - @Test void execute_hedging_honorResponseTimeout() throws TException { when(server1.getServiceHandler().hello(anyString())).thenReturn("server1"); @@ -420,7 +417,7 @@ void execute_hedging_honorResponseTimeout() throws TException { ) .maxTotalAttempts(3) .responseTimeoutMillisForEachAttempt(300) - .hedgingBackoff(Backoff.fixed(20)) + .hedgingDelayMillis(20) .build(), 300 + 100 // Lets give the client 100ms to schedule the second attempt. ); @@ -444,7 +441,8 @@ void execute_hedging_honorResponseTimeout() throws TException { { assertThat(cause.getCause()).isInstanceOf(TTransportException.class); - assertThat(cause.getCause().getCause()).isInstanceOf(ResponseTimeoutException.class); + assertThat(cause.getCause().getCause()).isInstanceOf( + ResponseTimeoutException.class); } ); @@ -468,7 +466,7 @@ void execute_hedging_honorResponseTimeout() throws TException { }); } - private CompletableFuture asyncHelloWith(HelloService.AsyncIface client) throws TException { + private CompletableFuture asyncHelloWith(HelloService.AsyncIface client) throws TException { final CompletableFuture future = new CompletableFuture<>(); try { client.hello("hello", new AsyncMethodCallback() { @@ -511,6 +509,7 @@ private static HelloService.AsyncIface helloClientThreeEndpoints(RetryConfig {} + private static final RequestLogVerifier VERIFY_REQUEST_CANCELLED = log -> { assertThat(log.responseCause()).isInstanceOf(TTransportException.class); From 835b268a386c4541f8ae7f3e6bca27101920f6be Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Tue, 10 Jun 2025 13:08:10 +0200 Subject: [PATCH 08/36] [WIP] test: add `RetryScheduler` tests --- .../armeria/client/retry/RetryScheduler.java | 4 +- .../client/retry/RetrySchedulerTest.java | 810 ++++++++++++++++++ .../retry/RetryingClientWithHedgingTest.java | 3 +- 3 files changed, 813 insertions(+), 4 deletions(-) create mode 100644 core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java index e813fbb6b0e..bd2888eff61 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java @@ -219,8 +219,8 @@ private synchronized void scheduleNextRetryTask(Runnable retryRunnable, long ret } } - public void close() { - cancelCurrentRetryTask(); + public boolean close() { + return cancelCurrentRetryTask(); } // todo(szymon): Remove dependency on nextEarliestNextRetryTimeNanos. Users should simply set it before diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java new file mode 100644 index 00000000000..59416e4c844 --- /dev/null +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java @@ -0,0 +1,810 @@ +package com.linecorp.armeria.client.retry; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.fail; +import static org.awaitility.Awaitility.await; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import com.google.common.collect.ImmutableList; + +import com.linecorp.armeria.internal.testing.AnticipatedException; + +import io.netty.channel.DefaultEventLoop; +import io.netty.channel.EventLoop; +import io.netty.util.concurrent.ScheduledFuture; + +// todo(szymon): clean up the test cases +class RetrySchedulerTest { + private static final long SCHEDULING_TOLERANCE_NANOS = TimeUnit.MILLISECONDS.toNanos(10); + private static final long SCHEDULING_TOLERANCE_MILLIS = TimeUnit.NANOSECONDS.toMillis( + SCHEDULING_TOLERANCE_NANOS); + + private ArgumentCaptor scheduleDelayArgumentCaptor; + private ArgumentCaptor scheduleTimeUnitArgumentCaptor; + private EventLoop eventLoop; + + private RetryScheduler scheduler; + + @BeforeEach + void setUp() { + eventLoop = spy(new DefaultEventLoop()); + scheduleDelayArgumentCaptor = ArgumentCaptor.forClass(Long.class); + scheduleTimeUnitArgumentCaptor = ArgumentCaptor.forClass(TimeUnit.class); + scheduler = new RetryScheduler(eventLoop); + } + + @AfterEach + void tearDown() throws Exception { + assertThat(scheduler.close()).isTrue(); + eventLoop.shutdownGracefully().sync(); + } + + @Test + void testEarlierRetryTaskOvertakesLaterOne() throws Exception { + final Runnable task1 = mock(Runnable.class); + + final long task1SchedulingTime = System.nanoTime(); + final long expectedTask1RunTime = task1SchedulingTime + TimeUnit.MILLISECONDS.toNanos(200); + final Consumer task1ExceptionHandler = mock(Consumer.class); + scheduler.schedule(task1, expectedTask1RunTime, task1ExceptionHandler); + + final Runnable task2 = mock(Runnable.class); + final long task2SchedulingTime = System.nanoTime(); + final long expectedTask2RunTime = task2SchedulingTime + TimeUnit.MILLISECONDS.toNanos(100); + final Consumer task2ExceptionHandler = mock(Consumer.class); + scheduler.schedule(task2, expectedTask2RunTime, task2ExceptionHandler); + + Thread.sleep(200 + SCHEDULING_TOLERANCE_MILLIS); + + verify(task1, times(0)).run(); + verify(task2, times(1)).run(); + verifyNoMoreInteractions(task1ExceptionHandler); + verifyNoMoreInteractions(task2ExceptionHandler); + verifyEventLoopSchedules( + ImmutableList.of( + EventLoopScheduleCall.of(task1SchedulingTime, expectedTask1RunTime), + EventLoopScheduleCall.of(task2SchedulingTime, expectedTask2RunTime) + ) + ); + } + + @Test + void testMultipleRetryTasksBeingOvertaken() throws Exception { + // Rationale of this test: + // - Schedule 10 tasks with decreasing run times (1000ms, 900ms, ..., 100ms). + // - Expect that the first 9 tasks are not executed as they are overtaken by the next task as + // next task is scheduled earlier. + // - Only the last task (100ms) should be executed. + + final List tasks = new ArrayList<>(); + final List schedulingTimes = new ArrayList<>(); + final List expectedRunTimes = new ArrayList<>(); + final List> exceptionHandlers = new ArrayList<>(); + + for (int taskNo = 0; taskNo < 10; taskNo++) { + final Runnable task = mock(Runnable.class); + tasks.add(task); + final Consumer exceptionHandler = mock(Consumer.class); + exceptionHandlers.add(exceptionHandler); + + final long schedulingTime = System.nanoTime(); + schedulingTimes.add(schedulingTime); + final long expectedRunTime = schedulingTime + TimeUnit.MILLISECONDS.toNanos(1000 - taskNo * 100); + expectedRunTimes.add(expectedRunTime); + + scheduler.schedule(task, expectedRunTime, exceptionHandler); + } + + // Wait for the tasks to be scheduled + Thread.sleep(1000 + SCHEDULING_TOLERANCE_MILLIS); + + for (int taskNo = 0; taskNo < 9; taskNo++) { + final Runnable task = tasks.get(taskNo); + verify(task, times(0)).run(); + verifyNoMoreInteractions(exceptionHandlers.get(taskNo)); + } + + // Verify that the last task was executed + verify(tasks.get(9), times(1)).run(); + verifyNoMoreInteractions(exceptionHandlers.get(9)); + + verifyEventLoopSchedules( + ImmutableList.copyOf( + tasks.stream() + .map(task -> EventLoopScheduleCall.of(schedulingTimes.get(tasks.indexOf(task)), + expectedRunTimes.get(tasks.indexOf(task)))) + .collect(Collectors.toList()) + ) + ); + } + + @Test + void testLaterRetryTaskDoesNotOvertakeEarlierOne() throws Exception { + final Runnable task1 = mock(Runnable.class); + final long task1SchedulingTime = System.nanoTime(); + final long expectedTask1RunTime = task1SchedulingTime + TimeUnit.MILLISECONDS.toNanos(200); + final Consumer task1ExceptionHandler = mock(Consumer.class); + scheduler.schedule(task1, expectedTask1RunTime, task1ExceptionHandler); + + // Schedule a new task with a later time than the current one + final Runnable task2 = mock(Runnable.class); + final long task2SchedulingTime = System.nanoTime(); + final long expectedTask2RunTime = task2SchedulingTime + TimeUnit.MILLISECONDS.toNanos(300); + final Consumer task2ExceptionHandler = mock(Consumer.class); + scheduler.schedule(task2, expectedTask2RunTime, task2ExceptionHandler); + + Thread.sleep(300 + SCHEDULING_TOLERANCE_MILLIS); + + // Verify that the first task was executed + verify(task1, times(1)).run(); + verifyNoMoreInteractions(task1ExceptionHandler); + + // Verify that the second task was not executed + verify(task2, times(0)).run(); + verify(task2ExceptionHandler, times(1)).accept(any(IllegalStateException.class)); + + verifyEventLoopSchedules( + ImmutableList.of( + EventLoopScheduleCall.of(task1SchedulingTime, expectedTask1RunTime) + // EventLoopScheduleCall.of(task2SchedulingTime, expectedTask2RunTime) + ) + ); + } + + @Test + void testRescheduleTaskWhenEarliestNextRetryTimeUpdated() throws Exception { + + // Set the earliest next retry time to 200ms from now + final long now = System.nanoTime(); + final long earliestTime = now + TimeUnit.MILLISECONDS.toNanos(200); + scheduler.addEarliestNextRetryTimeNanos(earliestTime); + + // Schedule a task to run after 300ms + final long task1SchedulingTime = System.nanoTime(); + final long taskTime = task1SchedulingTime + TimeUnit.MILLISECONDS.toNanos(300); + final Runnable task1 = mock(Runnable.class); + final Consumer exceptionHandler = mock(Consumer.class); + + scheduler.schedule(task1, taskTime, exceptionHandler); + + // Move the earliest next retry time to 100ms from now + final long earliestTimeUpdateTime = System.nanoTime(); + final long newEarliestTimeNanos = now + TimeUnit.MILLISECONDS.toNanos(400); + scheduler.addEarliestNextRetryTimeNanos(newEarliestTimeNanos); + + scheduler.rescheduleCurrentRetryTaskIfTooEarly(); + + Thread.sleep(400 + SCHEDULING_TOLERANCE_MILLIS); + + verify(task1, times(1)).run(); + verifyEventLoopSchedules( + ImmutableList.of( + EventLoopScheduleCall.of(task1SchedulingTime, taskTime), + EventLoopScheduleCall.of(earliestTimeUpdateTime, newEarliestTimeNanos) + ) + ); + verifyNoMoreInteractions(exceptionHandler); + } + + /** + * Test plan 5: Verify that when a task is scheduled and then the scheduler is closed, + * the task is cancelled and not executed. + */ + @Test + void testCloseSchedulerCancelsTask() throws Exception { + final AtomicBoolean taskExecuted = new AtomicBoolean(); + final Consumer exceptionHandler = mock(Consumer.class); + + // Schedule a task to run after 200ms + final long now = System.nanoTime(); + final long taskTime = now + TimeUnit.MILLISECONDS.toNanos(200); + + scheduler.schedule(() -> { + taskExecuted.set(true); + }, taskTime, exceptionHandler); + + // Close the scheduler immediately + scheduler.close(); + + // Wait a bit to ensure the task would have run if not cancelled + // Use Awaitility to wait for a reasonable time + await().pollDelay(300, TimeUnit.MILLISECONDS) + .atMost(400, TimeUnit.MILLISECONDS) + .until(() -> true); // Just wait + + // Verify that the task was not executed + assertThat(taskExecuted.get()).isFalse(); + + // Verify that the exception handler was not called + // The scheduler mutes cancellation notifications when closed + verifyNoMoreInteractions(exceptionHandler); + } + + /** + * Test with a large number of tasks (100) overtaking each other to verify that + * the scheduler can handle a large number of tasks and that only the earliest one is executed. + */ + @Test + void testManyOvertakingTasks() throws Exception { + + final int numTasks = 100; + final AtomicInteger executedTaskIndex = new AtomicInteger(-1); + final AtomicLong executionTime = new AtomicLong(); + + // Create an array of mock exception handlers + @SuppressWarnings("unchecked") + final Consumer[] exceptionHandlers = new Consumer[numTasks]; + for (int i = 0; i < numTasks; i++) { + exceptionHandlers[i] = mock(Consumer.class); + } + + // Schedule tasks with random times, but make sure the last one is the earliest + final long now = System.nanoTime(); + final long baseTime = now + TimeUnit.MILLISECONDS.toNanos(200); + + // Schedule tasks in reverse order (except the last one) to ensure they're all scheduled + // before the earliest one executes + for (int i = numTasks - 1; i >= 0; i--) { + final int taskIndex = i; + final long taskTime; + + if (i == numTasks - 1) { + // Make the last task the earliest + taskTime = baseTime; + } else { + // Random time between baseTime + 50ms and baseTime + 500ms + taskTime = baseTime + TimeUnit.MILLISECONDS.toNanos(50 + (i * 5L)); + } + + scheduler.schedule(() -> { + executedTaskIndex.set(taskIndex); + executionTime.set(System.nanoTime()); + }, taskTime, exceptionHandlers[i]); + } + + // Use Awaitility to wait for a task to execute + await().atMost(1, TimeUnit.SECONDS) + .until(() -> executedTaskIndex.get() >= 0); + + // Verify that only the last task (index numTasks-1) was executed + assertThat(executedTaskIndex.get()).isEqualTo(numTasks - 1); + + // Verify that the task was executed at the expected time (not too early) + final long taskDelay = executionTime.get() - now; + assertThat(taskDelay).isGreaterThanOrEqualTo( + TimeUnit.MILLISECONDS.toNanos(190)); // Allow some timing flexibility + + // In this test, we can't make specific assertions about which exception handlers were called + // because the behavior depends on the exact order of task scheduling, which can vary. + // We only verify that at least one task was executed. + } + + /** + * Test that scheduling a task before the earliestNextRetryTimeNanos fails + * and calls the exception handler with the expected exception. + */ + @Test + void testScheduleBeforeEarliestNextRetryTime() throws Exception { + + // Set the earliest next retry time + final long now = System.nanoTime(); + final long earliestTime = now + TimeUnit.MILLISECONDS.toNanos(200); + scheduler.addEarliestNextRetryTimeNanos(earliestTime); + + // Try to schedule a task before the earliest next retry time + final long taskTime = now + TimeUnit.MILLISECONDS.toNanos(100); + final AtomicBoolean taskExecuted = new AtomicBoolean(); + final Consumer exceptionHandler = mock(Consumer.class); + + scheduler.schedule(() -> { + taskExecuted.set(true); + }, taskTime, exceptionHandler); + + // Use Awaitility to wait a bit to ensure the exception handler is called + await().pollDelay(100, TimeUnit.MILLISECONDS) + .atMost(200, TimeUnit.MILLISECONDS) + .until(() -> true); // Just wait + + // Verify that the task was not executed + assertThat(taskExecuted.get()).isFalse(); + + // Verify that the exception handler was called with the expected exception + verify(exceptionHandler, times(1)).accept(any(IllegalStateException.class)); + } + + /** + * Test that scheduling a task after the latestNextRetryTimeNanos fails + * and calls the exception handler with the expected exception. + */ + @Test + void testScheduleAfterLatestNextRetryTime() throws Exception { + // Create a scheduler with a limited latest next retry time + final long now = System.nanoTime(); + final long latestTime = now + TimeUnit.MILLISECONDS.toNanos(200); + final RetryScheduler scheduler = new RetryScheduler(eventLoop, latestTime); + + // Try to schedule a task after the latest next retry time + final long taskTime = now + TimeUnit.MILLISECONDS.toNanos(300); + final AtomicBoolean taskExecuted = new AtomicBoolean(); + final Consumer exceptionHandler = mock(Consumer.class); + + scheduler.schedule(() -> { + taskExecuted.set(true); + }, taskTime, exceptionHandler); + + // Use Awaitility to wait a bit to ensure the exception handler is called + await().pollDelay(100, TimeUnit.MILLISECONDS) + .atMost(200, TimeUnit.MILLISECONDS) + .until(() -> true); // Just wait + + // Verify that the task was not executed + assertThat(taskExecuted.get()).isFalse(); + + // Verify that the exception handler was called with the expected exception + verify(exceptionHandler, times(1)).accept(any(IllegalStateException.class)); + } + + /** + * Test that tasks can be scheduled concurrently from multiple threads + * and the scheduler correctly handles the concurrency. + */ + @Test + void testConcurrentScheduling() throws Exception { + + final int numThreads = 10; + final int tasksPerThread = 10; + final AtomicInteger executedTasks = new AtomicInteger(); + final AtomicLong earliestExecutionTime = new AtomicLong(Long.MAX_VALUE); + + // Create a latch to synchronize the start of all threads + final CountDownLatch startLatch = new CountDownLatch(1); + // Create a latch to wait for all threads to finish scheduling + final CountDownLatch schedulingDoneLatch = new CountDownLatch(numThreads); + + // Create a list to store all exception handlers + final List> exceptionHandlers = new ArrayList<>(); + + // Create and start threads + final ExecutorService executorService = Executors.newFixedThreadPool(numThreads); + final long now = System.nanoTime(); + + for (int i = 0; i < numThreads; i++) { + final int threadIndex = i; + executorService.submit(() -> { + try { + // Wait for the start signal + startLatch.await(); + + // Each thread schedules multiple tasks + for (int j = 0; j < tasksPerThread; j++) { + final int taskIndex = threadIndex * tasksPerThread + j; + final Consumer exceptionHandler = mock(Consumer.class); + + synchronized (exceptionHandlers) { + exceptionHandlers.add(exceptionHandler); + } + + // Calculate a task time - make them all different + // The earliest task will be the one with the smallest time + final long taskTime = now + TimeUnit.MILLISECONDS.toNanos(200 + taskIndex * 5); + + scheduler.schedule(() -> { + executedTasks.incrementAndGet(); + earliestExecutionTime.updateAndGet( + current -> Math.min(current, System.nanoTime())); + }, taskTime, exceptionHandler); + } + } catch (Exception e) { + fail("Exception in test thread: " + e.getMessage()); + } finally { + schedulingDoneLatch.countDown(); + } + }); + } + + // Start all threads simultaneously + startLatch.countDown(); + + // Wait for all threads to finish scheduling + schedulingDoneLatch.await(); + + // Shutdown the executor service + executorService.shutdown(); + executorService.awaitTermination(1, TimeUnit.SECONDS); + + // Use Awaitility to wait for a task to execute + await().atMost(1, TimeUnit.SECONDS) + .until(() -> executedTasks.get() > 0); + + // Verify that exactly one task was executed + assertThat(executedTasks.get()).isEqualTo(1); + + // Verify that the task was executed at the expected time (not too early) + final long taskDelay = earliestExecutionTime.get() - now; + assertThat(taskDelay).isGreaterThanOrEqualTo( + TimeUnit.MILLISECONDS.toNanos(190)); // Allow some timing flexibility + + // Verify that at least some exception handlers were called + // (since most tasks will be rejected due to earlier tasks being scheduled) + int exceptionHandlerCallCount = 0; + for (Consumer handler : exceptionHandlers) { + try { + verify(handler, times(0)).accept(any(Throwable.class)); + } catch (AssertionError e) { + exceptionHandlerCallCount++; + } + } + + // We expect most tasks to be rejected, but we can't know exactly how many + // due to the concurrent nature of the test + assertThat(exceptionHandlerCallCount).isGreaterThan(0); + } + + /** + * Test that retry tasks that raise exceptions call the exception handler + * with exactly the thrown exception. + */ + @Test + void testRetryTaskRaisesException() throws Exception { + + // Use a CountDownLatch to wait for the exception handler to be called + final CountDownLatch exceptionLatch = new CountDownLatch(1); + final AtomicReference caughtException = new AtomicReference<>(); + + final AnticipatedException expectedException = new AnticipatedException("Test exception"); + + // Schedule a task that throws an exception + final long taskTime = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(100); + + scheduler.schedule(() -> { + throw expectedException; + }, taskTime, ex -> { + caughtException.set(ex); + exceptionLatch.countDown(); + }); + + // Wait for the exception handler to be called + assertThat(exceptionLatch.await(500, TimeUnit.MILLISECONDS)).isTrue(); + + // Verify that the exception handler was called with exactly the thrown exception + assertThat(caughtException.get()).isSameAs(expectedException); + } + + /** + * Test that an exception during the eventLoop.schedule call is handled properly. + */ + @Test + void testExceptionDuringEventLoopSchedule() throws Exception { + // Create a mock EventLoop that throws an exception when schedule is called + final EventLoop mockEventLoop = mock(EventLoop.class); + final RuntimeException expectedException = new RuntimeException("Schedule exception"); + when(mockEventLoop.schedule(any(Runnable.class), any(Long.class), any(TimeUnit.class))) + .thenThrow(expectedException); + + final RetryScheduler scheduler = new RetryScheduler(mockEventLoop); + final Consumer exceptionHandler = mock(Consumer.class); + + // Schedule a task (which should fail) + final long taskTime = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(100); + + scheduler.schedule(() -> { + fail("This task should not be executed"); + }, taskTime, exceptionHandler); + + // Verify that the exception handler was called with the expected exception + verify(exceptionHandler, times(1)).accept(expectedException); + } + + /** + * Test for hasAlreadyRetryScheduledBefore method with various scenarios. + */ + @Test + void testHasAlreadyRetryScheduledBefore() throws Exception { + + // Case 1: No current retry task + assertThat(scheduler.hasAlreadyRetryScheduledBefore(100, 0)).isFalse(); + + // Schedule a task + final long now = System.nanoTime(); + final long taskTime = now + TimeUnit.MILLISECONDS.toNanos(200); + + final CountDownLatch taskLatch = new CountDownLatch(1); + scheduler.schedule(taskLatch::countDown, taskTime, ex -> { + }); + + // Case 2: Current retry task exists but is scheduled after the next retry time + assertThat(scheduler.hasAlreadyRetryScheduledBefore(taskTime + 100, 0)).isTrue(); + + // Case 3: Current retry task exists but is scheduled before the next retry time + assertThat(scheduler.hasAlreadyRetryScheduledBefore(taskTime - 100, 0)).isFalse(); + + // Case 4: Current retry task exists and is scheduled at exactly the next retry time + assertThat(scheduler.hasAlreadyRetryScheduledBefore(taskTime, 0)).isTrue(); + + // Case 5: With a non-zero earliestNextRetryTimeNanos + final long earliestTime = taskTime + 50; + assertThat(scheduler.hasAlreadyRetryScheduledBefore(earliestTime + 100, earliestTime)).isTrue(); + assertThat(scheduler.hasAlreadyRetryScheduledBefore(earliestTime - 100, earliestTime)).isFalse(); + + // Wait for the task to complete to avoid interference with other tests + taskLatch.await(500, TimeUnit.MILLISECONDS); + } + + /** + * Test for negative checkState conditions. + */ + @Test + void testNegativeCheckStateConditions() throws Exception { + // Create a scheduler with a limited latest next retry time + final long latestTime = 1000; + final RetryScheduler scheduler = new RetryScheduler(eventLoop, latestTime); + + // Test that nextEarliestNextRetryTimeNanos > latestNextRetryTimeNanos throws IllegalStateException + assertThatThrownBy(() -> scheduler.hasAlreadyRetryScheduledBefore(0, latestTime + 1)) + .isInstanceOf(IllegalStateException.class); + + // Test that earliestNextRetryTimeNanos > latestNextRetryTimeNanos throws IllegalStateException + assertThatThrownBy(() -> scheduler.addEarliestNextRetryTimeNanos(latestTime + 1)) + .isInstanceOf(IllegalStateException.class); + } + + /** + * Test rescheduleCurrentRetryTaskIfTooEarly with no retry task. + */ + @Test + void testRescheduleCurrentRetryTaskIfTooEarlyWithNoTask() throws Exception { + + // Set the earliest next retry time + final long earliestTime = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(200); + scheduler.addEarliestNextRetryTimeNanos(earliestTime); + + // Call rescheduleCurrentRetryTaskIfTooEarly with no current retry task + // This should not throw an exception + scheduler.rescheduleCurrentRetryTaskIfTooEarly(); + + // Verify that there is still no scheduled retry task + assertThat(scheduler.hasScheduledRetryTask()).isFalse(); + } + + /** + * Test rescheduleCurrentRetryTaskIfTooEarly with a retry task that doesn't need to be rescheduled. + */ + @Test + void testRescheduleCurrentRetryTaskIfTooEarlyWithTaskNotNeedingReschedule() throws Exception { + + // Set the earliest next retry time + final long now = System.nanoTime(); + final long earliestTime = now + TimeUnit.MILLISECONDS.toNanos(100); + scheduler.addEarliestNextRetryTimeNanos(earliestTime); + + // Schedule a task after the earliest next retry time + final long taskTime = now + TimeUnit.MILLISECONDS.toNanos(200); + final AtomicInteger taskExecutions = new AtomicInteger(); + final AtomicLong taskExecutionTime = new AtomicLong(); + final Consumer exceptionHandler = mock(Consumer.class); + + scheduler.schedule(() -> { + taskExecutionTime.set(System.nanoTime()); + taskExecutions.incrementAndGet(); + }, taskTime, exceptionHandler); + + // Call rescheduleCurrentRetryTaskIfTooEarly + // The task should not be rescheduled because it's already scheduled after the earliest time + scheduler.rescheduleCurrentRetryTaskIfTooEarly(); + + // Wait for the task to execute + await() + .until(() -> taskExecutions.get() == 1); + + // Verify that the task was executed once + assertThat(taskExecutions.get()).isEqualTo(1); + + // Verify that the task was executed at the expected time (not too early) + final long taskDelay = taskExecutionTime.get() - now; + assertThat(taskDelay).isGreaterThanOrEqualTo( + TimeUnit.MILLISECONDS.toNanos(190)); // Allow some timing flexibility + + // Verify that the exception handler was not called + verifyNoMoreInteractions(exceptionHandler); + } + + /** + * Test that changing the earliest retry time without rescheduling allows + * a task to execute before the new earliest time. + */ + @Test + void testChangeEarliestRetryTimeWithoutRescheduling() throws Exception { + + final AtomicInteger taskExecutions = new AtomicInteger(); + final AtomicLong taskExecutionTime = new AtomicLong(); + final Consumer exceptionHandler = mock(Consumer.class); + + // Schedule a task to run after 100ms + final long now = System.nanoTime(); + final long taskTime = now + TimeUnit.MILLISECONDS.toNanos(100); + + scheduler.schedule(() -> { + taskExecutionTime.set(System.nanoTime()); + taskExecutions.incrementAndGet(); + }, taskTime, exceptionHandler); + + // Update the earliest next retry time to be after the scheduled task + final long newEarliestTime = now + TimeUnit.MILLISECONDS.toNanos(200); + scheduler.addEarliestNextRetryTimeNanos(newEarliestTime); + + // Do NOT call rescheduleCurrentRetryTaskIfTooEarly() + + // Use Awaitility to wait for the task to execute + await() + .until(() -> taskExecutions.get() == 1); + + // Verify that the task was executed once + assertThat(taskExecutions.get()).isEqualTo(1); + + // Verify that the task was executed at the original time (not delayed to the new earliest time) + final long taskDelay = taskExecutionTime.get() - now; + assertThat(taskDelay).isGreaterThanOrEqualTo( + TimeUnit.MILLISECONDS.toNanos(90)); // Allow some timing flexibility + assertThat(taskDelay).isLessThan( + TimeUnit.MILLISECONDS.toNanos(190)); // Should execute before new earliest time + + // Verify that the exception handler was not called + verifyNoMoreInteractions(exceptionHandler); + } + + /** + * Test for task cancellation by user (not by scheduler). + * Since we can't directly access the RetryTaskHandle, we'll test the behavior indirectly. + */ + @Test + void testTaskCancellationByUser() throws Exception { + // Use a CountDownLatch to wait for the exception handler to be called + final CountDownLatch exceptionLatch = new CountDownLatch(1); + final AtomicReference caughtException = new AtomicReference<>(); + + // Create a scheduler with a real EventLoop + + final AtomicBoolean taskExecuted = new AtomicBoolean(); + + // Schedule a task with a long delay + final long taskTime = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(10000); // 10 seconds + + // Use reflection to access the private currentRetryTask field + final Field currentRetryTaskField; + try { + currentRetryTaskField = RetryScheduler.class.getDeclaredField("currentRetryTask"); + currentRetryTaskField.setAccessible(true); + } catch (NoSuchFieldException e) { + throw new RuntimeException("Could not access currentRetryTask field", e); + } + + // Schedule the task + scheduler.schedule(() -> { + taskExecuted.set(true); + }, taskTime, ex -> { + caughtException.set(ex); + exceptionLatch.countDown(); + }); + + // Get the currentRetryTask field + final Object currentRetryTask; + try { + currentRetryTask = currentRetryTaskField.get(scheduler); + assertThat(currentRetryTask).isNotNull(); + } catch (IllegalAccessException e) { + throw new RuntimeException("Could not access currentRetryTask", e); + } + + // Use reflection to access the scheduledFuture field + final Field scheduledFutureField; + try { + // Get the RetryTaskHandle class (inner class of RetryScheduler) + final Class retryTaskHandleClass = Class.forName( + "com.linecorp.armeria.client.retry.RetryScheduler$RetryTaskHandle"); + scheduledFutureField = retryTaskHandleClass.getDeclaredField("scheduledFuture"); + scheduledFutureField.setAccessible(true); + } catch (NoSuchFieldException | ClassNotFoundException e) { + throw new RuntimeException("Could not access scheduledFuture field", e); + } + + // Get the scheduledFuture + final ScheduledFuture scheduledFuture; + try { + scheduledFuture = (ScheduledFuture) scheduledFutureField.get(currentRetryTask); + assertThat((Object) scheduledFuture).isNotNull(); + } catch (IllegalAccessException e) { + throw new RuntimeException("Could not access scheduledFuture", e); + } + + // Cancel the future directly + scheduledFuture.cancel(false); + + // Wait for the exception handler to be called + assertThat(exceptionLatch.await(500, TimeUnit.MILLISECONDS)).isTrue(); + + // Verify that the exception handler was called with the expected exception + assertThat(caughtException.get()).isInstanceOf(IllegalStateException.class); + assertThat(caughtException.get().getMessage()).contains("cancelled by the user"); + + // Verify that the task was not executed + assertThat(taskExecuted.get()).isFalse(); + } + + private static class EventLoopScheduleCall { + private final long delayNanos; + + private EventLoopScheduleCall(long scheduledTimeNanos) { + this(System.nanoTime(), scheduledTimeNanos); + } + + private EventLoopScheduleCall(long schedulingTimeNanos, long scheduledTimeNanos) { + assert schedulingTimeNanos <= scheduledTimeNanos : "Scheduling time must be before scheduled time"; + delayNanos = scheduledTimeNanos - schedulingTimeNanos; + } + + public static EventLoopScheduleCall of(long scheduledTimeNanos) { + return new EventLoopScheduleCall(scheduledTimeNanos); + } + + public static EventLoopScheduleCall of(long schedulingTimeNanos, + long scheduledTimeNanos) { + return new EventLoopScheduleCall(schedulingTimeNanos, scheduledTimeNanos); + } + + public long delayNanos() { + return delayNanos; + } + } + + private void verifyEventLoopSchedules(List expectedSchedules) { + final int expectedNumCalls = expectedSchedules.size(); + + verify(eventLoop, times(expectedNumCalls)).schedule(any(Runnable.class), + scheduleDelayArgumentCaptor.capture(), + scheduleTimeUnitArgumentCaptor.capture()); + + final List actualDelays = scheduleDelayArgumentCaptor.getAllValues(); + final List actualTimeUnits = scheduleTimeUnitArgumentCaptor.getAllValues(); + + assertThat(actualDelays).hasSize(expectedNumCalls); + assertThat(actualTimeUnits).hasSize(expectedNumCalls); + + // for simplicity, we assume all time units are NANOSECONDS + assertThat(actualTimeUnits).allMatch(unit -> unit == TimeUnit.NANOSECONDS); + + for (int i = 0; i < expectedSchedules.size(); i++) { + final EventLoopScheduleCall expected = expectedSchedules.get(i); + final long actualDelayNanos = actualDelays.get(i); + + assertThat(actualDelayNanos) + .isBetween(expected.delayNanos() - SCHEDULING_TOLERANCE_NANOS, + expected.delayNanos() + SCHEDULING_TOLERANCE_NANOS); + } + } +} \ No newline at end of file diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java index a3addfb8b6a..61e266f1468 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java @@ -40,7 +40,6 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.RepeatedTest; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -444,7 +443,7 @@ void loosesAfterNonRetriableResponse() throws Exception { }); } - @RepeatedTest(5) + @Test void loosesAfterResponseTimeout() throws Exception { final RetryConfig config = RetryConfig .builder(NO_RETRY_RULE) From 5fe93c72f244f3d690c33a99ddd65cebb6a6738a Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Sat, 14 Jun 2025 15:52:50 +0200 Subject: [PATCH 09/36] [WIP] fix, test: tackle concurrency issues in RetryingClient, simplify its interface and add tests for RetryingScheduler --- .../client/retry/AbstractRetryingClient.java | 492 +++++----- .../armeria/client/retry/Backoff.java | 6 +- .../armeria/client/retry/RetryScheduler.java | 183 ++-- .../retry/RetrySchedulingException.java | 51 + .../armeria/client/retry/RetryingClient.java | 153 ++- .../client/retry/RetryingRpcClient.java | 113 ++- .../client/retry/RetrySchedulerTest.java | 873 +++++++----------- .../retry/RetryingClientWithHedgingTest.java | 13 +- .../server/ServiceRequestContextCaptor.java | 4 + 9 files changed, 859 insertions(+), 1029 deletions(-) create mode 100644 core/src/main/java/com/linecorp/armeria/client/retry/RetrySchedulingException.java diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java index c1196dfc9d9..40ccf268bc0 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java @@ -15,12 +15,14 @@ */ package com.linecorp.armeria.client.retry; -import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static java.util.Objects.requireNonNull; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; import java.util.function.Consumer; import org.slf4j.Logger; @@ -30,7 +32,6 @@ import com.linecorp.armeria.client.ClientRequestContext; import com.linecorp.armeria.client.Endpoint; import com.linecorp.armeria.client.SimpleDecoratingClient; -import com.linecorp.armeria.client.retry.AbstractRetryingClient.RetrySchedulabilityDecision.Rationale; import com.linecorp.armeria.common.HttpHeaderNames; import com.linecorp.armeria.common.HttpRequest; import com.linecorp.armeria.common.Request; @@ -51,8 +52,7 @@ */ public abstract class AbstractRetryingClient extends SimpleDecoratingClient { - - protected static final Logger logger = LoggerFactory.getLogger(AbstractRetryingClient.class); + private static final Logger logger = LoggerFactory.getLogger(AbstractRetryingClient.class); /** * The header which indicates the retry count of a {@link Request}. @@ -60,7 +60,7 @@ public abstract class AbstractRetryingClient STATE = + private static final AttributeKey> STATE = AttributeKey.valueOf(AbstractRetryingClient.class, "STATE"); private final RetryConfigMapping mapping; @@ -80,67 +80,27 @@ public abstract class AbstractRetryingClient rfp = getResponseFuturePair(ctx); - - if (!ctx.eventLoop().inEventLoop()) { - ctx.eventLoop().execute(() -> { - try { - doFirstExecute(ctx, req, rfp.response(), rfp.responseFuture()); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - } else { - doFirstExecute(ctx, req, rfp.response(), rfp.responseFuture()); - } - - return rfp.response(); - } - - private void doFirstExecute(ClientRequestContext ctx, I req, O res, CompletableFuture resFuture) - throws Exception { final RetryConfig config = mapping.get(ctx, req); requireNonNull(config, "mapping.get() returned null"); - final State state; + final State state; if (ctx.responseTimeoutMillis() <= 0 || ctx.responseTimeoutMillis() == Long.MAX_VALUE) { final RetryScheduler scheduler = new RetryScheduler(ctx.eventLoop()); - state = new State(config, scheduler); + state = new State<>(config, scheduler); } else { final long responseTimeoutTimeNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(ctx.responseTimeoutMillis()); final RetryScheduler scheduler = new RetryScheduler(ctx.eventLoop(), responseTimeoutTimeNanos); - state = new State(config, scheduler, responseTimeoutTimeNanos); + state = new State<>(config, scheduler, responseTimeoutTimeNanos); } - ctx.setAttr(STATE, state); - state.whenRetryingComplete().handle((f, t) -> { - state.scheduler().close(); + state.scheduler().shutdown(); return null; }); - doExecute(ctx, req, res, resFuture); - } - - abstract ResponseFuturePair getResponseFuturePair(ClientRequestContext ctx); - - protected static class ResponseFuturePair { - private final O response; - private final CompletableFuture responseFuture; - - ResponseFuturePair(O response, CompletableFuture responseFuture) { - this.response = requireNonNull(response, "response"); - this.responseFuture = requireNonNull(responseFuture, "responseFuture"); - } - - public O response() { - return response; - } - - public CompletableFuture responseFuture() { - return responseFuture; - } + ctx.setAttr(STATE, state); + return doExecute(ctx, req); } /** @@ -154,23 +114,19 @@ protected final RetryConfigMapping mapping() { * Invoked by {@link #execute(ClientRequestContext, Request)} * after the deadline for response timeout is set. */ - protected abstract void doExecute(ClientRequestContext ctx, I req, O res, CompletableFuture resFuture) + protected abstract O doExecute(ClientRequestContext ctx, I req) throws Exception; /** * This should be called when retrying is finished. */ - protected static void onRetryingComplete(ClientRequestContext ctx, - ClientRequestContext attemptCtx) { - logger.debug("onRetryingComplete: {}", attemptCtx); - ctx.logBuilder().endResponseWithChild(attemptCtx.log()); - state(ctx).complete(attemptCtx); + protected static void completeRetryingIfNoPendingAttempts(ClientRequestContext ctx) { + state(ctx).completeIfNoPendingAttempts(); } - protected static void onRetryingCompleteExceptionally(ClientRequestContext ctx, - Throwable cause) { + protected static void completeRetryingExceptionally(ClientRequestContext ctx, + Throwable cause) { logger.debug("onRetryingCompleteExceptionally: {}", ctx, cause); - ctx.logBuilder().endResponse(cause); state(ctx).completeExceptionally(cause); } @@ -208,90 +164,139 @@ protected final RetryRuleWithContent retryRuleWithContent() { return retryRuleWithContent; } - protected static void onAttemptStarted(ClientRequestContext ctx, - ClientRequestContext attemptCtx, - Consumer<@Nullable Throwable> onAttemptAbortedHandler + protected void startRetryAttempt(ClientRequestContext ctx, + ClientRequestContext attemptCtx, + BiConsumer onAcceptHandler, + BiConsumer onAttemptAbortedHandler ) { requireNonNull(ctx, "ctx"); requireNonNull(attemptCtx, "attemptCtx"); - // todo(szymon) [Q]: should we track the attemptCtxs and check if we have multiple attempts started? - // todo(szymon): do we want to wait for acquisition when we schedule it? - state(ctx).incrementPendingAttemptCount(); - state(ctx).whenRetryingComplete().handle((winningAttemptCtx, cause) -> { + requireNonNull(onAttemptAbortedHandler, "onAttemptAbortedHandler"); + final State state = state(ctx); + + state.startAttempt(attemptCtx); + state.whenRetryingComplete().handle((winningAttempt, cause) -> { + if (winningAttempt == null) { + // If the retrying is complete exceptionally, we need to call the onAttemptAbortedHandler. + onAttemptAbortedHandler.accept(winningAttempt.ctx(), cause); + return null; + } + + final ClientRequestContext winningAttemptCtx = winningAttempt.ctx(); + final O winningAttemptRes = winningAttempt.res(); + assert winningAttemptRes != null; + if (attemptCtx == winningAttemptCtx) { + onAcceptHandler.accept(winningAttemptCtx, winningAttemptRes); return null; } - onAttemptAbortedHandler.accept(cause); + onAttemptAbortedHandler.accept(attemptCtx, cause); return null; }); - logger.debug("onAttemptStarted: {}. numRemainingPendingAttempts = {}, hasScheduledRetryTask={}", - ctx, state(ctx).numPendingAttempts, state(ctx).scheduler().hasScheduledRetryTask()); + logger.debug("onAttemptStarted: {}", ctx); } - protected boolean onAttemptEnded(ClientRequestContext ctx) { - final int numRemainingPendingAttempts = state(ctx).decrementPendingAttemptCount(); - logger.debug("onAttemptEnded: {}. numRemainingPendingAttempts = {}, hasScheduledRetryTask={}", - ctx, numRemainingPendingAttempts, state(ctx).scheduler().hasScheduledRetryTask()); - return hasPendingOrScheduledAttempts(ctx); - } + protected static void completeRetryAttempt(ClientRequestContext ctx, + ClientRequestContext attemptCtx, + O attemptRes, boolean isWinning) { - protected static boolean hasPendingOrScheduledAttempts(ClientRequestContext ctx) { - return state(ctx).numPendingAttempts() > 0 || state(ctx).scheduler().hasScheduledRetryTask(); - } + // todo(szymon): needs to be checked atomically + state(ctx).completeAttempt(attemptCtx, attemptRes); - // an attempt did not trigger a retry. when does the attempt end? + if (isWinning) { + state(ctx).complete(); + } + } protected static boolean isRetryingComplete(ClientRequestContext ctx) { requireNonNull(ctx, "ctx"); return state(ctx).whenRetryingComplete().isDone(); } - protected static void scheduleNextRetry(ClientRequestContext ctx, - Runnable retryTask, - long retryTimeNanos, - Backoff responsibleBackoff, - Consumer actionOnException) { - requireNonNull(ctx, "ctx"); - requireNonNull(actionOnException, "actionOnException"); - requireNonNull(retryTask, "retryTask"); - - final RetryScheduler scheduler = state(ctx).scheduler(); - - scheduleNextRetry(ctx, retryTask, retryTimeNanos, responsibleBackoff, - scheduler.getEarliestNextRetryTimeNanos(), actionOnException); + protected void scheduleNextRetry(ClientRequestContext ctx, + Runnable retryTask, + Backoff backoff, + Consumer actionOnException) { + scheduleNextRetry(ctx, retryTask, backoff, -1, actionOnException); } protected static void scheduleNextRetry(ClientRequestContext ctx, Runnable retryTask, - long retryTimeNanos, Backoff backoff, - long earliestNextRetryTimeFromServerNanos, + long retryDelayFromServerMillis, Consumer actionOnException) { requireNonNull(ctx, "ctx"); - requireNonNull(actionOnException, "actionOnException"); requireNonNull(retryTask, "retryTask"); + requireNonNull(backoff, "backoff"); + requireNonNull(actionOnException, "actionOnException"); - final State state = state(ctx); + final State state = state(ctx); final RetryScheduler scheduler = state.scheduler(); - // todo(szymon): remove - scheduler.addEarliestNextRetryTimeNanos(earliestNextRetryTimeFromServerNanos); + final long nowTimeNanos = System.nanoTime(); + + final long earliestRetryTimeNanos = retryDelayFromServerMillis >= 0 ? + (nowTimeNanos + TimeUnit.MILLISECONDS.toNanos( + retryDelayFromServerMillis)) : + Long.MIN_VALUE; + if (state.timeoutForWholeRetryEnabled() && + earliestRetryTimeNanos > state.responseTimeoutTimeNanos()) { + actionOnException.accept( + new RetrySchedulingException( + RetrySchedulingException.Type.DELAY_FROM_BACKOFF_EXCEEDS_RESPONSE_TIMEOUT)); + return; + } + + // Even when we cannot schedule the retry task, we want to respect the + // minimum retry delay from the server. + scheduler.addEarliestNextRetryTimeNanos(earliestRetryTimeNanos); + + final int attemptNoWithBackoff = state.nextAttemptNoWithBackoff(backoff); + if (attemptNoWithBackoff < 0) { + scheduler.rescheduleCurrentRetryTaskIfTooEarly(); + actionOnException.accept( + new RetrySchedulingException(RetrySchedulingException.Type.NO_MORE_ATTEMPTS_IN_RETRY)); + return; + } + + final long retryDelayMillis = backoff.nextDelayMillis(attemptNoWithBackoff); + if (retryDelayMillis < 0) { + scheduler.rescheduleCurrentRetryTaskIfTooEarly(); + actionOnException.accept( + new RetrySchedulingException(RetrySchedulingException.Type.NO_MORE_ATTEMPTS_IN_BACKOFF)); + return; + } + + final long retryTimeNanos = Math.max(nowTimeNanos + TimeUnit.MILLISECONDS.toNanos(retryDelayMillis), + earliestRetryTimeNanos); + if (state.timeoutForWholeRetryEnabled() && retryTimeNanos > state.responseTimeoutTimeNanos()) { + scheduler.rescheduleCurrentRetryTaskIfTooEarly(); + actionOnException.accept( + new RetrySchedulingException( + RetrySchedulingException.Type.DELAY_FROM_SERVER_EXCEEDS_RESPONSE_TIMEOUT)); + return; + } + + state.startRetryTask(); scheduler.schedule(() -> { - checkState(!isRetryingComplete(ctx)); + if (isRetryingComplete(ctx)) { + state.completeRetryTask(); + return; + } + state.acquireAttemptNoWithCurrentBackoff(backoff); retryTask.run(); - }, retryTimeNanos, actionOnException); + state.completeRetryTask(); + completeRetryingIfNoPendingAttempts(ctx); + }, retryTimeNanos, earliestRetryTimeNanos, cause -> { + state.completeRetryTask(); + actionOnException.accept(cause); + }); } - protected static void addEarliestNextRetryTimeNanos(ClientRequestContext ctx, - long earliestNextRetryTimeNanos) { - requireNonNull(ctx, "ctx"); - final RetryScheduler scheduler = state(ctx).scheduler(); - scheduler.addEarliestNextRetryTimeNanos(earliestNextRetryTimeNanos); - scheduler.rescheduleCurrentRetryTaskIfTooEarly(); - } + // todo(szymon): improve documentation for this method /** * Resets the {@link ClientRequestContext#responseTimeoutMillis()}. @@ -299,7 +304,7 @@ protected static void addEarliestNextRetryTimeNanos(ClientRequestContext ctx, * @return {@code true} if the response timeout is set, {@code false} if it can't be set due to the timeout */ @SuppressWarnings("MethodMayBeStatic") // Intentionally left non-static for better user experience. - protected final boolean updateResponseTimeout(ClientRequestContext ctx) { + protected final boolean setResponseTimeout(ClientRequestContext ctx) { requireNonNull(ctx, "ctx"); final long responseTimeoutMillis = state(ctx).responseTimeoutMillisForAttempt(); if (responseTimeoutMillis < 0) { @@ -313,109 +318,12 @@ protected final boolean updateResponseTimeout(ClientRequestContext ctx) { } } - protected final RetrySchedulabilityDecision canScheduleWith(ClientRequestContext ctx, Backoff backoff) { - return canScheduleWith(ctx, backoff, -1); - } - - protected final RetrySchedulabilityDecision canScheduleWith(ClientRequestContext ctx, Backoff backoff, - long millisFromServer) { - requireNonNull(ctx, "ctx"); - requireNonNull(backoff, "backoff"); - final State state = state(ctx); - final RetryScheduler scheduler = state.scheduler(); - final long nowTimeNanos = System.nanoTime(); - - final long earliestNextRetryTimeNanos = Math.max(scheduler.getEarliestNextRetryTimeNanos(), - millisFromServer < 0 ? - scheduler.getEarliestNextRetryTimeNanos() : - nowTimeNanos + TimeUnit.MILLISECONDS.toNanos( - millisFromServer)); - - if (state.timeoutForWholeRetryEnabled()) { - if (earliestNextRetryTimeNanos > state.responseTimeoutTimeNanos()) { - logger.debug("The earliest next retry time {} is after the response timeout time {}. " - + "Not scheduling a retry.", - earliestNextRetryTimeNanos, state.responseTimeoutTimeNanos()); - return new RetrySchedulabilityDecision(Rationale.EXCEEDS_RESPONSE_TIMEOUT, Long.MAX_VALUE, - scheduler.getEarliestNextRetryTimeNanos()); - } - } - - final int nextAttemptNo = state.nextAttemptNoWithBackoff(backoff); - if (nextAttemptNo < 0) { - logger.debug("Exceeded the default number of max attempt: {}", state.config.maxTotalAttempts()); - return new RetrySchedulabilityDecision(Rationale.NO_MORE_ATTEMPTS, Long.MAX_VALUE, - earliestNextRetryTimeNanos); - } - - final long nextDelay = backoff.nextDelayMillis(nextAttemptNo); - if (nextDelay < 0) { - logger.debug("Exceeded the number of max attempts in the backoff: {}", backoff); - return new RetrySchedulabilityDecision(Rationale.NO_MORE_ATTEMPTS_IN_BACKOFF, Long.MAX_VALUE, - earliestNextRetryTimeNanos); - } - - final long nextRetryTimeNanos = Math.max(nowTimeNanos + TimeUnit.MILLISECONDS.toNanos(nextDelay), - earliestNextRetryTimeNanos); - - if (state(ctx).timeoutForWholeRetryEnabled()) { - if (nextRetryTimeNanos > state(ctx).responseTimeoutTimeNanos()) { - logger.debug("The next retry time {} is after the response timeout time {}. " - + "Not scheduling a retry.", - nextRetryTimeNanos, state(ctx).responseTimeoutTimeNanos()); - return new RetrySchedulabilityDecision(Rationale.EXCEEDS_RESPONSE_TIMEOUT, Long.MAX_VALUE, - earliestNextRetryTimeNanos); - } - } - - if (scheduler.hasAlreadyRetryScheduledBefore(nextRetryTimeNanos, earliestNextRetryTimeNanos)) { - return new RetrySchedulabilityDecision(Rationale.HAS_EARLIER_RETRY, - nextRetryTimeNanos, earliestNextRetryTimeNanos); - } - - return new RetrySchedulabilityDecision(Rationale.SCHEDULABLE, nextRetryTimeNanos, - earliestNextRetryTimeNanos); - } - - /** - * Returns the next delay which retry will be made after. The delay will be: - * - *

{@code Math.min(responseTimeoutMillis, Backoff.nextDelayMillis(int))} - * - * @return the number of milliseconds to wait for before attempting a retry. -1 if the - * {@code currentAttemptNo} exceeds the {@code maxAttempts} or the {@code nextDelay} is after - * the moment which timeout happens. - */ - protected final long getNextDelay(ClientRequestContext ctx, Backoff backoff) { - return getNextDelay(ctx, backoff, -1); - } - - /** - * Returns the next delay which retry will be made after. The delay will be: - * - *

{@code Math.min(responseTimeoutMillis, Math.max(Backoff.nextDelayMillis(int), - * millisFromServer))} - *

- * If delay is non-negative, we expect a retry to be issued and so a retry attempt is consumed. - * - * @return the number of milliseconds to wait for before attempting a retry. -1 if either - * - {@code currentAttemptNo} exceeds the {@code maxAttempts} or - * - the {@code nextDelay} is after the moment which timeout happens or - * - there is a pending retry task that is shorter than the next delay. - */ - @SuppressWarnings("MethodMayBeStatic") // Intentionally left non-static for better user experience. - protected final long getNextDelay(ClientRequestContext ctx, Backoff backoff, - long millisFromServer) { - // todo(szymon): map to canschedulewith. - return -1; - } - /** * Returns the total number of attempts of the current request represented by the specified * {@link ClientRequestContext}. */ protected static int getTotalAttempts(ClientRequestContext ctx) { - final State state = ctx.attr(STATE); + final State state = ctx.attr(STATE); if (state == null) { return 0; } @@ -433,76 +341,62 @@ protected static ClientRequestContext newAttemptContext(ClientRequestContext ctx return ClientUtil.newDerivedContext(ctx, req, rpcReq, initialAttempt); } - private static State state(ClientRequestContext ctx) { - final State state = ctx.attr(STATE); + @SuppressWarnings("unchecked") + private static State state(ClientRequestContext ctx) { + final State state = (State) ctx.attr(STATE); assert state != null; return state; } - protected static final class RetrySchedulabilityDecision { - enum Rationale { - SCHEDULABLE(true), - NO_MORE_ATTEMPTS(false), - NO_MORE_ATTEMPTS_IN_BACKOFF(false), - EXCEEDS_RESPONSE_TIMEOUT(false), - HAS_EARLIER_RETRY(false); + private static final class State { + static class Attempt { + private final ClientRequestContext attemptCtx; + private @Nullable O attemptRes; - private final boolean canSchedule; + private boolean isCompleted; - Rationale(boolean canSchedule) { - this.canSchedule = canSchedule; + Attempt(ClientRequestContext attemptCtx, @Nullable O attemptRes) { + this.attemptCtx = requireNonNull(attemptCtx, "attemptCtx"); + this.attemptRes = attemptRes; + isCompleted = this.attemptRes != null; } - public boolean canSchedule() { - return canSchedule; + public ClientRequestContext ctx() { + return attemptCtx; } - } - - private final Rationale outcome; - private final long nextRetryTimeNanos; - private final long earliestNextRetryTimeNanos; - - RetrySchedulabilityDecision(Rationale outcome, long nextRetryTimeNanos, - long earliestNextRetryTimeNanos) { - requireNonNull(outcome, "outcome"); - checkArgument(earliestNextRetryTimeNanos <= nextRetryTimeNanos); - - this.outcome = outcome; - this.nextRetryTimeNanos = nextRetryTimeNanos; - this.earliestNextRetryTimeNanos = earliestNextRetryTimeNanos; - } - - long nextRetryTimeNanos() { - return nextRetryTimeNanos; - } - long earliestNextRetryTimeNanos() { - return earliestNextRetryTimeNanos; - } - - boolean canSchedule() { - return outcome.canSchedule(); - } + @Nullable + public O res() { + return attemptRes; + } - @Override - public String toString() { - return "RetrySchedulabilityDecision{" + - "outcome=" + outcome + - ", nextRetryTimeNanos=" + nextRetryTimeNanos + - ", earliestNextRetryTimeNanos=" + earliestNextRetryTimeNanos + - '}'; + public void setRes(O res) { + checkState(!isCompleted); + isCompleted = true; + attemptRes = requireNonNull(res, "res"); + } } - } - private static final class State { private final RetryConfig config; private final long deadlineNanos; private final boolean isTimeoutEnabled; + private final Map> activeAttempts = new HashMap<>(); + private final RetryScheduler retryScheduler; - private int numPendingAttempts; - private final CompletableFuture retryingCompleteFuture; + // An attempt is considered scheduled between two points: + // - right before its retry task is scheduled + // - right after the retry task finishes execution + // + // As a consequence, the retry task must call startAttempt() synchronously. + // If it does not, we may assume there are no active or scheduled attempts + // once the task ends, and stop retrying too early. + private int numScheduledAttempts; + + private @Nullable Attempt lastAttempt; + + private final CompletableFuture> retryingCompleteFuture; @Nullable private Backoff lastBackoff; @@ -572,6 +466,44 @@ long responseTimeoutTimeNanos() { return deadlineNanos; } + void startAttempt(ClientRequestContext attemptCtx) { + checkState(!retryingCompleteFuture.isDone()); + + logger.debug("Attempt started: {}, num attempts pending = {}, num retry task scheduled = {}", + attemptCtx, activeAttempts.size(), numScheduledAttempts); + checkState(!activeAttempts.containsKey(attemptCtx), + "Attempt %s already exists in active attempts.", attemptCtx); + final Attempt attempt = new Attempt<>(attemptCtx, null); + activeAttempts.put(attempt.ctx(), attempt); + } + + void completeAttempt(ClientRequestContext attemptCtx, O attemptRes) { + if (retryingCompleteFuture.isDone()) { + return; + } + + checkState(activeAttempts.containsKey(attemptCtx), + "Attempt %s not found in active attempts: %s", attemptCtx, activeAttempts); + + lastAttempt = activeAttempts.get(attemptCtx); + lastAttempt.setRes(attemptRes); + activeAttempts.remove(attemptCtx); + + completeIfNoPendingAttempts(); + } + + void startRetryTask() { + // This can get (temporarily) above max total attempts as there is a moment between scheduling + // a new retry task and cancelling the previous one. This is because startRetryTask() needs to be + // increment before we call the RetryScheduler.schedule() method. + numScheduledAttempts++; + } + + void completeRetryTask() { + checkState(numScheduledAttempts > 0); + numScheduledAttempts--; + } + int nextAttemptNoWithBackoff(Backoff backoff) { if (totalAttemptNo >= config.maxTotalAttempts()) { return -1; @@ -580,10 +512,12 @@ int nextAttemptNoWithBackoff(Backoff backoff) { if (lastBackoff != backoff) { return 1; } + return currentAttemptNoWithLastBackoff + 1; } void acquireAttemptNoWithCurrentBackoff(Backoff backoff) { + checkState(!retryingCompleteFuture.isDone()); checkState((totalAttemptNo + 1) <= config.maxTotalAttempts(), "Exceeded the maximum number of attempts: %s", config.maxTotalAttempts()); @@ -598,29 +532,47 @@ void acquireAttemptNoWithCurrentBackoff(Backoff backoff) { currentAttemptNoWithLastBackoff++; } - void incrementPendingAttemptCount() { - numPendingAttempts++; + int numPendingAttempts() { + return activeAttempts.size() + (numScheduledAttempts > 0 ? 1 : 0); } - int decrementPendingAttemptCount() { - checkArgument(numPendingAttempts > 0, "numPendingAttempts must be greater than 0. did you call " - + "incrementPendingAttemptCount() before?"); - return --numPendingAttempts; - } + void completeIfNoPendingAttempts() { + if (retryingCompleteFuture.isDone()) { + return; + } - int numPendingAttempts() { - return numPendingAttempts; + if (numPendingAttempts() > 0) { + return; + } + + // If there are no pending attempts, we can complete the retrying. + complete(); } - void complete(ClientRequestContext winningAttemptCtx) { - retryingCompleteFuture.complete(winningAttemptCtx); + void complete() { + if (retryingCompleteFuture.isDone()) { + return; + } + + if (lastAttempt == null) { + completeExceptionally(new IllegalStateException("completed retrying " + + "without a single " + + "successful" + + "attempt")); + } else { + retryingCompleteFuture.complete(lastAttempt); + } } void completeExceptionally(Throwable cause) { + if (retryingCompleteFuture.isDone()) { + return; + } + retryingCompleteFuture.completeExceptionally(cause); } - CompletableFuture whenRetryingComplete() { + CompletableFuture> whenRetryingComplete() { return retryingCompleteFuture; } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/Backoff.java b/core/src/main/java/com/linecorp/armeria/client/retry/Backoff.java index 5b8a289fd33..dd5963e8774 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/Backoff.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/Backoff.java @@ -1,7 +1,7 @@ /* - * Copyright 2017 LINE Corporation + * Copyright 2025 LY Corporation * - * LINE Corporation licenses this file to you under the Apache License, + * LY Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * @@ -187,7 +187,7 @@ static RandomBackoffBuilder builderForRandom() { } /** - * Returns the number of milliseconds to wait for before attempting a retry. + * Returns the number of milliseconds to wait for before attempting a retry. This method is idempotent. * * @param numAttemptsSoFar the number of attempts made by a client so far, including the first attempt and * its following retries. diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java index bd2888eff61..8c9a4c78f6e 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java @@ -16,6 +16,7 @@ package com.linecorp.armeria.client.retry; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static java.util.Objects.requireNonNull; @@ -25,6 +26,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.linecorp.armeria.client.retry.RetrySchedulingException.Type; import com.linecorp.armeria.common.annotation.Nullable; import io.netty.channel.EventLoop; @@ -32,48 +34,61 @@ class RetryScheduler { private static class RetryTaskHandle { + enum State { + SCHEDULED, + OVERTAKEN, + RESCHEDULED + } + private final Runnable retryTask; private final ScheduledFuture scheduledFuture; - private boolean isCancellationMuted; + private State state; - private final Consumer<@Nullable ? super Throwable> onExceptionHandler; + private final Consumer<@Nullable ? super Throwable> exceptionHandler; private final long retryTimeNanos; RetryTaskHandle(ScheduledFuture scheduledFuture, Runnable retryTask, long retryTimeNanos, - Consumer<@Nullable ? super Throwable> onExceptionHandler) { + Consumer<@Nullable ? super Throwable> exceptionHandler) { this.retryTask = retryTask; this.scheduledFuture = scheduledFuture; - this.onExceptionHandler = onExceptionHandler; + this.exceptionHandler = exceptionHandler; this.retryTimeNanos = retryTimeNanos; + state = State.SCHEDULED; } - public Runnable getRetryTaskRunnable() { + Runnable getRetryTaskRunnable() { return retryTask; } - public long retryTimeNanos() { + long retryTimeNanos() { return retryTimeNanos; } - public boolean isCancellationMuted() { - return isCancellationMuted; + void markRescheduled() { + assert state == State.SCHEDULED; + state = State.RESCHEDULED; } - public void muteCancellation() { - isCancellationMuted = true; + void markOvertaken() { + assert state == State.SCHEDULED; + state = State.OVERTAKEN; } - public void unmuteCancellation() { - isCancellationMuted = false; + boolean isRescheduled() { + return state == State.RESCHEDULED; } - public Consumer getOnExceptionHandler() { - return onExceptionHandler; + boolean isOvertaken() { + return state == State.OVERTAKEN; } - public ScheduledFuture getFuture() { + Consumer getexceptionHandler() { + return exceptionHandler; + } + + ScheduledFuture getFuture() { return scheduledFuture; } } @@ -84,6 +99,11 @@ public ScheduledFuture getFuture() { private final long latestNextRetryTimeNanos; private long earliestNextRetryTimeNanos; + + // The retry task that is about to be executed next. + // It is possible that the delay of this task is shorter than the `earliestNextRetryTimeNanos`, + // because of calls to `addEarliestNextRetryTimeNanos` with a pending call to `schedule` or + // `rescheduleCurrentRetryTaskIfTooEarly`. private @Nullable RetryTaskHandle currentRetryTask; RetryScheduler(EventLoop eventLoop) { @@ -98,46 +118,41 @@ public ScheduledFuture getFuture() { } public synchronized void schedule(Runnable retryTask, long nextRetryTimeNanos, - Consumer onRetryTaskFailedHandler) { + long earliestNextRetryTimeNanos, + Consumer exceptionHandler) { requireNonNull(retryTask, "retryTask"); - requireNonNull(onRetryTaskFailedHandler, "onRetryTaskFailedHandler"); - - if (nextRetryTimeNanos < earliestNextRetryTimeNanos) { - // The next retry time is before the earliestNextRetryTimeNanos. - // We need to update the earliestNextRetryTimeNanos. - onRetryTaskFailedHandler.accept( - new IllegalStateException( - "nextRetryTimeNanos is before the earliestNextRetryTimeNanos: " + - nextRetryTimeNanos + " < " + earliestNextRetryTimeNanos)); - return; - } - - if (nextRetryTimeNanos > latestNextRetryTimeNanos) { - // The next retry time is after the latestNextRetryTimeNanos. - onRetryTaskFailedHandler.accept( - new IllegalStateException("nextRetryTimeNanos is after the latestNextRetryTimeNanos: " + - nextRetryTimeNanos + " > " + latestNextRetryTimeNanos)); - return; + checkArgument(nextRetryTimeNanos >= earliestNextRetryTimeNanos, + "nextRetryTimeNanos: %s (expected: >= %s)", + nextRetryTimeNanos, earliestNextRetryTimeNanos); + requireNonNull(exceptionHandler, "exceptionHandler"); + + addEarliestNextRetryTimeNanos(earliestNextRetryTimeNanos); + nextRetryTimeNanos = Math.max(nextRetryTimeNanos, this.earliestNextRetryTimeNanos); + + // Math.max because we could have added an earliestNextRetryTimeNanos that is later than the + // currently scheduled retry task (even not by us in this method). + if (currentRetryTask == null || nextRetryTimeNanos < Math.max(this.earliestNextRetryTimeNanos, + currentRetryTask.retryTimeNanos())) { + scheduleNextRetryTask(retryTask, nextRetryTimeNanos, exceptionHandler, false); + } else { + // Make sure the current retry task is not scheduled too early. + rescheduleCurrentRetryTaskIfTooEarly(); + + exceptionHandler.accept(new RetrySchedulingException( + RetrySchedulingException.Type.RETRY_TASK_OVERTAKEN)); } - - if (currentRetryTask == null || nextRetryTimeNanos < currentRetryTask.retryTimeNanos()) { - scheduleNextRetryTask(retryTask, nextRetryTimeNanos, onRetryTaskFailedHandler); - return; - } - - onRetryTaskFailedHandler.accept( - new IllegalStateException("A retry task is already scheduled at " + - currentRetryTask.retryTimeNanos() + ". " + - "nextRetryTimeNanos: " + nextRetryTimeNanos)); } - private synchronized boolean cancelCurrentRetryTask() { + private synchronized boolean cancelCurrentRetryTask(boolean cancelForRescheduling) { if (currentRetryTask != null) { final ScheduledFuture retryTaskFuture = currentRetryTask.getFuture(); - currentRetryTask.muteCancellation(); + if (cancelForRescheduling) { + currentRetryTask.markRescheduled(); + } else { + currentRetryTask.markOvertaken(); + } if (!retryTaskFuture.cancel(false)) { - currentRetryTask.unmuteCancellation(); return false; } else { clearCurrentRetryTask(); @@ -157,16 +172,25 @@ private synchronized void handleRetryTaskCompletion(RetryTaskHandle retryTaskHan } if (retryTaskFuture.isCancelled()) { - if (!retryTaskHandle.isCancellationMuted()) { - // The retry task was cancelled by the user, not by the scheduler. - retryTaskHandle.getOnExceptionHandler().accept( - new IllegalStateException("Retry task was cancelled by the user.")); + if (retryTaskHandle.isRescheduled()) { + return; + } + + if (retryTaskHandle.isOvertaken()) { + retryTaskHandle.getexceptionHandler().accept( + new RetrySchedulingException(Type.RETRY_TASK_OVERTAKEN) + ); + return; } + + // The retry task was cancelled by the user, not by the scheduler. + retryTaskHandle.getexceptionHandler().accept( + new RetrySchedulingException(Type.RETRY_TASK_CANCELLED)); return; } if (!retryTaskFuture.isSuccess()) { - retryTaskHandle.getOnExceptionHandler().accept(retryTaskFuture.cause()); + retryTaskHandle.getexceptionHandler().accept(retryTaskFuture.cause()); } } @@ -176,24 +200,33 @@ private synchronized void clearCurrentRetryTask() { } private synchronized void scheduleNextRetryTask(Runnable retryRunnable, long retryTimeNanos, - Consumer onExceptionHandler) { + Consumer exceptionHandler, + boolean isReschedule) { assert earliestNextRetryTimeNanos <= retryTimeNanos; - assert retryTimeNanos <= latestNextRetryTimeNanos; - if (!cancelCurrentRetryTask()) { - onExceptionHandler.accept( - new IllegalStateException("Could not cancel the current retry task.")); + if (!cancelCurrentRetryTask(isReschedule)) { + exceptionHandler.accept(new IllegalStateException("Current retry task could not be cancelled.")); return; } assert currentRetryTask == null; try { - final long delayNanos = Math.max(retryTimeNanos - System.nanoTime(), 0); + final long nowNanos = System.nanoTime(); + final long delayNanos; + if (retryTimeNanos <= nowNanos) { + delayNanos = 0; + } else { + delayNanos = retryTimeNanos - nowNanos; + } + final Runnable wrappedRetryRunnable = () -> { logger.debug("Retry task starting. Resetting..."); // todo(szymon): do sanity check that we are clearing this task (very bad otherwise). clearCurrentRetryTask(); + + // todo(szymon): Q should we check whether we are too early here? + retryRunnable.run(); }; @@ -210,32 +243,13 @@ private synchronized void scheduleNextRetryTask(Runnable retryRunnable, long ret // rescheduled multiple times. final RetryTaskHandle nextRetryTask = new RetryTaskHandle(nextRetryTaskFuture, retryRunnable, retryTimeNanos, - onExceptionHandler); + exceptionHandler); nextRetryTaskFuture.addListener(f -> handleRetryTaskCompletion(nextRetryTask)); currentRetryTask = nextRetryTask; } catch (Throwable t) { - onExceptionHandler.accept(t); - } - } - - public boolean close() { - return cancelCurrentRetryTask(); - } - - // todo(szymon): Remove dependency on nextEarliestNextRetryTimeNanos. Users should simply set it before - // calling this method. - public synchronized boolean hasAlreadyRetryScheduledBefore(long nextRetryTimeNanos, - long nextEarliestNextRetryTimeNanos) { - checkState(nextEarliestNextRetryTimeNanos <= latestNextRetryTimeNanos); - earliestNextRetryTimeNanos = Math.max(earliestNextRetryTimeNanos, nextEarliestNextRetryTimeNanos); - - if (currentRetryTask == null) { - return false; + exceptionHandler.accept(t); } - - return Math.max(currentRetryTask.retryTimeNanos(), earliestNextRetryTimeNanos) - <= nextRetryTimeNanos; } public synchronized void addEarliestNextRetryTimeNanos(long earliestNextRetryTimeNanos) { @@ -244,23 +258,20 @@ public synchronized void addEarliestNextRetryTimeNanos(long earliestNextRetryTim earliestNextRetryTimeNanos); } - public synchronized long getEarliestNextRetryTimeNanos() { - return earliestNextRetryTimeNanos; - } - public synchronized void rescheduleCurrentRetryTaskIfTooEarly() { if (currentRetryTask != null) { if (currentRetryTask.retryTimeNanos() < earliestNextRetryTimeNanos) { // Current retry task is going to be executed before the earliestNextRetryTimeNanos so // we need to reschedule it. + scheduleNextRetryTask(currentRetryTask.getRetryTaskRunnable(), earliestNextRetryTimeNanos, - currentRetryTask.getOnExceptionHandler()); + currentRetryTask.getexceptionHandler(), true); } } } - public synchronized boolean hasScheduledRetryTask() { - return currentRetryTask != null; + public boolean shutdown() { + return cancelCurrentRetryTask(true); } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetrySchedulingException.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetrySchedulingException.java new file mode 100644 index 00000000000..1bfecc2f6ac --- /dev/null +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetrySchedulingException.java @@ -0,0 +1,51 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.client.retry; + +class RetrySchedulingException extends RuntimeException { + private static final long serialVersionUID = 1L; + private final Type type; + + enum Type { + NO_MORE_ATTEMPTS_IN_RETRY("No more attempts available in retry"), + NO_MORE_ATTEMPTS_IN_BACKOFF("No more attempts available in backoff"), + DELAY_FROM_BACKOFF_EXCEEDS_RESPONSE_TIMEOUT("Delay from backoff exceeds response timeout"), + DELAY_FROM_SERVER_EXCEEDS_RESPONSE_TIMEOUT("Delay from server exceeds response timeout"), + RETRY_TASK_OVERTAKEN("Has earlier retry"), + RETRY_TASK_CANCELLED("Retry task cancelled without outside of rescheduling."); + + private final String message; + + Type(String message) { + this.message = message; + } + + public String getMessage() { + return message; + } + } + + RetrySchedulingException(Type type) { + super(type.getMessage()); + this.type = type; + } + + Type getType() { + return type; + } +} + diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index c37605e1157..8a544c41504 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -29,6 +29,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.linecorp.armeria.client.ClientFactory; import com.linecorp.armeria.client.ClientRequestContext; import com.linecorp.armeria.client.HttpClient; import com.linecorp.armeria.client.ResponseTimeoutException; @@ -241,16 +242,10 @@ public static Function newDecorator(RetryRul } @Override - protected ResponseFuturePair getResponseFuturePair( - ClientRequestContext ctx) { + protected HttpResponse doExecute(ClientRequestContext ctx, HttpRequest req) throws Exception { final CompletableFuture responseFuture = new CompletableFuture<>(); final HttpResponse res = HttpResponse.of(responseFuture, ctx.eventLoop()); - return new ResponseFuturePair<>(res, responseFuture); - } - @Override - protected void doExecute(ClientRequestContext ctx, HttpRequest req, HttpResponse res, - CompletableFuture responseFuture) throws Exception { if (ctx.exchangeType().isRequestStreaming()) { final HttpRequestDuplicator reqDuplicator = req.toDuplicator(ctx.eventLoop().withoutContext(), 0); doExecute0(new RetryingContext(mappedRetryConfig(ctx), ctx, reqDuplicator, req, res, @@ -259,7 +254,7 @@ protected void doExecute(ClientRequestContext ctx, HttpRequest req, HttpResponse req.aggregate(AggregationOptions.usePooledObjects(ctx.alloc(), ctx.eventLoop())) .handle((agg, cause) -> { if (cause != null) { - handleException(ctx, null, responseFuture, cause, true); + completeRetryingExceptionally(ctx, null, responseFuture, cause, true); } else { final HttpRequestDuplicator reqDuplicator = new AggregatedHttpRequestDuplicator(agg); doExecute0(new RetryingContext(mappedRetryConfig(ctx), ctx, reqDuplicator, req, res, @@ -268,23 +263,27 @@ protected void doExecute(ClientRequestContext ctx, HttpRequest req, HttpResponse return null; }); } + + return res; } private void doExecute0(RetryingContext retryingContext) { + final RetryConfig config = retryingContext.config(); final ClientRequestContext ctx = retryingContext.ctx(); final HttpRequestDuplicator rootReqDuplicator = retryingContext.reqDuplicator(); final HttpRequest originalReq = retryingContext.req(); final HttpResponse returnedRes = retryingContext.res(); + // todo(szymon): we need to inject the attempt number as there may be concurrent attempts + // starting and acquiring an attempt number. final int totalAttempts = getTotalAttempts(ctx); - logger.trace("doExecute0: {}", totalAttempts); final boolean initialAttempt = totalAttempts <= 1; // The request or attemptRes has been aborted by the client before it receives a attemptRes, // so stop retrying. if (originalReq.whenComplete().isCompletedExceptionally()) { originalReq.whenComplete().handle((unused, cause) -> { - handleException(retryingContext, cause, initialAttempt); + completeRetryingExceptionally(retryingContext, cause, initialAttempt); return null; }); return; @@ -297,14 +296,14 @@ private void doExecute0(RetryingContext retryingContext) { } else { abortCause = AbortedStreamException.get(); } - handleException(retryingContext, abortCause, initialAttempt); + completeRetryingExceptionally(retryingContext, abortCause, initialAttempt); return null; }); return; } - if (!updateResponseTimeout(ctx)) { - handleException(retryingContext, ResponseTimeoutException.get(), initialAttempt); + if (!setResponseTimeout(ctx)) { + completeRetryingExceptionally(retryingContext, ResponseTimeoutException.get(), initialAttempt); return; } @@ -321,7 +320,7 @@ private void doExecute0(RetryingContext retryingContext) { try { attemptCtx = newAttemptContext(ctx, attemptReq, ctx.rpcRequest(), initialAttempt); } catch (Throwable t) { - handleException(retryingContext, t, initialAttempt); + completeRetryingExceptionally(retryingContext, t, initialAttempt); return; } @@ -342,8 +341,20 @@ private void doExecute0(RetryingContext retryingContext) { false); } - onAttemptStarted(ctx, attemptCtx, (@Nullable Throwable cause) -> abortAttempt(attemptCtx, - cause)); + startRetryAttempt(ctx, attemptCtx, (winningAttemptCtx, + winningAttemptRes) -> { + logger.debug("accepting win for attempt: {}, winningAttemptCtx: {}, ", + winningAttemptCtx, winningAttemptRes); + retryingContext.reqDuplicator().close(); + + retryingContext.ctx().logBuilder().endResponseWithChild(winningAttemptCtx.log()); + retryingContext.resFuture().complete(winningAttemptRes); + }, + (abortingAttemptCtx, cause) -> { + abortAttempt(abortingAttemptCtx, + cause); + } + ); if (!ctx.exchangeType().isResponseStreaming() || config.requiresResponseTrailers()) { attemptRes.aggregate().handle((attemptAggResponse, cause) -> { @@ -385,17 +396,31 @@ private void doExecute0(RetryingContext retryingContext) { if (hedgingDelayMillis >= 0) { final Backoff hedgingDelayBackoff = Backoff.fixed(hedgingDelayMillis); - final RetrySchedulabilityDecision retrySchedulabilityDecision = canScheduleWith(ctx, - hedgingDelayBackoff, - -1); - if (retrySchedulabilityDecision.canSchedule()) { - logger.debug("Scheduling hedging with backoff: {}", hedgingDelayBackoff); - scheduleNextRetry(ctx, () -> doExecute0(retryingContext), - retrySchedulabilityDecision.nextRetryTimeNanos(), - hedgingDelayBackoff, - cause -> handleException(retryingContext, cause, false)); + logger.debug("Scheduling hedging with backoff: {}", hedgingDelayBackoff); + scheduleNextRetry(ctx, () -> doExecute0(retryingContext), + hedgingDelayBackoff, + cause -> handleExceptionAfterScheduling(retryingContext, cause)); + } + } + + private void handleExceptionAfterScheduling( + RetryingContext retryingContext, Throwable cause) { + if (cause instanceof RetrySchedulingException) { + switch (((RetrySchedulingException) cause).getType()) { + case NO_MORE_ATTEMPTS_IN_RETRY: + case NO_MORE_ATTEMPTS_IN_BACKOFF: + case DELAY_FROM_BACKOFF_EXCEEDS_RESPONSE_TIMEOUT: + case DELAY_FROM_SERVER_EXCEEDS_RESPONSE_TIMEOUT: + case RETRY_TASK_OVERTAKEN: + completeRetryingIfNoPendingAttempts(retryingContext.ctx()); + return; + case RETRY_TASK_CANCELLED: + cause = new IllegalStateException( + ClientFactory.class.getSimpleName() + " has been closed.", cause); + break; } } + completeRetryingExceptionally(retryingContext, cause, false); } private void handleResponseWithoutContent(RetryingContext retryingContext, @@ -418,7 +443,7 @@ private void handleResponseWithoutContent(RetryingContext retryingContext, return null; }); } catch (Throwable cause) { - handleException(retryingContext, cause, false); + completeRetryingExceptionally(retryingContext, cause, false); } } @@ -485,7 +510,7 @@ private void handleStreamingResponse(RetryingContext retryingContext, }); } catch (Throwable cause) { attemptResDuplicator.abort(cause); - handleException(retryingContext, cause, false); + completeRetryingExceptionally(retryingContext, cause, false); } } else { final HttpResponse attemptUnsplitRes; @@ -523,7 +548,7 @@ private void handleAggregatedResponse(RetryingContext retryingContext, return null; }); } catch (Throwable cause) { - handleException(retryingContext, cause, false); + completeRetryingExceptionally(retryingContext, cause, false); } return; } @@ -577,17 +602,18 @@ private static void warnIfExceptionIsRaised(Object retryRule, @Nullable Throwabl } } - private static void handleException(RetryingContext retryingContext, Throwable cause, - boolean endRequestLog) { - handleException( + private static void completeRetryingExceptionally(RetryingContext retryingContext, Throwable cause, + boolean endRequestLog) { + completeRetryingExceptionally( retryingContext.ctx(), retryingContext.reqDuplicator(), retryingContext.resFuture(), cause, endRequestLog); } - private static void handleException(ClientRequestContext ctx, - @Nullable HttpRequestDuplicator rootReqDuplicator, - CompletableFuture returnedResFuture, Throwable cause, - boolean endRequestLog) { + private static void completeRetryingExceptionally(ClientRequestContext ctx, + @Nullable HttpRequestDuplicator rootReqDuplicator, + CompletableFuture returnedResFuture, + Throwable cause, + boolean endRequestLog) { if (isRetryingComplete(ctx)) { return; } @@ -600,56 +626,31 @@ private static void handleException(ClientRequestContext ctx, ctx.logBuilder().endRequest(cause); } + ctx.logBuilder().endResponse(cause); returnedResFuture.completeExceptionally(cause); - onRetryingCompleteExceptionally(ctx, cause); + completeRetryingExceptionally(ctx, cause); } private void handleRetryDecision(RetryingContext retryingContext, @Nullable RetryDecision decision, ClientRequestContext attemptCtx, HttpResponse attemptRes) { final Backoff backoff = decision != null ? decision.backoff() : null; - final boolean shouldContinueRetry; if (backoff != null) { - shouldContinueRetry = true; final long millisAfter = useRetryAfter ? getRetryAfterMillis(attemptCtx) : -1; - final RetrySchedulabilityDecision schedulabilityDecision = canScheduleWith(retryingContext.ctx(), - backoff, - millisAfter); - - if (schedulabilityDecision.canSchedule()) { - logger.debug("Scheduling next retry for {} with backoff: {}, " - + "schedulabilityDecision: {}", - retryingContext.ctx(), backoff, schedulabilityDecision); - scheduleNextRetry(retryingContext.ctx(), - () -> doExecute0(retryingContext), - schedulabilityDecision.nextRetryTimeNanos(), - backoff, - schedulabilityDecision.earliestNextRetryTimeNanos(), - cause -> handleException(retryingContext, cause, false)); - } else { - logger.debug("Not scheduling next retry for {} with backoff: {}, " - + "schedulabilityDecision: {}", - retryingContext.ctx(), backoff, schedulabilityDecision); - addEarliestNextRetryTimeNanos(retryingContext.ctx(), - schedulabilityDecision.earliestNextRetryTimeNanos()); - } - } else { - shouldContinueRetry = false; + scheduleNextRetry(retryingContext.ctx(), + () -> doExecute0(retryingContext), + backoff, + millisAfter, + cause -> handleExceptionAfterScheduling(retryingContext, cause)); } - final boolean isOtherAttemptInProgress = onAttemptEnded(retryingContext.ctx()); - - if (!shouldContinueRetry || !isOtherAttemptInProgress) { - onRetryingComplete(retryingContext, attemptCtx, attemptRes); - } else { - logger.debug("Retrying is not complete for {} with decision: {}", - retryingContext.ctx(), decision); - } + completeRetryAttempt(retryingContext.ctx(), attemptCtx, attemptRes, backoff == null); } private static void abortAttempt(ClientRequestContext attemptCtx, @Nullable Throwable cause) { + logger.debug("aborting attempt. attemptCtx: {}, cause: {}", attemptCtx, cause); // Set response content with null to make sure that the log is complete. final RequestLogBuilder logBuilder = attemptCtx.logBuilder(); logBuilder.responseContent(null, null); @@ -703,20 +704,6 @@ private static RetryRule retryRule(RetryConfig retryConfig) { } } - void onRetryingComplete(RetryingContext retryingContext, - ClientRequestContext attemptCtx, - HttpResponse attemptRes) { - if (isRetryingComplete(retryingContext.ctx())) { - return; - } - - logger.debug("Completing retrying"); - - retryingContext.reqDuplicator().close(); - retryingContext.resFuture().complete(attemptRes); - onRetryingComplete(retryingContext.ctx(), attemptCtx); - } - private static class RetryingContext { private final ClientRequestContext ctx; private final HttpRequestDuplicator reqDuplicator; diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java index ff46f09e52e..ac4554b81f9 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java @@ -22,15 +22,16 @@ import java.util.concurrent.CompletableFuture; import java.util.function.Function; +import com.linecorp.armeria.client.ClientFactory; import com.linecorp.armeria.client.ClientRequestContext; import com.linecorp.armeria.client.ResponseTimeoutException; import com.linecorp.armeria.client.RpcClient; import com.linecorp.armeria.client.endpoint.EndpointGroup; import com.linecorp.armeria.common.HttpRequest; import com.linecorp.armeria.common.Request; +import com.linecorp.armeria.common.RequestContext; import com.linecorp.armeria.common.RpcRequest; import com.linecorp.armeria.common.RpcResponse; -import com.linecorp.armeria.common.annotation.Nullable; import com.linecorp.armeria.internal.client.ClientPendingThrowableUtil; import com.linecorp.armeria.internal.client.ClientRequestContextExtension; import com.linecorp.armeria.internal.common.util.StringUtil; @@ -142,17 +143,11 @@ public static RetryingRpcClientBuilder builder(RetryConfigMapping m } @Override - protected ResponseFuturePair getResponseFuturePair( - ClientRequestContext ctx) { - final CompletableFuture responseFuture = new CompletableFuture<>(); - final RpcResponse res = RpcResponse.from(responseFuture); - return new ResponseFuturePair<>(res, responseFuture); - } - - @Override - protected void doExecute(ClientRequestContext ctx, RpcRequest req, RpcResponse res, - CompletableFuture returnedResFuture) throws Exception { + protected RpcResponse doExecute(ClientRequestContext ctx, RpcRequest req) throws Exception { + final CompletableFuture returnedResFuture = new CompletableFuture<>(); + final RpcResponse res = RpcResponse.from(returnedResFuture); doExecute0(ctx, req, res, returnedResFuture); + return res; } private void doExecute0(ClientRequestContext ctx, RpcRequest req, @@ -161,12 +156,13 @@ private void doExecute0(ClientRequestContext ctx, RpcRequest req, final boolean initialAttempt = totalAttempts <= 1; if (returnedRes.isDone()) { // The response has been cancelled by the client before it receives a response, so stop retrying. - handleException(ctx, returnedResFuture, new CancellationException( + completeRetryingExceptionally(ctx, returnedResFuture, new CancellationException( "the response returned to the client has been cancelled"), initialAttempt); return; } - if (!updateResponseTimeout(ctx)) { - handleException(ctx, returnedResFuture, ResponseTimeoutException.get(), initialAttempt); + if (!setResponseTimeout(ctx)) { + completeRetryingExceptionally(ctx, returnedResFuture, ResponseTimeoutException.get(), + initialAttempt); return; } @@ -195,9 +191,15 @@ private void doExecute0(ClientRequestContext ctx, RpcRequest req, req, true); } - onAttemptStarted(ctx, attemptCtx, (@Nullable Throwable cause) -> { - attemptCtx.cancel(); - }); + startRetryAttempt(ctx, attemptCtx, (winningAttemptCtx, winningAttemptRes) -> { + final HttpRequest actualHttpReq = winningAttemptCtx.request(); + + if (actualHttpReq != null) { + ctx.updateRequest(actualHttpReq); + } + + returnedResFuture.complete(winningAttemptRes); + }, RequestContext::cancel); final RetryConfig retryConfig = mappedRetryConfig(ctx); final RetryRuleWithContent retryRule = @@ -210,28 +212,18 @@ private void doExecute0(ClientRequestContext ctx, RpcRequest req, final Backoff backoff = decision != null ? decision.backoff() : null; if (backoff != null) { - final RetrySchedulabilityDecision schedulabilityDecision = canScheduleWith(ctx, - backoff); - - if (schedulabilityDecision.canSchedule()) { - scheduleNextRetry(ctx, - () -> doExecute0(ctx, req, returnedRes, returnedResFuture), - schedulabilityDecision.nextRetryTimeNanos(), - backoff, - cause0 -> handleException(ctx, returnedResFuture, cause0, false)); - } - } - - final boolean isOtherAttemptInProgress = onAttemptEnded(ctx); - - if (!isOtherAttemptInProgress) { - onRetryingComplete(ctx, returnedResFuture, attemptCtx, attemptRes); + scheduleNextRetry(ctx, + () -> doExecute0(ctx, req, returnedRes, returnedResFuture), + backoff, + cause0 -> completeRetryingExceptionally(ctx, returnedResFuture, + cause0, false)); } + completeRetryAttempt(ctx, attemptCtx, attemptRes, backoff == null); return null; }); } catch (Throwable t) { - handleException(ctx, returnedResFuture, t, false); + completeRetryingExceptionally(ctx, returnedResFuture, t, false); } return null; }); @@ -240,20 +232,35 @@ private void doExecute0(ClientRequestContext ctx, RpcRequest req, if (hedgingDelayMillis >= 0) { final Backoff hedgingBackoff = Backoff.fixed(hedgingDelayMillis); - final RetrySchedulabilityDecision schedulabilityDecision = - canScheduleWith(ctx, hedgingBackoff); - if (schedulabilityDecision.canSchedule()) { - scheduleNextRetry(ctx, () -> doExecute0(ctx, req, returnedRes, returnedResFuture), - schedulabilityDecision.nextRetryTimeNanos(), - hedgingBackoff, - cause -> handleException(ctx, returnedResFuture, cause, false)); + scheduleNextRetry(ctx, () -> doExecute0(ctx, req, returnedRes, returnedResFuture), + hedgingBackoff, + cause -> handleExceptionAfterScheduling(ctx, returnedResFuture, cause)); + } + } + + private void handleExceptionAfterScheduling( + ClientRequestContext ctx, CompletableFuture returnedResFuture, Throwable cause) { + if (cause instanceof RetrySchedulingException) { + switch (((RetrySchedulingException) cause).getType()) { + case NO_MORE_ATTEMPTS_IN_RETRY: + case NO_MORE_ATTEMPTS_IN_BACKOFF: + case DELAY_FROM_BACKOFF_EXCEEDS_RESPONSE_TIMEOUT: + case DELAY_FROM_SERVER_EXCEEDS_RESPONSE_TIMEOUT: + case RETRY_TASK_OVERTAKEN: + completeRetryingIfNoPendingAttempts(ctx); + return; + case RETRY_TASK_CANCELLED: + cause = new IllegalStateException( + ClientFactory.class.getSimpleName() + " has been closed.", cause); + break; } } + completeRetryingExceptionally(ctx, returnedResFuture, cause, false); } - private static void handleException(ClientRequestContext ctx, - CompletableFuture returnedResFuture, - Throwable cause, boolean endRequestLog) { + private static void completeRetryingExceptionally(ClientRequestContext ctx, + CompletableFuture returnedResFuture, + Throwable cause, boolean endRequestLog) { if (isRetryingComplete(ctx)) { return; } @@ -262,23 +269,9 @@ private static void handleException(ClientRequestContext ctx, if (endRequestLog) { ctx.logBuilder().endRequest(cause); } - onRetryingCompleteExceptionally(ctx, cause); - } - void onRetryingComplete(ClientRequestContext ctx, - CompletableFuture returnedResFuture, - ClientRequestContext attemptCtx, - RpcResponse attemptRes) { - if (isRetryingComplete(ctx)) { - return; - } - - final HttpRequest actualHttpReq = attemptCtx.request(); - if (actualHttpReq != null) { - ctx.updateRequest(actualHttpReq); - } + ctx.logBuilder().endResponse(cause); - returnedResFuture.complete(attemptRes); - onRetryingComplete(ctx, attemptCtx); + completeRetryingExceptionally(ctx, cause); } } diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java index 59416e4c844..fb719881e4f 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java @@ -1,52 +1,60 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package com.linecorp.armeria.client.retry; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.assertj.core.api.Assertions.fail; -import static org.awaitility.Awaitility.await; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; -import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.mockito.ArgumentCaptor; import com.google.common.collect.ImmutableList; +import com.linecorp.armeria.client.retry.RetrySchedulingException.Type; import com.linecorp.armeria.internal.testing.AnticipatedException; import io.netty.channel.DefaultEventLoop; import io.netty.channel.EventLoop; -import io.netty.util.concurrent.ScheduledFuture; -// todo(szymon): clean up the test cases class RetrySchedulerTest { - private static final long SCHEDULING_TOLERANCE_NANOS = TimeUnit.MILLISECONDS.toNanos(10); + private static final long SCHEDULING_TOLERANCE_NANOS = TimeUnit.MILLISECONDS.toNanos(50); private static final long SCHEDULING_TOLERANCE_MILLIS = TimeUnit.NANOSECONDS.toMillis( SCHEDULING_TOLERANCE_NANOS); - private ArgumentCaptor scheduleDelayArgumentCaptor; - private ArgumentCaptor scheduleTimeUnitArgumentCaptor; private EventLoop eventLoop; private RetryScheduler scheduler; @@ -54,15 +62,160 @@ class RetrySchedulerTest { @BeforeEach void setUp() { eventLoop = spy(new DefaultEventLoop()); - scheduleDelayArgumentCaptor = ArgumentCaptor.forClass(Long.class); - scheduleTimeUnitArgumentCaptor = ArgumentCaptor.forClass(TimeUnit.class); scheduler = new RetryScheduler(eventLoop); } @AfterEach void tearDown() throws Exception { - assertThat(scheduler.close()).isTrue(); - eventLoop.shutdownGracefully().sync(); + assertThat(scheduler.shutdown()).isTrue(); + eventLoop.shutdownGracefully(); + } + + @ParameterizedTest + @MethodSource("scheduleTaskParameters") + void testScheduleTask(int taskDelayMs, int minDelayMs, int prevDelayMs, int expectedDelayMs) + throws Exception { + // Convert all delays to nanos + final long now = System.nanoTime(); + final long taskScheduledTimeNanos = now + TimeUnit.MILLISECONDS.toNanos(taskDelayMs); + final long newEarliestNextRetryTimeNanos = + minDelayMs >= 0 ? now + TimeUnit.MILLISECONDS.toNanos(minDelayMs) : Long.MIN_VALUE; + final long previousEarliestNextRetryTimeNanos = + prevDelayMs >= 0 ? now + TimeUnit.MILLISECONDS.toNanos(prevDelayMs) : Long.MIN_VALUE; + + final long expectedScheduledTimeNanos = now + TimeUnit.MILLISECONDS.toNanos(expectedDelayMs); + + // Set the previous delay if it exists + if (prevDelayMs >= 0) { + scheduler.addEarliestNextRetryTimeNanos(previousEarliestNextRetryTimeNanos); + } + + // Schedule the task + final Runnable task = mock(Runnable.class); + final Consumer exceptionHandler = mock(Consumer.class); + + scheduler.schedule(task, taskScheduledTimeNanos, newEarliestNextRetryTimeNanos, exceptionHandler); + + verifyEventLoopScheduleCalls( + EventLoopScheduleCall.of(now, expectedScheduledTimeNanos) + ); + + Thread.sleep(expectedDelayMs + SCHEDULING_TOLERANCE_MILLIS); + + verify(task, times(1)).run(); + verifyNoMoreInteractions(exceptionHandler); + verifyEventLoopScheduleCalls( + EventLoopScheduleCall.of(now, expectedScheduledTimeNanos) + ); + } + + private static Stream scheduleTaskParameters() { + return Stream.of( + // taskDelay, minDelay, prevDelay, expectedDelay + + // No previous delay, no minimum delay + Arguments.of(0, -1, -1, 0), + Arguments.of(200, -1, -1, 200), + + Arguments.of(100, 100, -1, 100), + Arguments.of(150, 100, -1, 150), + + // With previous delay, no minimum delay + Arguments.of(50, -1, 100, 100), + Arguments.of(100, -1, 100, 100), + Arguments.of(150, -1, 100, 150), + + // With both previous and minimum delay + Arguments.of(100, 100, 200, 200), + Arguments.of(150, 100, 200, 200), + Arguments.of(200, 100, 200, 200), + Arguments.of(250, 100, 200, 250) + ); + } + + @Test + void testScheduleTaskInThePast() throws Exception { + final Runnable task = mock(Runnable.class); + final long taskSchedulingTime = System.nanoTime(); + final long expectedTaskRunTime = taskSchedulingTime; + final Consumer exceptionHandler = mock(Consumer.class); + + scheduler.schedule(task, expectedTaskRunTime - 1_000_000, Long.MIN_VALUE, exceptionHandler); + + verifyEventLoopScheduleCalls( + EventLoopScheduleCall.of(taskSchedulingTime, expectedTaskRunTime) + ); + + Thread.sleep(SCHEDULING_TOLERANCE_MILLIS); + + verify(task, times(1)).run(); + verifyNoMoreInteractions(exceptionHandler); + verifyEventLoopScheduleCalls( + EventLoopScheduleCall.of(taskSchedulingTime, expectedTaskRunTime) + ); + + final Runnable task2 = mock(Runnable.class); + final long task2SchedulingTime = System.nanoTime(); + final long expectedTask2RunTime = task2SchedulingTime; + final Consumer task2ExceptionHandler = mock(Consumer.class); + + scheduler.schedule(task2, Long.MIN_VALUE, Long.MIN_VALUE, task2ExceptionHandler); + + verifyEventLoopScheduleCalls( + EventLoopScheduleCall.of(taskSchedulingTime, expectedTaskRunTime), + EventLoopScheduleCall.of(task2SchedulingTime, expectedTask2RunTime) + ); + + Thread.sleep(SCHEDULING_TOLERANCE_MILLIS); + + verify(task2, times(1)).run(); + verifyNoMoreInteractions(task2ExceptionHandler); + verifyEventLoopScheduleCalls( + EventLoopScheduleCall.of(taskSchedulingTime, expectedTaskRunTime), + EventLoopScheduleCall.of(task2SchedulingTime, expectedTask2RunTime) + ); + } + + @Test + void testFailingScheduleWithInvalidArguments() { + final Runnable task = mock(Runnable.class); + final Consumer exceptionHandler = mock(Consumer.class); + + assertThatThrownBy(() -> scheduler.schedule(null, 200, 200, + exceptionHandler)).isInstanceOf( + NullPointerException.class).hasMessageContaining("retryTask"); + + assertThatThrownBy(() -> scheduler.schedule(task, 200, 200, null)).isInstanceOf( + NullPointerException.class).hasMessageContaining("exceptionHandler"); + + assertThatThrownBy(() -> scheduler.schedule(task, Long.MAX_VALUE - 1, Long.MAX_VALUE, + exceptionHandler)).isInstanceOf( + IllegalArgumentException.class).hasMessageContaining("nextRetryTimeNanos"); + + verifyNoMoreInteractions(task, exceptionHandler); + verifyEventLoopScheduleCalls(); + } + + @Test + void testScheduleTaskWithException() throws Exception { + final Runnable task1 = mock(Runnable.class); + doThrow(new AnticipatedException()).when(task1).run(); + + final long task1SchedulingTime = System.nanoTime(); + final long expectedTask1RunTime = task1SchedulingTime + TimeUnit.MILLISECONDS.toNanos(100); + final Consumer task1ExceptionHandler = mock(Consumer.class); + scheduler.schedule(task1, expectedTask1RunTime, Long.MIN_VALUE, task1ExceptionHandler); + + verifyEventLoopScheduleCalls( + EventLoopScheduleCall.of(task1SchedulingTime, expectedTask1RunTime) + ); + + Thread.sleep(100 + SCHEDULING_TOLERANCE_MILLIS); + + verify(task1, times(1)).run(); + final ArgumentCaptor exceptionCaptor = ArgumentCaptor.forClass(Throwable.class); + verify(task1ExceptionHandler, times(1)).accept(exceptionCaptor.capture()); + assertThat(exceptionCaptor.getValue()).isInstanceOf(AnticipatedException.class); } @Test @@ -72,25 +225,26 @@ void testEarlierRetryTaskOvertakesLaterOne() throws Exception { final long task1SchedulingTime = System.nanoTime(); final long expectedTask1RunTime = task1SchedulingTime + TimeUnit.MILLISECONDS.toNanos(200); final Consumer task1ExceptionHandler = mock(Consumer.class); - scheduler.schedule(task1, expectedTask1RunTime, task1ExceptionHandler); + scheduler.schedule(task1, expectedTask1RunTime, Long.MIN_VALUE, task1ExceptionHandler); final Runnable task2 = mock(Runnable.class); final long task2SchedulingTime = System.nanoTime(); final long expectedTask2RunTime = task2SchedulingTime + TimeUnit.MILLISECONDS.toNanos(100); final Consumer task2ExceptionHandler = mock(Consumer.class); - scheduler.schedule(task2, expectedTask2RunTime, task2ExceptionHandler); + scheduler.schedule(task2, expectedTask2RunTime, Long.MIN_VALUE, task2ExceptionHandler); Thread.sleep(200 + SCHEDULING_TOLERANCE_MILLIS); verify(task1, times(0)).run(); verify(task2, times(1)).run(); - verifyNoMoreInteractions(task1ExceptionHandler); + + verifyExceptionHandlerCatchedSchedulingException(task1ExceptionHandler, + RetrySchedulingException.Type.RETRY_TASK_OVERTAKEN); + verifyNoMoreInteractions(task2ExceptionHandler); - verifyEventLoopSchedules( - ImmutableList.of( - EventLoopScheduleCall.of(task1SchedulingTime, expectedTask1RunTime), - EventLoopScheduleCall.of(task2SchedulingTime, expectedTask2RunTime) - ) + verifyEventLoopScheduleCalls( + EventLoopScheduleCall.of(task1SchedulingTime, expectedTask1RunTime), + EventLoopScheduleCall.of(task2SchedulingTime, expectedTask2RunTime) ); } @@ -118,7 +272,7 @@ void testMultipleRetryTasksBeingOvertaken() throws Exception { final long expectedRunTime = schedulingTime + TimeUnit.MILLISECONDS.toNanos(1000 - taskNo * 100); expectedRunTimes.add(expectedRunTime); - scheduler.schedule(task, expectedRunTime, exceptionHandler); + scheduler.schedule(task, expectedRunTime, Long.MIN_VALUE, exceptionHandler); } // Wait for the tasks to be scheduled @@ -127,14 +281,15 @@ void testMultipleRetryTasksBeingOvertaken() throws Exception { for (int taskNo = 0; taskNo < 9; taskNo++) { final Runnable task = tasks.get(taskNo); verify(task, times(0)).run(); - verifyNoMoreInteractions(exceptionHandlers.get(taskNo)); + verifyExceptionHandlerCatchedSchedulingException(exceptionHandlers.get(taskNo), + RetrySchedulingException.Type.RETRY_TASK_OVERTAKEN); } // Verify that the last task was executed verify(tasks.get(9), times(1)).run(); verifyNoMoreInteractions(exceptionHandlers.get(9)); - verifyEventLoopSchedules( + verifyEventLoopScheduleCalls( ImmutableList.copyOf( tasks.stream() .map(task -> EventLoopScheduleCall.of(schedulingTimes.get(tasks.indexOf(task)), @@ -150,14 +305,14 @@ void testLaterRetryTaskDoesNotOvertakeEarlierOne() throws Exception { final long task1SchedulingTime = System.nanoTime(); final long expectedTask1RunTime = task1SchedulingTime + TimeUnit.MILLISECONDS.toNanos(200); final Consumer task1ExceptionHandler = mock(Consumer.class); - scheduler.schedule(task1, expectedTask1RunTime, task1ExceptionHandler); + scheduler.schedule(task1, expectedTask1RunTime, Long.MIN_VALUE, task1ExceptionHandler); // Schedule a new task with a later time than the current one final Runnable task2 = mock(Runnable.class); final long task2SchedulingTime = System.nanoTime(); final long expectedTask2RunTime = task2SchedulingTime + TimeUnit.MILLISECONDS.toNanos(300); final Consumer task2ExceptionHandler = mock(Consumer.class); - scheduler.schedule(task2, expectedTask2RunTime, task2ExceptionHandler); + scheduler.schedule(task2, expectedTask2RunTime, Long.MIN_VALUE, task2ExceptionHandler); Thread.sleep(300 + SCHEDULING_TOLERANCE_MILLIS); @@ -167,19 +322,16 @@ void testLaterRetryTaskDoesNotOvertakeEarlierOne() throws Exception { // Verify that the second task was not executed verify(task2, times(0)).run(); - verify(task2ExceptionHandler, times(1)).accept(any(IllegalStateException.class)); + verifyExceptionHandlerCatchedSchedulingException( + task2ExceptionHandler, RetrySchedulingException.Type.RETRY_TASK_OVERTAKEN); - verifyEventLoopSchedules( - ImmutableList.of( - EventLoopScheduleCall.of(task1SchedulingTime, expectedTask1RunTime) - // EventLoopScheduleCall.of(task2SchedulingTime, expectedTask2RunTime) - ) + verifyEventLoopScheduleCalls( + EventLoopScheduleCall.of(task1SchedulingTime, expectedTask1RunTime) ); } @Test void testRescheduleTaskWhenEarliestNextRetryTimeUpdated() throws Exception { - // Set the earliest next retry time to 200ms from now final long now = System.nanoTime(); final long earliestTime = now + TimeUnit.MILLISECONDS.toNanos(200); @@ -191,569 +343,223 @@ void testRescheduleTaskWhenEarliestNextRetryTimeUpdated() throws Exception { final Runnable task1 = mock(Runnable.class); final Consumer exceptionHandler = mock(Consumer.class); - scheduler.schedule(task1, taskTime, exceptionHandler); + scheduler.schedule(task1, taskTime, Long.MIN_VALUE, exceptionHandler); // Move the earliest next retry time to 100ms from now final long earliestTimeUpdateTime = System.nanoTime(); final long newEarliestTimeNanos = now + TimeUnit.MILLISECONDS.toNanos(400); scheduler.addEarliestNextRetryTimeNanos(newEarliestTimeNanos); - scheduler.rescheduleCurrentRetryTaskIfTooEarly(); Thread.sleep(400 + SCHEDULING_TOLERANCE_MILLIS); verify(task1, times(1)).run(); - verifyEventLoopSchedules( - ImmutableList.of( - EventLoopScheduleCall.of(task1SchedulingTime, taskTime), - EventLoopScheduleCall.of(earliestTimeUpdateTime, newEarliestTimeNanos) - ) + verifyEventLoopScheduleCalls( + EventLoopScheduleCall.of(task1SchedulingTime, taskTime), + EventLoopScheduleCall.of(earliestTimeUpdateTime, newEarliestTimeNanos) ); verifyNoMoreInteractions(exceptionHandler); } - /** - * Test plan 5: Verify that when a task is scheduled and then the scheduler is closed, - * the task is cancelled and not executed. - */ - @Test - void testCloseSchedulerCancelsTask() throws Exception { - final AtomicBoolean taskExecuted = new AtomicBoolean(); - final Consumer exceptionHandler = mock(Consumer.class); - - // Schedule a task to run after 200ms - final long now = System.nanoTime(); - final long taskTime = now + TimeUnit.MILLISECONDS.toNanos(200); - - scheduler.schedule(() -> { - taskExecuted.set(true); - }, taskTime, exceptionHandler); - - // Close the scheduler immediately - scheduler.close(); - - // Wait a bit to ensure the task would have run if not cancelled - // Use Awaitility to wait for a reasonable time - await().pollDelay(300, TimeUnit.MILLISECONDS) - .atMost(400, TimeUnit.MILLISECONDS) - .until(() -> true); // Just wait - - // Verify that the task was not executed - assertThat(taskExecuted.get()).isFalse(); - - // Verify that the exception handler was not called - // The scheduler mutes cancellation notifications when closed - verifyNoMoreInteractions(exceptionHandler); - } - - /** - * Test with a large number of tasks (100) overtaking each other to verify that - * the scheduler can handle a large number of tasks and that only the earliest one is executed. - */ @Test - void testManyOvertakingTasks() throws Exception { - - final int numTasks = 100; - final AtomicInteger executedTaskIndex = new AtomicInteger(-1); - final AtomicLong executionTime = new AtomicLong(); + void testRescheduleWithNoTasks() throws InterruptedException { + scheduler.rescheduleCurrentRetryTaskIfTooEarly(); + verifyEventLoopScheduleCalls(); - // Create an array of mock exception handlers - @SuppressWarnings("unchecked") - final Consumer[] exceptionHandlers = new Consumer[numTasks]; - for (int i = 0; i < numTasks; i++) { - exceptionHandlers[i] = mock(Consumer.class); - } + scheduler.addEarliestNextRetryTimeNanos(System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(-100)); + scheduler.addEarliestNextRetryTimeNanos(System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(0)); + scheduler.addEarliestNextRetryTimeNanos(System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(100)); - // Schedule tasks with random times, but make sure the last one is the earliest - final long now = System.nanoTime(); - final long baseTime = now + TimeUnit.MILLISECONDS.toNanos(200); - - // Schedule tasks in reverse order (except the last one) to ensure they're all scheduled - // before the earliest one executes - for (int i = numTasks - 1; i >= 0; i--) { - final int taskIndex = i; - final long taskTime; - - if (i == numTasks - 1) { - // Make the last task the earliest - taskTime = baseTime; - } else { - // Random time between baseTime + 50ms and baseTime + 500ms - taskTime = baseTime + TimeUnit.MILLISECONDS.toNanos(50 + (i * 5L)); - } - - scheduler.schedule(() -> { - executedTaskIndex.set(taskIndex); - executionTime.set(System.nanoTime()); - }, taskTime, exceptionHandlers[i]); - } + verifyEventLoopScheduleCalls(); - // Use Awaitility to wait for a task to execute - await().atMost(1, TimeUnit.SECONDS) - .until(() -> executedTaskIndex.get() >= 0); + scheduler.rescheduleCurrentRetryTaskIfTooEarly(); - // Verify that only the last task (index numTasks-1) was executed - assertThat(executedTaskIndex.get()).isEqualTo(numTasks - 1); + verifyEventLoopScheduleCalls(); - // Verify that the task was executed at the expected time (not too early) - final long taskDelay = executionTime.get() - now; - assertThat(taskDelay).isGreaterThanOrEqualTo( - TimeUnit.MILLISECONDS.toNanos(190)); // Allow some timing flexibility + final long taskScheduledTime = System.nanoTime(); + final long expectedTaskTime = taskScheduledTime + TimeUnit.MILLISECONDS.toNanos(100); - // In this test, we can't make specific assertions about which exception handlers were called - // because the behavior depends on the exact order of task scheduling, which can vary. - // We only verify that at least one task was executed. - } + scheduler.schedule( + () -> { + // note that we inherit the earliest next retry time from the calls above + }, taskScheduledTime + TimeUnit.MILLISECONDS.toNanos(50), Long.MIN_VALUE, t -> { + // do nothing + }); - /** - * Test that scheduling a task before the earliestNextRetryTimeNanos fails - * and calls the exception handler with the expected exception. - */ - @Test - void testScheduleBeforeEarliestNextRetryTime() throws Exception { + Thread.sleep(100 + SCHEDULING_TOLERANCE_MILLIS); - // Set the earliest next retry time - final long now = System.nanoTime(); - final long earliestTime = now + TimeUnit.MILLISECONDS.toNanos(200); - scheduler.addEarliestNextRetryTimeNanos(earliestTime); - - // Try to schedule a task before the earliest next retry time - final long taskTime = now + TimeUnit.MILLISECONDS.toNanos(100); - final AtomicBoolean taskExecuted = new AtomicBoolean(); - final Consumer exceptionHandler = mock(Consumer.class); + verifyEventLoopScheduleCalls( + EventLoopScheduleCall.of(taskScheduledTime, expectedTaskTime) + ); - scheduler.schedule(() -> { - taskExecuted.set(true); - }, taskTime, exceptionHandler); + scheduler.addEarliestNextRetryTimeNanos(System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(1)); + scheduler.addEarliestNextRetryTimeNanos(System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(100)); - // Use Awaitility to wait a bit to ensure the exception handler is called - await().pollDelay(100, TimeUnit.MILLISECONDS) - .atMost(200, TimeUnit.MILLISECONDS) - .until(() -> true); // Just wait + verifyEventLoopScheduleCalls( + EventLoopScheduleCall.of(taskScheduledTime, expectedTaskTime) + ); - // Verify that the task was not executed - assertThat(taskExecuted.get()).isFalse(); + scheduler.rescheduleCurrentRetryTaskIfTooEarly(); - // Verify that the exception handler was called with the expected exception - verify(exceptionHandler, times(1)).accept(any(IllegalStateException.class)); + verifyEventLoopScheduleCalls( + EventLoopScheduleCall.of(taskScheduledTime, expectedTaskTime) + ); } - /** - * Test that scheduling a task after the latestNextRetryTimeNanos fails - * and calls the exception handler with the expected exception. - */ @Test - void testScheduleAfterLatestNextRetryTime() throws Exception { - // Create a scheduler with a limited latest next retry time - final long now = System.nanoTime(); - final long latestTime = now + TimeUnit.MILLISECONDS.toNanos(200); - final RetryScheduler scheduler = new RetryScheduler(eventLoop, latestTime); - - // Try to schedule a task after the latest next retry time - final long taskTime = now + TimeUnit.MILLISECONDS.toNanos(300); - final AtomicBoolean taskExecuted = new AtomicBoolean(); + void testIdempotentReschedule() throws Exception { + // Schedule a task to run after 200ms + final Runnable task = mock(Runnable.class); + final long taskScheduledTime = System.nanoTime(); + final long taskTime = taskScheduledTime + TimeUnit.MILLISECONDS.toNanos(100); final Consumer exceptionHandler = mock(Consumer.class); - scheduler.schedule(() -> { - taskExecuted.set(true); - }, taskTime, exceptionHandler); - - // Use Awaitility to wait a bit to ensure the exception handler is called - await().pollDelay(100, TimeUnit.MILLISECONDS) - .atMost(200, TimeUnit.MILLISECONDS) - .until(() -> true); // Just wait - - // Verify that the task was not executed - assertThat(taskExecuted.get()).isFalse(); - - // Verify that the exception handler was called with the expected exception - verify(exceptionHandler, times(1)).accept(any(IllegalStateException.class)); - } - - /** - * Test that tasks can be scheduled concurrently from multiple threads - * and the scheduler correctly handles the concurrency. - */ - @Test - void testConcurrentScheduling() throws Exception { - - final int numThreads = 10; - final int tasksPerThread = 10; - final AtomicInteger executedTasks = new AtomicInteger(); - final AtomicLong earliestExecutionTime = new AtomicLong(Long.MAX_VALUE); - - // Create a latch to synchronize the start of all threads - final CountDownLatch startLatch = new CountDownLatch(1); - // Create a latch to wait for all threads to finish scheduling - final CountDownLatch schedulingDoneLatch = new CountDownLatch(numThreads); - - // Create a list to store all exception handlers - final List> exceptionHandlers = new ArrayList<>(); - - // Create and start threads - final ExecutorService executorService = Executors.newFixedThreadPool(numThreads); - final long now = System.nanoTime(); - - for (int i = 0; i < numThreads; i++) { - final int threadIndex = i; - executorService.submit(() -> { - try { - // Wait for the start signal - startLatch.await(); - - // Each thread schedules multiple tasks - for (int j = 0; j < tasksPerThread; j++) { - final int taskIndex = threadIndex * tasksPerThread + j; - final Consumer exceptionHandler = mock(Consumer.class); - - synchronized (exceptionHandlers) { - exceptionHandlers.add(exceptionHandler); - } - - // Calculate a task time - make them all different - // The earliest task will be the one with the smallest time - final long taskTime = now + TimeUnit.MILLISECONDS.toNanos(200 + taskIndex * 5); - - scheduler.schedule(() -> { - executedTasks.incrementAndGet(); - earliestExecutionTime.updateAndGet( - current -> Math.min(current, System.nanoTime())); - }, taskTime, exceptionHandler); - } - } catch (Exception e) { - fail("Exception in test thread: " + e.getMessage()); - } finally { - schedulingDoneLatch.countDown(); - } - }); - } + scheduler.schedule(task, taskTime, Long.MIN_VALUE, exceptionHandler); - // Start all threads simultaneously - startLatch.countDown(); - - // Wait for all threads to finish scheduling - schedulingDoneLatch.await(); - - // Shutdown the executor service - executorService.shutdown(); - executorService.awaitTermination(1, TimeUnit.SECONDS); - - // Use Awaitility to wait for a task to execute - await().atMost(1, TimeUnit.SECONDS) - .until(() -> executedTasks.get() > 0); - - // Verify that exactly one task was executed - assertThat(executedTasks.get()).isEqualTo(1); - - // Verify that the task was executed at the expected time (not too early) - final long taskDelay = earliestExecutionTime.get() - now; - assertThat(taskDelay).isGreaterThanOrEqualTo( - TimeUnit.MILLISECONDS.toNanos(190)); // Allow some timing flexibility - - // Verify that at least some exception handlers were called - // (since most tasks will be rejected due to earlier tasks being scheduled) - int exceptionHandlerCallCount = 0; - for (Consumer handler : exceptionHandlers) { - try { - verify(handler, times(0)).accept(any(Throwable.class)); - } catch (AssertionError e) { - exceptionHandlerCallCount++; - } - } - - // We expect most tasks to be rejected, but we can't know exactly how many - // due to the concurrent nature of the test - assertThat(exceptionHandlerCallCount).isGreaterThan(0); - } - - /** - * Test that retry tasks that raise exceptions call the exception handler - * with exactly the thrown exception. - */ - @Test - void testRetryTaskRaisesException() throws Exception { - - // Use a CountDownLatch to wait for the exception handler to be called - final CountDownLatch exceptionLatch = new CountDownLatch(1); - final AtomicReference caughtException = new AtomicReference<>(); - - final AnticipatedException expectedException = new AnticipatedException("Test exception"); + verifyEventLoopScheduleCalls( + EventLoopScheduleCall.of(taskScheduledTime, taskTime) + ); - // Schedule a task that throws an exception - final long taskTime = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(100); + final long earliestTime1 = taskScheduledTime + TimeUnit.MILLISECONDS.toNanos(200); - scheduler.schedule(() -> { - throw expectedException; - }, taskTime, ex -> { - caughtException.set(ex); - exceptionLatch.countDown(); - }); + scheduler.addEarliestNextRetryTimeNanos(earliestTime1 - 100); + scheduler.addEarliestNextRetryTimeNanos(earliestTime1); - // Wait for the exception handler to be called - assertThat(exceptionLatch.await(500, TimeUnit.MILLISECONDS)).isTrue(); + verifyEventLoopScheduleCalls( + EventLoopScheduleCall.of(taskScheduledTime, taskTime) + ); - // Verify that the exception handler was called with exactly the thrown exception - assertThat(caughtException.get()).isSameAs(expectedException); - } + final long rescheduleTime1 = System.nanoTime(); + scheduler.rescheduleCurrentRetryTaskIfTooEarly(); - /** - * Test that an exception during the eventLoop.schedule call is handled properly. - */ - @Test - void testExceptionDuringEventLoopSchedule() throws Exception { - // Create a mock EventLoop that throws an exception when schedule is called - final EventLoop mockEventLoop = mock(EventLoop.class); - final RuntimeException expectedException = new RuntimeException("Schedule exception"); - when(mockEventLoop.schedule(any(Runnable.class), any(Long.class), any(TimeUnit.class))) - .thenThrow(expectedException); - - final RetryScheduler scheduler = new RetryScheduler(mockEventLoop); - final Consumer exceptionHandler = mock(Consumer.class); + verifyEventLoopScheduleCalls( + EventLoopScheduleCall.of(taskScheduledTime, taskTime), + EventLoopScheduleCall.of(rescheduleTime1, earliestTime1) + ); - // Schedule a task (which should fail) - final long taskTime = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(100); + scheduler.rescheduleCurrentRetryTaskIfTooEarly(); + scheduler.rescheduleCurrentRetryTaskIfTooEarly(); - scheduler.schedule(() -> { - fail("This task should not be executed"); - }, taskTime, exceptionHandler); + verifyEventLoopScheduleCalls( + EventLoopScheduleCall.of(taskScheduledTime, taskTime), + EventLoopScheduleCall.of(rescheduleTime1, earliestTime1) + ); - // Verify that the exception handler was called with the expected exception - verify(exceptionHandler, times(1)).accept(expectedException); - } + final long earliestTime2 = taskScheduledTime + TimeUnit.MILLISECONDS.toNanos(300); - /** - * Test for hasAlreadyRetryScheduledBefore method with various scenarios. - */ - @Test - void testHasAlreadyRetryScheduledBefore() throws Exception { + scheduler.addEarliestNextRetryTimeNanos(earliestTime2 - 100); + scheduler.addEarliestNextRetryTimeNanos(earliestTime2 - 200); + scheduler.addEarliestNextRetryTimeNanos(earliestTime2); - // Case 1: No current retry task - assertThat(scheduler.hasAlreadyRetryScheduledBefore(100, 0)).isFalse(); + verifyEventLoopScheduleCalls( + EventLoopScheduleCall.of(taskScheduledTime, taskTime), + EventLoopScheduleCall.of(rescheduleTime1, earliestTime1) + ); - // Schedule a task - final long now = System.nanoTime(); - final long taskTime = now + TimeUnit.MILLISECONDS.toNanos(200); + Thread.sleep(20); - final CountDownLatch taskLatch = new CountDownLatch(1); - scheduler.schedule(taskLatch::countDown, taskTime, ex -> { - }); + final long rescheduleTime2 = System.nanoTime(); + scheduler.rescheduleCurrentRetryTaskIfTooEarly(); - // Case 2: Current retry task exists but is scheduled after the next retry time - assertThat(scheduler.hasAlreadyRetryScheduledBefore(taskTime + 100, 0)).isTrue(); + verifyEventLoopScheduleCalls( + EventLoopScheduleCall.of(taskScheduledTime, taskTime), + EventLoopScheduleCall.of(rescheduleTime1, earliestTime1), + EventLoopScheduleCall.of(rescheduleTime2, earliestTime2) + ); - // Case 3: Current retry task exists but is scheduled before the next retry time - assertThat(scheduler.hasAlreadyRetryScheduledBefore(taskTime - 100, 0)).isFalse(); + scheduler.rescheduleCurrentRetryTaskIfTooEarly(); + scheduler.rescheduleCurrentRetryTaskIfTooEarly(); - // Case 4: Current retry task exists and is scheduled at exactly the next retry time - assertThat(scheduler.hasAlreadyRetryScheduledBefore(taskTime, 0)).isTrue(); + verifyEventLoopScheduleCalls( + EventLoopScheduleCall.of(taskScheduledTime, taskTime), + EventLoopScheduleCall.of(rescheduleTime1, earliestTime1), + EventLoopScheduleCall.of(rescheduleTime2, earliestTime2) + ); - // Case 5: With a non-zero earliestNextRetryTimeNanos - final long earliestTime = taskTime + 50; - assertThat(scheduler.hasAlreadyRetryScheduledBefore(earliestTime + 100, earliestTime)).isTrue(); - assertThat(scheduler.hasAlreadyRetryScheduledBefore(earliestTime - 100, earliestTime)).isFalse(); + Thread.sleep(300 + SCHEDULING_TOLERANCE_MILLIS); - // Wait for the task to complete to avoid interference with other tests - taskLatch.await(500, TimeUnit.MILLISECONDS); - } + verify(task, times(1)).run(); + verifyNoMoreInteractions(exceptionHandler); - /** - * Test for negative checkState conditions. - */ - @Test - void testNegativeCheckStateConditions() throws Exception { - // Create a scheduler with a limited latest next retry time - final long latestTime = 1000; - final RetryScheduler scheduler = new RetryScheduler(eventLoop, latestTime); - - // Test that nextEarliestNextRetryTimeNanos > latestNextRetryTimeNanos throws IllegalStateException - assertThatThrownBy(() -> scheduler.hasAlreadyRetryScheduledBefore(0, latestTime + 1)) - .isInstanceOf(IllegalStateException.class); - - // Test that earliestNextRetryTimeNanos > latestNextRetryTimeNanos throws IllegalStateException - assertThatThrownBy(() -> scheduler.addEarliestNextRetryTimeNanos(latestTime + 1)) - .isInstanceOf(IllegalStateException.class); + verifyEventLoopScheduleCalls( + EventLoopScheduleCall.of(taskScheduledTime, taskTime), + EventLoopScheduleCall.of(rescheduleTime1, earliestTime1), + EventLoopScheduleCall.of(rescheduleTime2, earliestTime2) + ); } - /** - * Test rescheduleCurrentRetryTaskIfTooEarly with no retry task. - */ @Test - void testRescheduleCurrentRetryTaskIfTooEarlyWithNoTask() throws Exception { - - // Set the earliest next retry time - final long earliestTime = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(200); - scheduler.addEarliestNextRetryTimeNanos(earliestTime); - - // Call rescheduleCurrentRetryTaskIfTooEarly with no current retry task - // This should not throw an exception - scheduler.rescheduleCurrentRetryTaskIfTooEarly(); - - // Verify that there is still no scheduled retry task - assertThat(scheduler.hasScheduledRetryTask()).isFalse(); + void testSchedulerShutdownWithoutTask() throws Exception { + // Shutdown the scheduler without scheduling any tasks + assertThat(scheduler.shutdown()).isTrue(); } - /** - * Test rescheduleCurrentRetryTaskIfTooEarly with a retry task that doesn't need to be rescheduled. - */ @Test - void testRescheduleCurrentRetryTaskIfTooEarlyWithTaskNotNeedingReschedule() throws Exception { - - // Set the earliest next retry time - final long now = System.nanoTime(); - final long earliestTime = now + TimeUnit.MILLISECONDS.toNanos(100); - scheduler.addEarliestNextRetryTimeNanos(earliestTime); - - // Schedule a task after the earliest next retry time - final long taskTime = now + TimeUnit.MILLISECONDS.toNanos(200); - final AtomicInteger taskExecutions = new AtomicInteger(); - final AtomicLong taskExecutionTime = new AtomicLong(); + void testSchedulerShutdownCancelsTask() throws Exception { + // Schedule a task that should run after 200ms + final Runnable task = mock(Runnable.class); + final long taskSchedulingTime = System.nanoTime(); + final long expectedTaskRunTime = taskSchedulingTime + TimeUnit.MILLISECONDS.toNanos(200); final Consumer exceptionHandler = mock(Consumer.class); + scheduler.schedule(task, expectedTaskRunTime, Long.MIN_VALUE, exceptionHandler); - scheduler.schedule(() -> { - taskExecutionTime.set(System.nanoTime()); - taskExecutions.incrementAndGet(); - }, taskTime, exceptionHandler); - - // Call rescheduleCurrentRetryTaskIfTooEarly - // The task should not be rescheduled because it's already scheduled after the earliest time - scheduler.rescheduleCurrentRetryTaskIfTooEarly(); + // Shutdown the scheduler + assertThat(scheduler.shutdown()).isTrue(); - // Wait for the task to execute - await() - .until(() -> taskExecutions.get() == 1); + Thread.sleep(SCHEDULING_TOLERANCE_MILLIS); - // Verify that the task was executed once - assertThat(taskExecutions.get()).isEqualTo(1); + verifyEventLoopScheduleCalls( + EventLoopScheduleCall.of(taskSchedulingTime, expectedTaskRunTime) + ); - // Verify that the task was executed at the expected time (not too early) - final long taskDelay = taskExecutionTime.get() - now; - assertThat(taskDelay).isGreaterThanOrEqualTo( - TimeUnit.MILLISECONDS.toNanos(190)); // Allow some timing flexibility + Thread.sleep(400); - // Verify that the exception handler was not called + verify(task, times(0)).run(); verifyNoMoreInteractions(exceptionHandler); } - /** - * Test that changing the earliest retry time without rescheduling allows - * a task to execute before the new earliest time. - */ @Test - void testChangeEarliestRetryTimeWithoutRescheduling() throws Exception { - - final AtomicInteger taskExecutions = new AtomicInteger(); - final AtomicLong taskExecutionTime = new AtomicLong(); + void testEventLoopShutdownCancelsTask() throws Exception { + // Schedule a task that should run after 200ms + final Runnable task = mock(Runnable.class); + final long taskSchedulingTime = System.nanoTime(); + final long expectedTaskRunTime = taskSchedulingTime + TimeUnit.MILLISECONDS.toNanos(200); final Consumer exceptionHandler = mock(Consumer.class); + scheduler.schedule(task, expectedTaskRunTime, Long.MIN_VALUE, exceptionHandler); - // Schedule a task to run after 100ms - final long now = System.nanoTime(); - final long taskTime = now + TimeUnit.MILLISECONDS.toNanos(100); - - scheduler.schedule(() -> { - taskExecutionTime.set(System.nanoTime()); - taskExecutions.incrementAndGet(); - }, taskTime, exceptionHandler); - - // Update the earliest next retry time to be after the scheduled task - final long newEarliestTime = now + TimeUnit.MILLISECONDS.toNanos(200); - scheduler.addEarliestNextRetryTimeNanos(newEarliestTime); - - // Do NOT call rescheduleCurrentRetryTaskIfTooEarly() + // Shutdown the event loop + eventLoop.shutdownGracefully().sync(); - // Use Awaitility to wait for the task to execute - await() - .until(() -> taskExecutions.get() == 1); + Thread.sleep(SCHEDULING_TOLERANCE_MILLIS); - // Verify that the task was executed once - assertThat(taskExecutions.get()).isEqualTo(1); + verifyEventLoopScheduleCalls( + EventLoopScheduleCall.of(taskSchedulingTime, expectedTaskRunTime) + ); - // Verify that the task was executed at the original time (not delayed to the new earliest time) - final long taskDelay = taskExecutionTime.get() - now; - assertThat(taskDelay).isGreaterThanOrEqualTo( - TimeUnit.MILLISECONDS.toNanos(90)); // Allow some timing flexibility - assertThat(taskDelay).isLessThan( - TimeUnit.MILLISECONDS.toNanos(190)); // Should execute before new earliest time + Thread.sleep(400); - // Verify that the exception handler was not called - verifyNoMoreInteractions(exceptionHandler); + verify(task, times(0)).run(); + verifyExceptionHandlerCatchedSchedulingException( + exceptionHandler, Type.RETRY_TASK_CANCELLED); } - /** - * Test for task cancellation by user (not by scheduler). - * Since we can't directly access the RetryTaskHandle, we'll test the behavior indirectly. - */ @Test - void testTaskCancellationByUser() throws Exception { - // Use a CountDownLatch to wait for the exception handler to be called - final CountDownLatch exceptionLatch = new CountDownLatch(1); - final AtomicReference caughtException = new AtomicReference<>(); - - // Create a scheduler with a real EventLoop - - final AtomicBoolean taskExecuted = new AtomicBoolean(); - - // Schedule a task with a long delay - final long taskTime = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(10000); // 10 seconds - - // Use reflection to access the private currentRetryTask field - final Field currentRetryTaskField; - try { - currentRetryTaskField = RetryScheduler.class.getDeclaredField("currentRetryTask"); - currentRetryTaskField.setAccessible(true); - } catch (NoSuchFieldException e) { - throw new RuntimeException("Could not access currentRetryTask field", e); - } - - // Schedule the task - scheduler.schedule(() -> { - taskExecuted.set(true); - }, taskTime, ex -> { - caughtException.set(ex); - exceptionLatch.countDown(); - }); - - // Get the currentRetryTask field - final Object currentRetryTask; - try { - currentRetryTask = currentRetryTaskField.get(scheduler); - assertThat(currentRetryTask).isNotNull(); - } catch (IllegalAccessException e) { - throw new RuntimeException("Could not access currentRetryTask", e); - } - - // Use reflection to access the scheduledFuture field - final Field scheduledFutureField; - try { - // Get the RetryTaskHandle class (inner class of RetryScheduler) - final Class retryTaskHandleClass = Class.forName( - "com.linecorp.armeria.client.retry.RetryScheduler$RetryTaskHandle"); - scheduledFutureField = retryTaskHandleClass.getDeclaredField("scheduledFuture"); - scheduledFutureField.setAccessible(true); - } catch (NoSuchFieldException | ClassNotFoundException e) { - throw new RuntimeException("Could not access scheduledFuture field", e); - } - - // Get the scheduledFuture - final ScheduledFuture scheduledFuture; - try { - scheduledFuture = (ScheduledFuture) scheduledFutureField.get(currentRetryTask); - assertThat((Object) scheduledFuture).isNotNull(); - } catch (IllegalAccessException e) { - throw new RuntimeException("Could not access scheduledFuture", e); - } - - // Cancel the future directly - scheduledFuture.cancel(false); + void testScheduledOnShutdownEventLoop() throws InterruptedException { + eventLoop.shutdownGracefully().sync(); - // Wait for the exception handler to be called - assertThat(exceptionLatch.await(500, TimeUnit.MILLISECONDS)).isTrue(); + final Runnable task = mock(Runnable.class); + final long taskSchedulingTime = System.nanoTime(); + final long expectedTaskRunTime = taskSchedulingTime + TimeUnit.MILLISECONDS.toNanos(200); + final Consumer exceptionHandler = mock(Consumer.class); + scheduler.schedule(task, expectedTaskRunTime, Long.MIN_VALUE, exceptionHandler); - // Verify that the exception handler was called with the expected exception - assertThat(caughtException.get()).isInstanceOf(IllegalStateException.class); - assertThat(caughtException.get().getMessage()).contains("cancelled by the user"); + final ArgumentCaptor exceptionCaptor = ArgumentCaptor.forClass(Throwable.class); + verify(exceptionHandler, times(1)).accept(exceptionCaptor.capture()); + final Throwable capturedException = exceptionCaptor.getValue(); - // Verify that the task was not executed - assertThat(taskExecuted.get()).isFalse(); + verify(task, times(0)).run(); + assertThat(capturedException).isInstanceOf(RejectedExecutionException.class); + assertThat(capturedException.getMessage()).contains("event executor terminated"); } private static class EventLoopScheduleCall { @@ -782,9 +588,26 @@ public long delayNanos() { } } - private void verifyEventLoopSchedules(List expectedSchedules) { + private static void verifyExceptionHandlerCatchedSchedulingException( + Consumer exceptionHandler, RetrySchedulingException.Type expectedType) { + final ArgumentCaptor exceptionCaptor = ArgumentCaptor.forClass(Throwable.class); + verify(exceptionHandler, times(1)).accept(exceptionCaptor.capture()); + + final Throwable capturedException = exceptionCaptor.getValue(); + assertThat(capturedException).isInstanceOf(RetrySchedulingException.class); + assertThat(((RetrySchedulingException) capturedException).getType()).isEqualTo(expectedType); + } + + private void verifyEventLoopScheduleCalls(EventLoopScheduleCall... expectedSchedules) { + verifyEventLoopScheduleCalls(ImmutableList.copyOf(expectedSchedules)); + } + + private void verifyEventLoopScheduleCalls(List expectedSchedules) { final int expectedNumCalls = expectedSchedules.size(); + final ArgumentCaptor scheduleDelayArgumentCaptor = ArgumentCaptor.forClass(Long.class); + final ArgumentCaptor scheduleTimeUnitArgumentCaptor = ArgumentCaptor.forClass(TimeUnit.class); + verify(eventLoop, times(expectedNumCalls)).schedule(any(Runnable.class), scheduleDelayArgumentCaptor.capture(), scheduleTimeUnitArgumentCaptor.capture()); diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java index 61e266f1468..4c4e7094f0f 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java @@ -42,6 +42,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.linecorp.armeria.client.ClientFactory; import com.linecorp.armeria.client.ClientRequestContext; @@ -53,6 +55,7 @@ import com.linecorp.armeria.client.WebClientBuilder; import com.linecorp.armeria.client.endpoint.EndpointGroup; import com.linecorp.armeria.client.endpoint.EndpointSelectionStrategy; +import com.linecorp.armeria.client.logging.LoggingClient; import com.linecorp.armeria.common.AggregatedHttpResponse; import com.linecorp.armeria.common.ExchangeType; import com.linecorp.armeria.common.HttpRequest; @@ -75,13 +78,15 @@ // todo(szymon): change tests that we wait for the response and demand that immediately after we see all the // logs in the appropriate state. class RetryingClientWithHedgingTest { - private static final long LOOSING_SERVER_RESPONSE_DELAY_MILLIS = 300; + private static final long LOOSING_SERVER_RESPONSE_DELAY_MILLIS = 1000; private static final String SERVER1_RESPONSE = "s1"; private static final String SERVER2_RESPONSE = "s2#"; private static final String SERVER3_RESPONSE = "s3##"; private static class TestServer extends ServerExtension { + private static final Logger logger = LoggerFactory.getLogger(TestServer.class); + private CountDownLatch responseLatch = new CountDownLatch(1); private final AtomicInteger numRequests = new AtomicInteger(); private HttpService helloService = mock(HttpService.class); @@ -93,7 +98,7 @@ private static class TestServer extends ServerExtension { @Override protected void configure(ServerBuilder sb) throws Exception { sb.decorator(LoggingService.newDecorator()); - sb.blockingTaskExecutor(1); + sb.blockingTaskExecutor(5); sb.service("/hello", new HttpService() { @Override @@ -108,12 +113,15 @@ public HttpResponse serve(ServiceRequestContext ctx, HttpRequest req) ctx.blockingTaskExecutor().execute(() -> { numRequests.incrementAndGet(); try { + logger.debug("Got a request. Waiting for the latch to be released."); responseLatch.await(); } catch (InterruptedException e) { responseFuture.completeExceptionally(e); fail(e); } + logger.debug("Latch released. Returning response."); + try { responseFuture.complete(helloService.serve(ctx, req)); } catch (Exception e) { @@ -510,6 +518,7 @@ private static WebClientBuilder clientBuilder() { server1.httpEndpoint(), server2.httpEndpoint(), server3.httpEndpoint())) + .decorator(LoggingClient.newDecorator()) .factory(clientFactory); } diff --git a/junit5/src/main/java/com/linecorp/armeria/testing/server/ServiceRequestContextCaptor.java b/junit5/src/main/java/com/linecorp/armeria/testing/server/ServiceRequestContextCaptor.java index 7cc44b45ce5..a3f600e55ee 100644 --- a/junit5/src/main/java/com/linecorp/armeria/testing/server/ServiceRequestContextCaptor.java +++ b/junit5/src/main/java/com/linecorp/armeria/testing/server/ServiceRequestContextCaptor.java @@ -151,6 +151,10 @@ public ServiceRequestContext take() throws InterruptedException { return serviceContexts.take(); } + /** + * Retrieves, but does not remove, the first captured {@link ServiceRequestContext}, or returns + * {@code null} if there are no captured contexts. + */ @Nullable public ServiceRequestContext peek() { return serviceContexts.peek(); From d7de984b420ba702549cbc37d8646d27e35d876e Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Sat, 14 Jun 2025 16:09:26 +0200 Subject: [PATCH 10/36] [WIP] style: fix checkstyle errors --- .../RetryingRpcClientWithHedgingTest.java | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java index 291240f33b2..82554084bce 100644 --- a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java +++ b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java @@ -182,8 +182,8 @@ void execute_hedging_lastWins() throws Exception { when(server3.getServiceHandler().hello(anyString())).thenReturn("server3"); final HelloService.AsyncIface client = helloClientThreeEndpoints( - RetryConfig. - builderForRpc( + RetryConfig + .builderForRpc( RetryRule .builder() .onTimeoutException() @@ -234,8 +234,8 @@ void execute_hedging_thirdWinsEventAfterPerAttemptTimeout() throws Exception { when(server3.getServiceHandler().hello(anyString())).thenReturn("server3"); final HelloService.AsyncIface client = helloClientThreeEndpoints( - RetryConfig. - builderForRpc( + RetryConfig + .builderForRpc( RetryRule .builder() .onTimeoutException() @@ -292,8 +292,8 @@ void execute_hedging_thirdWinsEvenWhenFirstErrors() throws Exception { when(server3.getServiceHandler().hello(anyString())).thenReturn("server3"); final HelloService.AsyncIface client = helloClientThreeEndpoints( - RetryConfig. - builderForRpc( + RetryConfig + .builderForRpc( RetryRule .builder() .onTimeoutException() @@ -343,8 +343,8 @@ void execute_hedging_returnErrorWhenSecondErrors() throws Exception { when(server3.getServiceHandler().hello(anyString())).thenReturn("server3"); final HelloService.AsyncIface client = helloClientThreeEndpoints( - RetryConfig. - builderForRpc( + RetryConfig + .builderForRpc( RetryRule .builder() .onTimeoutException() @@ -408,8 +408,8 @@ void execute_hedging_honorResponseTimeout() throws TException { when(server2.getServiceHandler().hello(anyString())).thenReturn("server2"); final HelloService.AsyncIface client = helloClientThreeEndpoints( - RetryConfig. - builderForRpc( + RetryConfig + .builderForRpc( RetryRule .builder() .onTimeoutException() @@ -437,9 +437,7 @@ void execute_hedging_honorResponseTimeout() throws TException { assertThatExceptionOfType(ExecutionException.class) .isThrownBy(result::get) - .satisfies(cause -> - { - + .satisfies(cause -> { assertThat(cause.getCause()).isInstanceOf(TTransportException.class); assertThat(cause.getCause().getCause()).isInstanceOf( ResponseTimeoutException.class); @@ -607,5 +605,4 @@ private void assertValidServerRequestContext(ServerExtension server, int attempt private void assertNoServerRequestContext(ServerExtension server) { assertThat(server.requestContextCaptor().size()).isEqualTo(0); } - } From 7862f2842ce26ff70bd61a02c6af869e989af0cf Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Sat, 14 Jun 2025 16:28:20 +0200 Subject: [PATCH 11/36] [WIP] style: fix checkstyle errors --- .../client/retry/AbstractRetryingClient.java | 29 ++++++++++++++----- .../armeria/client/retry/RetryConfig.java | 3 ++ .../client/retry/RetryConfigBuilder.java | 6 ++++ .../armeria/client/retry/RetryScheduler.java | 4 --- .../armeria/client/retry/RetryingClient.java | 8 ++--- .../common/logging/RequestLogBuilder.java | 4 +++ .../client/retry/RetrySchedulerTest.java | 10 ++++--- .../client/retry/RetryingClientTest.java | 4 +-- .../retry/RetryingClientWithHedgingTest.java | 3 +- 9 files changed, 48 insertions(+), 23 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java index 40ccf268bc0..b4ca8c9bc15 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java @@ -124,6 +124,9 @@ protected static void completeRetryingIfNoPendingAttempts(ClientRequestContext c state(ctx).completeIfNoPendingAttempts(); } + /** + * todo(szymon): [doc]. + */ protected static void completeRetryingExceptionally(ClientRequestContext ctx, Throwable cause) { logger.debug("onRetryingCompleteExceptionally: {}", ctx, cause); @@ -164,6 +167,9 @@ protected final RetryRuleWithContent retryRuleWithContent() { return retryRuleWithContent; } + /** + * todo(szymon): [doc]. + */ protected void startRetryAttempt(ClientRequestContext ctx, ClientRequestContext attemptCtx, BiConsumer onAcceptHandler, @@ -198,6 +204,9 @@ protected void startRetryAttempt(ClientRequestContext ctx, logger.debug("onAttemptStarted: {}", ctx); } + /** + * todo(szymon): [doc]. + */ protected static void completeRetryAttempt(ClientRequestContext ctx, ClientRequestContext attemptCtx, O attemptRes, boolean isWinning) { @@ -210,11 +219,17 @@ protected static void completeRetryAttempt(ClientRequestCon } } + /** + * todo(szymon): [doc]. + */ protected static boolean isRetryingComplete(ClientRequestContext ctx) { requireNonNull(ctx, "ctx"); return state(ctx).whenRetryingComplete().isDone(); } + /** + * todo(szymon): [doc]. + */ protected void scheduleNextRetry(ClientRequestContext ctx, Runnable retryTask, Backoff backoff, @@ -222,6 +237,9 @@ protected void scheduleNextRetry(ClientRequestContext ctx, scheduleNextRetry(ctx, retryTask, backoff, -1, actionOnException); } + /** + * todo(szymon): [doc]. + */ protected static void scheduleNextRetry(ClientRequestContext ctx, Runnable retryTask, Backoff backoff, @@ -239,8 +257,8 @@ protected static void scheduleNextRetry(ClientRequestContext ctx, final long earliestRetryTimeNanos = retryDelayFromServerMillis >= 0 ? (nowTimeNanos + TimeUnit.MILLISECONDS.toNanos( - retryDelayFromServerMillis)) : - Long.MIN_VALUE; + retryDelayFromServerMillis)) + : Long.MIN_VALUE; if (state.timeoutForWholeRetryEnabled() && earliestRetryTimeNanos > state.responseTimeoutTimeNanos()) { actionOnException.accept( @@ -431,7 +449,7 @@ RetryScheduler scheduler() { * remaining {@link #responseTimeoutMillisForAttempt}. * * @return 0 if the response timeout for both of each request and whole retry is disabled or - * -1 if the elapsed time from the first request has passed {@code responseTimeoutMillis} + * -1 if the elapsed time from the first request has passed {@code responseTimeoutMillis} */ long responseTimeoutMillisForAttempt() { if (!timeoutForWholeRetryEnabled()) { @@ -555,10 +573,7 @@ void complete() { } if (lastAttempt == null) { - completeExceptionally(new IllegalStateException("completed retrying " - + "without a single " - + "successful" - + "attempt")); + completeExceptionally(new IllegalStateException("Completed retrying without any attempts.")); } else { retryingCompleteFuture.complete(lastAttempt); } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfig.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfig.java index 151b6bd1099..736056731ad 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfig.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfig.java @@ -197,6 +197,9 @@ public long responseTimeoutMillisForEachAttempt() { return responseTimeoutMillisForEachAttempt; } + /** + * todo(szymon): [doc]. + */ public long hedgingDelayMillis() { return hedgingDelayMillis; } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfigBuilder.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfigBuilder.java index 326de6d3ed5..05f9c81d9bf 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfigBuilder.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfigBuilder.java @@ -85,6 +85,9 @@ public RetryConfigBuilder maxTotalAttempts(int maxTotalAttempts) { return this; } + /** + * todo(szymon): [doc]. + */ public RetryConfigBuilder hedgingDelay(Duration hedgingDelay) { final long millis = requireNonNull(hedgingDelay, "hedgingDelay") @@ -97,6 +100,9 @@ public RetryConfigBuilder hedgingDelay(Duration hedgingDelay) { return this; } + /** + * todo(szymon): [doc]. + */ public RetryConfigBuilder hedgingDelayMillis(long hedgingDelayMillis) { checkArgument( hedgingDelayMillis >= 0, diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java index 8c9a4c78f6e..0402384e5c1 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java @@ -230,10 +230,6 @@ private synchronized void scheduleNextRetryTask(Runnable retryRunnable, long ret retryRunnable.run(); }; - logger.debug("Scheduling the retry task. delayNanos = {}, " - + "retryTimeNanos = {}, earliestNextRetryTimeNanos = {}", - delayNanos, retryTimeNanos, earliestNextRetryTimeNanos); - //noinspection unchecked final ScheduledFuture nextRetryTaskFuture = (ScheduledFuture) eventLoop.schedule(wrappedRetryRunnable, delayNanos, diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index 8a544c41504..48c3d8892e8 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -97,7 +97,7 @@ public static RetryingClientBuilder builder(RetryRuleWithContent r * it will hand over the stream to the client. * * @throws IllegalArgumentException if the specified {@code maxContentLength} is equal to or - * less than {@code 0} + * less than {@code 0} */ public static RetryingClientBuilder builder(RetryRuleWithContent retryRuleWithContent, int maxContentLength) { @@ -171,7 +171,7 @@ public static Function newDecorator(RetryRul * @param retryRule the retry rule * @param maxTotalAttempts the maximum number of total attempts * @param responseTimeoutMillisForEachAttempt response timeout for each attempt. {@code 0} disables - * the timeout + * the timeout * * @deprecated Use {@link #newDecorator(RetryConfig)} instead. */ @@ -190,7 +190,7 @@ public static Function newDecorator(RetryRul * @param retryRuleWithContent the retry rule * @param maxTotalAttempts the maximum number of total attempts * @param responseTimeoutMillisForEachAttempt response timeout for each attempt. {@code 0} disables - * the timeout + * the timeout * * @deprecated Use {@link #newDecorator(RetryConfig)} instead. */ @@ -220,7 +220,7 @@ public static Function newDecorator(RetryRul * requests. * * @param mapping the mapping that returns a {@link RetryConfig} for a given {@link ClientRequestContext} - * and {@link Request}. + * and {@link Request}. */ public static Function newDecoratorWithMapping(RetryConfigMapping mapping) { diff --git a/core/src/main/java/com/linecorp/armeria/common/logging/RequestLogBuilder.java b/core/src/main/java/com/linecorp/armeria/common/logging/RequestLogBuilder.java index c450bbca547..7b7a09fc817 100644 --- a/core/src/main/java/com/linecorp/armeria/common/logging/RequestLogBuilder.java +++ b/core/src/main/java/com/linecorp/armeria/common/logging/RequestLogBuilder.java @@ -437,6 +437,10 @@ void session(@Nullable Channel channel, SessionProtocol sessionProtocol, @Nullab */ void addChild(RequestLogAccess child); + /** + * Fills the response-side logs from the specified child. Note that already collected properties + * in the child log will be propagated immediately. + */ void endResponseWithChild(RequestLogAccess child); /** diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java index fb719881e4f..2173998cad7 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java @@ -281,8 +281,10 @@ void testMultipleRetryTasksBeingOvertaken() throws Exception { for (int taskNo = 0; taskNo < 9; taskNo++) { final Runnable task = tasks.get(taskNo); verify(task, times(0)).run(); - verifyExceptionHandlerCatchedSchedulingException(exceptionHandlers.get(taskNo), - RetrySchedulingException.Type.RETRY_TASK_OVERTAKEN); + verifyExceptionHandlerCatchedSchedulingException( + exceptionHandlers.get(taskNo), + RetrySchedulingException.Type.RETRY_TASK_OVERTAKEN + ); } // Verify that the last task was executed @@ -562,7 +564,7 @@ void testScheduledOnShutdownEventLoop() throws InterruptedException { assertThat(capturedException.getMessage()).contains("event executor terminated"); } - private static class EventLoopScheduleCall { + private static final class EventLoopScheduleCall { private final long delayNanos; private EventLoopScheduleCall(long scheduledTimeNanos) { @@ -630,4 +632,4 @@ private void verifyEventLoopScheduleCalls(List expectedSc expected.delayNanos() + SCHEDULING_TOLERANCE_NANOS); } } -} \ No newline at end of file +} diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java index c632ed74de0..e08b0f714d3 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java @@ -663,8 +663,8 @@ void shouldGetExceptionWhenFactoryIsClosed() { } assertThat(t).isInstanceOf(IllegalStateException.class) .satisfies(cause -> assertThat(cause.getMessage()).matches( - "(?i).*(factory has been closed|not accepting a task|factory is closing or " - + "closed).*")); + "(?i).*(factory has been closed|not accepting a task" + + "|factory is closing or closed).*")); } @Test diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java index 4c4e7094f0f..bafb4c8cc00 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java @@ -166,7 +166,7 @@ public void unlatchResponse() { private static final TestServer server3 = new TestServer(); private static ClientFactory clientFactory; - private final RetryRule NO_RETRY_RULE = RetryRule.builder().thenNoRetry(); + private static final RetryRule NO_RETRY_RULE = RetryRule.builder().thenNoRetry(); @BeforeAll static void beforeAll() { @@ -616,7 +616,6 @@ private static void assertValidServerRequestContext(ServerExtension server, int } assertThat(slog.requestHeaders().path()).contains("hello"); - } private static void assertNoServerRequestContext(ServerExtension server) { From 4d97a1bb1b79cbb44bbf3f4347073e18f55eb67d Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Sat, 14 Jun 2025 16:53:39 +0200 Subject: [PATCH 12/36] [WIP] fix: fix exception message for RETRY_TASK_CANCELLED --- .../linecorp/armeria/client/retry/RetrySchedulingException.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetrySchedulingException.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetrySchedulingException.java index 1bfecc2f6ac..5d5ab2953fd 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetrySchedulingException.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetrySchedulingException.java @@ -26,7 +26,7 @@ enum Type { DELAY_FROM_BACKOFF_EXCEEDS_RESPONSE_TIMEOUT("Delay from backoff exceeds response timeout"), DELAY_FROM_SERVER_EXCEEDS_RESPONSE_TIMEOUT("Delay from server exceeds response timeout"), RETRY_TASK_OVERTAKEN("Has earlier retry"), - RETRY_TASK_CANCELLED("Retry task cancelled without outside of rescheduling."); + RETRY_TASK_CANCELLED("Retry task cancelled unexpectedly."); private final String message; From 4e2b5404ade19ed0374ff753b275923d2c4c439f Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Sat, 14 Jun 2025 17:02:22 +0200 Subject: [PATCH 13/36] [WIP] test: add firstServerWins to RetryingClientWithHedgingTest --- .../retry/RetryingClientWithHedgingTest.java | 65 +++++++++++++++---- 1 file changed, 54 insertions(+), 11 deletions(-) diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java index bafb4c8cc00..408bd0ee8df 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java @@ -78,12 +78,6 @@ // todo(szymon): change tests that we wait for the response and demand that immediately after we see all the // logs in the appropriate state. class RetryingClientWithHedgingTest { - private static final long LOOSING_SERVER_RESPONSE_DELAY_MILLIS = 1000; - - private static final String SERVER1_RESPONSE = "s1"; - private static final String SERVER2_RESPONSE = "s2#"; - private static final String SERVER3_RESPONSE = "s3##"; - private static class TestServer extends ServerExtension { private static final Logger logger = LoggerFactory.getLogger(TestServer.class); @@ -158,6 +152,12 @@ public void unlatchResponse() { } } + private static final long LOOSING_SERVER_RESPONSE_DELAY_MILLIS = 1000; + + private static final String SERVER1_RESPONSE = "s1"; + private static final String SERVER2_RESPONSE = "s2#"; + private static final String SERVER3_RESPONSE = "s3##"; + @RegisterExtension private static final TestServer server1 = new TestServer(); @RegisterExtension @@ -195,7 +195,43 @@ void afterEach() { } @Test - void letSecondServerWins() throws Exception { + void firstServerWins() throws Exception { + when(server1.getHelloService().serve(any(), any())).thenReturn(HttpResponse.of(SERVER1_RESPONSE)); + + final RetryConfig hedgingNoRetryConfig = RetryConfig + .builder(NO_RETRY_RULE) + .maxTotalAttempts(3) + .hedgingDelayMillis(100) + .build(); + + final WebClient client = client(hedgingNoRetryConfig); + + final CompletableFuture responseFuture; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + responseFuture = client.get("/hello").aggregate(); + ctx = captor.get(); + } + + server1.unlatchResponse(); + server2.unlatchResponse(); + server3.unlatchResponse(); + + await().untilAsserted(() -> { + assertThat(server1.getNumRequests()).isEqualTo(1); + assertValidServerRequestContext(server1, 1, false); + assertNoServerRequestContext(server2); + assertNoServerRequestContext(server3); + + assertValidAggregatedResponse(responseFuture, SERVER1_RESPONSE); + assertValidClientRequestContext( + ctx, GET_VERIFY_RESPONSE_HAS_CONTENT.apply(SERVER1_RESPONSE), + GET_VERIFY_RESPONSE_HAS_CONTENT.apply(SERVER1_RESPONSE), null, null); + }); + } + + @Test + void secondServerWins() throws Exception { when(server1.getHelloService().serve(any(), any())).thenReturn(HttpResponse.of(SERVER1_RESPONSE)); when(server2.getHelloService().serve(any(), any())).thenReturn(HttpResponse.of(SERVER2_RESPONSE)); when(server3.getHelloService().serve(any(), any())).thenReturn(HttpResponse.of(SERVER3_RESPONSE)); @@ -243,7 +279,7 @@ void letSecondServerWins() throws Exception { } @Test - void letThirdServerWin() throws Exception { + void thirdServerWin() throws Exception { when(server1.getHelloService().serve(any(), any())).thenReturn(HttpResponse.of(SERVER1_RESPONSE)); when(server2.getHelloService().serve(any(), any())).thenReturn(HttpResponse.of(SERVER2_RESPONSE)); when(server3.getHelloService().serve(any(), any())).thenReturn(HttpResponse.of(SERVER3_RESPONSE)); @@ -562,12 +598,19 @@ private static void assertValidRootClientRequestContext(ClientRequestContext ctx private static void assertValidClientRequestContext(ClientRequestContext ctx, RequestLogVerifier logVerifierCtx, RequestLogVerifier logVerifierServer1, - RequestLogVerifier logVerifierServer2, + @Nullable RequestLogVerifier logVerifierServer2, @Nullable RequestLogVerifier logVerifierServer3 ) { - assertValidRootClientRequestContext(ctx, logVerifierCtx, logVerifierServer3 == null ? 2 : 3); + + final int expectedNumChildren = 1 + (logVerifierServer2 == null ? 0 : 1) + + (logVerifierServer3 == null ? 0 : 1); + + assertValidRootClientRequestContext(ctx, logVerifierCtx, expectedNumChildren); assertValidChildLog(ctx.log().children().get(0), 1, logVerifierServer1); - assertValidChildLog(ctx.log().children().get(1), 2, logVerifierServer2); + if (logVerifierServer2 != null) { + assertValidChildLog(ctx.log().children().get(1), 2, logVerifierServer2); + } + if (logVerifierServer3 != null) { assertValidChildLog(ctx.log().children().get(2), 3, logVerifierServer3); } From f0c63b3133ae4573c141abb7b6ef409129f95256 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Sun, 15 Jun 2025 11:45:41 +0200 Subject: [PATCH 14/36] [WIP] fix: RetryingRpcClient scheduling exception handling --- .../client/retry/AbstractRetryingClient.java | 137 +++++++++++++----- .../armeria/client/retry/RetryScheduler.java | 75 ++++++++-- .../retry/RetrySchedulingException.java | 1 + .../armeria/client/retry/RetryingClient.java | 35 ++--- .../client/retry/RetryingRpcClient.java | 10 +- .../retry/RetryingClientWithHedgingTest.java | 5 + 6 files changed, 185 insertions(+), 78 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java index b4ca8c9bc15..fbe0157d07a 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java @@ -32,6 +32,7 @@ import com.linecorp.armeria.client.ClientRequestContext; import com.linecorp.armeria.client.Endpoint; import com.linecorp.armeria.client.SimpleDecoratingClient; +import com.linecorp.armeria.client.retry.RetrySchedulingException.Type; import com.linecorp.armeria.common.HttpHeaderNames; import com.linecorp.armeria.common.HttpRequest; import com.linecorp.armeria.common.Request; @@ -121,7 +122,10 @@ protected abstract O doExecute(ClientRequestContext ctx, I req) * This should be called when retrying is finished. */ protected static void completeRetryingIfNoPendingAttempts(ClientRequestContext ctx) { - state(ctx).completeIfNoPendingAttempts(); + final State state = state(ctx); + synchronized (state) { + state.completeIfNoPendingAttempts(); + } } /** @@ -130,7 +134,10 @@ protected static void completeRetryingIfNoPendingAttempts(ClientRequestContext c protected static void completeRetryingExceptionally(ClientRequestContext ctx, Throwable cause) { logger.debug("onRetryingCompleteExceptionally: {}", ctx, cause); - state(ctx).completeExceptionally(cause); + final State state = state(ctx); + synchronized (state) { + state.completeExceptionally(cause); + } } /** @@ -173,35 +180,43 @@ protected final RetryRuleWithContent retryRuleWithContent() { protected void startRetryAttempt(ClientRequestContext ctx, ClientRequestContext attemptCtx, BiConsumer onAcceptHandler, - BiConsumer onAttemptAbortedHandler + BiConsumer onAttemptAbortedHandler ) { requireNonNull(ctx, "ctx"); requireNonNull(attemptCtx, "attemptCtx"); + requireNonNull(onAcceptHandler, "onAcceptHandler"); requireNonNull(onAttemptAbortedHandler, "onAttemptAbortedHandler"); + final State state = state(ctx); - state.startAttempt(attemptCtx); - state.whenRetryingComplete().handle((winningAttempt, cause) -> { - if (winningAttempt == null) { - // If the retrying is complete exceptionally, we need to call the onAttemptAbortedHandler. - onAttemptAbortedHandler.accept(winningAttempt.ctx(), cause); - return null; + synchronized (state) { + if (isRetryingComplete(ctx)) { + onAttemptAbortedHandler.accept(attemptCtx, + new IllegalStateException("Retrying is already complete.")); + return; } - final ClientRequestContext winningAttemptCtx = winningAttempt.ctx(); - final O winningAttemptRes = winningAttempt.res(); - assert winningAttemptRes != null; + state.startAttempt(attemptCtx); + state.whenRetryingComplete().handle((winningAttempt, cause) -> { + if (winningAttempt == null) { + // If the retrying is complete exceptionally, we need to call the onAttemptAbortedHandler. + onAttemptAbortedHandler.accept(winningAttempt.ctx(), cause); + return null; + } - if (attemptCtx == winningAttemptCtx) { - onAcceptHandler.accept(winningAttemptCtx, winningAttemptRes); - return null; - } + final ClientRequestContext winningAttemptCtx = winningAttempt.ctx(); + final O winningAttemptRes = winningAttempt.res(); + assert winningAttemptRes != null; - onAttemptAbortedHandler.accept(attemptCtx, cause); - return null; - }); + if (attemptCtx == winningAttemptCtx) { + onAcceptHandler.accept(winningAttemptCtx, winningAttemptRes); + return null; + } - logger.debug("onAttemptStarted: {}", ctx); + onAttemptAbortedHandler.accept(attemptCtx, cause); + return null; + }); + } } /** @@ -210,12 +225,18 @@ protected void startRetryAttempt(ClientRequestContext ctx, protected static void completeRetryAttempt(ClientRequestContext ctx, ClientRequestContext attemptCtx, O attemptRes, boolean isWinning) { + if (isRetryingComplete(ctx)) { + // The complete handler of this attempt was already executed (provided that `attemptCtx` + // was registered with `startRetryAttempt` as by the contract of `completeRetryAttempt`. + return; + } - // todo(szymon): needs to be checked atomically - state(ctx).completeAttempt(attemptCtx, attemptRes); + synchronized (state(ctx)) { + state(ctx).completeAttempt(attemptCtx, attemptRes); - if (isWinning) { - state(ctx).complete(); + if (isWinning) { + state(ctx).complete(); + } } } @@ -224,7 +245,12 @@ protected static void completeRetryAttempt(ClientRequestCon */ protected static boolean isRetryingComplete(ClientRequestContext ctx) { requireNonNull(ctx, "ctx"); - return state(ctx).whenRetryingComplete().isDone(); + + final State state = state(ctx); + + synchronized (state) { + return state.whenRetryingComplete().isDone(); + } } /** @@ -250,26 +276,32 @@ protected static void scheduleNextRetry(ClientRequestContext ctx, requireNonNull(backoff, "backoff"); requireNonNull(actionOnException, "actionOnException"); + if (isRetryingComplete(ctx)) { + actionOnException.accept(new RetrySchedulingException(Type.RETRYING_ALREADY_COMPLETED)); + return; + } + final State state = state(ctx); final RetryScheduler scheduler = state.scheduler(); final long nowTimeNanos = System.nanoTime(); - final long earliestRetryTimeNanos = retryDelayFromServerMillis >= 0 ? - (nowTimeNanos + TimeUnit.MILLISECONDS.toNanos( - retryDelayFromServerMillis)) - : Long.MIN_VALUE; + long earliestRetryTimeNanos = retryDelayFromServerMillis >= 0 ? + (nowTimeNanos + TimeUnit.MILLISECONDS.toNanos( + retryDelayFromServerMillis)) + : Long.MIN_VALUE; + if (state.timeoutForWholeRetryEnabled() && earliestRetryTimeNanos > state.responseTimeoutTimeNanos()) { actionOnException.accept( new RetrySchedulingException( - RetrySchedulingException.Type.DELAY_FROM_BACKOFF_EXCEEDS_RESPONSE_TIMEOUT)); + Type.DELAY_FROM_SERVER_EXCEEDS_RESPONSE_TIMEOUT)); return; } // Even when we cannot schedule the retry task, we want to respect the // minimum retry delay from the server. - scheduler.addEarliestNextRetryTimeNanos(earliestRetryTimeNanos); + earliestRetryTimeNanos = scheduler.addEarliestNextRetryTimeNanos(earliestRetryTimeNanos); final int attemptNoWithBackoff = state.nextAttemptNoWithBackoff(backoff); if (attemptNoWithBackoff < 0) { @@ -291,25 +323,37 @@ protected static void scheduleNextRetry(ClientRequestContext ctx, earliestRetryTimeNanos); if (state.timeoutForWholeRetryEnabled() && retryTimeNanos > state.responseTimeoutTimeNanos()) { scheduler.rescheduleCurrentRetryTaskIfTooEarly(); + // This cannot be the minimum delay from the server, because we already checked it above. actionOnException.accept( new RetrySchedulingException( - RetrySchedulingException.Type.DELAY_FROM_SERVER_EXCEEDS_RESPONSE_TIMEOUT)); + Type.DELAY_FROM_BACKOFF_EXCEEDS_RESPONSE_TIMEOUT)); return; } state.startRetryTask(); scheduler.schedule(() -> { - if (isRetryingComplete(ctx)) { - state.completeRetryTask(); - return; + // *, see comment above. + synchronized (state) { + if (isRetryingComplete(ctx)) { + state.completeRetryTask(); + actionOnException.accept(new RetrySchedulingException(Type.RETRYING_ALREADY_COMPLETED)); + return; + } + + state.acquireAttemptNoWithCurrentBackoff(backoff); } - state.acquireAttemptNoWithCurrentBackoff(backoff); retryTask.run(); - state.completeRetryTask(); + + synchronized (state) { + state.completeRetryTask(); + } + completeRetryingIfNoPendingAttempts(ctx); }, retryTimeNanos, earliestRetryTimeNanos, cause -> { - state.completeRetryTask(); + synchronized (state) { + state.completeRetryTask(); + } actionOnException.accept(cause); }); } @@ -323,6 +367,7 @@ protected static void scheduleNextRetry(ClientRequestContext ctx, */ @SuppressWarnings("MethodMayBeStatic") // Intentionally left non-static for better user experience. protected final boolean setResponseTimeout(ClientRequestContext ctx) { + // We do not need to acquire a lock on the state as this method body is thread-safe. requireNonNull(ctx, "ctx"); final long responseTimeoutMillis = state(ctx).responseTimeoutMillisForAttempt(); if (responseTimeoutMillis < 0) { @@ -341,6 +386,7 @@ protected final boolean setResponseTimeout(ClientRequestContext ctx) { * {@link ClientRequestContext}. */ protected static int getTotalAttempts(ClientRequestContext ctx) { + // We do not need to acquire a lock on the state as this method body is thread-safe. final State state = ctx.attr(STATE); if (state == null) { return 0; @@ -446,7 +492,7 @@ RetryScheduler scheduler() { /** * Returns the smaller value between {@link RetryConfig#responseTimeoutMillisForEachAttempt()} and - * remaining {@link #responseTimeoutMillisForAttempt}. + * remaining {@link #responseTimeoutMillisForAttempt}. This method is thread-safe. * * @return 0 if the response timeout for both of each request and whole retry is disabled or * -1 if the elapsed time from the first request has passed {@code responseTimeoutMillis} @@ -474,6 +520,7 @@ boolean timeoutForWholeRetryEnabled() { return isTimeoutEnabled; } + // Is thread-safe. long responseTimeoutMillis() { assert isTimeoutEnabled; return Math.max(TimeUnit.NANOSECONDS.toMillis(deadlineNanos - System.nanoTime()), -1); @@ -555,11 +602,22 @@ int numPendingAttempts() { } void completeIfNoPendingAttempts() { + logger.debug( + "completeIfNoPendingAttempts: {}, num attempts pending = {}, num retry task scheduled = {}", + lastAttempt != null ? lastAttempt.ctx() : "null", numPendingAttempts(), + numScheduledAttempts); if (retryingCompleteFuture.isDone()) { + logger.debug("completeIfNoPendingAttempts: Retrying already completed: {}", + lastAttempt != null ? lastAttempt.ctx() : "null"); return; } if (numPendingAttempts() > 0) { + logger.debug( + "completeIfNoPendingAttempts: Have pending attempts," + + " num attempts pending = {}, num retry task scheduled = {}", + numPendingAttempts(), numScheduledAttempts + ); return; } @@ -575,6 +633,7 @@ void complete() { if (lastAttempt == null) { completeExceptionally(new IllegalStateException("Completed retrying without any attempts.")); } else { + logger.debug("Completing..."); retryingCompleteFuture.complete(lastAttempt); } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java index 0402384e5c1..37286524d9a 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java @@ -84,7 +84,7 @@ boolean isOvertaken() { return state == State.OVERTAKEN; } - Consumer getexceptionHandler() { + Consumer getExceptionHandler() { return exceptionHandler; } @@ -93,6 +93,12 @@ ScheduledFuture getFuture() { } } + // todo(szymon) [Q]: make this configurable? + // Number of nanoseconds that we allow the retry task to be scheduled earlier than + // the earliestNextRetryTimeNanos. + // This should avoid unnecessary rescheduling. + private static final long RESCHEDULING_OVERTAKING_TOLERANCE_NANOS = TimeUnit.MICROSECONDS.toNanos(500); + private static final Logger logger = LoggerFactory.getLogger(RetryScheduler.class); private final EventLoop eventLoop; @@ -177,20 +183,20 @@ private synchronized void handleRetryTaskCompletion(RetryTaskHandle retryTaskHan } if (retryTaskHandle.isOvertaken()) { - retryTaskHandle.getexceptionHandler().accept( + retryTaskHandle.getExceptionHandler().accept( new RetrySchedulingException(Type.RETRY_TASK_OVERTAKEN) ); return; } // The retry task was cancelled by the user, not by the scheduler. - retryTaskHandle.getexceptionHandler().accept( + retryTaskHandle.getExceptionHandler().accept( new RetrySchedulingException(Type.RETRY_TASK_CANCELLED)); return; } if (!retryTaskFuture.isSuccess()) { - retryTaskHandle.getexceptionHandler().accept(retryTaskFuture.cause()); + retryTaskHandle.getExceptionHandler().accept(retryTaskFuture.cause()); } } @@ -223,6 +229,41 @@ private synchronized void scheduleNextRetryTask(Runnable retryRunnable, long ret final Runnable wrappedRetryRunnable = () -> { logger.debug("Retry task starting. Resetting..."); // todo(szymon): do sanity check that we are clearing this task (very bad otherwise). + + synchronized (this) { + if (currentRetryTask == null) { + clearCurrentRetryTask(); + exceptionHandler.accept( + new IllegalStateException( + "Currently executing retry task was cleared." + + " Likely a bug in the retry scheduler." + ) + ); + return; + } + + final long taskRunTimeNanos = System.nanoTime(); + + // max to be robust against overflows. + if (Math.max(taskRunTimeNanos, taskRunTimeNanos + RESCHEDULING_OVERTAKING_TOLERANCE_NANOS) < + earliestNextRetryTimeNanos) { + // We are too early to execute the retry task. + logger.debug("Retry task is too early. Rescheduling..."); + + final Runnable currentRetryRunnable = currentRetryTask.getRetryTaskRunnable(); + final Consumer currentExceptionHandler = + currentRetryTask.getExceptionHandler(); + + clearCurrentRetryTask(); + // do not invoke rescheduleCurrentRetryTaskIfTooEarly here as it will not be able + // to cancel us. + scheduleNextRetryTask(currentRetryRunnable, + earliestNextRetryTimeNanos, + currentExceptionHandler, false); + return; + } + } + clearCurrentRetryTask(); // todo(szymon): Q should we check whether we are too early here? @@ -248,22 +289,30 @@ private synchronized void scheduleNextRetryTask(Runnable retryRunnable, long ret } } - public synchronized void addEarliestNextRetryTimeNanos(long earliestNextRetryTimeNanos) { + public synchronized long addEarliestNextRetryTimeNanos(long earliestNextRetryTimeNanos) { checkState(earliestNextRetryTimeNanos <= latestNextRetryTimeNanos); this.earliestNextRetryTimeNanos = Math.max(this.earliestNextRetryTimeNanos, earliestNextRetryTimeNanos); + + return this.earliestNextRetryTimeNanos; } public synchronized void rescheduleCurrentRetryTaskIfTooEarly() { - if (currentRetryTask != null) { - if (currentRetryTask.retryTimeNanos() < earliestNextRetryTimeNanos) { - // Current retry task is going to be executed before the earliestNextRetryTimeNanos so - // we need to reschedule it. + if (currentRetryTask == null) { + return; + } - scheduleNextRetryTask(currentRetryTask.getRetryTaskRunnable(), - earliestNextRetryTimeNanos, - currentRetryTask.getexceptionHandler(), true); - } + if ( + Math.max(currentRetryTask.retryTimeNanos(), + currentRetryTask.retryTimeNanos() + RESCHEDULING_OVERTAKING_TOLERANCE_NANOS) < + earliestNextRetryTimeNanos + ) { + // Current retry task is going to be executed before the earliestNextRetryTimeNanos so + // we need to reschedule it. + + scheduleNextRetryTask(currentRetryTask.getRetryTaskRunnable(), + earliestNextRetryTimeNanos, + currentRetryTask.getExceptionHandler(), true); } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetrySchedulingException.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetrySchedulingException.java index 5d5ab2953fd..bb020446537 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetrySchedulingException.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetrySchedulingException.java @@ -21,6 +21,7 @@ class RetrySchedulingException extends RuntimeException { private final Type type; enum Type { + RETRYING_ALREADY_COMPLETED("Retrying completed"), NO_MORE_ATTEMPTS_IN_RETRY("No more attempts available in retry"), NO_MORE_ATTEMPTS_IN_BACKOFF("No more attempts available in backoff"), DELAY_FROM_BACKOFF_EXCEEDS_RESPONSE_TIMEOUT("Delay from backoff exceeds response timeout"), diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index 48c3d8892e8..89adb9bbd4b 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -343,16 +343,20 @@ private void doExecute0(RetryingContext retryingContext) { startRetryAttempt(ctx, attemptCtx, (winningAttemptCtx, winningAttemptRes) -> { - logger.debug("accepting win for attempt: {}, winningAttemptCtx: {}, ", - winningAttemptCtx, winningAttemptRes); retryingContext.reqDuplicator().close(); - retryingContext.ctx().logBuilder().endResponseWithChild(winningAttemptCtx.log()); retryingContext.resFuture().complete(winningAttemptRes); }, (abortingAttemptCtx, cause) -> { - abortAttempt(abortingAttemptCtx, - cause); + final RequestLogBuilder logBuilder = abortingAttemptCtx.logBuilder(); + logBuilder.responseContent(null, null); + logBuilder.responseContentPreview(null); + + if (cause != null) { + attemptCtx.cancel(cause); + } else { + attemptCtx.cancel(); + } } ); @@ -396,17 +400,19 @@ private void doExecute0(RetryingContext retryingContext) { if (hedgingDelayMillis >= 0) { final Backoff hedgingDelayBackoff = Backoff.fixed(hedgingDelayMillis); - logger.debug("Scheduling hedging with backoff: {}", hedgingDelayBackoff); scheduleNextRetry(ctx, () -> doExecute0(retryingContext), hedgingDelayBackoff, cause -> handleExceptionAfterScheduling(retryingContext, cause)); } } - private void handleExceptionAfterScheduling( + private static void handleExceptionAfterScheduling( RetryingContext retryingContext, Throwable cause) { + logger.debug("handleExceptionAfterScheduling", cause); if (cause instanceof RetrySchedulingException) { switch (((RetrySchedulingException) cause).getType()) { + case RETRYING_ALREADY_COMPLETED: + return; case NO_MORE_ATTEMPTS_IN_RETRY: case NO_MORE_ATTEMPTS_IN_BACKOFF: case DELAY_FROM_BACKOFF_EXCEEDS_RESPONSE_TIMEOUT: @@ -648,21 +654,6 @@ private void handleRetryDecision(RetryingContext retryingContext, @Nullable Retr completeRetryAttempt(retryingContext.ctx(), attemptCtx, attemptRes, backoff == null); } - private static void abortAttempt(ClientRequestContext attemptCtx, - @Nullable Throwable cause) { - logger.debug("aborting attempt. attemptCtx: {}, cause: {}", attemptCtx, cause); - // Set response content with null to make sure that the log is complete. - final RequestLogBuilder logBuilder = attemptCtx.logBuilder(); - logBuilder.responseContent(null, null); - logBuilder.responseContentPreview(null); - - if (cause != null) { - attemptCtx.cancel(cause); - } else { - attemptCtx.cancel(); - } - } - private static long getRetryAfterMillis(ClientRequestContext ctx) { final RequestLogAccess log = ctx.log(); final String value; diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java index ac4554b81f9..152fb74a2d6 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java @@ -192,8 +192,8 @@ private void doExecute0(ClientRequestContext ctx, RpcRequest req, } startRetryAttempt(ctx, attemptCtx, (winningAttemptCtx, winningAttemptRes) -> { + ctx.logBuilder().endResponseWithChild(winningAttemptCtx.log()); final HttpRequest actualHttpReq = winningAttemptCtx.request(); - if (actualHttpReq != null) { ctx.updateRequest(actualHttpReq); } @@ -215,8 +215,8 @@ private void doExecute0(ClientRequestContext ctx, RpcRequest req, scheduleNextRetry(ctx, () -> doExecute0(ctx, req, returnedRes, returnedResFuture), backoff, - cause0 -> completeRetryingExceptionally(ctx, returnedResFuture, - cause0, false)); + cause0 -> handleExceptionAfterScheduling(ctx, returnedResFuture, + cause0)); } completeRetryAttempt(ctx, attemptCtx, attemptRes, backoff == null); @@ -238,10 +238,12 @@ private void doExecute0(ClientRequestContext ctx, RpcRequest req, } } - private void handleExceptionAfterScheduling( + private static void handleExceptionAfterScheduling( ClientRequestContext ctx, CompletableFuture returnedResFuture, Throwable cause) { if (cause instanceof RetrySchedulingException) { switch (((RetrySchedulingException) cause).getType()) { + case RETRYING_ALREADY_COMPLETED: + return; case NO_MORE_ATTEMPTS_IN_RETRY: case NO_MORE_ATTEMPTS_IN_BACKOFF: case DELAY_FROM_BACKOFF_EXCEEDS_RESPONSE_TIMEOUT: diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java index 408bd0ee8df..bba907e382b 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java @@ -326,6 +326,11 @@ void thirdServerWin() throws Exception { }); } + @Test + void noOneWinsPickLastResponse() { + // todo(szymon): implement + } + @Test void thirdWinsEvenAfterPerAttemptTimeout() throws Exception { when(server1.getHelloService().serve(any(), any())).thenReturn(HttpResponse.of(SERVER1_RESPONSE)); From 072e3e42c6a7e803a6785df7203913b2ff916e74 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Sun, 15 Jun 2025 11:57:04 +0200 Subject: [PATCH 15/36] [WIP] style: remove comment in ClientUtil.newDerivedContext --- .../java/com/linecorp/armeria/internal/client/ClientUtil.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java b/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java index bfd7ed3ec08..7e7b3a95418 100644 --- a/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java +++ b/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java @@ -239,7 +239,6 @@ public static ClientRequestContext newDerivedContext(ClientRequestContext ctx, // we are deriving from. // For that we copy over all log properties from the parent log to the derived log // and add future actions to copy over content (previews). - // We are doing this because final RequestLogAccess parentLog = ctx.log(); final RequestLog partial = parentLog.partial(); final RequestLogBuilder logBuilder = derived.logBuilder(); From 785c3846356c8ce6b5d759349a0088afab730b22 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Sun, 15 Jun 2025 11:57:24 +0200 Subject: [PATCH 16/36] [WIP] style: remove comment in RetryScheduler --- .../java/com/linecorp/armeria/client/retry/RetryScheduler.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java index 37286524d9a..4df318c0b3e 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java @@ -266,8 +266,6 @@ private synchronized void scheduleNextRetryTask(Runnable retryRunnable, long ret clearCurrentRetryTask(); - // todo(szymon): Q should we check whether we are too early here? - retryRunnable.run(); }; From 17b9645add47e25797ef00fed5959a63a88e3dce Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Sun, 15 Jun 2025 15:44:11 +0200 Subject: [PATCH 17/36] [WIP] test: add reminder in RetrySchedulerTest for testing reschedule upon execution --- .../client/retry/RetryingRpcClient.java | 9 +- .../client/retry/RetrySchedulerTest.java | 3 + .../retry/RetryingClientWithHedgingTest.java | 59 +- .../RetryingRpcClientWithHedgingTest.java | 564 +++++++++--------- 4 files changed, 325 insertions(+), 310 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java index 152fb74a2d6..9f7ca51ae05 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java @@ -29,7 +29,6 @@ import com.linecorp.armeria.client.endpoint.EndpointGroup; import com.linecorp.armeria.common.HttpRequest; import com.linecorp.armeria.common.Request; -import com.linecorp.armeria.common.RequestContext; import com.linecorp.armeria.common.RpcRequest; import com.linecorp.armeria.common.RpcResponse; import com.linecorp.armeria.internal.client.ClientPendingThrowableUtil; @@ -199,7 +198,13 @@ private void doExecute0(ClientRequestContext ctx, RpcRequest req, } returnedResFuture.complete(winningAttemptRes); - }, RequestContext::cancel); + }, (abortingAttemptCtx, cause) -> { + if (cause != null) { + abortingAttemptCtx.cancel(cause); + } else { + abortingAttemptCtx.cancel(); + } + }); final RetryConfig retryConfig = mappedRetryConfig(ctx); final RetryRuleWithContent retryRule = diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java index 2173998cad7..3c72575980e 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java @@ -564,6 +564,9 @@ void testScheduledOnShutdownEventLoop() throws InterruptedException { assertThat(capturedException.getMessage()).contains("event executor terminated"); } + // todo(szymon): add test that verifies that when a task is about to run it gets rescheduled when the + // earliest next retry time was set to something later in the meantime. + private static final class EventLoopScheduleCall { private final long delayNanos; diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java index bba907e382b..7a8a306a976 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java @@ -93,6 +93,7 @@ private static class TestServer extends ServerExtension { protected void configure(ServerBuilder sb) throws Exception { sb.decorator(LoggingService.newDecorator()); sb.blockingTaskExecutor(5); + sb.service("/hello", new HttpService() { @Override @@ -201,7 +202,7 @@ void firstServerWins() throws Exception { final RetryConfig hedgingNoRetryConfig = RetryConfig .builder(NO_RETRY_RULE) .maxTotalAttempts(3) - .hedgingDelayMillis(100) + .hedgingDelayMillis(500) .build(); final WebClient client = client(hedgingNoRetryConfig); @@ -214,32 +215,27 @@ void firstServerWins() throws Exception { } server1.unlatchResponse(); - server2.unlatchResponse(); - server3.unlatchResponse(); await().untilAsserted(() -> { - assertThat(server1.getNumRequests()).isEqualTo(1); - assertValidServerRequestContext(server1, 1, false); - assertNoServerRequestContext(server2); - assertNoServerRequestContext(server3); - assertValidAggregatedResponse(responseFuture, SERVER1_RESPONSE); assertValidClientRequestContext( ctx, GET_VERIFY_RESPONSE_HAS_CONTENT.apply(SERVER1_RESPONSE), GET_VERIFY_RESPONSE_HAS_CONTENT.apply(SERVER1_RESPONSE), null, null); + + assertValidServerRequestContext(server1, 1, false); + assertNoServerRequestContext(server2); + assertNoServerRequestContext(server3); }); } @Test void secondServerWins() throws Exception { - when(server1.getHelloService().serve(any(), any())).thenReturn(HttpResponse.of(SERVER1_RESPONSE)); when(server2.getHelloService().serve(any(), any())).thenReturn(HttpResponse.of(SERVER2_RESPONSE)); - when(server3.getHelloService().serve(any(), any())).thenReturn(HttpResponse.of(SERVER3_RESPONSE)); final RetryConfig hedgingNoRetryConfig = RetryConfig .builder(NO_RETRY_RULE) .maxTotalAttempts(3) - .hedgingDelayMillis(100) + .hedgingDelayMillis(500) .build(); final WebClient client = client(hedgingNoRetryConfig); @@ -253,41 +249,36 @@ void secondServerWins() throws Exception { await().untilAsserted(() -> { assertThat(server1.getNumRequests()).isOne(); - assertThat(server2.getNumRequests()).isOne(); assertThat(server3.getNumRequests()).isOne(); }); server2.unlatchResponse(); - Thread.sleep(LOOSING_SERVER_RESPONSE_DELAY_MILLIS); - server1.unlatchResponse(); - server3.unlatchResponse(); await() .untilAsserted(() -> { - assertValidServerRequestContext(server1, 1, true); - assertValidServerRequestContext(server2, 2, false); - assertValidServerRequestContext(server3, 3, true); - assertValidAggregatedResponse(responseFuture, SERVER2_RESPONSE); + assertValidClientRequestContext( ctx, GET_VERIFY_RESPONSE_HAS_CONTENT.apply(SERVER2_RESPONSE), VERIFY_REQUEST_CANCELLED, GET_VERIFY_RESPONSE_HAS_CONTENT.apply(SERVER2_RESPONSE), VERIFY_REQUEST_CANCELLED ); + + assertValidServerRequestContext(server1, 1, true); + assertValidServerRequestContext(server2, 2, false); + assertValidServerRequestContext(server3, 3, true); }); } @Test - void thirdServerWin() throws Exception { - when(server1.getHelloService().serve(any(), any())).thenReturn(HttpResponse.of(SERVER1_RESPONSE)); - when(server2.getHelloService().serve(any(), any())).thenReturn(HttpResponse.of(SERVER2_RESPONSE)); + void thirdServerWins() throws Exception { when(server3.getHelloService().serve(any(), any())).thenReturn(HttpResponse.of(SERVER3_RESPONSE)); final RetryConfig hedgingNoRetryConfig = RetryConfig .builder(NO_RETRY_RULE) .maxTotalAttempts(3) - .hedgingDelayMillis(10) + .hedgingDelayMillis(500) .build(); final WebClient client = client(hedgingNoRetryConfig); @@ -302,20 +293,12 @@ void thirdServerWin() throws Exception { await().untilAsserted(() -> { assertThat(server1.getNumRequests()).isOne(); assertThat(server2.getNumRequests()).isOne(); - assertThat(server3.getNumRequests()).isOne(); }); server3.unlatchResponse(); - Thread.sleep(LOOSING_SERVER_RESPONSE_DELAY_MILLIS); - server1.unlatchResponse(); - server2.unlatchResponse(); await() .untilAsserted(() -> { - assertValidServerRequestContext(server1, 1, true); - assertValidServerRequestContext(server2, 2, true); - assertValidServerRequestContext(server3, 3, false); - assertValidAggregatedResponse(responseFuture, SERVER3_RESPONSE); assertValidClientRequestContext( ctx, GET_VERIFY_RESPONSE_HAS_CONTENT.apply(SERVER3_RESPONSE), @@ -323,16 +306,20 @@ void thirdServerWin() throws Exception { VERIFY_REQUEST_CANCELLED, GET_VERIFY_RESPONSE_HAS_CONTENT.apply(SERVER3_RESPONSE) ); + + assertValidServerRequestContext(server1, 1, true); + assertValidServerRequestContext(server2, 2, true); + assertValidServerRequestContext(server3, 3, false); }); } @Test - void noOneWinsPickLastResponse() { + void allServerLosePickLastResponse() { // todo(szymon): implement } @Test - void thirdWinsEvenAfterPerAttemptTimeout() throws Exception { + void thirdServerWinsEvenAfterPerAttemptTimeout() throws Exception { when(server1.getHelloService().serve(any(), any())).thenReturn(HttpResponse.of(SERVER1_RESPONSE)); when(server2.getHelloService().serve(any(), any())).thenReturn(HttpResponse.of(SERVER2_RESPONSE)); when(server3.getHelloService().serve(any(), any())).thenReturn(HttpResponse.of(SERVER3_RESPONSE)); @@ -454,7 +441,9 @@ void loosesAfterNonRetriableResponse() throws Exception { final RetryConfig config = RetryConfig .builder(NO_RETRY_RULE) .maxTotalAttempts(3) - .hedgingDelayMillis(100) + // Should be long enough so we can complete the second request before we continue issuing + // a third request to the third server. + .hedgingDelayMillis(500) .build(); final WebClient client = client(config); @@ -466,6 +455,8 @@ void loosesAfterNonRetriableResponse() throws Exception { ctx = captor.get(); } + await().untilAsserted(() -> assertThat(server1.getNumRequests()).isEqualTo(1)); + server2.unlatchResponse(); await().untilAsserted(() -> { diff --git a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java index 82554084bce..d00e09aaafd 100644 --- a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java +++ b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java @@ -17,18 +17,20 @@ import static com.linecorp.armeria.client.retry.AbstractRetryingClient.ARMERIA_RETRY_COUNT; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.fail; import static org.awaitility.Awaitility.await; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; @@ -52,6 +54,7 @@ import com.linecorp.armeria.client.retry.Backoff; import com.linecorp.armeria.client.retry.RetryConfig; import com.linecorp.armeria.client.retry.RetryRule; +import com.linecorp.armeria.client.retry.RetryRuleWithContent; import com.linecorp.armeria.client.retry.RetryingRpcClient; import com.linecorp.armeria.client.thrift.ThriftClients; import com.linecorp.armeria.common.RpcRequest; @@ -61,6 +64,9 @@ import com.linecorp.armeria.common.logging.RequestLog; import com.linecorp.armeria.common.logging.RequestLogAccess; import com.linecorp.armeria.common.logging.RequestLogProperty; +import com.linecorp.armeria.common.stream.ClosedStreamException; +import com.linecorp.armeria.common.util.Exceptions; +import com.linecorp.armeria.internal.testing.AnticipatedException; import com.linecorp.armeria.server.ServerBuilder; import com.linecorp.armeria.server.ServiceRequestContext; import com.linecorp.armeria.server.logging.LoggingService; @@ -71,11 +77,10 @@ import testing.thrift.main.HelloService.Iface; class RetryingRpcClientWithHedgingTest { - private static final long LOOSING_SERVER_RESPONSE_DELAY_MILLIS = 50; - private static class TestServer extends ServerExtension { private CountDownLatch responseLatch = new CountDownLatch(1); - private CountDownLatch requestLatch = new CountDownLatch(1); + private final AtomicInteger numRequests = new AtomicInteger(); + private volatile HelloService.Iface serviceHandler; TestServer() { @@ -84,49 +89,64 @@ private static class TestServer extends ServerExtension { @Override protected void configure(ServerBuilder sb) throws Exception { - sb.service("/thrift", THttpService.of((Iface) name -> getServiceHandler().hello(name)) - .decorate( - (delegate, ctx, req) -> { - getRequestLatch().countDown(); - getResponseLatch().await(); - return delegate.serve(ctx, req); - } - ) + sb.blockingTaskExecutor(5); + sb.service("/thrift", THttpService.builder().useBlockingTaskExecutor(true).addService( + (Iface) name -> { + numRequests.incrementAndGet(); + try { + responseLatch.await(); + } catch (InterruptedException e) { + fail(e); + } + + return getServiceHandler().hello(name); + }).build() .decorate(LoggingService.newDecorator()) ); } private void reset() { + requestContextCaptor().clear(); serviceHandler = mock(HelloService.Iface.class); responseLatch = new CountDownLatch(1); - requestLatch = new CountDownLatch(1); + numRequests.set(0); } public HelloService.Iface getServiceHandler() { return serviceHandler; } - public CountDownLatch getResponseLatch() { - return responseLatch; - } - - public CountDownLatch getRequestLatch() { - return requestLatch; + public int getNumRequests() { + return numRequests.get(); } public void unlatchResponse() { responseLatch.countDown(); } - - public void waitForFirstRequest() { - try { - requestLatch.await(); - } catch (InterruptedException e) { - fail(e); - } - } } + private static final long LOOSING_SERVER_RESPONSE_DELAY_MILLIS = 50; + + private static final String RETRIABLE_RESPONSE = "please-retry-thanks"; + private static final String SERVER1_RESPONSE = "s1"; + private static final String SERVER2_RESPONSE = "s2#"; + private static final String SERVER3_RESPONSE = "s3##"; + + private static final RetryRule NO_RETRY_RULE = RetryRule.builder().thenNoRetry(); + private static final RetryRuleWithContent RETRY_RETRIABLE_RESPONSES_RULE = + RetryRuleWithContent + .builder() + .onResponse((ctx, response) -> { + return response.whenComplete().handle( + (responseData, cause) -> { + assertThat(cause).isNull(); + assertThat(responseData).isInstanceOf(String.class); + return responseData.equals(RETRIABLE_RESPONSE); + } + ); + }) + .thenBackoff(Backoff.withoutDelay()); + @RegisterExtension private static final TestServer server1 = new TestServer(); @RegisterExtension @@ -149,322 +169,287 @@ void afterEach() { server3.unlatchResponse(); } - /* - todo(szymon): Tests for hedging. - todo(szymon): Sometimes returnErrorWhenSecondErrors blocks in a second iteration. - Each test: - - are connections closed on the client and server side? - - are decorators before invoked with the right request context? - - are decorators after invoked for every attempt? Are they invoked when we abort pending attempts? - - the order and timing of the request send out (via client request contexts). Do we respect the - per-attempt timeouts? - - does a server receive a cancellation signal when it is lost? - Test cases: - Positive outcome: - - First server wins, before the per attempt timeout - - First server wins, after the per attempt timeout - - Third server wins, before the per attempt timeout - - Third server wins, after the per attempt timeout - - First, second and third requests are issued in a row (Backoff.fixed(0)). - - First, second and third requests are issued with backoff 0, 100, 0ms (non-monotonic). - - First, second and third request each arrive earlier than the timeout. - Negative outcome: - - Request times out even before first server answers - - Request times out shortly before third server, who should win, answers - - First, second and third requests are issued; second request errors out; should abort every - other request - - Interaction with CircuitBreaker? - */ @Test - void execute_hedging_lastWins() throws Exception { - when(server1.getServiceHandler().hello(anyString())).thenReturn("server1"); - when(server2.getServiceHandler().hello(anyString())).thenReturn("server2"); - when(server3.getServiceHandler().hello(anyString())).thenReturn("server3"); + void firstServerWins() throws Exception { + when(server1.getServiceHandler().hello(anyString())).thenReturn(SERVER1_RESPONSE); - final HelloService.AsyncIface client = helloClientThreeEndpoints( + final HelloService.AsyncIface client = client( RetryConfig - .builderForRpc( - RetryRule - .builder() - .onTimeoutException() - .thenBackoff(Backoff.withoutDelay()) - ) + .builderForRpc(NO_RETRY_RULE) .maxTotalAttempts(3) - .responseTimeoutMillisForEachAttempt(50) - .hedgingDelayMillis(20) + .hedgingDelayMillis(500) .build() ); - final CompletableFuture result; + final CompletableFuture responseFuture; final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { - result = asyncHelloWith(client); + responseFuture = asyncHelloWith(client); ctx = captor.get(); } - server1.waitForFirstRequest(); - server2.waitForFirstRequest(); - server3.waitForFirstRequest(); - - // Let server 3 win. - server3.unlatchResponse(); - Thread.sleep(LOOSING_SERVER_RESPONSE_DELAY_MILLIS); + // Let server 1 win. server1.unlatchResponse(); + + await().untilAsserted(() -> { + assertThat(responseFuture.get()).isEqualTo(SERVER1_RESPONSE); + + assertValidServerRequestContext(server1, 1, false); + assertNoServerRequestContext(server2); + assertNoServerRequestContext(server3); + + assertValidClientRequestContext( + ctx, GET_VERIFY_RESPONSE_HAS_CONTENT.apply(SERVER1_RESPONSE), + GET_VERIFY_RESPONSE_HAS_CONTENT.apply(SERVER1_RESPONSE), + null, + null + ); + }); + } + + @Test + void secondServerWins() throws Exception { + when(server2.getServiceHandler().hello(anyString())).thenReturn(SERVER2_RESPONSE); + + final RetryConfig hedgingNoRetryConfig = RetryConfig + .builderForRpc(NO_RETRY_RULE) + .maxTotalAttempts(3) + .hedgingDelayMillis(100) + .build(); + + final HelloService.AsyncIface client = client(hedgingNoRetryConfig); + + final CompletableFuture responseFuture; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + responseFuture = asyncHelloWith(client); + ctx = captor.get(); + } + + await().untilAsserted(() -> { + assertThat(server1.getNumRequests()).isEqualTo(1); + assertThat(server3.getNumRequests()).isEqualTo(1); + }); + server2.unlatchResponse(); await() .untilAsserted(() -> { - assertValidServerRequestContext(server1, 1); - assertValidServerRequestContext(server2, 2); - assertValidServerRequestContext(server3, 3); + assertThat(responseFuture.get()).isEqualTo(SERVER2_RESPONSE); - assertThat(result.get()).isEqualTo("server3"); assertValidClientRequestContext( - ctx, GET_VERIFY_RESPONSE_HAS_CONTENT.apply("server3"), VERIFY_REQUEST_CANCELLED, + ctx, GET_VERIFY_RESPONSE_HAS_CONTENT.apply(SERVER2_RESPONSE), VERIFY_REQUEST_CANCELLED, - GET_VERIFY_RESPONSE_HAS_CONTENT.apply("server3") + GET_VERIFY_RESPONSE_HAS_CONTENT.apply(SERVER2_RESPONSE), + VERIFY_REQUEST_CANCELLED ); + + assertValidServerRequestContext(server1, 1, true); + assertValidServerRequestContext(server2, 2, false); + assertValidServerRequestContext(server3, 3, true); }); } @Test - void execute_hedging_thirdWinsEventAfterPerAttemptTimeout() throws Exception { - when(server1.getServiceHandler().hello(anyString())).thenReturn("server1"); - when(server2.getServiceHandler().hello(anyString())).thenReturn("server2"); - when(server3.getServiceHandler().hello(anyString())).thenReturn("server3"); + void thirdServerWins() throws Exception { + when(server3.getServiceHandler().hello(anyString())).thenReturn(SERVER3_RESPONSE); - final HelloService.AsyncIface client = helloClientThreeEndpoints( + final HelloService.AsyncIface client = client( RetryConfig - .builderForRpc( - RetryRule - .builder() - .onTimeoutException() - .thenBackoff(Backoff.withoutDelay()) - ) + .builderForRpc(NO_RETRY_RULE) .maxTotalAttempts(3) - .responseTimeoutMillisForEachAttempt(50) - .hedgingDelayMillis(20) + .hedgingDelayMillis(10) .build() ); - final CompletableFuture result; + final CompletableFuture responseFuture; final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { - result = asyncHelloWith(client); + responseFuture = asyncHelloWith(client); ctx = captor.get(); } - // After 3 * , we should have called the - // per-attempt timeout handler a third time. It should not cancel - // any request but should just wait for some request to finish (or for the request timeout). - Thread.sleep(3 * 50 + 100); + // Let server 3 win. + server3.unlatchResponse(); - server1.waitForFirstRequest(); - server2.waitForFirstRequest(); - server3.waitForFirstRequest(); + await().untilAsserted(() -> { + assertThat(responseFuture.get()).isEqualTo(SERVER3_RESPONSE); - // Let the third server win. - server3.unlatchResponse(); - Thread.sleep(LOOSING_SERVER_RESPONSE_DELAY_MILLIS); - server1.unlatchResponse(); - server2.unlatchResponse(); + assertValidServerRequestContext(server1, 1, true); + assertValidServerRequestContext(server2, 2, true); + assertValidServerRequestContext(server3, 3, false); - await() - .untilAsserted(() -> { - assertValidServerRequestContext(server1, 1); - assertValidServerRequestContext(server2, 2); - assertValidServerRequestContext(server3, 3); - - assertThat(result.get()).isEqualTo("server3"); - assertValidClientRequestContext(ctx, GET_VERIFY_RESPONSE_HAS_CONTENT.apply("server3"), - VERIFY_REQUEST_CANCELLED, VERIFY_REQUEST_CANCELLED, - GET_VERIFY_RESPONSE_HAS_CONTENT.apply("server3")); - }); + assertValidClientRequestContext( + ctx, GET_VERIFY_RESPONSE_HAS_CONTENT.apply(SERVER3_RESPONSE), + VERIFY_REQUEST_CANCELLED, + VERIFY_REQUEST_CANCELLED, + GET_VERIFY_RESPONSE_HAS_CONTENT.apply(SERVER3_RESPONSE) + ); + }); } @Test - void execute_hedging_thirdWinsEvenWhenFirstErrors() throws Exception { - final String errorMessage = "it's a me! non-retried error!"; - when(server1.getServiceHandler().hello(anyString())).thenThrow( - new TApplicationException(TApplicationException.INTERNAL_ERROR, errorMessage)); + void allServerLosePickLastResponse() { + // todo(szymon): implement + } - when(server2.getServiceHandler().hello(anyString())).thenReturn("server2"); - when(server3.getServiceHandler().hello(anyString())).thenReturn("server3"); + @Test + void thirdServerWinsEvenAfterPerAttemptTimeout() throws Exception { + // todo(szymon): implement + } - final HelloService.AsyncIface client = helloClientThreeEndpoints( - RetryConfig - .builderForRpc( - RetryRule - .builder() - .onTimeoutException() - .thenBackoff(Backoff.withoutDelay()) - ) - .maxTotalAttempts(3) - .responseTimeoutMillisForEachAttempt(1) - .hedgingDelayMillis(20) - .build() - ); + @Test + void thirdServerWinsEvenAfterRetriableResponse() throws Exception { + when(server1.getServiceHandler().hello(anyString())).thenReturn(RETRIABLE_RESPONSE); + when(server2.getServiceHandler().hello(anyString())).thenReturn(RETRIABLE_RESPONSE); + when(server3.getServiceHandler().hello(anyString())).thenReturn(SERVER3_RESPONSE); - final CompletableFuture result; + final RetryConfig config = RetryConfig.builderForRpc(RETRY_RETRIABLE_RESPONSES_RULE) + .maxTotalAttempts(3) + .hedgingDelayMillis(10) + .build(); + + final HelloService.AsyncIface client = client(config); + + final CompletableFuture responseFuture; final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { - result = asyncHelloWith(client); + responseFuture = asyncHelloWith(client); ctx = captor.get(); } - server1.waitForFirstRequest(); - server2.waitForFirstRequest(); - server3.waitForFirstRequest(); - - server3.unlatchResponse(); - Thread.sleep(LOOSING_SERVER_RESPONSE_DELAY_MILLIS); server1.unlatchResponse(); server2.unlatchResponse(); + server3.unlatchResponse(); await().untilAsserted(() -> { - assertValidServerRequestContext(server1, 1); - assertValidServerRequestContext(server2, 2); - assertValidServerRequestContext(server3, 3); + assertThat(responseFuture.get()).isEqualTo(SERVER3_RESPONSE); + + assertValidServerRequestContext(server1, 1, false); + assertValidServerRequestContext(server2, 2, false); + assertValidServerRequestContext(server3, 3, false); - assertThat(result.get()).isEqualTo("server3"); assertValidClientRequestContext( - ctx, GET_VERIFY_RESPONSE_HAS_CONTENT.apply("server3"), VERIFY_REQUEST_CANCELLED, - VERIFY_REQUEST_CANCELLED, GET_VERIFY_RESPONSE_HAS_CONTENT.apply("server3") + ctx, GET_VERIFY_RESPONSE_HAS_CONTENT.apply(SERVER3_RESPONSE), + GET_VERIFY_RESPONSE_HAS_CONTENT.apply(RETRIABLE_RESPONSE), + GET_VERIFY_RESPONSE_HAS_CONTENT.apply(RETRIABLE_RESPONSE), + GET_VERIFY_RESPONSE_HAS_CONTENT.apply(SERVER3_RESPONSE) ); }); } @Test - void execute_hedging_returnErrorWhenSecondErrors() throws Exception { - when(server1.getServiceHandler().hello(anyString())).thenReturn("server1"); - final String errorMessage = "it's a me! non-retried error!"; - when(server2.getServiceHandler().hello(anyString())).thenThrow( - new TApplicationException(TApplicationException.INTERNAL_ERROR, errorMessage)); - when(server3.getServiceHandler().hello(anyString())).thenReturn("server3"); - - final HelloService.AsyncIface client = helloClientThreeEndpoints( - RetryConfig - .builderForRpc( - RetryRule - .builder() - .onTimeoutException() - .thenBackoff(Backoff.withoutDelay()) - ) - .maxTotalAttempts(3) - .responseTimeoutMillisForEachAttempt(1) - .hedgingDelayMillis(20) - .build() - ); + void loosesAfterNonRetriableResponse() throws TException { + when(server2.getServiceHandler().hello(anyString())).thenThrow(new AnticipatedException("Aachen")); + + final RetryConfig config = RetryConfig + .builderForRpc(NO_RETRY_RULE) + .maxTotalAttempts(3) + // Should be long enough so we can complete the second request before we continue issuing + // a third request to the third server. + .hedgingDelayMillis(500) + .build(); + + final HelloService.AsyncIface client = client(config); - final CompletableFuture result; + final CompletableFuture responseFuture; final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { - result = asyncHelloWith(client); + responseFuture = asyncHelloWith(client); ctx = captor.get(); } - server1.waitForFirstRequest(); - server2.waitForFirstRequest(); - server3.waitForFirstRequest(); + await().untilAsserted(() -> assertThat(server1.getNumRequests()).isEqualTo(1)); - // Let the second server win. server2.unlatchResponse(); - Thread.sleep(LOOSING_SERVER_RESPONSE_DELAY_MILLIS); - server1.unlatchResponse(); - server3.unlatchResponse(); await().untilAsserted(() -> { - assertValidServerRequestContext(server1, 1); - assertValidServerRequestContext(server2, 2); - assertValidServerRequestContext(server3, 3); - - assertThat(result) - .isCompletedExceptionally(); - - assertThatExceptionOfType(ExecutionException.class) - .isThrownBy(result::get) - .satisfies(e -> - assertThat(e.getCause()) - .isInstanceOf(TApplicationException.class) - .hasMessageContaining(errorMessage) - .satisfies(cause -> assertThat( - ((TApplicationException) cause).getType()) - .isEqualTo(TApplicationException.INTERNAL_ERROR))); + assertThatThrownBy(responseFuture::get) + .satisfies(throwable -> { + final Throwable peeledThrowable = Exceptions.peel(throwable); + assertThat(peeledThrowable).isInstanceOf(TApplicationException.class); + }); assertValidClientRequestContext( ctx, GET_VERIFY_RESPONSE_HAS_APPLICATION_EXCEPTION.apply(TApplicationException.INTERNAL_ERROR, - errorMessage), + "Aachen"), VERIFY_REQUEST_CANCELLED, GET_VERIFY_RESPONSE_HAS_APPLICATION_EXCEPTION.apply(TApplicationException.INTERNAL_ERROR, - errorMessage), - VERIFY_REQUEST_CANCELLED); + "Aachen"), + null + ); + + assertValidServerRequestContext(server1, 1, true); + assertValidServerRequestContext(server2, 2, false, AnticipatedException.class); + assertNoServerRequestContext(server3); }); } @Test - void execute_hedging_honorResponseTimeout() throws TException { - when(server1.getServiceHandler().hello(anyString())).thenReturn("server1"); - when(server2.getServiceHandler().hello(anyString())).thenReturn("server2"); + void loosesAfterResponseTimeout() { + final RetryConfig config = RetryConfig + .builderForRpc(NO_RETRY_RULE) + .maxTotalAttempts(3) + .hedgingDelayMillis(0) + .build(); - final HelloService.AsyncIface client = helloClientThreeEndpoints( - RetryConfig - .builderForRpc( - RetryRule - .builder() - .onTimeoutException() - .thenBackoff(Backoff.withoutDelay()) - ) - .maxTotalAttempts(3) - .responseTimeoutMillisForEachAttempt(300) - .hedgingDelayMillis(20) - .build(), 300 + 100 // Lets give the client 100ms to schedule the second attempt. - ); + final HelloService.AsyncIface client = client(config, 500); - final CompletableFuture result; + final CompletableFuture responseFuture; final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { - result = asyncHelloWith(client); + responseFuture = asyncHelloWith(client); ctx = captor.get(); } - // The first request is issued "immediately". - // The second request issued after the per-attempt request timeout which is 300ms. - // The third request would be issued after the per-attempt request timeout which is 300ms. - // However, the request timeout is set to 400ms, so we should never call this. - await().untilAsserted(() -> { - assertThat(result) - .isCompletedExceptionally(); - - assertThatExceptionOfType(ExecutionException.class) - .isThrownBy(result::get) - .satisfies(cause -> { - assertThat(cause.getCause()).isInstanceOf(TTransportException.class); - assertThat(cause.getCause().getCause()).isInstanceOf( - ResponseTimeoutException.class); - } - ); - // The first request timed out and the second was cancelled when it was detected. - assertValidClientRequestContext( - ctx, VERIFY_RESPONSE_TIMEOUT, VERIFY_RESPONSE_TIMEOUT, VERIFY_REQUEST_CANCELLED, null); + await().atLeast(400, TimeUnit.MILLISECONDS).atMost(600, TimeUnit.MILLISECONDS).untilAsserted(() -> { + assertThat(responseFuture).isCompletedExceptionally(); + assertThatThrownBy(responseFuture::get).satisfies(throwable -> { + final Throwable peeledThrowable = Exceptions.peel(throwable); + assertThat(peeledThrowable).isInstanceOf(TTransportException.class); + assertThat(peeledThrowable.getCause()).isInstanceOf(ResponseTimeoutException.class); + }); }); - server1.unlatchResponse(); - server2.unlatchResponse(); - server3.unlatchResponse(); + final List childLogExceptions = new ArrayList<>(); + + final RequestLogVerifier catchException = log -> { + assertThat(log.responseCause()).isInstanceOf(TTransportException.class); + final TTransportException cause = (TTransportException) log.responseCause(); + assertThat(cause.getCause()).isNotNull(); + childLogExceptions.add(cause.getCause()); + }; + + assertValidClientRequestContext(ctx, + VERIFY_RESPONSE_TIMEOUT, + catchException, + catchException, + catchException); + + int numTimeouts = 0; + int numCancelled = 0; + + for (final @Nullable Throwable childException : childLogExceptions) { + if (childException instanceof ResponseTimeoutException) { + numTimeouts++; + } else if (childException instanceof ResponseCancellationException) { + numCancelled++; + } else { + fail("Unexpected exception: " + childException); + } + } - await().untilAsserted(() -> { - assertValidServerRequestContext(server1, 1); - assertValidServerRequestContext(server2, 2); - assertNoServerRequestContext(server3); - }); + assertThat(numTimeouts + numCancelled).isEqualTo(3); + // At least one attempt needs to time out. + assertThat(numTimeouts).isPositive(); - await().pollDelay(1000, TimeUnit.MILLISECONDS).untilAsserted(() -> { - assertNoServerRequestContext(server3); - }); + assertValidServerRequestContext(server1, 1, true); + assertValidServerRequestContext(server2, 2, true); + assertValidServerRequestContext(server3, 3, true); } - private CompletableFuture asyncHelloWith(HelloService.AsyncIface client) throws TException { + private CompletableFuture asyncHelloWith(HelloService.AsyncIface client) { final CompletableFuture future = new CompletableFuture<>(); try { client.hello("hello", new AsyncMethodCallback() { @@ -485,19 +470,19 @@ public void onError(Exception exception) { return future; } - private static HelloService.AsyncIface helloClientThreeEndpoints(RetryConfig config) { - return helloClientThreeEndpoints(config, 10000); + private static HelloService.AsyncIface client(RetryConfig config) { + return client(config, 10000); } - private static HelloService.AsyncIface helloClientThreeEndpoints(RetryConfig config, - long responseTimeoutMillis) { + private static HelloService.AsyncIface client(RetryConfig config, + long responseTimeoutMillis) { return ThriftClients.builder( - SessionProtocol.HTTP, + SessionProtocol.H2C, EndpointGroup.of( EndpointSelectionStrategy.roundRobin(), - server1.endpoint(SessionProtocol.HTTP), - server2.endpoint(SessionProtocol.HTTP), - server3.endpoint(SessionProtocol.HTTP) + server1.endpoint(SessionProtocol.H2C), + server2.endpoint(SessionProtocol.H2C), + server3.endpoint(SessionProtocol.H2C) ) ) .responseTimeoutMillis(responseTimeoutMillis) @@ -536,21 +521,33 @@ private interface RequestLogVerifier extends Consumer {} assertThat(cause.getMessage()).contains(expectedMessage); }; - private void assertValidClientRequestContext(ClientRequestContext ctx, - RequestLogVerifier logVerifierCtx, - RequestLogVerifier logVerifierServer1, - RequestLogVerifier logVerifierServer2, - @Nullable RequestLogVerifier logVerifierServer3 - ) { + private static void assertValidRootClientRequestContext(ClientRequestContext ctx, + RequestLogVerifier logVerifierCtx, + int expectedNumChildren) { assertThat(ctx.log().isComplete()).isTrue(); - assertThat(ctx.log().children()).hasSize(logVerifierServer3 == null ? 2 : 3); + assertThat(ctx.log().children()).hasSize(expectedNumChildren); final RequestLog log = ctx.log().getIfAvailable(RequestLogProperty.RESPONSE_CONTENT, RequestLogProperty.RESPONSE_CAUSE, RequestLogProperty.REQUEST_HEADERS); assertThat(log).isNotNull(); logVerifierCtx.accept(log); + } + + private void assertValidClientRequestContext(ClientRequestContext ctx, + RequestLogVerifier logVerifierCtx, + RequestLogVerifier logVerifierServer1, + @Nullable RequestLogVerifier logVerifierServer2, + @Nullable RequestLogVerifier logVerifierServer3 + ) { + final int expectedNumChildren = 1 + (logVerifierServer2 == null ? 0 : 1) + + (logVerifierServer3 == null ? 0 : 1); + + assertValidRootClientRequestContext(ctx, logVerifierCtx, expectedNumChildren); assertValidChildLog(ctx.log().children().get(0), 1, logVerifierServer1); - assertValidChildLog(ctx.log().children().get(1), 2, logVerifierServer2); + if (logVerifierServer2 != null) { + assertValidChildLog(ctx.log().children().get(1), 2, logVerifierServer2); + } + if (logVerifierServer3 != null) { assertValidChildLog(ctx.log().children().get(2), 3, logVerifierServer3); } @@ -574,19 +571,28 @@ void assertValidChildLog(RequestLogAccess logAccess, int attemptNumber, requestLogVerifier.accept(log); } - private void assertValidServerRequestContext(ServerExtension server, int attemptNumber) { + private void assertValidServerRequestContext(ServerExtension server, int attemptNumber, + boolean expectCancelled) { + assertValidServerRequestContext(server, attemptNumber, expectCancelled, null); + } + + private void assertValidServerRequestContext(ServerExtension server, int attemptNumber, + boolean expectCancelled, + @Nullable Class expectedResponseException) { assertThat(server.requestContextCaptor().size()).isEqualTo(1); final ServiceRequestContext sctx; + sctx = server.requestContextCaptor().peek(); + assertThat(sctx).isNotNull(); + assertThat(sctx.log().isComplete()).isTrue(); - try { - sctx = server.requestContextCaptor().take(); - } catch (InterruptedException e) { - fail(e); - return; - } + assertThat(sctx.isCancelled()).isEqualTo(expectCancelled); - assertThat(sctx.log().isComplete()).isTrue(); + if (expectCancelled) { + assertThat(sctx.cancellationCause()).isInstanceOf(ClosedStreamException.class); + assertThat(sctx.cancellationCause().getMessage()) + .contains("received a RST_STREAM frame: CANCEL"); + } final RequestLog slog = sctx.log().getIfAvailable(RequestLogProperty.REQUEST_HEADERS, RequestLogProperty.REQUEST_CONTENT); @@ -600,6 +606,16 @@ private void assertValidServerRequestContext(ServerExtension server, int attempt assertThat(slog.requestContent()).isInstanceOf(RpcRequest.class); assertThat(((RpcRequest) slog.requestContent()).params().get(0)).isEqualTo("hello"); + + if (expectedResponseException != null) { + assertThat(slog.responseCause()).isInstanceOf(expectedResponseException); + } else if (expectCancelled) { + assertThat(slog.responseCause()).isInstanceOf(ClosedStreamException.class); + assertThat(slog.responseCause().getMessage()) + .contains("received a RST_STREAM frame: CANCEL"); + } else { + assertThat(slog.responseCause()).isNull(); + } } private void assertNoServerRequestContext(ServerExtension server) { From a4f302c84468c07d4344469559846f262c0da279 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Mon, 16 Jun 2025 18:38:57 +0200 Subject: [PATCH 18/36] [WIP] test: add TODO comments for tests in RetryingClient --- .../retry/RetryingClientWithHedgingTest.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java index 7a8a306a976..0d9b8e04960 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java @@ -544,6 +544,20 @@ void loosesAfterResponseTimeout() throws Exception { assertValidServerRequestContext(server3, 3, true); } + // todo(szymon): add test to verify that we are respecting the maximum number of attempts + + /* + todo(szymon): + Add a test verifing following behaviour given by gRPC: + "If server pushback that specifies not to retry is received in response to a hedged request, + no further hedged requests should be issued for the call." + For us the Retry-Header as our pushback mechanism. Negative or not parsable it should be interpreted + as an explicit do not retry (like gRPC does for grpc-retry-pushback-ms): + https://grpc.io/docs/guides/request-hedging/#server-pushback + */ + + // todo(szymon): test being able to set different hedging delays for different servers + private static WebClientBuilder clientBuilder() { return WebClient.builder(SessionProtocol.H2C, EndpointGroup.of(EndpointSelectionStrategy.roundRobin(), From 6a3db6738ebbb983b4e50ce397d55a1cad2d77cf Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Tue, 17 Jun 2025 12:47:09 +0200 Subject: [PATCH 19/36] [WIP] test: add thirdServerWinsEvenAfterPerAttemptTimeout to RetryingRpcClientWithHedgingTest --- .../retry/RetryingClientWithHedgingTest.java | 13 ++--- .../RetryingRpcClientWithHedgingTest.java | 47 +++++++++++++++++-- 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java index 0d9b8e04960..58df8076022 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java @@ -153,8 +153,6 @@ public void unlatchResponse() { } } - private static final long LOOSING_SERVER_RESPONSE_DELAY_MILLIS = 1000; - private static final String SERVER1_RESPONSE = "s1"; private static final String SERVER2_RESPONSE = "s2#"; private static final String SERVER3_RESPONSE = "s3##"; @@ -346,15 +344,14 @@ void thirdServerWinsEvenAfterPerAttemptTimeout() throws Exception { } await().pollInterval(25, TimeUnit.MILLISECONDS).untilAsserted(() -> { - assertThat(server1.getNumRequests()).isOne(); - assertThat(server2.getNumRequests()).isOne(); assertThat(server3.getNumRequests()).isOne(); }); - // Let the third server win - server1.unlatchResponse(); // issued at T - server2.unlatchResponse(); // issued at T + 200 (request 1 timed out) - server3.unlatchResponse(); // issued at T + 400 (request 2 timed out) + // As we know that the third server received a request, we know that + // we are at >= T + 400. This means that: + server1.unlatchResponse(); // issued at T (timed out) + server2.unlatchResponse(); // issued at T + 200 (timed out) + server3.unlatchResponse(); // issued at T + 400 (hopefully not timed out yet) await() .untilAsserted(() -> { diff --git a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java index d00e09aaafd..ae49b93b7f9 100644 --- a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java +++ b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java @@ -125,8 +125,6 @@ public void unlatchResponse() { } } - private static final long LOOSING_SERVER_RESPONSE_DELAY_MILLIS = 50; - private static final String RETRIABLE_RESPONSE = "please-retry-thanks"; private static final String SERVER1_RESPONSE = "s1"; private static final String SERVER2_RESPONSE = "s2#"; @@ -295,7 +293,50 @@ void allServerLosePickLastResponse() { @Test void thirdServerWinsEvenAfterPerAttemptTimeout() throws Exception { - // todo(szymon): implement + when(server3.getServiceHandler().hello(anyString())).thenReturn(SERVER3_RESPONSE); + + final RetryConfig hedgingNoRetryConfig = RetryConfig + .builderForRpc( + RetryRule.builder().onTimeoutException().thenBackoff(Backoff.fixed(10_000))) // should + // be always overtaken by hedging task + .maxTotalAttempts(3) + .responseTimeoutMillisForEachAttempt(100) + .hedgingDelayMillis(200) + .build(); + + final HelloService.AsyncIface client = client(hedgingNoRetryConfig); + + final CompletableFuture responseFuture; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + responseFuture = asyncHelloWith(client); + ctx = captor.get(); + } + + await().pollInterval(25, TimeUnit.MILLISECONDS).untilAsserted(() -> { + assertThat(server3.getNumRequests()).isOne(); + }); + + // As we know that the third server received a request, we know that + // we are at >= T + 400. This means that: + server1.unlatchResponse(); // issued at T (timed out) + server2.unlatchResponse(); // issued at T + 200 (timed out) + server3.unlatchResponse(); // issued at T + 400 (hopefully not timed out yet) + + await().untilAsserted(() -> { + assertThat(responseFuture.get()).isEqualTo(SERVER3_RESPONSE); + + assertValidServerRequestContext(server1, 1, true); + assertValidServerRequestContext(server2, 2, true); + assertValidServerRequestContext(server3, 3, false); + + assertValidClientRequestContext( + ctx, GET_VERIFY_RESPONSE_HAS_CONTENT.apply(SERVER3_RESPONSE), + VERIFY_RESPONSE_TIMEOUT, + VERIFY_RESPONSE_TIMEOUT, + GET_VERIFY_RESPONSE_HAS_CONTENT.apply(SERVER3_RESPONSE) + ); + }); } @Test From 4db95252ca7b5849107d22e208c3296f9c2a9f2a Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Tue, 17 Jun 2025 16:08:25 +0200 Subject: [PATCH 20/36] [WIP] feat, test: make sure each attempt is started with a consistent attempt number Also add more tests to Retrying(Rpc)Client --- .../client/retry/AbstractRetryingClient.java | 28 +-- .../armeria/client/retry/RetryingClient.java | 15 +- .../client/retry/RetryingRpcClient.java | 17 +- .../retry/RetryingClientWithHedgingTest.java | 172 ++++++++++++++++-- .../RetryingRpcClientWithHedgingTest.java | 138 ++++++++++++-- 5 files changed, 312 insertions(+), 58 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java index fbe0157d07a..22cecce3180 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java @@ -180,7 +180,8 @@ protected final RetryRuleWithContent retryRuleWithContent() { protected void startRetryAttempt(ClientRequestContext ctx, ClientRequestContext attemptCtx, BiConsumer onAcceptHandler, - BiConsumer onAttemptAbortedHandler + BiConsumer + onAttemptAbortedHandler ) { requireNonNull(ctx, "ctx"); requireNonNull(attemptCtx, "attemptCtx"); @@ -231,11 +232,13 @@ protected static void completeRetryAttempt(ClientRequestCon return; } - synchronized (state(ctx)) { - state(ctx).completeAttempt(attemptCtx, attemptRes); + final State state = state(ctx); + + synchronized (state) { + state.completeAttempt(attemptCtx, attemptRes); if (isWinning) { - state(ctx).complete(); + state.complete(); } } } @@ -257,7 +260,7 @@ protected static boolean isRetryingComplete(ClientRequestContext ctx) { * todo(szymon): [doc]. */ protected void scheduleNextRetry(ClientRequestContext ctx, - Runnable retryTask, + Consumer retryTask, Backoff backoff, Consumer actionOnException) { scheduleNextRetry(ctx, retryTask, backoff, -1, actionOnException); @@ -267,7 +270,7 @@ protected void scheduleNextRetry(ClientRequestContext ctx, * todo(szymon): [doc]. */ protected static void scheduleNextRetry(ClientRequestContext ctx, - Runnable retryTask, + Consumer retryTask, Backoff backoff, long retryDelayFromServerMillis, Consumer actionOnException) { @@ -332,6 +335,8 @@ protected static void scheduleNextRetry(ClientRequestContext ctx, state.startRetryTask(); scheduler.schedule(() -> { + + final int thisAttemptNo; // *, see comment above. synchronized (state) { if (isRetryingComplete(ctx)) { @@ -340,10 +345,10 @@ protected static void scheduleNextRetry(ClientRequestContext ctx, return; } - state.acquireAttemptNoWithCurrentBackoff(backoff); + thisAttemptNo = state.acquireAttemptNoWithCurrentBackoff(backoff); } - retryTask.run(); + retryTask.accept(thisAttemptNo); synchronized (state) { state.completeRetryTask(); @@ -578,10 +583,10 @@ int nextAttemptNoWithBackoff(Backoff backoff) { return 1; } - return currentAttemptNoWithLastBackoff + 1; + return currentAttemptNoWithLastBackoff; } - void acquireAttemptNoWithCurrentBackoff(Backoff backoff) { + int acquireAttemptNoWithCurrentBackoff(Backoff backoff) { checkState(!retryingCompleteFuture.isDone()); checkState((totalAttemptNo + 1) <= config.maxTotalAttempts(), "Exceeded the maximum number of attempts: %s", config.maxTotalAttempts()); @@ -591,10 +596,11 @@ void acquireAttemptNoWithCurrentBackoff(Backoff backoff) { if (lastBackoff != backoff) { lastBackoff = backoff; currentAttemptNoWithLastBackoff = 1; - return; } currentAttemptNoWithLastBackoff++; + + return totalAttemptNo; } int numPendingAttempts() { diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index 89adb9bbd4b..06a3ac8a6e3 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -249,7 +249,7 @@ protected HttpResponse doExecute(ClientRequestContext ctx, HttpRequest req) thro if (ctx.exchangeType().isRequestStreaming()) { final HttpRequestDuplicator reqDuplicator = req.toDuplicator(ctx.eventLoop().withoutContext(), 0); doExecute0(new RetryingContext(mappedRetryConfig(ctx), ctx, reqDuplicator, req, res, - responseFuture)); + responseFuture), 1); } else { req.aggregate(AggregationOptions.usePooledObjects(ctx.alloc(), ctx.eventLoop())) .handle((agg, cause) -> { @@ -258,7 +258,7 @@ protected HttpResponse doExecute(ClientRequestContext ctx, HttpRequest req) thro } else { final HttpRequestDuplicator reqDuplicator = new AggregatedHttpRequestDuplicator(agg); doExecute0(new RetryingContext(mappedRetryConfig(ctx), ctx, reqDuplicator, req, res, - responseFuture)); + responseFuture), 1); } return null; }); @@ -267,7 +267,7 @@ protected HttpResponse doExecute(ClientRequestContext ctx, HttpRequest req) thro return res; } - private void doExecute0(RetryingContext retryingContext) { + private void doExecute0(RetryingContext retryingContext, int attemptNo) { final RetryConfig config = retryingContext.config(); final ClientRequestContext ctx = retryingContext.ctx(); @@ -277,8 +277,7 @@ private void doExecute0(RetryingContext retryingContext) { // todo(szymon): we need to inject the attempt number as there may be concurrent attempts // starting and acquiring an attempt number. - final int totalAttempts = getTotalAttempts(ctx); - final boolean initialAttempt = totalAttempts <= 1; + final boolean initialAttempt = attemptNo <= 1; // The request or attemptRes has been aborted by the client before it receives a attemptRes, // so stop retrying. if (originalReq.whenComplete().isCompletedExceptionally()) { @@ -312,7 +311,7 @@ private void doExecute0(RetryingContext retryingContext) { attemptReq = rootReqDuplicator.duplicate(); } else { final RequestHeadersBuilder newHeaders = originalReq.headers().toBuilder(); - newHeaders.setInt(ARMERIA_RETRY_COUNT, totalAttempts - 1); + newHeaders.setInt(ARMERIA_RETRY_COUNT, attemptNo - 1); attemptReq = rootReqDuplicator.duplicate(newHeaders.build()); } @@ -400,7 +399,7 @@ private void doExecute0(RetryingContext retryingContext) { if (hedgingDelayMillis >= 0) { final Backoff hedgingDelayBackoff = Backoff.fixed(hedgingDelayMillis); - scheduleNextRetry(ctx, () -> doExecute0(retryingContext), + scheduleNextRetry(ctx, hedgingAttemptNo -> doExecute0(retryingContext, hedgingAttemptNo), hedgingDelayBackoff, cause -> handleExceptionAfterScheduling(retryingContext, cause)); } @@ -645,7 +644,7 @@ private void handleRetryDecision(RetryingContext retryingContext, @Nullable Retr if (backoff != null) { final long millisAfter = useRetryAfter ? getRetryAfterMillis(attemptCtx) : -1; scheduleNextRetry(retryingContext.ctx(), - () -> doExecute0(retryingContext), + attemptNo -> doExecute0(retryingContext, attemptNo), backoff, millisAfter, cause -> handleExceptionAfterScheduling(retryingContext, cause)); diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java index 9f7ca51ae05..716886389bd 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java @@ -145,14 +145,14 @@ public static RetryingRpcClientBuilder builder(RetryConfigMapping m protected RpcResponse doExecute(ClientRequestContext ctx, RpcRequest req) throws Exception { final CompletableFuture returnedResFuture = new CompletableFuture<>(); final RpcResponse res = RpcResponse.from(returnedResFuture); - doExecute0(ctx, req, res, returnedResFuture); + doExecute0(ctx, req, res, returnedResFuture, 1); return res; } private void doExecute0(ClientRequestContext ctx, RpcRequest req, - RpcResponse returnedRes, CompletableFuture returnedResFuture) { - final int totalAttempts = getTotalAttempts(ctx); - final boolean initialAttempt = totalAttempts <= 1; + RpcResponse returnedRes, CompletableFuture returnedResFuture, + int attemptNo) { + final boolean initialAttempt = attemptNo <= 1; if (returnedRes.isDone()) { // The response has been cancelled by the client before it receives a response, so stop retrying. completeRetryingExceptionally(ctx, returnedResFuture, new CancellationException( @@ -169,7 +169,7 @@ private void doExecute0(ClientRequestContext ctx, RpcRequest req, if (!initialAttempt) { attemptCtx.mutateAdditionalRequestHeaders( - mutator -> mutator.add(ARMERIA_RETRY_COUNT, StringUtil.toString(totalAttempts - 1))); + mutator -> mutator.add(ARMERIA_RETRY_COUNT, StringUtil.toString(attemptNo - 1))); } final RpcResponse attemptRes; @@ -218,7 +218,8 @@ private void doExecute0(ClientRequestContext ctx, RpcRequest req, if (backoff != null) { scheduleNextRetry(ctx, - () -> doExecute0(ctx, req, returnedRes, returnedResFuture), + nextAttemptNo -> doExecute0(ctx, req, returnedRes, + returnedResFuture, nextAttemptNo), backoff, cause0 -> handleExceptionAfterScheduling(ctx, returnedResFuture, cause0)); @@ -237,7 +238,9 @@ private void doExecute0(ClientRequestContext ctx, RpcRequest req, if (hedgingDelayMillis >= 0) { final Backoff hedgingBackoff = Backoff.fixed(hedgingDelayMillis); - scheduleNextRetry(ctx, () -> doExecute0(ctx, req, returnedRes, returnedResFuture), + scheduleNextRetry(ctx, hedgingAttemptNo -> + doExecute0(ctx, req, returnedRes, returnedResFuture, + hedgingAttemptNo), hedgingBackoff, cause -> handleExceptionAfterScheduling(ctx, returnedResFuture, cause)); } diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java index 58df8076022..aecbaf832c1 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java @@ -23,6 +23,9 @@ import static org.awaitility.Awaitility.await; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.nio.charset.Charset; @@ -106,9 +109,13 @@ public HttpResponse serve(ServiceRequestContext ctx, HttpRequest req) // We are using a blocking task executor to not block the event loop so we are // able to receive request cancellations. ctx.blockingTaskExecutor().execute(() -> { - numRequests.incrementAndGet(); + if (numRequests.incrementAndGet() != 1) { + responseFuture.completeExceptionally(new IllegalStateException( + "Expected only one request, but got: " + numRequests.get())); + return; + } + try { - logger.debug("Got a request. Waiting for the latch to be released."); responseLatch.await(); } catch (InterruptedException e) { responseFuture.completeExceptionally(e); @@ -312,8 +319,153 @@ void thirdServerWins() throws Exception { } @Test - void allServerLosePickLastResponse() { - // todo(szymon): implement + void respectsShorterBackoffTriggeringFasterRetry() throws Exception { + when(server1.getHelloService().serve(any(), any())) + .thenReturn(HttpResponse.of(HttpStatus.TOO_MANY_REQUESTS, MediaType.PLAIN_TEXT, + SERVER1_RESPONSE)); + when(server2.getHelloService().serve(any(), any())) + .thenReturn(HttpResponse.of(HttpStatus.TOO_MANY_REQUESTS, MediaType.PLAIN_TEXT, + SERVER2_RESPONSE)); + when(server3.getHelloService().serve(any(), any())) + .thenReturn(HttpResponse.of(HttpStatus.TOO_MANY_REQUESTS, MediaType.PLAIN_TEXT, + SERVER3_RESPONSE)); + + class SpyableAttemptLimitingBackoff implements Backoff { + private final Backoff delegate = Backoff.withoutDelay().withMaxAttempts(2); + + @Override + public long nextDelayMillis(int numAttemptsSoFar) { + return delegate.nextDelayMillis(numAttemptsSoFar); + } + } + + final Backoff backoff = spy(new SpyableAttemptLimitingBackoff()); + + final RetryConfig config = RetryConfig + .builder(RetryRule.of( + RetryRule + .builder() + .onStatus(HttpStatus.TOO_MANY_REQUESTS) + .thenBackoff(backoff), + NO_RETRY_RULE + )) + .maxTotalAttempts(3) + .hedgingDelayMillis(500) + .build(); + + final WebClient client = client(config); + final CompletableFuture responseFuture; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + responseFuture = client.get("/hello").aggregate(); + ctx = captor.get(); + } + + server2.unlatchResponse(); + server3.unlatchResponse(); + + // 500ms for the hedging request to server 2 and 250ms tolerance. + await().atMost(500 + 250, TimeUnit.MILLISECONDS).untilAsserted(() -> { + assertThat(server1.getNumRequests()).isOne(); + // Server 1 is blocking, after 500ms hedging request to server 2 will come to the rescue. + assertThat(server2.getNumRequests()).isOne(); + // Server 2 responds immediately, backoff will order retry immediately. + // This cancels the hedging request that came with the request to server 2. + assertThat(server3.getNumRequests()).isOne(); + // The hedged request to server 3 will not be issued because we are out of total attempts. + // Server 3 responds immediately, but we will not continue as we are out of attempts. + + verify(backoff, times(1)).nextDelayMillis(1); + }); + + // Should be enough for the client to receive and process the two responses from server + // 2 and server 3. + Thread.sleep(500); + // Server 1 answers, again with TOO_MANY_REQUESTS, and it tries to retry. + // We exceeded the number of attempts _of the backoff_ so we should stop retrying and take the last + // response which is the response from server 1. + server1.unlatchResponse(); + + await().untilAsserted(() -> { + assertValidAggregatedResponse(responseFuture, HttpStatus.TOO_MANY_REQUESTS, SERVER1_RESPONSE); + assertValidClientRequestContext( + ctx, GET_VERIFY_RESPONSE_HAS_CONTENT_AND_STATUS.apply(SERVER1_RESPONSE, + HttpStatus.TOO_MANY_REQUESTS), + GET_VERIFY_RESPONSE_HAS_CONTENT_AND_STATUS.apply(SERVER1_RESPONSE, + HttpStatus.TOO_MANY_REQUESTS), + GET_VERIFY_RESPONSE_HAS_CONTENT_AND_STATUS.apply(SERVER2_RESPONSE, + HttpStatus.TOO_MANY_REQUESTS), + GET_VERIFY_RESPONSE_HAS_CONTENT_AND_STATUS.apply(SERVER3_RESPONSE, + HttpStatus.TOO_MANY_REQUESTS) + ); + + assertValidServerRequestContext(server1, 1, false); + assertValidServerRequestContext(server2, 2, false); + assertValidServerRequestContext(server3, 3, false); + }); + } + + @Test + void allServerLosePickLastResponse() throws Exception { + when(server1.getHelloService().serve(any(), any())) + .thenReturn(HttpResponse.of(HttpStatus.TOO_MANY_REQUESTS, MediaType.PLAIN_TEXT, + SERVER1_RESPONSE)); + when(server2.getHelloService().serve(any(), any())) + .thenReturn(HttpResponse.of(HttpStatus.TOO_MANY_REQUESTS, MediaType.PLAIN_TEXT, + SERVER2_RESPONSE)); + when(server3.getHelloService().serve(any(), any())) + .thenReturn(HttpResponse.of(HttpStatus.TOO_MANY_REQUESTS, MediaType.PLAIN_TEXT, + SERVER3_RESPONSE)); + + final RetryConfig config = RetryConfig + .builder(RetryRule.of( + RetryRule + .builder() + .onStatus(HttpStatus.TOO_MANY_REQUESTS) + .thenBackoff(Backoff.fixed(10_000)), + NO_RETRY_RULE + )) + .maxTotalAttempts(3) + .hedgingDelayMillis(500) + .build(); + + final WebClient client = client(config); + final CompletableFuture responseFuture; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + responseFuture = client.get("/hello").aggregate(); + ctx = captor.get(); + } + + server1.unlatchResponse(); + server2.unlatchResponse(); + server3.unlatchResponse(); + + await().untilAsserted(() -> { + assertValidAggregatedResponse(responseFuture, HttpStatus.TOO_MANY_REQUESTS, SERVER3_RESPONSE); + assertValidClientRequestContext(ctx, + GET_VERIFY_RESPONSE_HAS_CONTENT_AND_STATUS.apply( + SERVER3_RESPONSE, + HttpStatus.TOO_MANY_REQUESTS + ), + GET_VERIFY_RESPONSE_HAS_CONTENT_AND_STATUS.apply( + SERVER1_RESPONSE, + HttpStatus.TOO_MANY_REQUESTS + ), + GET_VERIFY_RESPONSE_HAS_CONTENT_AND_STATUS.apply( + SERVER2_RESPONSE, + HttpStatus.TOO_MANY_REQUESTS + ), + GET_VERIFY_RESPONSE_HAS_CONTENT_AND_STATUS.apply( + SERVER3_RESPONSE, + HttpStatus.TOO_MANY_REQUESTS + ) + ); + + assertValidServerRequestContext(server1, 1, false); + assertValidServerRequestContext(server2, 2, false); + assertValidServerRequestContext(server3, 3, false); + }); } @Test @@ -541,18 +693,6 @@ void loosesAfterResponseTimeout() throws Exception { assertValidServerRequestContext(server3, 3, true); } - // todo(szymon): add test to verify that we are respecting the maximum number of attempts - - /* - todo(szymon): - Add a test verifing following behaviour given by gRPC: - "If server pushback that specifies not to retry is received in response to a hedged request, - no further hedged requests should be issued for the call." - For us the Retry-Header as our pushback mechanism. Negative or not parsable it should be interpreted - as an explicit do not retry (like gRPC does for grpc-retry-pushback-ms): - https://grpc.io/docs/guides/request-hedging/#server-pushback - */ - // todo(szymon): test being able to set different hedging delays for different servers private static WebClientBuilder clientBuilder() { diff --git a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java index ae49b93b7f9..385110d85ec 100644 --- a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java +++ b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java @@ -43,6 +43,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import org.mockito.Mockito; import com.linecorp.armeria.client.ClientRequestContext; import com.linecorp.armeria.client.ClientRequestContextCaptor; @@ -131,19 +132,6 @@ public void unlatchResponse() { private static final String SERVER3_RESPONSE = "s3##"; private static final RetryRule NO_RETRY_RULE = RetryRule.builder().thenNoRetry(); - private static final RetryRuleWithContent RETRY_RETRIABLE_RESPONSES_RULE = - RetryRuleWithContent - .builder() - .onResponse((ctx, response) -> { - return response.whenComplete().handle( - (responseData, cause) -> { - assertThat(cause).isNull(); - assertThat(responseData).isInstanceOf(String.class); - return responseData.equals(RETRIABLE_RESPONSE); - } - ); - }) - .thenBackoff(Backoff.withoutDelay()); @RegisterExtension private static final TestServer server1 = new TestServer(); @@ -287,8 +275,107 @@ void thirdServerWins() throws Exception { } @Test - void allServerLosePickLastResponse() { - // todo(szymon): implement + void respectsShorterBackoffTriggeringFasterRetry() throws Exception { + when(server1.getServiceHandler().hello(anyString())).thenReturn(RETRIABLE_RESPONSE); + when(server2.getServiceHandler().hello(anyString())).thenReturn(RETRIABLE_RESPONSE); + when(server3.getServiceHandler().hello(anyString())).thenReturn(RETRIABLE_RESPONSE); + + class SpyableAttemptLimitingBackoff implements Backoff { + private final Backoff delegate = Backoff.withoutDelay().withMaxAttempts(2); + + @Override + public long nextDelayMillis(int numAttemptsSoFar) { + return delegate.nextDelayMillis(numAttemptsSoFar); + } + } + + final Backoff backoff = Mockito.spy(new SpyableAttemptLimitingBackoff()); + + final RetryConfig config = RetryConfig.builderForRpc( + getRetryRetriableResponsesRule(backoff)) + .maxTotalAttempts(3) + .hedgingDelayMillis(500) + .build(); + + final HelloService.AsyncIface client = client(config); + final CompletableFuture responseFuture; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + responseFuture = asyncHelloWith(client); + ctx = captor.get(); + } + + server2.unlatchResponse(); + server3.unlatchResponse(); + + // 500ms for the hedging request to server 2 and 250ms tolerance. + await().atMost(500 + 250, TimeUnit.MILLISECONDS).untilAsserted(() -> { + assertThat(server1.getNumRequests()).isEqualTo(1); + assertThat(server2.getNumRequests()).isEqualTo(1); + assertThat(server3.getNumRequests()).isEqualTo(1); + Mockito.verify(backoff, Mockito.times(1)).nextDelayMillis(1); + }); + + Thread.sleep(500); + server1.unlatchResponse(); + + await().untilAsserted(() -> { + assertThat(responseFuture.get()).isEqualTo(RETRIABLE_RESPONSE); + assertValidClientRequestContext( + ctx, + GET_VERIFY_RESPONSE_HAS_CONTENT.apply(RETRIABLE_RESPONSE), + GET_VERIFY_RESPONSE_HAS_CONTENT.apply(RETRIABLE_RESPONSE), + GET_VERIFY_RESPONSE_HAS_CONTENT.apply(RETRIABLE_RESPONSE), + GET_VERIFY_RESPONSE_HAS_CONTENT.apply(RETRIABLE_RESPONSE) + ); + assertValidServerRequestContext(server1, 1, false); + assertValidServerRequestContext(server2, 2, false); + assertValidServerRequestContext(server3, 3, false); + }); + } + + @Test + void allServerLosePickLastResponse() throws Exception { + when(server1.getServiceHandler().hello(anyString())).thenReturn(RETRIABLE_RESPONSE); + when(server2.getServiceHandler().hello(anyString())).thenReturn(RETRIABLE_RESPONSE); + when(server3.getServiceHandler().hello(anyString())).thenReturn(RETRIABLE_RESPONSE); + + final RetryConfig config = RetryConfig.builderForRpc( + getRetryRetriableResponsesRule( + Backoff.fixed(10_000) + ) + ) + .maxTotalAttempts(3) + .hedgingDelayMillis(0) + .build(); + + final HelloService.AsyncIface client = client(config); + final CompletableFuture responseFuture; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + responseFuture = asyncHelloWith(client); + ctx = captor.get(); + } + + server1.unlatchResponse(); + server2.unlatchResponse(); + server3.unlatchResponse(); + + await().untilAsserted(() -> { + assertThat(responseFuture.get()).isEqualTo(RETRIABLE_RESPONSE); + + assertValidServerRequestContext(server1, 1, false); + assertValidServerRequestContext(server2, 2, false); + assertValidServerRequestContext(server3, 3, false); + + assertValidClientRequestContext( + ctx, + GET_VERIFY_RESPONSE_HAS_CONTENT.apply(RETRIABLE_RESPONSE), + GET_VERIFY_RESPONSE_HAS_CONTENT.apply(RETRIABLE_RESPONSE), + GET_VERIFY_RESPONSE_HAS_CONTENT.apply(RETRIABLE_RESPONSE), + GET_VERIFY_RESPONSE_HAS_CONTENT.apply(RETRIABLE_RESPONSE) + ); + }); } @Test @@ -345,7 +432,11 @@ void thirdServerWinsEvenAfterRetriableResponse() throws Exception { when(server2.getServiceHandler().hello(anyString())).thenReturn(RETRIABLE_RESPONSE); when(server3.getServiceHandler().hello(anyString())).thenReturn(SERVER3_RESPONSE); - final RetryConfig config = RetryConfig.builderForRpc(RETRY_RETRIABLE_RESPONSES_RULE) + final RetryConfig config = RetryConfig.builderForRpc( + getRetryRetriableResponsesRule( + Backoff.withoutDelay() + ) + ) .maxTotalAttempts(3) .hedgingDelayMillis(10) .build(); @@ -532,6 +623,21 @@ private static HelloService.AsyncIface client(RetryConfig config, .build(HelloService.AsyncIface.class); } + private static RetryRuleWithContent getRetryRetriableResponsesRule(Backoff backoff) { + return RetryRuleWithContent + .builder() + .onResponse((ctx, response) -> { + return response.whenComplete().handle( + (responseData, cause) -> { + assertThat(cause).isNull(); + assertThat(responseData).isInstanceOf(String.class); + return responseData.equals(RETRIABLE_RESPONSE); + } + ); + }) + .thenBackoff(backoff); + } + private interface RequestLogVerifier extends Consumer {} private static final RequestLogVerifier VERIFY_REQUEST_CANCELLED = From 5625e751fac59ba7bdb1dd90c584a3bc4d688ffa Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Tue, 17 Jun 2025 17:54:01 +0200 Subject: [PATCH 21/36] [WIP] refactor: some renaming for consistency --- .../client/retry/AbstractRetryingClient.java | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java index 22cecce3180..94eacc02e53 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java @@ -179,13 +179,13 @@ protected final RetryRuleWithContent retryRuleWithContent() { */ protected void startRetryAttempt(ClientRequestContext ctx, ClientRequestContext attemptCtx, - BiConsumer onAcceptHandler, + BiConsumer onWinHandler, BiConsumer onAttemptAbortedHandler ) { requireNonNull(ctx, "ctx"); requireNonNull(attemptCtx, "attemptCtx"); - requireNonNull(onAcceptHandler, "onAcceptHandler"); + requireNonNull(onWinHandler, "onWinHandler"); requireNonNull(onAttemptAbortedHandler, "onAttemptAbortedHandler"); final State state = state(ctx); @@ -210,7 +210,7 @@ protected void startRetryAttempt(ClientRequestContext ctx, assert winningAttemptRes != null; if (attemptCtx == winningAttemptCtx) { - onAcceptHandler.accept(winningAttemptCtx, winningAttemptRes); + onWinHandler.accept(winningAttemptCtx, winningAttemptRes); return null; } @@ -223,9 +223,9 @@ protected void startRetryAttempt(ClientRequestContext ctx, /** * todo(szymon): [doc]. */ - protected static void completeRetryAttempt(ClientRequestContext ctx, - ClientRequestContext attemptCtx, - O attemptRes, boolean isWinning) { + protected void completeRetryAttempt(ClientRequestContext ctx, + ClientRequestContext attemptCtx, + O attemptRes, boolean isWinning) { if (isRetryingComplete(ctx)) { // The complete handler of this attempt was already executed (provided that `attemptCtx` // was registered with `startRetryAttempt` as by the contract of `completeRetryAttempt`. @@ -306,7 +306,7 @@ protected static void scheduleNextRetry(ClientRequestContext ctx, // minimum retry delay from the server. earliestRetryTimeNanos = scheduler.addEarliestNextRetryTimeNanos(earliestRetryTimeNanos); - final int attemptNoWithBackoff = state.nextAttemptNoWithBackoff(backoff); + final int attemptNoWithBackoff = state.numAttemptsSoFarWithBackoff(backoff); if (attemptNoWithBackoff < 0) { scheduler.rescheduleCurrentRetryTaskIfTooEarly(); actionOnException.accept( @@ -333,7 +333,9 @@ protected static void scheduleNextRetry(ClientRequestContext ctx, return; } - state.startRetryTask(); + synchronized (state) { + state.startRetryTask(); + } scheduler.schedule(() -> { final int thisAttemptNo; @@ -574,7 +576,7 @@ void completeRetryTask() { numScheduledAttempts--; } - int nextAttemptNoWithBackoff(Backoff backoff) { + int numAttemptsSoFarWithBackoff(Backoff backoff) { if (totalAttemptNo >= config.maxTotalAttempts()) { return -1; } From de478890263b927ce751f10a34d044ab99373f99 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Wed, 18 Jun 2025 10:02:53 +0200 Subject: [PATCH 22/36] test: remove unnecessary change from doNotRetryWhenResponseIsCancelled --- .../it/client/retry/RetryingRpcClientTest.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java index 41cab0ed0b8..4ff30f9f553 100644 --- a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java +++ b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java @@ -334,16 +334,9 @@ void doNotRetryWhenResponseIsCancelled() throws Exception { .path("/thrift") .rpcDecorator(RetryingRpcClient.builder(retryAlways).newDecorator()) .rpcDecorator((delegate, ctx, req) -> { + context.set(ctx); final RpcResponse res = delegate.execute(ctx, req); - - // We are not guarenteed that the retrying client will immediately retry but - // execute the first attempt in the event loop. It should be enough to just - // enqueue in the loop and only then cancel the response. - ctx.eventLoop().execute(() -> { - context.set(ctx); - res.cancel(true); - }); - + res.cancel(true); return res; }) .build(HelloService.Iface.class); From 9914f784bd68637fd2b440d9ca00f45ee4f7f8f8 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Wed, 18 Jun 2025 20:32:19 +0200 Subject: [PATCH 23/36] test: add log checks to RetryingClientTest --- .../client/retry/RetryingClientTest.java | 315 ++++++++++++++---- 1 file changed, 254 insertions(+), 61 deletions(-) diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java index e08b0f714d3..7f7cee02b27 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java @@ -21,6 +21,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.catchThrowable; +import static org.assertj.core.api.Assertions.fail; +import static org.awaitility.Awaitility.await; import java.time.Duration; import java.util.Arrays; @@ -31,6 +33,7 @@ import java.util.concurrent.CompletionStage; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -55,6 +58,8 @@ import com.linecorp.armeria.client.ClientFactory; import com.linecorp.armeria.client.ClientRequestContext; +import com.linecorp.armeria.client.ClientRequestContextCaptor; +import com.linecorp.armeria.client.Clients; import com.linecorp.armeria.client.HttpClient; import com.linecorp.armeria.client.ResponseTimeoutException; import com.linecorp.armeria.client.UnprocessedRequestException; @@ -331,16 +336,27 @@ void retryWhenContentMatched() { .factory(clientFactory) .decorator(retryingDecorator) .build(); - - final AggregatedHttpResponse res = client.get("/retry-content").aggregate().join(); + final AggregatedHttpResponse res; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + res = client.get("/retry-content").aggregate().join(); + ctx = captor.get(); + } assertThat(res.contentUtf8()).isEqualTo("Succeeded after retry"); + validateClientRequestLog(ctx, 3); } @Test void retryWhenStatusMatched() { final WebClient client = client(RetryRule.builder().onServerErrorStatus().onException().thenBackoff()); - final AggregatedHttpResponse res = client.get("/503-then-success").aggregate().join(); + final AggregatedHttpResponse res; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + res = client.get("/503-then-success").aggregate().join(); + ctx = captor.get(); + } assertThat(res.contentUtf8()).isEqualTo("Succeeded after retry"); + validateClientRequestLog(ctx, 2); } @Test @@ -349,8 +365,14 @@ void retryWhenStatusMatchedWithContent() { .onServerErrorStatus() .onException() .thenBackoff(), 10000, 0, 100); - final AggregatedHttpResponse res = client.get("/503-then-success").aggregate().join(); + final AggregatedHttpResponse res; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + res = client.get("/503-then-success").aggregate().join(); + ctx = captor.get(); + } assertThat(res.contentUtf8()).isEqualTo("Succeeded after retry"); + validateClientRequestLog(ctx, 2); } @Test @@ -361,8 +383,14 @@ void retryWhenTrailerMatched() { return trailers.getInt("grpc-status", -1) != 0; }) .thenBackoff()); - final AggregatedHttpResponse res = client.get("/trailers-then-success").aggregate().join(); + final AggregatedHttpResponse res; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + res = client.get("/trailers-then-success").aggregate().join(); + ctx = captor.get(); + } assertThat(res.contentUtf8()).isEqualTo("Succeeded after retry"); + validateClientRequestLog(ctx, 2); } @Test @@ -371,16 +399,28 @@ void retryWhenTotalDurationIsHigh() { client(RetryRule.builder() .onTotalDuration((unused, duration) -> duration.toNanos() > 100) .thenBackoff()); - final AggregatedHttpResponse res = client.get("/1sleep-then-success").aggregate().join(); + final AggregatedHttpResponse res; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + res = client.get("/1sleep-then-success").aggregate().join(); + ctx = captor.get(); + } assertThat(res.contentUtf8()).isEqualTo("Succeeded after retry"); + validateClientRequestLog(ctx); } @Test void disableResponseTimeout() { final WebClient client = client(RetryRule.failsafe(), 0, 0, 100); - final AggregatedHttpResponse res = client.get("/503-then-success").aggregate().join(); + final AggregatedHttpResponse res; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + res = client.get("/503-then-success").aggregate().join(); + ctx = captor.get(); + } assertThat(res.contentUtf8()).isEqualTo("Succeeded after retry"); // response timeout did not happen. + validateClientRequestLog(ctx, 2); } @Test @@ -388,10 +428,17 @@ void respectRetryAfter() { final WebClient client = client(RetryRule.failsafe()); final Stopwatch sw = Stopwatch.createStarted(); - final AggregatedHttpResponse res = client.get("/retry-after-1-second").aggregate().join(); + final AggregatedHttpResponse res; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + res = client.get("/retry-after-1-second").aggregate().join(); + ctx = captor.get(); + } + assertThat(res.contentUtf8()).isEqualTo("Succeeded after retry"); assertThat(sw.elapsed(TimeUnit.MILLISECONDS)).isGreaterThanOrEqualTo( (long) (TimeUnit.SECONDS.toMillis(1) * 0.9)); + validateClientRequestLog(ctx, 2); } @Test @@ -399,12 +446,18 @@ void respectRetryAfterWithHttpDate() { final WebClient client = client(RetryRule.failsafe()); final Stopwatch sw = Stopwatch.createStarted(); - final AggregatedHttpResponse res = client.get("/retry-after-with-http-date").aggregate().join(); - assertThat(res.contentUtf8()).isEqualTo("Succeeded after retry"); + final AggregatedHttpResponse res; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + res = client.get("/retry-after-with-http-date").aggregate().join(); + ctx = captor.get(); + } + assertThat(res.contentUtf8()).isEqualTo("Succeeded after retry"); // Since ZonedDateTime doesn't express exact time, // just check out whether it is retried after delayed some time. assertThat(sw.elapsed(TimeUnit.MILLISECONDS)).isGreaterThanOrEqualTo(1000); + validateClientRequestLog(ctx, 2); } @Test @@ -413,27 +466,44 @@ void propagateLastResponseWhenNextRetryIsAfterTimeout() { .onServerErrorStatus() .onException() .thenBackoff(Backoff.fixed(10000000))); - final AggregatedHttpResponse res = client.get("/service-unavailable").aggregate().join(); + final AggregatedHttpResponse res; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + res = client.get("/service-unavailable").aggregate().join(); + ctx = captor.get(); + } assertThat(res.status()).isSameAs(HttpStatus.SERVICE_UNAVAILABLE); + validateClientRequestLog(ctx, 1); } @Test void propagateLastResponseWhenExceedMaxAttempts() { final WebClient client = client( RetryRule.builder().onServerErrorStatus().onException().thenBackoff(Backoff.fixed(1)), 0, 0, 3); - final AggregatedHttpResponse res = client.get("/service-unavailable").aggregate().join(); + final AggregatedHttpResponse res; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + res = client.get("/service-unavailable").aggregate().join(); + ctx = captor.get(); + } assertThat(res.status()).isSameAs(HttpStatus.SERVICE_UNAVAILABLE); + validateClientRequestLog(ctx, 3); // equal to max attempts } @Test void retryAfterOneYear() { final WebClient client = client(RetryRule.failsafe()); - // The response will be the last response whose headers contains HttpHeaderNames.RETRY_AFTER // because next retry is after timeout - final ResponseHeaders headers = client.get("retry-after-one-year").aggregate().join().headers(); + final ResponseHeaders headers; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + headers = client.get("retry-after-one-year").aggregate().join().headers(); + ctx = captor.get(); + } assertThat(headers.status()).isSameAs(HttpStatus.SERVICE_UNAVAILABLE); assertThat(headers.get(HttpHeaderNames.RETRY_AFTER)).isNotNull(); + validateClientRequestLog(ctx, 1); } @Test @@ -448,8 +518,16 @@ void retryOnResponseTimeout() { }; final WebClient client = client(strategy, 0, 500, 100); - final AggregatedHttpResponse res = client.get("/1sleep-then-success").aggregate().join(); + + final AggregatedHttpResponse res; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + res = client.get("/1sleep-then-success").aggregate().join(); + ctx = captor.get(); + } + assertThat(res.contentUtf8()).isEqualTo("Succeeded after retry"); + validateClientRequestLog(ctx, 2); } @Test @@ -470,10 +548,16 @@ void retryWithContentOnResponseTimeout() { .onException(ResponseTimeoutException.class) .thenBackoff(backoff))); final WebClient client = client(strategy, 0, 500, 100); - final AggregatedHttpResponse res = client.get("/1sleep-then-success").aggregate().join(); + final AggregatedHttpResponse res; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + res = client.get("/1sleep-then-success").aggregate().join(); + ctx = captor.get(); + } assertThat(res.contentUtf8()).isEqualTo("Succeeded after retry"); // Make sure that all customized RetryRuleWithContents are called. assertThat(queue).containsExactly(1, 2, 3); + validateClientRequestLog(ctx, 2); } @Test @@ -509,20 +593,34 @@ void honorRetryMapping() { final WebClient client = client(mapping); + ClientRequestContext ctx; + Stopwatch stopwatch = Stopwatch.createStarted(); - assertThat(client.get("/500-always").aggregate().join().status()) - .isEqualTo(HttpStatus.valueOf(500)); + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + assertThat(client.get("/500-always").aggregate().join().status()) + .isEqualTo(HttpStatus.valueOf(500)); + ctx = captor.get(); + } assertThat(stopwatch.elapsed()).isBetween(Duration.ofSeconds(2), Duration.ofSeconds(6)); + validateClientRequestLog(ctx, 2); stopwatch = Stopwatch.createStarted(); - assertThat(client.get("/501-always").aggregate().join().status()) - .isEqualTo(HttpStatus.valueOf(501)); + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + assertThat(client.get("/501-always").aggregate().join().status()) + .isEqualTo(HttpStatus.valueOf(501)); + ctx = captor.get(); + } assertThat(stopwatch.elapsed()).isBetween(Duration.ofSeconds(14), Duration.ofSeconds(28)); + validateClientRequestLog(ctx, 8); stopwatch = Stopwatch.createStarted(); - assertThat(client.get("/502-always").aggregate().join().status()) - .isEqualTo(HttpStatus.valueOf(502)); + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + assertThat(client.get("/502-always").aggregate().join().status()) + .isEqualTo(HttpStatus.valueOf(502)); + ctx = captor.get(); + } assertThat(stopwatch.elapsed()).isBetween(Duration.ofSeconds(0), Duration.ofSeconds(2)); + validateClientRequestLog(ctx, 1); } @Test @@ -541,18 +639,27 @@ void evaluatesMappingOnce() { final WebClient client = client(mapping); - assertThat(client.get("/500-then-success").aggregate().join().status()) - .isEqualTo(HttpStatus.valueOf(200)); + ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + assertThat(client.get("/500-then-success").aggregate().join().status()) + .isEqualTo(HttpStatus.valueOf(200)); + ctx = captor.get(); + } // 1 logical request; 2 retries assertThat(evaluations.get()).isEqualTo(1); + validateClientRequestLog(ctx, 2); reqCount.set(0); - assertThat(client.get("/500-then-success").aggregate().join().status()) - .isEqualTo(HttpStatus.valueOf(200)); + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + assertThat(client.get("/500-then-success").aggregate().join().status()) + .isEqualTo(HttpStatus.valueOf(200)); + ctx = captor.get(); + } - // 2 logical requests; 4 retries + // 2 logical requests; 2 retries assertThat(evaluations.get()).isEqualTo(2); + validateClientRequestLog(ctx, 2); } @Test @@ -585,10 +692,15 @@ void retryWithContentOnUnprocessedException() { .decorator(retryingDecorator) .build(); final Stopwatch stopwatch = Stopwatch.createStarted(); - assertThatThrownBy(() -> client.get("/unprocessed-exception").aggregate().join()) - .isInstanceOf(CompletionException.class) - .hasCauseInstanceOf(UnprocessedRequestException.class); + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + assertThatThrownBy(() -> client.get("/unprocessed-exception").aggregate().join()) + .isInstanceOf(CompletionException.class) + .hasCauseInstanceOf(UnprocessedRequestException.class); + ctx = captor.get(); + } assertThat(stopwatch.elapsed()).isBetween(Duration.ofSeconds(7), Duration.ofSeconds(20)); + validateClientRequestLog(ctx, 5); // max attempts } } @@ -597,17 +709,27 @@ void retryWithContentOnUnprocessedException() { void differentBackoffBasedOnStatus(RetryRule retryRule) { final WebClient client = client(retryRule); + ClientRequestContext ctx; + AggregatedHttpResponse res; final Stopwatch sw = Stopwatch.createStarted(); - AggregatedHttpResponse res = client.get("/503-then-success").aggregate().join(); + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + res = client.get("/503-then-success").aggregate().join(); + ctx = captor.get(); + } assertThat(res.contentUtf8()).isEqualTo("Succeeded after retry"); assertThat(sw.elapsed(TimeUnit.MILLISECONDS)).isBetween((long) (10 * 0.9), (long) (1000 * 1.1)); + validateClientRequestLog(ctx, 2); reqCount.set(0); sw.reset().start(); - res = client.get("/500-then-success").aggregate().join(); + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + res = client.get("/500-then-success").aggregate().join(); + ctx = captor.get(); + } assertThat(res.contentUtf8()).isEqualTo("Succeeded after retry"); assertThat(sw.elapsed(TimeUnit.MILLISECONDS)).isGreaterThanOrEqualTo((long) (1000 * 0.9)); + validateClientRequestLog(ctx, 2); } @Test @@ -616,8 +738,14 @@ void retryWithRequestBody() { .onServerErrorStatus() .onException() .thenBackoff(Backoff.fixed(10))); - final AggregatedHttpResponse res = client.post("/post-ping-pong", "bar").aggregate().join(); + final AggregatedHttpResponse res; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + res = client.post("/post-ping-pong", "bar").aggregate().join(); + ctx = captor.get(); + } assertThat(res.contentUtf8()).isEqualTo("bar"); + validateClientRequestLog(ctx, 2); } @Test @@ -655,7 +783,13 @@ void shouldGetExceptionWhenFactoryIsClosed() { // // Peel CompletionException first. - Throwable t = peel(catchThrowable(() -> client.get("/service-unavailable").aggregate().join())); + final ClientRequestContext ctx; + Throwable t; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + t = peel(catchThrowable(() -> client.get("/service-unavailable").aggregate().join())); + ctx = captor.get(); + } + validateClientRequestLog(ctx, 1); // not able to schedule second retry. if (t instanceof UnprocessedRequestException) { final Throwable cause = t.getCause(); assertThat(cause).isInstanceOf(IllegalStateException.class); @@ -683,12 +817,18 @@ void doNotRetryWhenResponseIsAborted() throws Exception { .decorator(LoggingClient.newDecorator()) .build(); responseAbortServiceCallCounter.set(0); - final HttpResponse httpResponse = client.get("/response-abort"); + final ClientRequestContext ctx; + final HttpResponse httpResponse; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + httpResponse = client.get("/response-abort"); + ctx = captor.get(); + } if (abortCause == null) { httpResponse.abort(); } else { httpResponse.abort(abortCause); } + validateClientRequestLog(ctx, 1); final RequestLog log = context.get().log().whenComplete().join(); final Throwable requestCause = log.requestCause(); @@ -716,25 +856,32 @@ void doNotRetryWhenResponseIsAborted() throws Exception { @Test void doNotRetryWhenSubscriberIsCancelled() throws Exception { final WebClient client = client(retryAlways); - client.get("/subscriber-cancel").subscribe( - new Subscriber() { - @Override - public void onSubscribe(Subscription s) { - s.cancel(); // Cancel as soon as getting the subscription. - } + final ClientRequestContext ctx; + + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + client.get("/subscriber-cancel").subscribe( + new Subscriber() { + @Override + public void onSubscribe(Subscription s) { + s.cancel(); // Cancel as soon as getting the subscription. + } + + @Override + public void onNext(HttpObject httpObject) {} - @Override - public void onNext(HttpObject httpObject) {} + @Override + public void onError(Throwable t) {} - @Override - public void onError(Throwable t) {} + @Override + public void onComplete() {} + }); - @Override - public void onComplete() {} - }); + ctx = captor.get(); + } TimeUnit.SECONDS.sleep(1L); // Sleep to check if there's a retry. assertThat(subscriberCancelServiceCallCounter.get()).isEqualTo(1); + validateClientRequestLog(ctx, 1); } @Test @@ -759,11 +906,16 @@ void doNotRetryWhenRequestIsAborted() throws Exception { } else { req.abort(abortCause); } - client.execute(req).aggregate(); + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + client.execute(req).aggregate(); + ctx = captor.get(); + } TimeUnit.SECONDS.sleep(1); // No request is made. assertThat(responseAbortServiceCallCounter.get()).isZero(); + validateClientRequestLog(ctx, 0); final RequestLog log = context.get().log().whenComplete().join(); if (abortCause == null) { assertThat(log.requestCause()).isExactlyInstanceOf(AbortedStreamException.class); @@ -789,9 +941,15 @@ void exceptionInDecorator() { .decorator(RetryingClient.newDecorator(strategy, 5)) .build(); - assertThatThrownBy(() -> client.get("/").aggregate().join()) - .hasCauseExactlyInstanceOf(AnticipatedException.class); + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + assertThatThrownBy(() -> client.get("/").aggregate().join()) + .isInstanceOf(CompletionException.class) + .hasCauseExactlyInstanceOf(AnticipatedException.class); + ctx = captor.get(); + } assertThat(retryCounter.get()).isEqualTo(5); + validateClientRequestLog(ctx, 5); } @Test @@ -802,9 +960,14 @@ void exceptionInRule() { }; final WebClient client = client(rule); - assertThatThrownBy(client.get("/").aggregate()::join) - .isInstanceOf(CompletionException.class) - .hasCauseReference(exception); + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + assertThatThrownBy(client.get("/").aggregate()::join) + .isInstanceOf(CompletionException.class) + .hasCauseReference(exception); + ctx = captor.get(); + } + validateClientRequestLog(ctx, 1); } @Test @@ -815,9 +978,14 @@ void exceptionInRuleWithContent() { }; final WebClient client = client(rule, 10000, 0, 100); - assertThatThrownBy(client.get("/").aggregate()::join) - .isInstanceOf(CompletionException.class) - .hasCauseReference(exception); + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + assertThatThrownBy(client.get("/").aggregate()::join) + .isInstanceOf(CompletionException.class) + .hasCauseReference(exception); + ctx = captor.get(); + } + validateClientRequestLog(ctx, 1); } @Test @@ -832,11 +1000,16 @@ void useSameEventLoopWhenAggregate() throws InterruptedException { }) .decorator(RetryingClient.newDecorator(RetryRule.failsafe(), 2)) .build(); - client.get("/503-then-success").aggregate().whenComplete((unused, cause) -> { - assertThat(eventLoop.get().inEventLoop()).isTrue(); - latch.countDown(); - }); + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + client.get("/503-then-success").aggregate().whenComplete((unused, cause) -> { + assertThat(eventLoop.get().inEventLoop()).isTrue(); + latch.countDown(); + }); + ctx = captor.get(); + } latch.await(); + validateClientRequestLog(ctx, 2); } private WebClient client(RetryRule retryRule) { @@ -890,6 +1063,26 @@ private WebClient client(RetryRuleWithContent retryRuleWithContent .build(); } + private static void validateClientRequestLog(ClientRequestContext ctx) { + validateClientRequestLog(ctx, ctx.log().children().size()); + } + + private static void validateClientRequestLog(ClientRequestContext ctx, int expectedNumRequests) { + await().untilAsserted(() -> { + assertThat(ctx.log().isComplete()).isTrue(); + assertThat(ctx.log().children()).hasSize(expectedNumRequests); + ctx.log().children().forEach(childLogAccess -> { + try { + final RequestLog childLog = childLogAccess.whenComplete().get(); + assertThat(childLog).isNotNull(); + assertThat(childLog.isComplete()).isTrue(); + } catch (InterruptedException | ExecutionException e) { + fail(e); + } + }); + }); + } + private static class RetryIfContentMatch implements RetryRuleWithContent { private final String retryContent; private final RetryDecision decision = RetryDecision.retry(Backoff.fixed(100)); From 3853d9f2cdaa79348fd09e6ede8693d35b69fcc5 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Wed, 18 Jun 2025 20:35:07 +0200 Subject: [PATCH 24/36] refactor: rename variable in propagateResponseSideLog --- .../common/logging/DefaultRequestLog.java | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/common/logging/DefaultRequestLog.java b/core/src/main/java/com/linecorp/armeria/common/logging/DefaultRequestLog.java index f34c13b9804..cff87adfb62 100644 --- a/core/src/main/java/com/linecorp/armeria/common/logging/DefaultRequestLog.java +++ b/core/src/main/java/com/linecorp/armeria/common/logging/DefaultRequestLog.java @@ -613,58 +613,58 @@ public void endResponseWithLastChild() { propagateResponseSideLog(lastChild.partial()); } - private void propagateResponseSideLog(RequestLog lastChild) { - if (lastChild.isAvailable(RequestLogProperty.RESPONSE_CAUSE)) { + private void propagateResponseSideLog(RequestLog child) { + if (child.isAvailable(RequestLogProperty.RESPONSE_CAUSE)) { // Update responseCause first if available because callbacks of the other properties may need it // to retry or open circuit breakers. - final Throwable responseCause = lastChild.responseCause(); + final Throwable responseCause = child.responseCause(); if (responseCause != null) { responseCause(responseCause); } } - // Update the available properties without adding a callback if the lastChild already has them. - if (lastChild.isAvailable(RequestLogProperty.RESPONSE_START_TIME)) { - startResponse(lastChild.responseStartTimeNanos(), lastChild.responseStartTimeMicros(), true); + // Update the available properties without adding a callback if the child already has them. + if (child.isAvailable(RequestLogProperty.RESPONSE_START_TIME)) { + startResponse(child.responseStartTimeNanos(), child.responseStartTimeMicros(), true); } else { - lastChild.whenAvailable(RequestLogProperty.RESPONSE_START_TIME) - .thenAccept(log -> startResponse(log.responseStartTimeNanos(), - log.responseStartTimeMicros(), true)); + child.whenAvailable(RequestLogProperty.RESPONSE_START_TIME) + .thenAccept(log -> startResponse(log.responseStartTimeNanos(), + log.responseStartTimeMicros(), true)); } - if (lastChild.isAvailable(RequestLogProperty.RESPONSE_FIRST_BYTES_TRANSFERRED_TIME)) { - final Long timeNanos = lastChild.responseFirstBytesTransferredTimeNanos(); + if (child.isAvailable(RequestLogProperty.RESPONSE_FIRST_BYTES_TRANSFERRED_TIME)) { + final Long timeNanos = child.responseFirstBytesTransferredTimeNanos(); if (timeNanos != null) { responseFirstBytesTransferred(timeNanos); } } else { - lastChild.whenAvailable(RequestLogProperty.RESPONSE_FIRST_BYTES_TRANSFERRED_TIME) - .thenAccept(log -> { - final Long timeNanos = log.responseFirstBytesTransferredTimeNanos(); - if (timeNanos != null) { - responseFirstBytesTransferred(timeNanos); - } - }); + child.whenAvailable(RequestLogProperty.RESPONSE_FIRST_BYTES_TRANSFERRED_TIME) + .thenAccept(log -> { + final Long timeNanos = log.responseFirstBytesTransferredTimeNanos(); + if (timeNanos != null) { + responseFirstBytesTransferred(timeNanos); + } + }); } - if (lastChild.isAvailable(RequestLogProperty.RESPONSE_HEADERS)) { - responseHeaders(lastChild.responseHeaders()); + if (child.isAvailable(RequestLogProperty.RESPONSE_HEADERS)) { + responseHeaders(child.responseHeaders()); } else { - lastChild.whenAvailable(RequestLogProperty.RESPONSE_HEADERS) - .thenAccept(log -> responseHeaders(log.responseHeaders())); + child.whenAvailable(RequestLogProperty.RESPONSE_HEADERS) + .thenAccept(log -> responseHeaders(log.responseHeaders())); } - if (lastChild.isAvailable(RequestLogProperty.RESPONSE_TRAILERS)) { - responseTrailers(lastChild.responseTrailers()); + if (child.isAvailable(RequestLogProperty.RESPONSE_TRAILERS)) { + responseTrailers(child.responseTrailers()); } else { - lastChild.whenAvailable(RequestLogProperty.RESPONSE_TRAILERS) - .thenAccept(log -> responseTrailers(log.responseTrailers())); + child.whenAvailable(RequestLogProperty.RESPONSE_TRAILERS) + .thenAccept(log -> responseTrailers(log.responseTrailers())); } - if (lastChild.isComplete()) { - propagateResponseEndData(lastChild); + if (child.isComplete()) { + propagateResponseEndData(child); } else { - lastChild.whenComplete().thenAccept(this::propagateResponseEndData); + child.whenComplete().thenAccept(this::propagateResponseEndData); } } From 6d6044a2f0171109daa95cc553e1131f1fdae3c9 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Wed, 18 Jun 2025 20:37:22 +0200 Subject: [PATCH 25/36] docs: comment on why retry task scheduling should be correct --- .../client/retry/AbstractRetryingClient.java | 388 +++++++++++------- .../armeria/client/retry/RetryScheduler.java | 308 +++++++++----- .../armeria/client/retry/RetryingClient.java | 1 - .../client/retry/RetrySchedulerTest.java | 112 +++-- 4 files changed, 516 insertions(+), 293 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java index 94eacc02e53..74d3f53529c 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java @@ -22,6 +22,8 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -41,6 +43,7 @@ import com.linecorp.armeria.common.annotation.Nullable; import com.linecorp.armeria.common.util.TimeoutMode; import com.linecorp.armeria.internal.client.ClientUtil; +import com.linecorp.armeria.internal.common.util.ReentrantShortLock; import io.netty.util.AsciiString; import io.netty.util.AttributeKey; @@ -85,14 +88,16 @@ public final O execute(ClientRequestContext ctx, I req) throws Exception { requireNonNull(config, "mapping.get() returned null"); final State state; + final ReentrantShortLock retryLock = new ReentrantShortLock(); if (ctx.responseTimeoutMillis() <= 0 || ctx.responseTimeoutMillis() == Long.MAX_VALUE) { - final RetryScheduler scheduler = new RetryScheduler(ctx.eventLoop()); - state = new State<>(config, scheduler); + final RetryScheduler scheduler = new RetryScheduler(retryLock, ctx.eventLoop()); + state = new State<>(retryLock, config, scheduler); } else { final long responseTimeoutTimeNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(ctx.responseTimeoutMillis()); - final RetryScheduler scheduler = new RetryScheduler(ctx.eventLoop(), responseTimeoutTimeNanos); - state = new State<>(config, scheduler, responseTimeoutTimeNanos); + final RetryScheduler scheduler = new RetryScheduler(retryLock, ctx.eventLoop(), + responseTimeoutTimeNanos); + state = new State<>(retryLock, config, scheduler, responseTimeoutTimeNanos); } state.whenRetryingComplete().handle((f, t) -> { @@ -119,13 +124,11 @@ protected abstract O doExecute(ClientRequestContext ctx, I req) throws Exception; /** - * This should be called when retrying is finished. + * todo(szymon): [doc]. */ protected static void completeRetryingIfNoPendingAttempts(ClientRequestContext ctx) { final State state = state(ctx); - synchronized (state) { - state.completeIfNoPendingAttempts(); - } + state.completeIfNoPendingAttempts(); } /** @@ -133,11 +136,8 @@ protected static void completeRetryingIfNoPendingAttempts(ClientRequestContext c */ protected static void completeRetryingExceptionally(ClientRequestContext ctx, Throwable cause) { - logger.debug("onRetryingCompleteExceptionally: {}", ctx, cause); final State state = state(ctx); - synchronized (state) { - state.completeExceptionally(cause); - } + state.completeExceptionally(cause); } /** @@ -189,35 +189,40 @@ protected void startRetryAttempt(ClientRequestContext ctx, requireNonNull(onAttemptAbortedHandler, "onAttemptAbortedHandler"); final State state = state(ctx); + final ReentrantLock retryLock = state.getLock(); - synchronized (state) { - if (isRetryingComplete(ctx)) { - onAttemptAbortedHandler.accept(attemptCtx, - new IllegalStateException("Retrying is already complete.")); - return; - } + retryLock.lock(); - state.startAttempt(attemptCtx); - state.whenRetryingComplete().handle((winningAttempt, cause) -> { - if (winningAttempt == null) { - // If the retrying is complete exceptionally, we need to call the onAttemptAbortedHandler. - onAttemptAbortedHandler.accept(winningAttempt.ctx(), cause); - return null; - } + if (isRetryingComplete(ctx)) { + // This really should not happen. However, let us call the abort handler to free + // potentially expensive resources here. + retryLock.unlock(); + onAttemptAbortedHandler.accept(attemptCtx, + new IllegalStateException("Retrying is already complete.")); + return; + } - final ClientRequestContext winningAttemptCtx = winningAttempt.ctx(); - final O winningAttemptRes = winningAttempt.res(); - assert winningAttemptRes != null; + state.startAttempt(attemptCtx); + retryLock.unlock(); + state.whenRetryingComplete().handleAsync((winningAttempt, cause) -> { + if (winningAttempt == null) { + // If the retrying is complete exceptionally, we need to call the onAttemptAbortedHandler. + onAttemptAbortedHandler.accept(winningAttempt.ctx(), cause); + return null; + } - if (attemptCtx == winningAttemptCtx) { - onWinHandler.accept(winningAttemptCtx, winningAttemptRes); - return null; - } + final ClientRequestContext winningAttemptCtx = winningAttempt.ctx(); + final O winningAttemptRes = winningAttempt.res(); + assert winningAttemptRes != null; - onAttemptAbortedHandler.accept(attemptCtx, cause); + if (attemptCtx == winningAttemptCtx) { + onWinHandler.accept(winningAttemptCtx, winningAttemptRes); return null; - }); - } + } + + onAttemptAbortedHandler.accept(attemptCtx, cause); + return null; + }, attemptCtx.eventLoop()); } /** @@ -232,15 +237,7 @@ protected void completeRetryAttempt(ClientRequestContext ctx, return; } - final State state = state(ctx); - - synchronized (state) { - state.completeAttempt(attemptCtx, attemptRes); - - if (isWinning) { - state.complete(); - } - } + state(ctx).completeAttempt(attemptCtx, attemptRes, isWinning); } /** @@ -251,9 +248,7 @@ protected static boolean isRetryingComplete(ClientRequestContext ctx) { final State state = state(ctx); - synchronized (state) { - return state.whenRetryingComplete().isDone(); - } + return state.isRetryingComplete(); } /** @@ -285,7 +280,9 @@ protected static void scheduleNextRetry(ClientRequestContext ctx, } final State state = state(ctx); - final RetryScheduler scheduler = state.scheduler(); + final ReentrantLock retryLock = state.getLock(); + + retryLock.lock(); final long nowTimeNanos = System.nanoTime(); @@ -296,6 +293,7 @@ protected static void scheduleNextRetry(ClientRequestContext ctx, if (state.timeoutForWholeRetryEnabled() && earliestRetryTimeNanos > state.responseTimeoutTimeNanos()) { + retryLock.unlock(); actionOnException.accept( new RetrySchedulingException( Type.DELAY_FROM_SERVER_EXCEEDS_RESPONSE_TIMEOUT)); @@ -304,10 +302,12 @@ protected static void scheduleNextRetry(ClientRequestContext ctx, // Even when we cannot schedule the retry task, we want to respect the // minimum retry delay from the server. + final RetryScheduler scheduler = state.scheduler(); earliestRetryTimeNanos = scheduler.addEarliestNextRetryTimeNanos(earliestRetryTimeNanos); final int attemptNoWithBackoff = state.numAttemptsSoFarWithBackoff(backoff); if (attemptNoWithBackoff < 0) { + retryLock.unlock(); scheduler.rescheduleCurrentRetryTaskIfTooEarly(); actionOnException.accept( new RetrySchedulingException(RetrySchedulingException.Type.NO_MORE_ATTEMPTS_IN_RETRY)); @@ -316,15 +316,18 @@ protected static void scheduleNextRetry(ClientRequestContext ctx, final long retryDelayMillis = backoff.nextDelayMillis(attemptNoWithBackoff); if (retryDelayMillis < 0) { + retryLock.unlock(); scheduler.rescheduleCurrentRetryTaskIfTooEarly(); actionOnException.accept( - new RetrySchedulingException(RetrySchedulingException.Type.NO_MORE_ATTEMPTS_IN_BACKOFF)); + new RetrySchedulingException( + RetrySchedulingException.Type.NO_MORE_ATTEMPTS_IN_BACKOFF)); return; } final long retryTimeNanos = Math.max(nowTimeNanos + TimeUnit.MILLISECONDS.toNanos(retryDelayMillis), earliestRetryTimeNanos); if (state.timeoutForWholeRetryEnabled() && retryTimeNanos > state.responseTimeoutTimeNanos()) { + retryLock.unlock(); scheduler.rescheduleCurrentRetryTaskIfTooEarly(); // This cannot be the minimum delay from the server, because we already checked it above. actionOnException.accept( @@ -333,36 +336,50 @@ protected static void scheduleNextRetry(ClientRequestContext ctx, return; } - synchronized (state) { - state.startRetryTask(); - } - scheduler.schedule(() -> { + final AtomicBoolean exceptionDuringScheduling = new AtomicBoolean(true); + state.startRetryTask(); + scheduler.schedule(retryLock0 -> { + assert retryLock == retryLock0; + assert retryLock.isHeldByCurrentThread(); + assert retryLock.getHoldCount() == 1; - final int thisAttemptNo; - // *, see comment above. - synchronized (state) { - if (isRetryingComplete(ctx)) { - state.completeRetryTask(); - actionOnException.accept(new RetrySchedulingException(Type.RETRYING_ALREADY_COMPLETED)); - return; - } - - thisAttemptNo = state.acquireAttemptNoWithCurrentBackoff(backoff); + if (isRetryingComplete(ctx)) { + state.completeRetryTask(); + retryLock.unlock(); + actionOnException.accept(new RetrySchedulingException(Type.RETRYING_ALREADY_COMPLETED)); + return; } - retryTask.accept(thisAttemptNo); + final int thisAttemptNo = state.acquireAttemptNoWithCurrentBackoff(backoff); + assert thisAttemptNo >= 1; + + retryLock.unlock(); - synchronized (state) { + try { + assert !retryLock.isHeldByCurrentThread(); + retryTask.accept(thisAttemptNo); + } finally { state.completeRetryTask(); } completeRetryingIfNoPendingAttempts(ctx); }, retryTimeNanos, earliestRetryTimeNanos, cause -> { - synchronized (state) { - state.completeRetryTask(); + state.completeRetryTask(); + + if (exceptionDuringScheduling.get()) { + assert state(ctx).getLock().isHeldByCurrentThread(); + retryLock.unlock(); } + actionOnException.accept(cause); + + if (exceptionDuringScheduling.get()) { + retryLock.lock(); + } }); + + exceptionDuringScheduling.set(false); + retryLock.unlock(); } // todo(szymon): improve documentation for this method @@ -448,6 +465,8 @@ public void setRes(O res) { } } + private final ReentrantLock lock; + private boolean isRetryingComplete; private final RetryConfig config; private final long deadlineNanos; private final boolean isTimeoutEnabled; @@ -475,7 +494,9 @@ public void setRes(O res) { // Starting with 1 private int totalAttemptNo; - State(RetryConfig config, RetryScheduler retryScheduler) { + State(ReentrantLock retryingLock, RetryConfig config, RetryScheduler retryScheduler) { + lock = retryingLock; + isRetryingComplete = false; this.config = config; this.retryScheduler = retryScheduler; totalAttemptNo = 1; @@ -484,7 +505,10 @@ public void setRes(O res) { isTimeoutEnabled = false; } - State(RetryConfig config, RetryScheduler retryScheduler, long responseTimeoutTimeNanos) { + State(ReentrantLock retryingLock, RetryConfig config, RetryScheduler retryScheduler, + long responseTimeoutTimeNanos) { + lock = retryingLock; + isRetryingComplete = false; this.config = config; this.retryScheduler = retryScheduler; totalAttemptNo = 1; @@ -497,6 +521,10 @@ RetryScheduler scheduler() { return retryScheduler; } + ReentrantLock getLock() { + return lock; + } + /** * Returns the smaller value between {@link RetryConfig#responseTimeoutMillisForEachAttempt()} and * remaining {@link #responseTimeoutMillisForAttempt}. This method is thread-safe. @@ -505,22 +533,28 @@ RetryScheduler scheduler() { * -1 if the elapsed time from the first request has passed {@code responseTimeoutMillis} */ long responseTimeoutMillisForAttempt() { - if (!timeoutForWholeRetryEnabled()) { - return config.responseTimeoutMillisForEachAttempt(); - } + lock.lock(); - final long actualResponseTimeoutMillis = responseTimeoutMillis(); + try { + if (!timeoutForWholeRetryEnabled()) { + return config.responseTimeoutMillisForEachAttempt(); + } - // Consider 0 or less than 0 of actualResponseTimeoutMillis as timed out. - if (actualResponseTimeoutMillis <= 0) { - return -1; - } + final long actualResponseTimeoutMillis = responseTimeoutMillis(); - if (config.responseTimeoutMillisForEachAttempt() > 0) { - return Math.min(config.responseTimeoutMillisForEachAttempt(), actualResponseTimeoutMillis); - } + // Consider 0 or less than 0 of actualResponseTimeoutMillis as timed out. + if (actualResponseTimeoutMillis <= 0) { + return -1; + } - return actualResponseTimeoutMillis; + if (config.responseTimeoutMillisForEachAttempt() > 0) { + return Math.min(config.responseTimeoutMillisForEachAttempt(), actualResponseTimeoutMillis); + } + + return actualResponseTimeoutMillis; + } finally { + lock.unlock(); + } } boolean timeoutForWholeRetryEnabled() { @@ -539,121 +573,193 @@ long responseTimeoutTimeNanos() { } void startAttempt(ClientRequestContext attemptCtx) { - checkState(!retryingCompleteFuture.isDone()); - - logger.debug("Attempt started: {}, num attempts pending = {}, num retry task scheduled = {}", - attemptCtx, activeAttempts.size(), numScheduledAttempts); - checkState(!activeAttempts.containsKey(attemptCtx), - "Attempt %s already exists in active attempts.", attemptCtx); - final Attempt attempt = new Attempt<>(attemptCtx, null); - activeAttempts.put(attempt.ctx(), attempt); + lock.lock(); + try { + checkState(!isRetryingComplete()); + checkState(!activeAttempts.containsKey(attemptCtx), + "Attempt %s already is already pending.", attemptCtx); + final Attempt attempt = new Attempt<>(attemptCtx, null); + activeAttempts.put(attempt.ctx(), attempt); + } finally { + lock.unlock(); + } } - void completeAttempt(ClientRequestContext attemptCtx, O attemptRes) { - if (retryingCompleteFuture.isDone()) { - return; - } + void completeAttempt(ClientRequestContext attemptCtx, O attemptRes, boolean isWinning) { + lock.lock(); + try { + if (isRetryingComplete()) { + lock.unlock(); + return; + } - checkState(activeAttempts.containsKey(attemptCtx), - "Attempt %s not found in active attempts: %s", attemptCtx, activeAttempts); + checkState(activeAttempts.containsKey(attemptCtx), + "Attempt %s is not registered as pending.", attemptCtx); - lastAttempt = activeAttempts.get(attemptCtx); - lastAttempt.setRes(attemptRes); - activeAttempts.remove(attemptCtx); + lastAttempt = activeAttempts.get(attemptCtx); + lastAttempt.setRes(attemptRes); + activeAttempts.remove(attemptCtx); + } catch (Exception e) { + lock.unlock(); + throw e; + } - completeIfNoPendingAttempts(); + if (isWinning) { + // If this is the winning attempt, we can complete the retrying. + // transfers ownership of current lock + complete0(); + } else { + // transfers ownership of current lock + completeIfNoPendingAttempts0(); + } } void startRetryTask() { // This can get (temporarily) above max total attempts as there is a moment between scheduling // a new retry task and cancelling the previous one. This is because startRetryTask() needs to be // increment before we call the RetryScheduler.schedule() method. + lock.lock(); numScheduledAttempts++; + lock.unlock(); } void completeRetryTask() { - checkState(numScheduledAttempts > 0); - numScheduledAttempts--; + lock.lock(); + try { + checkState(numScheduledAttempts > 0); + numScheduledAttempts--; + } finally { + lock.unlock(); + } } int numAttemptsSoFarWithBackoff(Backoff backoff) { - if (totalAttemptNo >= config.maxTotalAttempts()) { - return -1; - } + lock.lock(); + try { + if (totalAttemptNo >= config.maxTotalAttempts()) { + return -1; + } - if (lastBackoff != backoff) { - return 1; - } + if (lastBackoff != backoff) { + return 1; + } - return currentAttemptNoWithLastBackoff; + return currentAttemptNoWithLastBackoff; + } finally { + lock.unlock(); + } } int acquireAttemptNoWithCurrentBackoff(Backoff backoff) { - checkState(!retryingCompleteFuture.isDone()); - checkState((totalAttemptNo + 1) <= config.maxTotalAttempts(), - "Exceeded the maximum number of attempts: %s", config.maxTotalAttempts()); + lock.lock(); + try { + checkState(!isRetryingComplete()); + checkState((totalAttemptNo + 1) <= config.maxTotalAttempts(), + "Exceeded the maximum number of attempts: %s", config.maxTotalAttempts()); - totalAttemptNo++; + totalAttemptNo++; - if (lastBackoff != backoff) { - lastBackoff = backoff; - currentAttemptNoWithLastBackoff = 1; - } + if (lastBackoff != backoff) { + lastBackoff = backoff; + currentAttemptNoWithLastBackoff = 1; + } - currentAttemptNoWithLastBackoff++; + currentAttemptNoWithLastBackoff++; - return totalAttemptNo; + return totalAttemptNo; + } finally { + lock.unlock(); + } } int numPendingAttempts() { - return activeAttempts.size() + (numScheduledAttempts > 0 ? 1 : 0); + lock.lock(); + try { + return activeAttempts.size() + (numScheduledAttempts > 0 ? 1 : 0); + } finally { + lock.unlock(); + } } void completeIfNoPendingAttempts() { - logger.debug( - "completeIfNoPendingAttempts: {}, num attempts pending = {}, num retry task scheduled = {}", - lastAttempt != null ? lastAttempt.ctx() : "null", numPendingAttempts(), - numScheduledAttempts); - if (retryingCompleteFuture.isDone()) { - logger.debug("completeIfNoPendingAttempts: Retrying already completed: {}", - lastAttempt != null ? lastAttempt.ctx() : "null"); + lock.lock(); + + if (isRetryingComplete()) { + lock.unlock(); return; } + // transfers ownership of current lock + completeIfNoPendingAttempts0(); + } + + // transfers ownership of current lock + private void completeIfNoPendingAttempts0() { + assert lock.isHeldByCurrentThread(); + assert !isRetryingComplete(); + if (numPendingAttempts() > 0) { - logger.debug( - "completeIfNoPendingAttempts: Have pending attempts," + - " num attempts pending = {}, num retry task scheduled = {}", - numPendingAttempts(), numScheduledAttempts - ); + lock.unlock(); return; } - // If there are no pending attempts, we can complete the retrying. - complete(); + // transfer ownership of current lock + complete0(); } - void complete() { - if (retryingCompleteFuture.isDone()) { - return; - } + // takes ownership of current lock + private void complete0() { + assert lock.isHeldByCurrentThread(); + assert !isRetryingComplete(); + + final boolean hasLastAttempt = lastAttempt != null; - if (lastAttempt == null) { - completeExceptionally(new IllegalStateException("Completed retrying without any attempts.")); + if (!hasLastAttempt) { + // takes ownership of current lock + completeExceptionally0( + new IllegalStateException("Completed retrying without any attempts.")); } else { - logger.debug("Completing..."); + isRetryingComplete = true; + // We do not want to call the handlers with the lock. + lock.unlock(); retryingCompleteFuture.complete(lastAttempt); } } void completeExceptionally(Throwable cause) { - if (retryingCompleteFuture.isDone()) { + lock.lock(); + + if (isRetryingComplete()) { + lock.unlock(); return; } + // takes ownership of current lock + completeExceptionally0(cause); + } + + // takes ownership of current lock + private void completeExceptionally0(Throwable cause) { + assert lock.isHeldByCurrentThread(); + assert !isRetryingComplete(); + + isRetryingComplete = true; + + // We do not want to call the handlers with the lock. + lock.unlock(); + retryingCompleteFuture.completeExceptionally(cause); } + boolean isRetryingComplete() { + try { + lock.lock(); + return isRetryingComplete; + } finally { + lock.unlock(); + } + } + CompletableFuture> whenRetryingComplete() { return retryingCompleteFuture; } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java index 4df318c0b3e..be191b4d662 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java @@ -21,6 +21,7 @@ import static java.util.Objects.requireNonNull; import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; import org.slf4j.Logger; @@ -34,54 +35,53 @@ class RetryScheduler { private static class RetryTaskHandle { - enum State { - SCHEDULED, + enum CancellationReason { OVERTAKEN, - RESCHEDULED + RESCHEDULED, + SHUTDOWN } - private final Runnable retryTask; + private final Consumer retryTask; private final ScheduledFuture scheduledFuture; - private State state; + private @Nullable CancellationReason cancellationReason; + private final long retryTaskId; private final Consumer<@Nullable ? super Throwable> exceptionHandler; private final long retryTimeNanos; - RetryTaskHandle(ScheduledFuture scheduledFuture, Runnable retryTask, + RetryTaskHandle(long retryTaskId, ScheduledFuture scheduledFuture, + Consumer retryTask, long retryTimeNanos, Consumer<@Nullable ? super Throwable> exceptionHandler) { + this.retryTaskId = retryTaskId; this.retryTask = retryTask; this.scheduledFuture = scheduledFuture; this.exceptionHandler = exceptionHandler; this.retryTimeNanos = retryTimeNanos; - state = State.SCHEDULED; + cancellationReason = null; } - Runnable getRetryTaskRunnable() { + Consumer getRetryTaskRunnable() { return retryTask; } - long retryTimeNanos() { - return retryTimeNanos; - } - - void markRescheduled() { - assert state == State.SCHEDULED; - state = State.RESCHEDULED; + long id() { + return retryTaskId; } - void markOvertaken() { - assert state == State.SCHEDULED; - state = State.OVERTAKEN; + long retryTimeNanos() { + return retryTimeNanos; } - boolean isRescheduled() { - return state == State.RESCHEDULED; + void setCancellationReason(CancellationReason cancellationReason) { + checkState(this.cancellationReason == null, "cancellationReason"); + this.cancellationReason = cancellationReason; } - boolean isOvertaken() { - return state == State.OVERTAKEN; + @Nullable + CancellationReason getCancellationReason() { + return cancellationReason; } Consumer getExceptionHandler() { @@ -101,10 +101,13 @@ ScheduledFuture getFuture() { private static final Logger logger = LoggerFactory.getLogger(RetryScheduler.class); + // for locking + private final ReentrantLock retryLock; private final EventLoop eventLoop; private final long latestNextRetryTimeNanos; private long earliestNextRetryTimeNanos; + private long retryTaskId; // The retry task that is about to be executed next. // It is possible that the delay of this task is shorter than the `earliestNextRetryTimeNanos`, @@ -112,26 +115,30 @@ ScheduledFuture getFuture() { // `rescheduleCurrentRetryTaskIfTooEarly`. private @Nullable RetryTaskHandle currentRetryTask; - RetryScheduler(EventLoop eventLoop) { - this(eventLoop, Long.MAX_VALUE); + RetryScheduler(ReentrantLock retryLock, EventLoop eventLoop) { + this(retryLock, eventLoop, Long.MAX_VALUE); } - RetryScheduler(EventLoop eventLoop, long latestNextRetryTimeNanos) { + RetryScheduler(ReentrantLock retryLock, EventLoop eventLoop, long latestNextRetryTimeNanos) { + this.retryLock = requireNonNull(retryLock, "retryLock"); this.eventLoop = requireNonNull(eventLoop, "eventLoop"); currentRetryTask = null; earliestNextRetryTimeNanos = Long.MIN_VALUE; this.latestNextRetryTimeNanos = latestNextRetryTimeNanos; + retryTaskId = 0; } - public synchronized void schedule(Runnable retryTask, long nextRetryTimeNanos, - long earliestNextRetryTimeNanos, - Consumer exceptionHandler) { + public void schedule(Consumer retryTask, long nextRetryTimeNanos, + long earliestNextRetryTimeNanos, + Consumer exceptionHandler) { requireNonNull(retryTask, "retryTask"); checkArgument(nextRetryTimeNanos >= earliestNextRetryTimeNanos, "nextRetryTimeNanos: %s (expected: >= %s)", nextRetryTimeNanos, earliestNextRetryTimeNanos); requireNonNull(exceptionHandler, "exceptionHandler"); + retryLock.lock(); + addEarliestNextRetryTimeNanos(earliestNextRetryTimeNanos); nextRetryTimeNanos = Math.max(nextRetryTimeNanos, this.earliestNextRetryTimeNanos); @@ -139,78 +146,115 @@ public synchronized void schedule(Runnable retryTask, long nextRetryTimeNanos, // currently scheduled retry task (even not by us in this method). if (currentRetryTask == null || nextRetryTimeNanos < Math.max(this.earliestNextRetryTimeNanos, currentRetryTask.retryTimeNanos())) { + // takes ownership of the acquired retry lock scheduleNextRetryTask(retryTask, nextRetryTimeNanos, exceptionHandler, false); } else { // Make sure the current retry task is not scheduled too early. - rescheduleCurrentRetryTaskIfTooEarly(); + // takes ownership of the acquired retry lock + rescheduleCurrentRetryTaskIfTooEarly0(); exceptionHandler.accept(new RetrySchedulingException( RetrySchedulingException.Type.RETRY_TASK_OVERTAKEN)); } } - private synchronized boolean cancelCurrentRetryTask(boolean cancelForRescheduling) { - if (currentRetryTask != null) { - final ScheduledFuture retryTaskFuture = currentRetryTask.getFuture(); + private boolean cancelCurrentRetryTask(RetryTaskHandle.CancellationReason cancellationReason) { + retryLock.lock(); - if (cancelForRescheduling) { - currentRetryTask.markRescheduled(); - } else { - currentRetryTask.markOvertaken(); - } - if (!retryTaskFuture.cancel(false)) { - return false; - } else { - clearCurrentRetryTask(); - return true; + try { + if (currentRetryTask != null) { + final ScheduledFuture retryTaskFuture = currentRetryTask.getFuture(); + + currentRetryTask.setCancellationReason(cancellationReason); + // This should not invoke the user-defined exception handler with our lock as + // we marked the task accordingly. + if (!retryTaskFuture.cancel(false)) { + return false; + } else { + clearCurrentRetryTask(); + return true; + } } - } - return true; + return true; + } finally { + retryLock.unlock(); + } } - private synchronized void handleRetryTaskCompletion(RetryTaskHandle retryTaskHandle) { - final ScheduledFuture retryTaskFuture = retryTaskHandle.getFuture(); - assert retryTaskFuture.isDone(); + private void handleRetryTaskCompletion(RetryTaskHandle retryTaskHandle) { + retryLock.lock(); - if (currentRetryTask == retryTaskHandle) { - clearCurrentRetryTask(); - } + try { + final ScheduledFuture retryTaskFuture = retryTaskHandle.getFuture(); + assert retryTaskFuture.isDone(); - if (retryTaskFuture.isCancelled()) { - if (retryTaskHandle.isRescheduled()) { - return; + if (currentRetryTask == retryTaskHandle) { + clearCurrentRetryTask(); } - if (retryTaskHandle.isOvertaken()) { + if (retryTaskFuture.isCancelled()) { + final RetryTaskHandle.CancellationReason cancellationReason = + retryTaskHandle.getCancellationReason(); + + if (cancellationReason == RetryTaskHandle.CancellationReason.RESCHEDULED) { + return; + } + + if (cancellationReason == RetryTaskHandle.CancellationReason.OVERTAKEN) { + retryTaskHandle.getExceptionHandler().accept( + new RetrySchedulingException(Type.RETRY_TASK_OVERTAKEN) + ); + return; + } + + if (cancellationReason == RetryTaskHandle.CancellationReason.SHUTDOWN) { + retryTaskHandle.getExceptionHandler().accept( + new RetrySchedulingException(Type.RETRYING_ALREADY_COMPLETED) + ); + return; + } + + assert cancellationReason == null; + + // The retry task was cancelled by the user, not by the scheduler. retryTaskHandle.getExceptionHandler().accept( - new RetrySchedulingException(Type.RETRY_TASK_OVERTAKEN) - ); + new RetrySchedulingException(Type.RETRY_TASK_CANCELLED)); return; } - // The retry task was cancelled by the user, not by the scheduler. - retryTaskHandle.getExceptionHandler().accept( - new RetrySchedulingException(Type.RETRY_TASK_CANCELLED)); - return; + if (!retryTaskFuture.isSuccess()) { + retryTaskHandle.getExceptionHandler().accept(retryTaskFuture.cause()); + } + } finally { + retryLock.unlock(); } + } - if (!retryTaskFuture.isSuccess()) { - retryTaskHandle.getExceptionHandler().accept(retryTaskFuture.cause()); + private void clearCurrentRetryTask() { + retryLock.lock(); + try { + currentRetryTask = null; + earliestNextRetryTimeNanos = Long.MIN_VALUE; + } finally { + retryLock.unlock(); } } - private synchronized void clearCurrentRetryTask() { - currentRetryTask = null; - earliestNextRetryTimeNanos = Long.MIN_VALUE; - } + // takes ownership of the acquired retry lock + private void scheduleNextRetryTask(Consumer retryRunnable, long retryTimeNanos, + Consumer exceptionHandler, + boolean isReschedule) { - private synchronized void scheduleNextRetryTask(Runnable retryRunnable, long retryTimeNanos, - Consumer exceptionHandler, - boolean isReschedule) { + assert retryLock.isHeldByCurrentThread(); assert earliestNextRetryTimeNanos <= retryTimeNanos; - if (!cancelCurrentRetryTask(isReschedule)) { + if (!cancelCurrentRetryTask( + isReschedule ? + RetryTaskHandle.CancellationReason.RESCHEDULED + : RetryTaskHandle.CancellationReason.OVERTAKEN + )) { + retryLock.unlock(); exceptionHandler.accept(new IllegalStateException("Current retry task could not be cancelled.")); return; } @@ -226,49 +270,57 @@ private synchronized void scheduleNextRetryTask(Runnable retryRunnable, long ret delayNanos = retryTimeNanos - nowNanos; } + final long thisRetryId = retryTaskId++; + final Runnable wrappedRetryRunnable = () -> { - logger.debug("Retry task starting. Resetting..."); - // todo(szymon): do sanity check that we are clearing this task (very bad otherwise). - - synchronized (this) { - if (currentRetryTask == null) { - clearCurrentRetryTask(); - exceptionHandler.accept( - new IllegalStateException( - "Currently executing retry task was cleared." + - " Likely a bug in the retry scheduler." - ) - ); - return; - } - - final long taskRunTimeNanos = System.nanoTime(); - - // max to be robust against overflows. - if (Math.max(taskRunTimeNanos, taskRunTimeNanos + RESCHEDULING_OVERTAKING_TOLERANCE_NANOS) < - earliestNextRetryTimeNanos) { - // We are too early to execute the retry task. - logger.debug("Retry task is too early. Rescheduling..."); - - final Runnable currentRetryRunnable = currentRetryTask.getRetryTaskRunnable(); - final Consumer currentExceptionHandler = - currentRetryTask.getExceptionHandler(); - - clearCurrentRetryTask(); - // do not invoke rescheduleCurrentRetryTaskIfTooEarly here as it will not be able - // to cancel us. - scheduleNextRetryTask(currentRetryRunnable, - earliestNextRetryTimeNanos, - currentExceptionHandler, false); - return; - } + retryLock.lock(); + assert retryLock.getHoldCount() == 1; + + // Let be defensive here. + if (currentRetryTask == null) { + clearCurrentRetryTask(); + retryLock.unlock(); + exceptionHandler.accept(new RetrySchedulingException(Type.RETRY_TASK_CANCELLED)); + return; + } + + if (currentRetryTask.id() != thisRetryId) { + retryLock.unlock(); + exceptionHandler.accept(new RetrySchedulingException(Type.RETRY_TASK_CANCELLED)); + return; + } + + final long taskRunTimeNanos = System.nanoTime(); + + // max to be robust against overflows. + if (Math.max(taskRunTimeNanos, taskRunTimeNanos + RESCHEDULING_OVERTAKING_TOLERANCE_NANOS) < + earliestNextRetryTimeNanos) { + + final Consumer currentRetryRunnable = + currentRetryTask.getRetryTaskRunnable(); + final Consumer currentExceptionHandler = + currentRetryTask.getExceptionHandler(); + + clearCurrentRetryTask(); + // do not invoke rescheduleCurrentRetryTaskIfTooEarly here as it will not be able + // to cancel us. + // takes ownership of the acquired retry lock + scheduleNextRetryTask(currentRetryRunnable, + earliestNextRetryTimeNanos, + currentExceptionHandler, false); + return; } clearCurrentRetryTask(); - retryRunnable.run(); + assert retryLock.isHeldByCurrentThread(); + assert retryLock.getHoldCount() == 1; + + // Run this with the lock. Expected to release the retry lock after consuming an attempt. + retryRunnable.accept(retryLock); }; + // todo(szymon): must not be executed immediately in this thread. //noinspection unchecked final ScheduledFuture nextRetryTaskFuture = (ScheduledFuture) eventLoop.schedule(wrappedRetryRunnable, delayNanos, @@ -276,27 +328,47 @@ private synchronized void scheduleNextRetryTask(Runnable retryRunnable, long ret // We are passing in the original to avoid multiple wrapping in case of the retry task being // rescheduled multiple times. - final RetryTaskHandle nextRetryTask = new RetryTaskHandle(nextRetryTaskFuture, retryRunnable, + final RetryTaskHandle nextRetryTask = new RetryTaskHandle(thisRetryId, nextRetryTaskFuture, + retryRunnable, retryTimeNanos, exceptionHandler); nextRetryTaskFuture.addListener(f -> handleRetryTaskCompletion(nextRetryTask)); currentRetryTask = nextRetryTask; + retryLock.unlock(); } catch (Throwable t) { + retryLock.unlock(); exceptionHandler.accept(t); } } - public synchronized long addEarliestNextRetryTimeNanos(long earliestNextRetryTimeNanos) { - checkState(earliestNextRetryTimeNanos <= latestNextRetryTimeNanos); - this.earliestNextRetryTimeNanos = Math.max(this.earliestNextRetryTimeNanos, - earliestNextRetryTimeNanos); + public long addEarliestNextRetryTimeNanos(long earliestNextRetryTimeNanos) { + retryLock.lock(); + + try { + checkState(earliestNextRetryTimeNanos <= latestNextRetryTimeNanos); + this.earliestNextRetryTimeNanos = Math.max(this.earliestNextRetryTimeNanos, + earliestNextRetryTimeNanos); + + return this.earliestNextRetryTimeNanos; + } finally { + retryLock.unlock(); + } + } - return this.earliestNextRetryTimeNanos; + // takes ownership of the acquired retry lock (if any) + public void rescheduleCurrentRetryTaskIfTooEarly() { + retryLock.lock(); + // takes ownership of the acquired retry lock + rescheduleCurrentRetryTaskIfTooEarly0(); } - public synchronized void rescheduleCurrentRetryTaskIfTooEarly() { + // takes ownership of the acquired retry lock + private void rescheduleCurrentRetryTaskIfTooEarly0() { + assert retryLock.isHeldByCurrentThread(); + if (currentRetryTask == null) { + retryLock.unlock(); return; } @@ -308,13 +380,21 @@ public synchronized void rescheduleCurrentRetryTaskIfTooEarly() { // Current retry task is going to be executed before the earliestNextRetryTimeNanos so // we need to reschedule it. + // Takes ownership of the acquired lock. scheduleNextRetryTask(currentRetryTask.getRetryTaskRunnable(), earliestNextRetryTimeNanos, currentRetryTask.getExceptionHandler(), true); + } else { + retryLock.unlock(); } } public boolean shutdown() { - return cancelCurrentRetryTask(true); + retryLock.lock(); + try { + return cancelCurrentRetryTask(RetryTaskHandle.CancellationReason.SHUTDOWN); + } finally { + retryLock.unlock(); + } } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index 06a3ac8a6e3..f0969dcac97 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -407,7 +407,6 @@ private void doExecute0(RetryingContext retryingContext, int attemptNo) { private static void handleExceptionAfterScheduling( RetryingContext retryingContext, Throwable cause) { - logger.debug("handleExceptionAfterScheduling", cause); if (cause instanceof RetrySchedulingException) { switch (((RetrySchedulingException) cause).getType()) { case RETRYING_ALREADY_COMPLETED: diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java index 3c72575980e..f1f4fe6674a 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java @@ -19,7 +19,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; @@ -30,6 +29,7 @@ import java.util.List; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -45,6 +45,7 @@ import com.google.common.collect.ImmutableList; import com.linecorp.armeria.client.retry.RetrySchedulingException.Type; +import com.linecorp.armeria.internal.common.util.ReentrantShortLock; import com.linecorp.armeria.internal.testing.AnticipatedException; import io.netty.channel.DefaultEventLoop; @@ -55,18 +56,51 @@ class RetrySchedulerTest { private static final long SCHEDULING_TOLERANCE_MILLIS = TimeUnit.NANOSECONDS.toMillis( SCHEDULING_TOLERANCE_NANOS); + private ReentrantLock lock; + private Consumer dummyRetryTask; + private Consumer dummyThrowingRetryTask; + private EventLoop eventLoop; private RetryScheduler scheduler; + private static class SpyableRetryTask implements Consumer { + private final Consumer delegate; + + SpyableRetryTask(Consumer delegate) { + this.delegate = delegate; + } + + @Override + public void accept(ReentrantLock retryLock) { + delegate.accept(retryLock); + } + } + @BeforeEach void setUp() { eventLoop = spy(new DefaultEventLoop()); - scheduler = new RetryScheduler(eventLoop); + lock = new ReentrantShortLock(); + dummyRetryTask = new SpyableRetryTask(retryLock -> { + assertThat(retryLock).isEqualTo(lock); + assertThat(retryLock.isHeldByCurrentThread()).isTrue(); + assertThat(retryLock.getHoldCount()).isOne(); + retryLock.unlock(); + }); + + dummyThrowingRetryTask = new SpyableRetryTask(retryLock -> { + assertThat(retryLock).isEqualTo(lock); + assertThat(retryLock.isHeldByCurrentThread()).isTrue(); + assertThat(retryLock.getHoldCount()).isOne(); + retryLock.unlock(); + throw new AnticipatedException(); + }); + scheduler = new RetryScheduler(lock, eventLoop); } @AfterEach void tearDown() throws Exception { + assertThat(lock.isLocked()).isFalse(); assertThat(scheduler.shutdown()).isTrue(); eventLoop.shutdownGracefully(); } @@ -91,7 +125,7 @@ void testScheduleTask(int taskDelayMs, int minDelayMs, int prevDelayMs, int expe } // Schedule the task - final Runnable task = mock(Runnable.class); + final Consumer task = spy(dummyRetryTask); final Consumer exceptionHandler = mock(Consumer.class); scheduler.schedule(task, taskScheduledTimeNanos, newEarliestNextRetryTimeNanos, exceptionHandler); @@ -102,7 +136,7 @@ void testScheduleTask(int taskDelayMs, int minDelayMs, int prevDelayMs, int expe Thread.sleep(expectedDelayMs + SCHEDULING_TOLERANCE_MILLIS); - verify(task, times(1)).run(); + verify(task, times(1)).accept(lock); verifyNoMoreInteractions(exceptionHandler); verifyEventLoopScheduleCalls( EventLoopScheduleCall.of(now, expectedScheduledTimeNanos) @@ -135,7 +169,7 @@ private static Stream scheduleTaskParameters() { @Test void testScheduleTaskInThePast() throws Exception { - final Runnable task = mock(Runnable.class); + final Consumer task = spy(dummyRetryTask); final long taskSchedulingTime = System.nanoTime(); final long expectedTaskRunTime = taskSchedulingTime; final Consumer exceptionHandler = mock(Consumer.class); @@ -148,13 +182,13 @@ void testScheduleTaskInThePast() throws Exception { Thread.sleep(SCHEDULING_TOLERANCE_MILLIS); - verify(task, times(1)).run(); + verify(task, times(1)).accept(lock); verifyNoMoreInteractions(exceptionHandler); verifyEventLoopScheduleCalls( EventLoopScheduleCall.of(taskSchedulingTime, expectedTaskRunTime) ); - final Runnable task2 = mock(Runnable.class); + final Consumer task2 = spy(dummyRetryTask); final long task2SchedulingTime = System.nanoTime(); final long expectedTask2RunTime = task2SchedulingTime; final Consumer task2ExceptionHandler = mock(Consumer.class); @@ -168,7 +202,7 @@ void testScheduleTaskInThePast() throws Exception { Thread.sleep(SCHEDULING_TOLERANCE_MILLIS); - verify(task2, times(1)).run(); + verify(task2, times(1)).accept(lock); verifyNoMoreInteractions(task2ExceptionHandler); verifyEventLoopScheduleCalls( EventLoopScheduleCall.of(taskSchedulingTime, expectedTaskRunTime), @@ -178,7 +212,7 @@ void testScheduleTaskInThePast() throws Exception { @Test void testFailingScheduleWithInvalidArguments() { - final Runnable task = mock(Runnable.class); + final Consumer task = spy(dummyRetryTask); final Consumer exceptionHandler = mock(Consumer.class); assertThatThrownBy(() -> scheduler.schedule(null, 200, 200, @@ -198,8 +232,7 @@ void testFailingScheduleWithInvalidArguments() { @Test void testScheduleTaskWithException() throws Exception { - final Runnable task1 = mock(Runnable.class); - doThrow(new AnticipatedException()).when(task1).run(); + final Consumer task1 = spy(dummyThrowingRetryTask); final long task1SchedulingTime = System.nanoTime(); final long expectedTask1RunTime = task1SchedulingTime + TimeUnit.MILLISECONDS.toNanos(100); @@ -212,7 +245,7 @@ void testScheduleTaskWithException() throws Exception { Thread.sleep(100 + SCHEDULING_TOLERANCE_MILLIS); - verify(task1, times(1)).run(); + verify(task1, times(1)).accept(lock); final ArgumentCaptor exceptionCaptor = ArgumentCaptor.forClass(Throwable.class); verify(task1ExceptionHandler, times(1)).accept(exceptionCaptor.capture()); assertThat(exceptionCaptor.getValue()).isInstanceOf(AnticipatedException.class); @@ -220,14 +253,14 @@ void testScheduleTaskWithException() throws Exception { @Test void testEarlierRetryTaskOvertakesLaterOne() throws Exception { - final Runnable task1 = mock(Runnable.class); + final Consumer task1 = spy(dummyRetryTask); final long task1SchedulingTime = System.nanoTime(); final long expectedTask1RunTime = task1SchedulingTime + TimeUnit.MILLISECONDS.toNanos(200); final Consumer task1ExceptionHandler = mock(Consumer.class); scheduler.schedule(task1, expectedTask1RunTime, Long.MIN_VALUE, task1ExceptionHandler); - final Runnable task2 = mock(Runnable.class); + final Consumer task2 = spy(dummyRetryTask); final long task2SchedulingTime = System.nanoTime(); final long expectedTask2RunTime = task2SchedulingTime + TimeUnit.MILLISECONDS.toNanos(100); final Consumer task2ExceptionHandler = mock(Consumer.class); @@ -235,8 +268,8 @@ void testEarlierRetryTaskOvertakesLaterOne() throws Exception { Thread.sleep(200 + SCHEDULING_TOLERANCE_MILLIS); - verify(task1, times(0)).run(); - verify(task2, times(1)).run(); + verify(task1, times(0)).accept(lock); + verify(task2, times(1)).accept(lock); verifyExceptionHandlerCatchedSchedulingException(task1ExceptionHandler, RetrySchedulingException.Type.RETRY_TASK_OVERTAKEN); @@ -256,13 +289,13 @@ void testMultipleRetryTasksBeingOvertaken() throws Exception { // next task is scheduled earlier. // - Only the last task (100ms) should be executed. - final List tasks = new ArrayList<>(); + final List> tasks = new ArrayList<>(); final List schedulingTimes = new ArrayList<>(); final List expectedRunTimes = new ArrayList<>(); final List> exceptionHandlers = new ArrayList<>(); for (int taskNo = 0; taskNo < 10; taskNo++) { - final Runnable task = mock(Runnable.class); + final Consumer task = spy(dummyRetryTask); tasks.add(task); final Consumer exceptionHandler = mock(Consumer.class); exceptionHandlers.add(exceptionHandler); @@ -279,8 +312,8 @@ void testMultipleRetryTasksBeingOvertaken() throws Exception { Thread.sleep(1000 + SCHEDULING_TOLERANCE_MILLIS); for (int taskNo = 0; taskNo < 9; taskNo++) { - final Runnable task = tasks.get(taskNo); - verify(task, times(0)).run(); + final Consumer task = tasks.get(taskNo); + verify(task, times(0)).accept(lock); verifyExceptionHandlerCatchedSchedulingException( exceptionHandlers.get(taskNo), RetrySchedulingException.Type.RETRY_TASK_OVERTAKEN @@ -288,7 +321,7 @@ void testMultipleRetryTasksBeingOvertaken() throws Exception { } // Verify that the last task was executed - verify(tasks.get(9), times(1)).run(); + verify(tasks.get(9), times(1)).accept(lock); verifyNoMoreInteractions(exceptionHandlers.get(9)); verifyEventLoopScheduleCalls( @@ -303,14 +336,14 @@ void testMultipleRetryTasksBeingOvertaken() throws Exception { @Test void testLaterRetryTaskDoesNotOvertakeEarlierOne() throws Exception { - final Runnable task1 = mock(Runnable.class); + final Consumer task1 = spy(dummyRetryTask); final long task1SchedulingTime = System.nanoTime(); final long expectedTask1RunTime = task1SchedulingTime + TimeUnit.MILLISECONDS.toNanos(200); final Consumer task1ExceptionHandler = mock(Consumer.class); scheduler.schedule(task1, expectedTask1RunTime, Long.MIN_VALUE, task1ExceptionHandler); // Schedule a new task with a later time than the current one - final Runnable task2 = mock(Runnable.class); + final Consumer task2 = spy(dummyRetryTask); final long task2SchedulingTime = System.nanoTime(); final long expectedTask2RunTime = task2SchedulingTime + TimeUnit.MILLISECONDS.toNanos(300); final Consumer task2ExceptionHandler = mock(Consumer.class); @@ -319,11 +352,11 @@ void testLaterRetryTaskDoesNotOvertakeEarlierOne() throws Exception { Thread.sleep(300 + SCHEDULING_TOLERANCE_MILLIS); // Verify that the first task was executed - verify(task1, times(1)).run(); + verify(task1, times(1)).accept(lock); verifyNoMoreInteractions(task1ExceptionHandler); // Verify that the second task was not executed - verify(task2, times(0)).run(); + verify(task2, times(0)).accept(lock); verifyExceptionHandlerCatchedSchedulingException( task2ExceptionHandler, RetrySchedulingException.Type.RETRY_TASK_OVERTAKEN); @@ -342,7 +375,7 @@ void testRescheduleTaskWhenEarliestNextRetryTimeUpdated() throws Exception { // Schedule a task to run after 300ms final long task1SchedulingTime = System.nanoTime(); final long taskTime = task1SchedulingTime + TimeUnit.MILLISECONDS.toNanos(300); - final Runnable task1 = mock(Runnable.class); + final Consumer task1 = spy(dummyRetryTask); final Consumer exceptionHandler = mock(Consumer.class); scheduler.schedule(task1, taskTime, Long.MIN_VALUE, exceptionHandler); @@ -355,7 +388,7 @@ void testRescheduleTaskWhenEarliestNextRetryTimeUpdated() throws Exception { Thread.sleep(400 + SCHEDULING_TOLERANCE_MILLIS); - verify(task1, times(1)).run(); + verify(task1, times(1)).accept(lock); verifyEventLoopScheduleCalls( EventLoopScheduleCall.of(task1SchedulingTime, taskTime), EventLoopScheduleCall.of(earliestTimeUpdateTime, newEarliestTimeNanos) @@ -382,7 +415,11 @@ void testRescheduleWithNoTasks() throws InterruptedException { final long expectedTaskTime = taskScheduledTime + TimeUnit.MILLISECONDS.toNanos(100); scheduler.schedule( - () -> { + retryLock0 -> { + assertThat(retryLock0).isEqualTo(lock); + assertThat(retryLock0.isHeldByCurrentThread()).isTrue(); + assertThat(retryLock0.getHoldCount()).isOne(); + retryLock0.unlock(); // note that we inherit the earliest next retry time from the calls above }, taskScheduledTime + TimeUnit.MILLISECONDS.toNanos(50), Long.MIN_VALUE, t -> { // do nothing @@ -411,7 +448,7 @@ void testRescheduleWithNoTasks() throws InterruptedException { @Test void testIdempotentReschedule() throws Exception { // Schedule a task to run after 200ms - final Runnable task = mock(Runnable.class); + final Consumer task = spy(dummyRetryTask); final long taskScheduledTime = System.nanoTime(); final long taskTime = taskScheduledTime + TimeUnit.MILLISECONDS.toNanos(100); final Consumer exceptionHandler = mock(Consumer.class); @@ -480,7 +517,7 @@ void testIdempotentReschedule() throws Exception { Thread.sleep(300 + SCHEDULING_TOLERANCE_MILLIS); - verify(task, times(1)).run(); + verify(task, times(1)).accept(lock); verifyNoMoreInteractions(exceptionHandler); verifyEventLoopScheduleCalls( @@ -499,7 +536,7 @@ void testSchedulerShutdownWithoutTask() throws Exception { @Test void testSchedulerShutdownCancelsTask() throws Exception { // Schedule a task that should run after 200ms - final Runnable task = mock(Runnable.class); + final Consumer task = spy(dummyRetryTask); final long taskSchedulingTime = System.nanoTime(); final long expectedTaskRunTime = taskSchedulingTime + TimeUnit.MILLISECONDS.toNanos(200); final Consumer exceptionHandler = mock(Consumer.class); @@ -516,14 +553,15 @@ void testSchedulerShutdownCancelsTask() throws Exception { Thread.sleep(400); - verify(task, times(0)).run(); - verifyNoMoreInteractions(exceptionHandler); + verify(task, times(0)).accept(any()); + verifyExceptionHandlerCatchedSchedulingException( + exceptionHandler, Type.RETRYING_ALREADY_COMPLETED); } @Test void testEventLoopShutdownCancelsTask() throws Exception { // Schedule a task that should run after 200ms - final Runnable task = mock(Runnable.class); + final Consumer task = spy(dummyRetryTask); final long taskSchedulingTime = System.nanoTime(); final long expectedTaskRunTime = taskSchedulingTime + TimeUnit.MILLISECONDS.toNanos(200); final Consumer exceptionHandler = mock(Consumer.class); @@ -540,7 +578,7 @@ void testEventLoopShutdownCancelsTask() throws Exception { Thread.sleep(400); - verify(task, times(0)).run(); + verify(task, times(0)).accept(any()); verifyExceptionHandlerCatchedSchedulingException( exceptionHandler, Type.RETRY_TASK_CANCELLED); } @@ -549,7 +587,7 @@ void testEventLoopShutdownCancelsTask() throws Exception { void testScheduledOnShutdownEventLoop() throws InterruptedException { eventLoop.shutdownGracefully().sync(); - final Runnable task = mock(Runnable.class); + final Consumer task = spy(dummyRetryTask); final long taskSchedulingTime = System.nanoTime(); final long expectedTaskRunTime = taskSchedulingTime + TimeUnit.MILLISECONDS.toNanos(200); final Consumer exceptionHandler = mock(Consumer.class); @@ -559,7 +597,7 @@ void testScheduledOnShutdownEventLoop() throws InterruptedException { verify(exceptionHandler, times(1)).accept(exceptionCaptor.capture()); final Throwable capturedException = exceptionCaptor.getValue(); - verify(task, times(0)).run(); + verify(task, times(0)).accept(lock); assertThat(capturedException).isInstanceOf(RejectedExecutionException.class); assertThat(capturedException.getMessage()).contains("event executor terminated"); } From 164aa2d8b6c105085f2b8f140673da2338592e00 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Thu, 19 Jun 2025 17:16:55 +0200 Subject: [PATCH 26/36] fix: leak failure in tests because of unfinished responses of aborted attempts --- .../client/retry/AbstractRetryingClient.java | 223 ++++++++++++------ .../armeria/client/retry/RetryScheduler.java | 8 +- .../armeria/client/retry/RetryingClient.java | 65 +++-- .../client/retry/RetryingRpcClient.java | 9 +- .../client/retry/RetrySchedulerTest.java | 40 ++-- .../client/retry/RetryingClientTest.java | 129 +++++----- .../retry/RetryingClientWithHedgingTest.java | 41 ++-- .../RetryingRpcClientWithHedgingTest.java | 16 ++ 8 files changed, 329 insertions(+), 202 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java index 74d3f53529c..2923f6e00a9 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java @@ -19,17 +19,20 @@ import static java.util.Objects.requireNonNull; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantLock; -import java.util.function.BiConsumer; import java.util.function.Consumer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; + import com.linecorp.armeria.client.Client; import com.linecorp.armeria.client.ClientRequestContext; import com.linecorp.armeria.client.Endpoint; @@ -42,6 +45,7 @@ import com.linecorp.armeria.common.RpcRequest; import com.linecorp.armeria.common.annotation.Nullable; import com.linecorp.armeria.common.util.TimeoutMode; +import com.linecorp.armeria.common.util.UnmodifiableFuture; import com.linecorp.armeria.internal.client.ClientUtil; import com.linecorp.armeria.internal.common.util.ReentrantShortLock; @@ -100,11 +104,6 @@ public final O execute(ClientRequestContext ctx, I req) throws Exception { state = new State<>(retryLock, config, scheduler, responseTimeoutTimeNanos); } - state.whenRetryingComplete().handle((f, t) -> { - state.scheduler().shutdown(); - return null; - }); - ctx.setAttr(STATE, state); return doExecute(ctx, req); } @@ -179,14 +178,15 @@ protected final RetryRuleWithContent retryRuleWithContent() { */ protected void startRetryAttempt(ClientRequestContext ctx, ClientRequestContext attemptCtx, - BiConsumer onWinHandler, - BiConsumer - onAttemptAbortedHandler + O attemptRes, + AttemptWinHandler onWinHandler, + AttemptAbortHandler + onAbortHandler ) { requireNonNull(ctx, "ctx"); requireNonNull(attemptCtx, "attemptCtx"); requireNonNull(onWinHandler, "onWinHandler"); - requireNonNull(onAttemptAbortedHandler, "onAttemptAbortedHandler"); + requireNonNull(onAbortHandler, "onAbortHandler"); final State state = state(ctx); final ReentrantLock retryLock = state.getLock(); @@ -197,32 +197,14 @@ protected void startRetryAttempt(ClientRequestContext ctx, // This really should not happen. However, let us call the abort handler to free // potentially expensive resources here. retryLock.unlock(); - onAttemptAbortedHandler.accept(attemptCtx, - new IllegalStateException("Retrying is already complete.")); + onAbortHandler.accept(attemptCtx, + attemptRes, + new IllegalStateException("Retrying is already complete.")); return; } - state.startAttempt(attemptCtx); + state.startAttempt(attemptCtx, attemptRes, onWinHandler, onAbortHandler); retryLock.unlock(); - state.whenRetryingComplete().handleAsync((winningAttempt, cause) -> { - if (winningAttempt == null) { - // If the retrying is complete exceptionally, we need to call the onAttemptAbortedHandler. - onAttemptAbortedHandler.accept(winningAttempt.ctx(), cause); - return null; - } - - final ClientRequestContext winningAttemptCtx = winningAttempt.ctx(); - final O winningAttemptRes = winningAttempt.res(); - assert winningAttemptRes != null; - - if (attemptCtx == winningAttemptCtx) { - onWinHandler.accept(winningAttemptCtx, winningAttemptRes); - return null; - } - - onAttemptAbortedHandler.accept(attemptCtx, cause); - return null; - }, attemptCtx.eventLoop()); } /** @@ -382,7 +364,7 @@ protected static void scheduleNextRetry(ClientRequestContext ctx, retryLock.unlock(); } - // todo(szymon): improve documentation for this method + // todo(szymon): [doc] /** * Resets the {@link ClientRequestContext#responseTimeoutMillis()}. @@ -429,39 +411,96 @@ protected static ClientRequestContext newAttemptContext(ClientRequestContext ctx return ClientUtil.newDerivedContext(ctx, req, rpcReq, initialAttempt); } + @FunctionalInterface + protected interface AttemptWinHandler { + /** + * todo(szymon): [doc]. + */ + void accept(ClientRequestContext attemptCtx, O attemptResOnComplete); + } + + @FunctionalInterface + protected interface AttemptAbortHandler { + /** + * todo(szymon): [doc]. + */ + CompletableFuture accept(ClientRequestContext attemptCtx, O attemptRes, + @Nullable Throwable abortCause); + } + + /** + * todo(szymon): [doc]. + */ + @VisibleForTesting @SuppressWarnings("unchecked") - private static State state(ClientRequestContext ctx) { + public static State state(ClientRequestContext ctx) { final State state = (State) ctx.attr(STATE); assert state != null; return state; } - private static final class State { - static class Attempt { + /** + * todo(szymon): [doc]. + */ + @VisibleForTesting + public static final class State { + /** + * todo(szymon): [doc]. + */ + @VisibleForTesting + public static class Attempt { private final ClientRequestContext attemptCtx; - private @Nullable O attemptRes; + private final O attemptRes; + private @Nullable O attemptResOnComplete; + + private final AttemptWinHandler onWinHandler; + private final AttemptAbortHandler + onAbortHandler; - private boolean isCompleted; + private boolean hasCalledHandler; - Attempt(ClientRequestContext attemptCtx, @Nullable O attemptRes) { - this.attemptCtx = requireNonNull(attemptCtx, "attemptCtx"); + Attempt(ClientRequestContext attemptCtx, O attemptRes, + AttemptWinHandler onWinHandler, + AttemptAbortHandler onAbortHandler) { + this.attemptCtx = attemptCtx; this.attemptRes = attemptRes; - isCompleted = this.attemptRes != null; + this.onWinHandler = onWinHandler; + this.onAbortHandler = onAbortHandler; + + hasCalledHandler = false; + attemptResOnComplete = null; } - public ClientRequestContext ctx() { + ClientRequestContext ctx() { return attemptCtx; } - @Nullable - public O res() { + /** + * todo(szymon): [doc]. + */ + public O attemptRes() { return attemptRes; } - public void setRes(O res) { - checkState(!isCompleted); - isCompleted = true; - attemptRes = requireNonNull(res, "res"); + void completeWithRes(O res) { + assert attemptResOnComplete == null; + attemptResOnComplete = res; + } + + CompletableFuture callHandler(boolean isWinning, @Nullable Throwable cause) { + if (hasCalledHandler) { + return UnmodifiableFuture.completedFuture(null); + } + + hasCalledHandler = true; + + if (isWinning) { + assert attemptResOnComplete != null; + onWinHandler.accept(attemptCtx, attemptResOnComplete); + return UnmodifiableFuture.completedFuture(null); + } else { + return onAbortHandler.accept(attemptCtx, attemptRes, cause); + } } } @@ -471,10 +510,12 @@ public void setRes(O res) { private final long deadlineNanos; private final boolean isTimeoutEnabled; - private final Map> activeAttempts = new HashMap<>(); + private final Map> startedAttempts = new HashMap<>(); private final RetryScheduler retryScheduler; + private int numPendingAttempts; + // An attempt is considered scheduled between two points: // - right before its retry task is scheduled // - right after the retry task finishes execution @@ -486,8 +527,6 @@ public void setRes(O res) { private @Nullable Attempt lastAttempt; - private final CompletableFuture> retryingCompleteFuture; - @Nullable private Backoff lastBackoff; private int currentAttemptNoWithLastBackoff; @@ -500,7 +539,9 @@ public void setRes(O res) { this.config = config; this.retryScheduler = retryScheduler; totalAttemptNo = 1; - retryingCompleteFuture = new CompletableFuture<>(); + numPendingAttempts = 0; + numScheduledAttempts = 0; + deadlineNanos = 0; isTimeoutEnabled = false; } @@ -512,7 +553,9 @@ public void setRes(O res) { this.config = config; this.retryScheduler = retryScheduler; totalAttemptNo = 1; - retryingCompleteFuture = new CompletableFuture<>(); + numPendingAttempts = 0; + numScheduledAttempts = 0; + deadlineNanos = responseTimeoutTimeNanos; isTimeoutEnabled = true; } @@ -572,14 +615,19 @@ long responseTimeoutTimeNanos() { return deadlineNanos; } - void startAttempt(ClientRequestContext attemptCtx) { + void startAttempt(ClientRequestContext attemptCtx, O attemptRes, + AttemptWinHandler onWinHandler, + AttemptAbortHandler + onAbortHandler) { lock.lock(); try { checkState(!isRetryingComplete()); - checkState(!activeAttempts.containsKey(attemptCtx), + checkState(!startedAttempts.containsKey(attemptCtx), "Attempt %s already is already pending.", attemptCtx); - final Attempt attempt = new Attempt<>(attemptCtx, null); - activeAttempts.put(attempt.ctx(), attempt); + final Attempt attempt = new Attempt<>(attemptCtx, attemptRes, onWinHandler, + onAbortHandler); + startedAttempts.put(attempt.ctx(), attempt); + numPendingAttempts++; } finally { lock.unlock(); } @@ -593,12 +641,12 @@ void completeAttempt(ClientRequestContext attemptCtx, O attemptRes, boolean isWi return; } - checkState(activeAttempts.containsKey(attemptCtx), + checkState(startedAttempts.containsKey(attemptCtx), "Attempt %s is not registered as pending.", attemptCtx); - lastAttempt = activeAttempts.get(attemptCtx); - lastAttempt.setRes(attemptRes); - activeAttempts.remove(attemptCtx); + lastAttempt = startedAttempts.get(attemptCtx); + lastAttempt.completeWithRes(attemptRes); + numPendingAttempts--; } catch (Exception e) { lock.unlock(); throw e; @@ -675,7 +723,7 @@ int acquireAttemptNoWithCurrentBackoff(Backoff backoff) { int numPendingAttempts() { lock.lock(); try { - return activeAttempts.size() + (numScheduledAttempts > 0 ? 1 : 0); + return numPendingAttempts + (numScheduledAttempts > 0 ? 1 : 0); } finally { lock.unlock(); } @@ -720,9 +768,11 @@ private void complete0() { new IllegalStateException("Completed retrying without any attempts.")); } else { isRetryingComplete = true; - // We do not want to call the handlers with the lock. + scheduler().shutdown(); lock.unlock(); - retryingCompleteFuture.complete(lastAttempt); + // We do not want to call the handlers with the lock. + // Save to reference lastAttempt as this cannot change anymore. + notifyAllAttemptHandlers(lastAttempt, null); } } @@ -738,20 +788,49 @@ void completeExceptionally(Throwable cause) { completeExceptionally0(cause); } + private void notifyAllAttemptHandlers(@Nullable Attempt winningAttempt, @Nullable Throwable cause) { + assert isRetryingComplete(); + + final CompletableFuture allOtherAttemptsAbortedFuture = CompletableFuture.allOf( + startedAttempts.values().stream() + .filter(attempt -> attempt != winningAttempt) + .map(attempt -> + attempt.callHandler(false, cause) + ) + .toArray(CompletableFuture[]::new) + ); + + allOtherAttemptsAbortedFuture.handle( + (unused, throwable) -> { + if (throwable != null) { + logger.warn("Failed to notify all aborted attempt handlers.", throwable); + } + + if (winningAttempt != null) { + winningAttempt.callHandler(true, cause); + } + return null; + }); + } + // takes ownership of current lock private void completeExceptionally0(Throwable cause) { assert lock.isHeldByCurrentThread(); assert !isRetryingComplete(); isRetryingComplete = true; + scheduler().shutdown(); // We do not want to call the handlers with the lock. lock.unlock(); - retryingCompleteFuture.completeExceptionally(cause); + notifyAllAttemptHandlers(null, cause); } - boolean isRetryingComplete() { + /** + * todo(szymon): [doc]. + */ + public boolean isRetryingComplete() { try { lock.lock(); return isRetryingComplete; @@ -760,8 +839,16 @@ boolean isRetryingComplete() { } } - CompletableFuture> whenRetryingComplete() { - return retryingCompleteFuture; + /** + * todo(szymon): [doc]. + */ + public List> startedAttempts() { + lock.lock(); + try { + return ImmutableList.copyOf(startedAttempts.values()); + } finally { + lock.unlock(); + } } } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java index be191b4d662..58f353b492c 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java @@ -24,9 +24,6 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.linecorp.armeria.client.retry.RetrySchedulingException.Type; import com.linecorp.armeria.common.annotation.Nullable; @@ -93,14 +90,12 @@ ScheduledFuture getFuture() { } } - // todo(szymon) [Q]: make this configurable? + // todo: should we make this a flag? // Number of nanoseconds that we allow the retry task to be scheduled earlier than // the earliestNextRetryTimeNanos. // This should avoid unnecessary rescheduling. private static final long RESCHEDULING_OVERTAKING_TOLERANCE_NANOS = TimeUnit.MICROSECONDS.toNanos(500); - private static final Logger logger = LoggerFactory.getLogger(RetryScheduler.class); - // for locking private final ReentrantLock retryLock; private final EventLoop eventLoop; @@ -320,7 +315,6 @@ private void scheduleNextRetryTask(Consumer retryRunnable, long r retryRunnable.accept(retryLock); }; - // todo(szymon): must not be executed immediately in this thread. //noinspection unchecked final ScheduledFuture nextRetryTaskFuture = (ScheduledFuture) eventLoop.schedule(wrappedRetryRunnable, delayNanos, diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index f0969dcac97..1bfe117f677 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -268,15 +268,12 @@ protected HttpResponse doExecute(ClientRequestContext ctx, HttpRequest req) thro } private void doExecute0(RetryingContext retryingContext, int attemptNo) { - final RetryConfig config = retryingContext.config(); final ClientRequestContext ctx = retryingContext.ctx(); final HttpRequestDuplicator rootReqDuplicator = retryingContext.reqDuplicator(); final HttpRequest originalReq = retryingContext.req(); final HttpResponse returnedRes = retryingContext.res(); - // todo(szymon): we need to inject the attempt number as there may be concurrent attempts - // starting and acquiring an attempt number. final boolean initialAttempt = attemptNo <= 1; // The request or attemptRes has been aborted by the client before it receives a attemptRes, // so stop retrying. @@ -340,22 +337,48 @@ private void doExecute0(RetryingContext retryingContext, int attemptNo) { false); } - startRetryAttempt(ctx, attemptCtx, (winningAttemptCtx, - winningAttemptRes) -> { - retryingContext.reqDuplicator().close(); - retryingContext.ctx().logBuilder().endResponseWithChild(winningAttemptCtx.log()); - retryingContext.resFuture().complete(winningAttemptRes); + startRetryAttempt(ctx, attemptCtx, attemptRes, (winningAttemptCtx, + winningAttemptRes) -> { + assert attemptCtx == winningAttemptCtx; + + winningAttemptCtx.eventLoop().execute(() -> { + retryingContext.reqDuplicator().close(); + retryingContext + .ctx() + .logBuilder() + .endResponseWithChild(winningAttemptCtx.log()); + retryingContext.resFuture().complete(winningAttemptRes); + }); }, - (abortingAttemptCtx, cause) -> { - final RequestLogBuilder logBuilder = abortingAttemptCtx.logBuilder(); - logBuilder.responseContent(null, null); - logBuilder.responseContentPreview(null); - - if (cause != null) { - attemptCtx.cancel(cause); - } else { - attemptCtx.cancel(); - } + (abortingAttemptCtx, + abortingAttemptRes, + cause + ) -> { + assert attemptCtx == abortingAttemptCtx; + assert attemptRes == abortingAttemptRes; + + final CompletableFuture abortCompletedFuture = new CompletableFuture<>(); + + attemptCtx.eventLoop().execute(() -> { + final RequestLogBuilder logBuilder = abortingAttemptCtx.logBuilder(); + logBuilder.responseContent(null, null); + logBuilder.responseContentPreview(null); + + if (cause != null) { + attemptCtx.cancel(cause); + abortingAttemptRes.abort(cause); + } else { + attemptCtx.cancel(); + abortingAttemptRes.abort(); + } + + abortingAttemptRes.whenComplete().handle((unused, unusedCause) -> { + abortCompletedFuture.complete(null); + return null; + }); + }); + + return abortCompletedFuture; } ); @@ -466,12 +489,6 @@ private void handleStreamingResponse(RetryingContext retryingContext, completeAttemptLogIfBytesNotTransferred(attemptCtx, attemptRes, headers, responseCause); - // see above - if (isRetryingComplete(retryingContext.ctx())) { - attemptSplitRes.body().abort(); - return null; - } - attemptCtx.log().whenAvailable(RequestLogProperty.RESPONSE_HEADERS).thenRun(() -> { // see above if (isRetryingComplete(retryingContext.ctx())) { diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java index 716886389bd..17982a7c0a6 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java @@ -31,6 +31,7 @@ import com.linecorp.armeria.common.Request; import com.linecorp.armeria.common.RpcRequest; import com.linecorp.armeria.common.RpcResponse; +import com.linecorp.armeria.common.util.UnmodifiableFuture; import com.linecorp.armeria.internal.client.ClientPendingThrowableUtil; import com.linecorp.armeria.internal.client.ClientRequestContextExtension; import com.linecorp.armeria.internal.common.util.StringUtil; @@ -190,7 +191,7 @@ private void doExecute0(ClientRequestContext ctx, RpcRequest req, req, true); } - startRetryAttempt(ctx, attemptCtx, (winningAttemptCtx, winningAttemptRes) -> { + startRetryAttempt(ctx, attemptCtx, attemptRes, (winningAttemptCtx, winningAttemptRes) -> { ctx.logBuilder().endResponseWithChild(winningAttemptCtx.log()); final HttpRequest actualHttpReq = winningAttemptCtx.request(); if (actualHttpReq != null) { @@ -198,12 +199,16 @@ private void doExecute0(ClientRequestContext ctx, RpcRequest req, } returnedResFuture.complete(winningAttemptRes); - }, (abortingAttemptCtx, cause) -> { + }, (abortingAttemptCtx, + abortingAttemptRes, + cause) -> { if (cause != null) { abortingAttemptCtx.cancel(cause); } else { abortingAttemptCtx.cancel(); } + abortingAttemptRes.cancel(false); + return UnmodifiableFuture.completedFuture(null); }); final RetryConfig retryConfig = mappedRetryConfig(ctx); diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java index f1f4fe6674a..37015784d33 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java @@ -56,7 +56,7 @@ class RetrySchedulerTest { private static final long SCHEDULING_TOLERANCE_MILLIS = TimeUnit.NANOSECONDS.toMillis( SCHEDULING_TOLERANCE_NANOS); - private ReentrantLock lock; + private ReentrantLock retryLock; private Consumer dummyRetryTask; private Consumer dummyThrowingRetryTask; @@ -80,27 +80,27 @@ public void accept(ReentrantLock retryLock) { @BeforeEach void setUp() { eventLoop = spy(new DefaultEventLoop()); - lock = new ReentrantShortLock(); + retryLock = new ReentrantShortLock(); dummyRetryTask = new SpyableRetryTask(retryLock -> { - assertThat(retryLock).isEqualTo(lock); + assertThat(retryLock).isEqualTo(this.retryLock); assertThat(retryLock.isHeldByCurrentThread()).isTrue(); assertThat(retryLock.getHoldCount()).isOne(); retryLock.unlock(); }); dummyThrowingRetryTask = new SpyableRetryTask(retryLock -> { - assertThat(retryLock).isEqualTo(lock); + assertThat(retryLock).isEqualTo(this.retryLock); assertThat(retryLock.isHeldByCurrentThread()).isTrue(); assertThat(retryLock.getHoldCount()).isOne(); retryLock.unlock(); throw new AnticipatedException(); }); - scheduler = new RetryScheduler(lock, eventLoop); + scheduler = new RetryScheduler(retryLock, eventLoop); } @AfterEach void tearDown() throws Exception { - assertThat(lock.isLocked()).isFalse(); + assertThat(retryLock.isLocked()).isFalse(); assertThat(scheduler.shutdown()).isTrue(); eventLoop.shutdownGracefully(); } @@ -136,7 +136,7 @@ void testScheduleTask(int taskDelayMs, int minDelayMs, int prevDelayMs, int expe Thread.sleep(expectedDelayMs + SCHEDULING_TOLERANCE_MILLIS); - verify(task, times(1)).accept(lock); + verify(task, times(1)).accept(retryLock); verifyNoMoreInteractions(exceptionHandler); verifyEventLoopScheduleCalls( EventLoopScheduleCall.of(now, expectedScheduledTimeNanos) @@ -182,7 +182,7 @@ void testScheduleTaskInThePast() throws Exception { Thread.sleep(SCHEDULING_TOLERANCE_MILLIS); - verify(task, times(1)).accept(lock); + verify(task, times(1)).accept(retryLock); verifyNoMoreInteractions(exceptionHandler); verifyEventLoopScheduleCalls( EventLoopScheduleCall.of(taskSchedulingTime, expectedTaskRunTime) @@ -202,7 +202,7 @@ void testScheduleTaskInThePast() throws Exception { Thread.sleep(SCHEDULING_TOLERANCE_MILLIS); - verify(task2, times(1)).accept(lock); + verify(task2, times(1)).accept(retryLock); verifyNoMoreInteractions(task2ExceptionHandler); verifyEventLoopScheduleCalls( EventLoopScheduleCall.of(taskSchedulingTime, expectedTaskRunTime), @@ -245,7 +245,7 @@ void testScheduleTaskWithException() throws Exception { Thread.sleep(100 + SCHEDULING_TOLERANCE_MILLIS); - verify(task1, times(1)).accept(lock); + verify(task1, times(1)).accept(retryLock); final ArgumentCaptor exceptionCaptor = ArgumentCaptor.forClass(Throwable.class); verify(task1ExceptionHandler, times(1)).accept(exceptionCaptor.capture()); assertThat(exceptionCaptor.getValue()).isInstanceOf(AnticipatedException.class); @@ -268,8 +268,8 @@ void testEarlierRetryTaskOvertakesLaterOne() throws Exception { Thread.sleep(200 + SCHEDULING_TOLERANCE_MILLIS); - verify(task1, times(0)).accept(lock); - verify(task2, times(1)).accept(lock); + verify(task1, times(0)).accept(retryLock); + verify(task2, times(1)).accept(retryLock); verifyExceptionHandlerCatchedSchedulingException(task1ExceptionHandler, RetrySchedulingException.Type.RETRY_TASK_OVERTAKEN); @@ -313,7 +313,7 @@ void testMultipleRetryTasksBeingOvertaken() throws Exception { for (int taskNo = 0; taskNo < 9; taskNo++) { final Consumer task = tasks.get(taskNo); - verify(task, times(0)).accept(lock); + verify(task, times(0)).accept(retryLock); verifyExceptionHandlerCatchedSchedulingException( exceptionHandlers.get(taskNo), RetrySchedulingException.Type.RETRY_TASK_OVERTAKEN @@ -321,7 +321,7 @@ void testMultipleRetryTasksBeingOvertaken() throws Exception { } // Verify that the last task was executed - verify(tasks.get(9), times(1)).accept(lock); + verify(tasks.get(9), times(1)).accept(retryLock); verifyNoMoreInteractions(exceptionHandlers.get(9)); verifyEventLoopScheduleCalls( @@ -352,11 +352,11 @@ void testLaterRetryTaskDoesNotOvertakeEarlierOne() throws Exception { Thread.sleep(300 + SCHEDULING_TOLERANCE_MILLIS); // Verify that the first task was executed - verify(task1, times(1)).accept(lock); + verify(task1, times(1)).accept(retryLock); verifyNoMoreInteractions(task1ExceptionHandler); // Verify that the second task was not executed - verify(task2, times(0)).accept(lock); + verify(task2, times(0)).accept(retryLock); verifyExceptionHandlerCatchedSchedulingException( task2ExceptionHandler, RetrySchedulingException.Type.RETRY_TASK_OVERTAKEN); @@ -388,7 +388,7 @@ void testRescheduleTaskWhenEarliestNextRetryTimeUpdated() throws Exception { Thread.sleep(400 + SCHEDULING_TOLERANCE_MILLIS); - verify(task1, times(1)).accept(lock); + verify(task1, times(1)).accept(retryLock); verifyEventLoopScheduleCalls( EventLoopScheduleCall.of(task1SchedulingTime, taskTime), EventLoopScheduleCall.of(earliestTimeUpdateTime, newEarliestTimeNanos) @@ -416,7 +416,7 @@ void testRescheduleWithNoTasks() throws InterruptedException { scheduler.schedule( retryLock0 -> { - assertThat(retryLock0).isEqualTo(lock); + assertThat(retryLock0).isEqualTo(retryLock); assertThat(retryLock0.isHeldByCurrentThread()).isTrue(); assertThat(retryLock0.getHoldCount()).isOne(); retryLock0.unlock(); @@ -517,7 +517,7 @@ void testIdempotentReschedule() throws Exception { Thread.sleep(300 + SCHEDULING_TOLERANCE_MILLIS); - verify(task, times(1)).accept(lock); + verify(task, times(1)).accept(retryLock); verifyNoMoreInteractions(exceptionHandler); verifyEventLoopScheduleCalls( @@ -597,7 +597,7 @@ void testScheduledOnShutdownEventLoop() throws InterruptedException { verify(exceptionHandler, times(1)).accept(exceptionCaptor.capture()); final Throwable capturedException = exceptionCaptor.getValue(); - verify(task, times(0)).accept(lock); + verify(task, times(0)).accept(retryLock); assertThat(capturedException).isInstanceOf(RejectedExecutionException.class); assertThat(capturedException.getMessage()).contains("event executor terminated"); } diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java index 7f7cee02b27..c068e21d4da 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java @@ -65,6 +65,8 @@ import com.linecorp.armeria.client.UnprocessedRequestException; import com.linecorp.armeria.client.WebClient; import com.linecorp.armeria.client.logging.LoggingClient; +import com.linecorp.armeria.client.retry.AbstractRetryingClient.State; +import com.linecorp.armeria.client.retry.AbstractRetryingClient.State.Attempt; import com.linecorp.armeria.common.AggregatedHttpResponse; import com.linecorp.armeria.common.HttpData; import com.linecorp.armeria.common.HttpHeaderNames; @@ -108,6 +110,9 @@ static void afterAll() { clientFactory.closeAsync(); } + @Nullable + ClientRequestContext ctx; + private final AtomicInteger responseAbortServiceCallCounter = new AtomicInteger(); private final AtomicInteger requestAbortServiceCallCounter = new AtomicInteger(); @@ -337,26 +342,24 @@ void retryWhenContentMatched() { .decorator(retryingDecorator) .build(); final AggregatedHttpResponse res; - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { res = client.get("/retry-content").aggregate().join(); ctx = captor.get(); } assertThat(res.contentUtf8()).isEqualTo("Succeeded after retry"); - validateClientRequestLog(ctx, 3); + awaitValidClientRequestContext(ctx, 3); } @Test void retryWhenStatusMatched() { final WebClient client = client(RetryRule.builder().onServerErrorStatus().onException().thenBackoff()); final AggregatedHttpResponse res; - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { res = client.get("/503-then-success").aggregate().join(); ctx = captor.get(); } assertThat(res.contentUtf8()).isEqualTo("Succeeded after retry"); - validateClientRequestLog(ctx, 2); + awaitValidClientRequestContext(ctx, 2); } @Test @@ -366,13 +369,12 @@ void retryWhenStatusMatchedWithContent() { .onException() .thenBackoff(), 10000, 0, 100); final AggregatedHttpResponse res; - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { res = client.get("/503-then-success").aggregate().join(); ctx = captor.get(); } assertThat(res.contentUtf8()).isEqualTo("Succeeded after retry"); - validateClientRequestLog(ctx, 2); + awaitValidClientRequestContext(ctx, 2); } @Test @@ -384,13 +386,12 @@ void retryWhenTrailerMatched() { }) .thenBackoff()); final AggregatedHttpResponse res; - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { res = client.get("/trailers-then-success").aggregate().join(); ctx = captor.get(); } assertThat(res.contentUtf8()).isEqualTo("Succeeded after retry"); - validateClientRequestLog(ctx, 2); + awaitValidClientRequestContext(ctx, 2); } @Test @@ -400,27 +401,25 @@ void retryWhenTotalDurationIsHigh() { .onTotalDuration((unused, duration) -> duration.toNanos() > 100) .thenBackoff()); final AggregatedHttpResponse res; - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { res = client.get("/1sleep-then-success").aggregate().join(); ctx = captor.get(); } assertThat(res.contentUtf8()).isEqualTo("Succeeded after retry"); - validateClientRequestLog(ctx); + awaitValidClientRequestContext(ctx); } @Test void disableResponseTimeout() { final WebClient client = client(RetryRule.failsafe(), 0, 0, 100); final AggregatedHttpResponse res; - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { res = client.get("/503-then-success").aggregate().join(); ctx = captor.get(); } assertThat(res.contentUtf8()).isEqualTo("Succeeded after retry"); // response timeout did not happen. - validateClientRequestLog(ctx, 2); + awaitValidClientRequestContext(ctx, 2); } @Test @@ -429,7 +428,6 @@ void respectRetryAfter() { final Stopwatch sw = Stopwatch.createStarted(); final AggregatedHttpResponse res; - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { res = client.get("/retry-after-1-second").aggregate().join(); ctx = captor.get(); @@ -438,7 +436,7 @@ void respectRetryAfter() { assertThat(res.contentUtf8()).isEqualTo("Succeeded after retry"); assertThat(sw.elapsed(TimeUnit.MILLISECONDS)).isGreaterThanOrEqualTo( (long) (TimeUnit.SECONDS.toMillis(1) * 0.9)); - validateClientRequestLog(ctx, 2); + awaitValidClientRequestContext(ctx, 2); } @Test @@ -447,7 +445,6 @@ void respectRetryAfterWithHttpDate() { final Stopwatch sw = Stopwatch.createStarted(); final AggregatedHttpResponse res; - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { res = client.get("/retry-after-with-http-date").aggregate().join(); ctx = captor.get(); @@ -457,7 +454,7 @@ void respectRetryAfterWithHttpDate() { // Since ZonedDateTime doesn't express exact time, // just check out whether it is retried after delayed some time. assertThat(sw.elapsed(TimeUnit.MILLISECONDS)).isGreaterThanOrEqualTo(1000); - validateClientRequestLog(ctx, 2); + awaitValidClientRequestContext(ctx, 2); } @Test @@ -467,13 +464,12 @@ void propagateLastResponseWhenNextRetryIsAfterTimeout() { .onException() .thenBackoff(Backoff.fixed(10000000))); final AggregatedHttpResponse res; - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { res = client.get("/service-unavailable").aggregate().join(); ctx = captor.get(); } assertThat(res.status()).isSameAs(HttpStatus.SERVICE_UNAVAILABLE); - validateClientRequestLog(ctx, 1); + awaitValidClientRequestContext(ctx, 1); } @Test @@ -481,13 +477,12 @@ void propagateLastResponseWhenExceedMaxAttempts() { final WebClient client = client( RetryRule.builder().onServerErrorStatus().onException().thenBackoff(Backoff.fixed(1)), 0, 0, 3); final AggregatedHttpResponse res; - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { res = client.get("/service-unavailable").aggregate().join(); ctx = captor.get(); } assertThat(res.status()).isSameAs(HttpStatus.SERVICE_UNAVAILABLE); - validateClientRequestLog(ctx, 3); // equal to max attempts + awaitValidClientRequestContext(ctx, 3); // equal to max attempts } @Test @@ -496,14 +491,13 @@ void retryAfterOneYear() { // The response will be the last response whose headers contains HttpHeaderNames.RETRY_AFTER // because next retry is after timeout final ResponseHeaders headers; - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { headers = client.get("retry-after-one-year").aggregate().join().headers(); ctx = captor.get(); } assertThat(headers.status()).isSameAs(HttpStatus.SERVICE_UNAVAILABLE); assertThat(headers.get(HttpHeaderNames.RETRY_AFTER)).isNotNull(); - validateClientRequestLog(ctx, 1); + awaitValidClientRequestContext(ctx, 1); } @Test @@ -520,14 +514,13 @@ void retryOnResponseTimeout() { final WebClient client = client(strategy, 0, 500, 100); final AggregatedHttpResponse res; - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { res = client.get("/1sleep-then-success").aggregate().join(); ctx = captor.get(); } assertThat(res.contentUtf8()).isEqualTo("Succeeded after retry"); - validateClientRequestLog(ctx, 2); + awaitValidClientRequestContext(ctx, 2); } @Test @@ -549,7 +542,6 @@ void retryWithContentOnResponseTimeout() { .thenBackoff(backoff))); final WebClient client = client(strategy, 0, 500, 100); final AggregatedHttpResponse res; - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { res = client.get("/1sleep-then-success").aggregate().join(); ctx = captor.get(); @@ -557,7 +549,7 @@ void retryWithContentOnResponseTimeout() { assertThat(res.contentUtf8()).isEqualTo("Succeeded after retry"); // Make sure that all customized RetryRuleWithContents are called. assertThat(queue).containsExactly(1, 2, 3); - validateClientRequestLog(ctx, 2); + awaitValidClientRequestContext(ctx, 2); } @Test @@ -593,8 +585,6 @@ void honorRetryMapping() { final WebClient client = client(mapping); - ClientRequestContext ctx; - Stopwatch stopwatch = Stopwatch.createStarted(); try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { assertThat(client.get("/500-always").aggregate().join().status()) @@ -602,7 +592,8 @@ void honorRetryMapping() { ctx = captor.get(); } assertThat(stopwatch.elapsed()).isBetween(Duration.ofSeconds(2), Duration.ofSeconds(6)); - validateClientRequestLog(ctx, 2); + awaitValidClientRequestContext(ctx, 2); + waitForEveryAttemptResponse(ctx); stopwatch = Stopwatch.createStarted(); try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { @@ -611,7 +602,8 @@ void honorRetryMapping() { ctx = captor.get(); } assertThat(stopwatch.elapsed()).isBetween(Duration.ofSeconds(14), Duration.ofSeconds(28)); - validateClientRequestLog(ctx, 8); + awaitValidClientRequestContext(ctx, 8); + waitForEveryAttemptResponse(ctx); stopwatch = Stopwatch.createStarted(); try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { @@ -620,7 +612,7 @@ void honorRetryMapping() { ctx = captor.get(); } assertThat(stopwatch.elapsed()).isBetween(Duration.ofSeconds(0), Duration.ofSeconds(2)); - validateClientRequestLog(ctx, 1); + awaitValidClientRequestContext(ctx, 1); } @Test @@ -639,7 +631,6 @@ void evaluatesMappingOnce() { final WebClient client = client(mapping); - ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { assertThat(client.get("/500-then-success").aggregate().join().status()) .isEqualTo(HttpStatus.valueOf(200)); @@ -648,7 +639,8 @@ void evaluatesMappingOnce() { // 1 logical request; 2 retries assertThat(evaluations.get()).isEqualTo(1); - validateClientRequestLog(ctx, 2); + awaitValidClientRequestContext(ctx, 2); + waitForEveryAttemptResponse(ctx); reqCount.set(0); try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { @@ -659,7 +651,7 @@ void evaluatesMappingOnce() { // 2 logical requests; 2 retries assertThat(evaluations.get()).isEqualTo(2); - validateClientRequestLog(ctx, 2); + awaitValidClientRequestContext(ctx, 2); } @Test @@ -692,7 +684,6 @@ void retryWithContentOnUnprocessedException() { .decorator(retryingDecorator) .build(); final Stopwatch stopwatch = Stopwatch.createStarted(); - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { assertThatThrownBy(() -> client.get("/unprocessed-exception").aggregate().join()) .isInstanceOf(CompletionException.class) @@ -700,7 +691,7 @@ void retryWithContentOnUnprocessedException() { ctx = captor.get(); } assertThat(stopwatch.elapsed()).isBetween(Duration.ofSeconds(7), Duration.ofSeconds(20)); - validateClientRequestLog(ctx, 5); // max attempts + awaitValidClientRequestContext(ctx, 5); // max attempts } } @@ -709,7 +700,6 @@ void retryWithContentOnUnprocessedException() { void differentBackoffBasedOnStatus(RetryRule retryRule) { final WebClient client = client(retryRule); - ClientRequestContext ctx; AggregatedHttpResponse res; final Stopwatch sw = Stopwatch.createStarted(); try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { @@ -718,7 +708,8 @@ void differentBackoffBasedOnStatus(RetryRule retryRule) { } assertThat(res.contentUtf8()).isEqualTo("Succeeded after retry"); assertThat(sw.elapsed(TimeUnit.MILLISECONDS)).isBetween((long) (10 * 0.9), (long) (1000 * 1.1)); - validateClientRequestLog(ctx, 2); + awaitValidClientRequestContext(ctx, 2); + waitForEveryAttemptResponse(ctx); reqCount.set(0); sw.reset().start(); @@ -729,7 +720,7 @@ void differentBackoffBasedOnStatus(RetryRule retryRule) { } assertThat(res.contentUtf8()).isEqualTo("Succeeded after retry"); assertThat(sw.elapsed(TimeUnit.MILLISECONDS)).isGreaterThanOrEqualTo((long) (1000 * 0.9)); - validateClientRequestLog(ctx, 2); + awaitValidClientRequestContext(ctx, 2); } @Test @@ -739,13 +730,12 @@ void retryWithRequestBody() { .onException() .thenBackoff(Backoff.fixed(10))); final AggregatedHttpResponse res; - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { res = client.post("/post-ping-pong", "bar").aggregate().join(); ctx = captor.get(); } assertThat(res.contentUtf8()).isEqualTo("bar"); - validateClientRequestLog(ctx, 2); + awaitValidClientRequestContext(ctx, 2); } @Test @@ -783,13 +773,12 @@ void shouldGetExceptionWhenFactoryIsClosed() { // // Peel CompletionException first. - final ClientRequestContext ctx; Throwable t; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { t = peel(catchThrowable(() -> client.get("/service-unavailable").aggregate().join())); ctx = captor.get(); } - validateClientRequestLog(ctx, 1); // not able to schedule second retry. + awaitValidClientRequestContext(ctx, 1); // not able to schedule second retry. if (t instanceof UnprocessedRequestException) { final Throwable cause = t.getCause(); assertThat(cause).isInstanceOf(IllegalStateException.class); @@ -817,7 +806,6 @@ void doNotRetryWhenResponseIsAborted() throws Exception { .decorator(LoggingClient.newDecorator()) .build(); responseAbortServiceCallCounter.set(0); - final ClientRequestContext ctx; final HttpResponse httpResponse; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { httpResponse = client.get("/response-abort"); @@ -828,7 +816,7 @@ void doNotRetryWhenResponseIsAborted() throws Exception { } else { httpResponse.abort(abortCause); } - validateClientRequestLog(ctx, 1); + awaitValidClientRequestContext(ctx, 1); final RequestLog log = context.get().log().whenComplete().join(); final Throwable requestCause = log.requestCause(); @@ -856,7 +844,6 @@ void doNotRetryWhenResponseIsAborted() throws Exception { @Test void doNotRetryWhenSubscriberIsCancelled() throws Exception { final WebClient client = client(retryAlways); - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { client.get("/subscriber-cancel").subscribe( @@ -881,7 +868,7 @@ public void onComplete() {} TimeUnit.SECONDS.sleep(1L); // Sleep to check if there's a retry. assertThat(subscriberCancelServiceCallCounter.get()).isEqualTo(1); - validateClientRequestLog(ctx, 1); + awaitValidClientRequestContext(ctx, 1); } @Test @@ -906,7 +893,6 @@ void doNotRetryWhenRequestIsAborted() throws Exception { } else { req.abort(abortCause); } - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { client.execute(req).aggregate(); ctx = captor.get(); @@ -915,7 +901,7 @@ void doNotRetryWhenRequestIsAborted() throws Exception { TimeUnit.SECONDS.sleep(1); // No request is made. assertThat(responseAbortServiceCallCounter.get()).isZero(); - validateClientRequestLog(ctx, 0); + awaitValidClientRequestContext(ctx, 0); final RequestLog log = context.get().log().whenComplete().join(); if (abortCause == null) { assertThat(log.requestCause()).isExactlyInstanceOf(AbortedStreamException.class); @@ -941,7 +927,6 @@ void exceptionInDecorator() { .decorator(RetryingClient.newDecorator(strategy, 5)) .build(); - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { assertThatThrownBy(() -> client.get("/").aggregate().join()) .isInstanceOf(CompletionException.class) @@ -949,7 +934,7 @@ void exceptionInDecorator() { ctx = captor.get(); } assertThat(retryCounter.get()).isEqualTo(5); - validateClientRequestLog(ctx, 5); + awaitValidClientRequestContext(ctx, 5); } @Test @@ -960,14 +945,13 @@ void exceptionInRule() { }; final WebClient client = client(rule); - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { assertThatThrownBy(client.get("/").aggregate()::join) .isInstanceOf(CompletionException.class) .hasCauseReference(exception); ctx = captor.get(); } - validateClientRequestLog(ctx, 1); + awaitValidClientRequestContext(ctx, 1); } @Test @@ -978,14 +962,13 @@ void exceptionInRuleWithContent() { }; final WebClient client = client(rule, 10000, 0, 100); - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { assertThatThrownBy(client.get("/").aggregate()::join) .isInstanceOf(CompletionException.class) .hasCauseReference(exception); ctx = captor.get(); } - validateClientRequestLog(ctx, 1); + awaitValidClientRequestContext(ctx, 1); } @Test @@ -1000,7 +983,6 @@ void useSameEventLoopWhenAggregate() throws InterruptedException { }) .decorator(RetryingClient.newDecorator(RetryRule.failsafe(), 2)) .build(); - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { client.get("/503-then-success").aggregate().whenComplete((unused, cause) -> { assertThat(eventLoop.get().inEventLoop()).isTrue(); @@ -1009,7 +991,7 @@ void useSameEventLoopWhenAggregate() throws InterruptedException { ctx = captor.get(); } latch.await(); - validateClientRequestLog(ctx, 2); + awaitValidClientRequestContext(ctx, 2); } private WebClient client(RetryRule retryRule) { @@ -1063,11 +1045,36 @@ private WebClient client(RetryRuleWithContent retryRuleWithContent .build(); } - private static void validateClientRequestLog(ClientRequestContext ctx) { - validateClientRequestLog(ctx, ctx.log().children().size()); + private static void waitForEveryAttemptResponse(ClientRequestContext ctx) { + final State state = AbstractRetryingClient.state(ctx); + final int numStartedAttempts = state.startedAttempts().size(); + assertThat(state.isRetryingComplete()).isTrue(); + + await().untilAsserted(() -> { + final List> startedAttempts = state.startedAttempts(); + assertThat(startedAttempts).hasSize(numStartedAttempts); + for (Attempt attempt : startedAttempts) { + assertThat(attempt.attemptRes().isComplete()).isTrue(); + } + }); } - private static void validateClientRequestLog(ClientRequestContext ctx, int expectedNumRequests) { + private static void assertValidRetryingState(ClientRequestContext ctx, int expectedAttempts) { + final State state = AbstractRetryingClient.state(ctx); + assertThat(state.isRetryingComplete()).isTrue(); + + final List> startedAttempts = state.startedAttempts(); + assertThat(startedAttempts).hasSize(expectedAttempts); + for (Attempt attempt : startedAttempts) { + assertThat(attempt.attemptRes().isComplete()).isTrue(); + } + } + + private static void awaitValidClientRequestContext(ClientRequestContext ctx) { + awaitValidClientRequestContext(ctx, ctx.log().children().size()); + } + + private static void awaitValidClientRequestContext(ClientRequestContext ctx, int expectedNumRequests) { await().untilAsserted(() -> { assertThat(ctx.log().isComplete()).isTrue(); assertThat(ctx.log().children()).hasSize(expectedNumRequests); @@ -1080,6 +1087,8 @@ private static void validateClientRequestLog(ClientRequestContext ctx, int expec fail(e); } }); + + assertValidRetryingState(ctx, expectedNumRequests); }); } diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java index aecbaf832c1..b27898550f4 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java @@ -45,8 +45,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.linecorp.armeria.client.ClientFactory; import com.linecorp.armeria.client.ClientRequestContext; @@ -59,6 +57,8 @@ import com.linecorp.armeria.client.endpoint.EndpointGroup; import com.linecorp.armeria.client.endpoint.EndpointSelectionStrategy; import com.linecorp.armeria.client.logging.LoggingClient; +import com.linecorp.armeria.client.retry.AbstractRetryingClient.State; +import com.linecorp.armeria.client.retry.AbstractRetryingClient.State.Attempt; import com.linecorp.armeria.common.AggregatedHttpResponse; import com.linecorp.armeria.common.ExchangeType; import com.linecorp.armeria.common.HttpRequest; @@ -78,12 +78,8 @@ import com.linecorp.armeria.server.logging.LoggingService; import com.linecorp.armeria.testing.junit5.server.ServerExtension; -// todo(szymon): change tests that we wait for the response and demand that immediately after we see all the -// logs in the appropriate state. class RetryingClientWithHedgingTest { private static class TestServer extends ServerExtension { - private static final Logger logger = LoggerFactory.getLogger(TestServer.class); - private CountDownLatch responseLatch = new CountDownLatch(1); private final AtomicInteger numRequests = new AtomicInteger(); private HttpService helloService = mock(HttpService.class); @@ -122,8 +118,6 @@ public HttpResponse serve(ServiceRequestContext ctx, HttpRequest req) fail(e); } - logger.debug("Latch released. Returning response."); - try { responseFuture.complete(helloService.serve(ctx, req)); } catch (Exception e) { @@ -171,6 +165,7 @@ public void unlatchResponse() { @RegisterExtension private static final TestServer server3 = new TestServer(); + private @Nullable ClientRequestContext ctx; private static ClientFactory clientFactory; private static final RetryRule NO_RETRY_RULE = RetryRule.builder().thenNoRetry(); @@ -213,7 +208,6 @@ void firstServerWins() throws Exception { final WebClient client = client(hedgingNoRetryConfig); final CompletableFuture responseFuture; - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { responseFuture = client.get("/hello").aggregate(); ctx = captor.get(); @@ -246,7 +240,6 @@ void secondServerWins() throws Exception { final WebClient client = client(hedgingNoRetryConfig); final CompletableFuture responseFuture; - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { responseFuture = client.get("/hello").aggregate(); ctx = captor.get(); @@ -289,7 +282,6 @@ void thirdServerWins() throws Exception { final WebClient client = client(hedgingNoRetryConfig); final CompletableFuture responseFuture; - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { responseFuture = client.get("/hello").aggregate(); ctx = captor.get(); @@ -355,7 +347,6 @@ public long nextDelayMillis(int numAttemptsSoFar) { final WebClient client = client(config); final CompletableFuture responseFuture; - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { responseFuture = client.get("/hello").aggregate(); ctx = captor.get(); @@ -431,7 +422,6 @@ void allServerLosePickLastResponse() throws Exception { final WebClient client = client(config); final CompletableFuture responseFuture; - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { responseFuture = client.get("/hello").aggregate(); ctx = captor.get(); @@ -489,7 +479,6 @@ void thirdServerWinsEvenAfterPerAttemptTimeout() throws Exception { .build(); final CompletableFuture responseFuture; - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { responseFuture = client.get("/hello").aggregate(); ctx = captor.get(); @@ -546,7 +535,6 @@ void thirdServerWinsEvenAfterRetriableResponse() throws Exception { final WebClient client = client(config); final CompletableFuture responseFuture; - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { responseFuture = client.get("/hello").aggregate(); ctx = captor.get(); @@ -598,7 +586,6 @@ void loosesAfterNonRetriableResponse() throws Exception { final WebClient client = client(config); final CompletableFuture responseFuture; - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { responseFuture = client.get("/hello").aggregate(); ctx = captor.get(); @@ -647,7 +634,6 @@ void loosesAfterResponseTimeout() throws Exception { ).build(); final CompletableFuture responseFuture; - final ClientRequestContext ctx; try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { responseFuture = client.get("/hello").aggregate(); ctx = captor.get(); @@ -701,6 +687,7 @@ private static WebClientBuilder clientBuilder() { server1.httpEndpoint(), server2.httpEndpoint(), server3.httpEndpoint())) + .requestAutoAbortDelayMillis(0) .decorator(LoggingClient.newDecorator()) .factory(clientFactory); } @@ -742,17 +729,27 @@ private static void assertValidRootClientRequestContext(ClientRequestContext ctx logVerifierCtx.accept(log); } + private static void assertValidRetryingState(ClientRequestContext ctx, int expectedAttempts) { + final State state = AbstractRetryingClient.state(ctx); + assertThat(state.isRetryingComplete()).isTrue(); + + final List> startedAttempts = state.startedAttempts(); + assertThat(startedAttempts).hasSize(expectedAttempts); + for (Attempt attempt : startedAttempts) { + assertThat(attempt.attemptRes().isComplete()).isTrue(); + } + } + private static void assertValidClientRequestContext(ClientRequestContext ctx, RequestLogVerifier logVerifierCtx, RequestLogVerifier logVerifierServer1, @Nullable RequestLogVerifier logVerifierServer2, @Nullable RequestLogVerifier logVerifierServer3 ) { + final int expectedAttempts = 1 + (logVerifierServer2 == null ? 0 : 1) + + (logVerifierServer3 == null ? 0 : 1); - final int expectedNumChildren = 1 + (logVerifierServer2 == null ? 0 : 1) + - (logVerifierServer3 == null ? 0 : 1); - - assertValidRootClientRequestContext(ctx, logVerifierCtx, expectedNumChildren); + assertValidRootClientRequestContext(ctx, logVerifierCtx, expectedAttempts); assertValidChildLog(ctx.log().children().get(0), 1, logVerifierServer1); if (logVerifierServer2 != null) { assertValidChildLog(ctx.log().children().get(1), 2, logVerifierServer2); @@ -761,6 +758,8 @@ private static void assertValidClientRequestContext(ClientRequestContext ctx, if (logVerifierServer3 != null) { assertValidChildLog(ctx.log().children().get(2), 3, logVerifierServer3); } + + assertValidRetryingState(ctx, expectedAttempts); } private static void assertValidChildLog(RequestLogAccess logAccess, int attemptNumber, diff --git a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java index 385110d85ec..c1009df249e 100644 --- a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java +++ b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java @@ -52,6 +52,9 @@ import com.linecorp.armeria.client.ResponseTimeoutException; import com.linecorp.armeria.client.endpoint.EndpointGroup; import com.linecorp.armeria.client.endpoint.EndpointSelectionStrategy; +import com.linecorp.armeria.client.retry.AbstractRetryingClient; +import com.linecorp.armeria.client.retry.AbstractRetryingClient.State; +import com.linecorp.armeria.client.retry.AbstractRetryingClient.State.Attempt; import com.linecorp.armeria.client.retry.Backoff; import com.linecorp.armeria.client.retry.RetryConfig; import com.linecorp.armeria.client.retry.RetryRule; @@ -668,6 +671,17 @@ private interface RequestLogVerifier extends Consumer {} assertThat(cause.getMessage()).contains(expectedMessage); }; + private static void assertValidRetryingState(ClientRequestContext ctx, int expectedAttempts) { + final State state = AbstractRetryingClient.state(ctx); + assertThat(state.isRetryingComplete()).isTrue(); + + final List> startedAttempts = state.startedAttempts(); + assertThat(startedAttempts).hasSize(expectedAttempts); + for (Attempt attempt : startedAttempts) { + assertThat(attempt.attemptRes().whenComplete().isDone()).isTrue(); + } + } + private static void assertValidRootClientRequestContext(ClientRequestContext ctx, RequestLogVerifier logVerifierCtx, int expectedNumChildren) { @@ -678,6 +692,8 @@ private static void assertValidRootClientRequestContext(ClientRequestContext ctx RequestLogProperty.REQUEST_HEADERS); assertThat(log).isNotNull(); logVerifierCtx.accept(log); + + assertValidRetryingState(ctx, expectedNumChildren); } private void assertValidClientRequestContext(ClientRequestContext ctx, From 5efaf4901edf16d2732a85948356064ee0529501 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Thu, 19 Jun 2025 19:00:58 +0200 Subject: [PATCH 27/36] fix: RetryingRpcClientWithHedgingTest --- .../client/retry/RetryingRpcClient.java | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java index 17982a7c0a6..b85298df04f 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java @@ -31,7 +31,6 @@ import com.linecorp.armeria.common.Request; import com.linecorp.armeria.common.RpcRequest; import com.linecorp.armeria.common.RpcResponse; -import com.linecorp.armeria.common.util.UnmodifiableFuture; import com.linecorp.armeria.internal.client.ClientPendingThrowableUtil; import com.linecorp.armeria.internal.client.ClientRequestContextExtension; import com.linecorp.armeria.internal.common.util.StringUtil; @@ -202,13 +201,22 @@ private void doExecute0(ClientRequestContext ctx, RpcRequest req, }, (abortingAttemptCtx, abortingAttemptRes, cause) -> { - if (cause != null) { - abortingAttemptCtx.cancel(cause); - } else { - abortingAttemptCtx.cancel(); - } - abortingAttemptRes.cancel(false); - return UnmodifiableFuture.completedFuture(null); + final CompletableFuture abortComplete = new CompletableFuture<>(); + + abortingAttemptCtx.eventLoop().execute(() -> { + if (cause != null) { + abortingAttemptCtx.cancel(cause); + } else { + abortingAttemptCtx.cancel(); + } + + abortingAttemptRes.toCompletableFuture().handle((unused, unusedCause) -> { + abortComplete.complete(null); + return null; + }); + }); + + return abortComplete; }); final RetryConfig retryConfig = mappedRetryConfig(ctx); From 42c870c8e2851491017af594f7288fb516872a34 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Thu, 19 Jun 2025 19:52:44 +0200 Subject: [PATCH 28/36] fix: give scheduler in RetrySchedulerTest more wiggleroom --- .../com/linecorp/armeria/client/retry/RetrySchedulerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java index 37015784d33..c86e72ba0dc 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java @@ -52,7 +52,7 @@ import io.netty.channel.EventLoop; class RetrySchedulerTest { - private static final long SCHEDULING_TOLERANCE_NANOS = TimeUnit.MILLISECONDS.toNanos(50); + private static final long SCHEDULING_TOLERANCE_NANOS = TimeUnit.MILLISECONDS.toNanos(200); private static final long SCHEDULING_TOLERANCE_MILLIS = TimeUnit.NANOSECONDS.toMillis( SCHEDULING_TOLERANCE_NANOS); From 9f849b0fc63fbd7a849fe135a7fc7f00926d0d90 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Thu, 19 Jun 2025 19:57:50 +0200 Subject: [PATCH 29/36] fix: RetryingRpcClientWithHedgingTest.loosesAfterResponseTimeout --- .../retry/RetryingClientWithHedgingTest.java | 48 +++++++------- .../RetryingRpcClientWithHedgingTest.java | 66 +++++++++---------- 2 files changed, 57 insertions(+), 57 deletions(-) diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java index b27898550f4..d26eb8d990f 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java @@ -645,38 +645,38 @@ void loosesAfterResponseTimeout() throws Exception { final Throwable rootCause = Exceptions.peel(throwable); assertThat(rootCause).isInstanceOf(ResponseTimeoutException.class); }); - }); - final List childLogExceptions = new ArrayList<>(); + final List childLogExceptions = new ArrayList<>(); - final RequestLogVerifier catchException = log -> { - assertThat(log.responseCause()).isNotNull(); - childLogExceptions.add(log.responseCause()); - }; + final RequestLogVerifier catchException = log -> { + assertThat(log.responseCause()).isNotNull(); + childLogExceptions.add(log.responseCause()); + }; - assertValidClientRequestContext(ctx, VERIFY_RESPONSE_TIMEOUT, catchException, catchException, - catchException); + assertValidClientRequestContext(ctx, VERIFY_RESPONSE_TIMEOUT, catchException, catchException, + catchException); - int numTimeouts = 0; - int numCancelled = 0; + int numTimeouts = 0; + int numCancelled = 0; - for (final @Nullable Throwable childException : childLogExceptions) { - if (childException instanceof ResponseTimeoutException) { - numTimeouts++; - } else if (childException instanceof ResponseCancellationException) { - numCancelled++; - } else { - fail("Unexpected exception: " + childException); + for (final @Nullable Throwable childException : childLogExceptions) { + if (childException instanceof ResponseTimeoutException) { + numTimeouts++; + } else if (childException instanceof ResponseCancellationException) { + numCancelled++; + } else { + fail("Unexpected exception: " + childException); + } } - } - assertThat(numTimeouts + numCancelled).isEqualTo(3); - // At least one attempt needs to time out. - assertThat(numTimeouts).isPositive(); + assertThat(numTimeouts + numCancelled).isEqualTo(3); + // At least one attempt needs to time out. + assertThat(numTimeouts).isPositive(); - assertValidServerRequestContext(server1, 1, true); - assertValidServerRequestContext(server2, 2, true); - assertValidServerRequestContext(server3, 3, true); + assertValidServerRequestContext(server1, 1, true); + assertValidServerRequestContext(server2, 2, true); + assertValidServerRequestContext(server3, 3, true); + }); } // todo(szymon): test being able to set different hedging delays for different servers diff --git a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java index c1009df249e..537586df49a 100644 --- a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java +++ b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientWithHedgingTest.java @@ -545,43 +545,43 @@ void loosesAfterResponseTimeout() { assertThat(peeledThrowable).isInstanceOf(TTransportException.class); assertThat(peeledThrowable.getCause()).isInstanceOf(ResponseTimeoutException.class); }); - }); - final List childLogExceptions = new ArrayList<>(); - - final RequestLogVerifier catchException = log -> { - assertThat(log.responseCause()).isInstanceOf(TTransportException.class); - final TTransportException cause = (TTransportException) log.responseCause(); - assertThat(cause.getCause()).isNotNull(); - childLogExceptions.add(cause.getCause()); - }; - - assertValidClientRequestContext(ctx, - VERIFY_RESPONSE_TIMEOUT, - catchException, - catchException, - catchException); - - int numTimeouts = 0; - int numCancelled = 0; - - for (final @Nullable Throwable childException : childLogExceptions) { - if (childException instanceof ResponseTimeoutException) { - numTimeouts++; - } else if (childException instanceof ResponseCancellationException) { - numCancelled++; - } else { - fail("Unexpected exception: " + childException); + final List childLogExceptions = new ArrayList<>(); + + final RequestLogVerifier catchException = log -> { + assertThat(log.responseCause()).isInstanceOf(TTransportException.class); + final TTransportException cause = (TTransportException) log.responseCause(); + assertThat(cause.getCause()).isNotNull(); + childLogExceptions.add(cause.getCause()); + }; + + assertValidClientRequestContext(ctx, + VERIFY_RESPONSE_TIMEOUT, + catchException, + catchException, + catchException); + + int numTimeouts = 0; + int numCancelled = 0; + + for (final @Nullable Throwable childException : childLogExceptions) { + if (childException instanceof ResponseTimeoutException) { + numTimeouts++; + } else if (childException instanceof ResponseCancellationException) { + numCancelled++; + } else { + fail("Unexpected exception: " + childException); + } } - } - assertThat(numTimeouts + numCancelled).isEqualTo(3); - // At least one attempt needs to time out. - assertThat(numTimeouts).isPositive(); + assertThat(numTimeouts + numCancelled).isEqualTo(3); + // At least one attempt needs to time out. + assertThat(numTimeouts).isPositive(); - assertValidServerRequestContext(server1, 1, true); - assertValidServerRequestContext(server2, 2, true); - assertValidServerRequestContext(server3, 3, true); + assertValidServerRequestContext(server1, 1, true); + assertValidServerRequestContext(server2, 2, true); + assertValidServerRequestContext(server3, 3, true); + }); } private CompletableFuture asyncHelloWith(HelloService.AsyncIface client) { From 431b927b509dc8aa8b408f49eccbecdb87a872e6 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Thu, 19 Jun 2025 19:58:23 +0200 Subject: [PATCH 30/36] fix: reduce state space by early exists in RetryingClient --- .../armeria/client/retry/RetryingClient.java | 40 ++----------------- 1 file changed, 4 insertions(+), 36 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index 3943c7e154f..e31f66fe59f 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -388,25 +388,12 @@ private void doExecute0(RetryingContext retryingContext, int attemptNo) { attemptCtx.logBuilder().endRequest(cause); attemptCtx.logBuilder().endResponse(cause); - // At that point we completed the request log for the attempt. - // What comes next investigates the attempt's response to decide - // on retry. We do not want to continue if we already completed - // the whole retrying process/ if we have a winning attempt. - if (isRetryingComplete(ctx)) { - return null; - } - handleResponseWithoutContent(retryingContext, attemptCtx, HttpResponse.ofFailure(cause), cause); } else { completeAttemptLogIfBytesNotTransferred(attemptCtx, attemptAggResponse); - // see above - if (isRetryingComplete(ctx)) { - return null; - } - attemptCtx.log().whenAvailable(RequestLogProperty.RESPONSE_END_TIME).thenRun(() -> { handleAggregatedResponse(retryingContext, attemptCtx, attemptAggResponse); }); @@ -461,10 +448,6 @@ private void handleResponseWithoutContent(RetryingContext retryingContext, final RetryRule retryRule = retryRule(retryingContext.config()); final CompletionStage f = retryRule.shouldRetry(attemptCtx, attemptResCause); f.handle((decision, shouldRetryCause) -> { - if (isRetryingComplete(retryingContext.ctx())) { - return null; - } - warnIfExceptionIsRaised(retryRule, shouldRetryCause); handleRetryDecision(retryingContext, decision, attemptCtx, attemptRes); return null; @@ -490,16 +473,6 @@ private void handleStreamingResponse(RetryingContext retryingContext, completeAttemptLogIfBytesNotTransferred(attemptCtx, attemptRes, headers, responseCause); attemptCtx.log().whenAvailable(RequestLogProperty.RESPONSE_HEADERS).thenRun(() -> { - // see above - if (isRetryingComplete(retryingContext.ctx())) { - if (responseCause != null) { - attemptSplitRes.body().abort(responseCause); - } else { - attemptSplitRes.body().abort(); - } - return; - } - if (retryingContext.config().needsContentInRule() && responseCause == null) { final HttpResponse attemptUnsplitRes = attemptSplitRes.unsplit(); final HttpResponseDuplicator attemptResDuplicator = @@ -520,11 +493,6 @@ private void handleStreamingResponse(RetryingContext retryingContext, warnIfExceptionIsRaised(ruleWithContent, cause); attemptTruncatedRes.abort(); - if (isRetryingComplete(retryingContext.ctx())) { - attemptResDuplicator.abort(); - return null; - } - handleRetryDecision(retryingContext, decision, attemptCtx, attemptDuplicatedRes); return null; @@ -560,10 +528,6 @@ private void handleAggregatedResponse(RetryingContext retryingContext, .handle((decision, cause) -> { warnIfExceptionIsRaised(ruleWithContent, cause); - if (isRetryingComplete(retryingContext.ctx())) { - return null; - } - handleRetryDecision(retryingContext, decision, attemptCtx, attemptAggRes.toHttpResponse()); return null; @@ -655,6 +619,10 @@ private static void completeRetryingExceptionally(ClientRequestContext ctx, private void handleRetryDecision(RetryingContext retryingContext, @Nullable RetryDecision decision, ClientRequestContext attemptCtx, HttpResponse attemptRes) { + if (isRetryingComplete(retryingContext.ctx())) { + return; + } + final Backoff backoff = decision != null ? decision.backoff() : null; if (backoff != null) { From b610027b2d6095c908399f381298f0529cd6f1c6 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Thu, 19 Jun 2025 20:30:57 +0200 Subject: [PATCH 31/36] fix: remove limit for `RetryingClientWithHedgingTest.loosesAfterResponseTimeout` --- .../armeria/client/retry/RetryingClientWithHedgingTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java index d26eb8d990f..bd967e91333 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java @@ -599,7 +599,7 @@ void loosesAfterNonRetriableResponse() throws Exception { assertThat(server2.getNumRequests()).isOne(); }); - Thread.sleep(10); + Thread.sleep(500); server1.unlatchResponse(); server2.unlatchResponse(); @@ -639,7 +639,7 @@ void loosesAfterResponseTimeout() throws Exception { ctx = captor.get(); } - await().atLeast(400, TimeUnit.MILLISECONDS).atMost(600, TimeUnit.MILLISECONDS).untilAsserted(() -> { + await().untilAsserted(() -> { assertThat(responseFuture).isCompletedExceptionally(); assertThatThrownBy(responseFuture::get).satisfies(throwable -> { final Throwable rootCause = Exceptions.peel(throwable); From 5e6a867580b9928a386232d6305351088f78142a Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Fri, 20 Jun 2025 15:55:46 +0200 Subject: [PATCH 32/36] fix: schedule winning handler on attempt event loop RetryingRpcClient --- .../armeria/client/retry/RetryingClient.java | 1 - .../armeria/client/retry/RetryingRpcClient.java | 14 ++++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index e31f66fe59f..aa0c7eac157 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -245,7 +245,6 @@ public static Function newDecorator(RetryRul protected HttpResponse doExecute(ClientRequestContext ctx, HttpRequest req) throws Exception { final CompletableFuture responseFuture = new CompletableFuture<>(); final HttpResponse res = HttpResponse.of(responseFuture, ctx.eventLoop()); - if (ctx.exchangeType().isRequestStreaming()) { final HttpRequestDuplicator reqDuplicator = req.toDuplicator(ctx.eventLoop().withoutContext(), 0); doExecute0(new RetryingContext(mappedRetryConfig(ctx), ctx, reqDuplicator, req, res, diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java index b85298df04f..0caf03c3c9d 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java @@ -191,13 +191,15 @@ private void doExecute0(ClientRequestContext ctx, RpcRequest req, } startRetryAttempt(ctx, attemptCtx, attemptRes, (winningAttemptCtx, winningAttemptRes) -> { - ctx.logBuilder().endResponseWithChild(winningAttemptCtx.log()); - final HttpRequest actualHttpReq = winningAttemptCtx.request(); - if (actualHttpReq != null) { - ctx.updateRequest(actualHttpReq); - } + winningAttemptCtx.eventLoop().execute(() -> { + ctx.logBuilder().endResponseWithChild(winningAttemptCtx.log()); + final HttpRequest actualHttpReq = winningAttemptCtx.request(); + if (actualHttpReq != null) { + ctx.updateRequest(actualHttpReq); + } - returnedResFuture.complete(winningAttemptRes); + returnedResFuture.complete(winningAttemptRes); + }); }, (abortingAttemptCtx, abortingAttemptRes, cause) -> { From b0631042cd4d0663ad5608f6c77c3cef9b1b0f09 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Fri, 20 Jun 2025 16:09:54 +0200 Subject: [PATCH 33/36] test: swap asserts to get more insight on failure --- .../armeria/client/retry/RetryingClientWithHedgingTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java index bd967e91333..daa10ae73fe 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithHedgingTest.java @@ -713,8 +713,8 @@ private static void assertValidAggregatedResponse(CompletableFuture Date: Fri, 20 Jun 2025 16:11:24 +0200 Subject: [PATCH 34/36] fix: doNotRetryWhenSubscriberIsCancelled --- .../com/linecorp/armeria/client/retry/RetryingClientTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java index c068e21d4da..97b151191a7 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java @@ -865,10 +865,9 @@ public void onComplete() {} ctx = captor.get(); } - + awaitValidClientRequestContext(ctx, 1); TimeUnit.SECONDS.sleep(1L); // Sleep to check if there's a retry. assertThat(subscriberCancelServiceCallCounter.get()).isEqualTo(1); - awaitValidClientRequestContext(ctx, 1); } @Test From c5d93e1ea4289e273340dfeea3265624e342ccef Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Wed, 25 Jun 2025 12:19:20 +0200 Subject: [PATCH 35/36] refactor: revert updates to copyright headers --- .../armeria/client/Http2ResponseDecoder.java | 5 +++- .../client/retry/AbstractRetryingClient.java | 23 +++++++-------- .../armeria/client/retry/Backoff.java | 6 ++-- .../armeria/client/retry/RetryConfig.java | 4 +-- .../client/retry/RetryConfigBuilder.java | 10 +++---- .../armeria/client/retry/RetryScheduler.java | 28 +++++++++---------- .../retry/RetrySchedulingException.java | 10 ++++--- .../armeria/client/retry/RetryingClient.java | 8 +++--- .../client/retry/RetryingRpcClient.java | 7 ++--- .../common/logging/DefaultRequestLog.java | 8 +++--- .../common/logging/RequestLogBuilder.java | 4 +-- .../armeria/internal/client/ClientUtil.java | 11 ++------ .../server/ServerHttp2ObjectEncoder.java | 2 +- .../client/retry/RetryingClientTest.java | 4 +-- .../RetryingClientWithContextAwareTest.java | 16 +++-------- .../server/ServiceRequestContextCaptor.java | 4 +-- 16 files changed, 68 insertions(+), 82 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/Http2ResponseDecoder.java b/core/src/main/java/com/linecorp/armeria/client/Http2ResponseDecoder.java index acb151c0d8a..409f3c105c2 100644 --- a/core/src/main/java/com/linecorp/armeria/client/Http2ResponseDecoder.java +++ b/core/src/main/java/com/linecorp/armeria/client/Http2ResponseDecoder.java @@ -301,7 +301,10 @@ public void onRstStreamRead(ChannelHandlerContext ctx, int streamId, long errorC keepAliveChannelRead(); final HttpResponseWrapper res = getResponse(streamIdToId(streamId)); if (res == null || !res.isOpen()) { - if (conn.streamMayHaveExisted(streamId)) { + final Http2Stream stream = conn.stream(streamId); + if (stream != null) { + // the stream was active, but will be closed now + } else if (conn.streamMayHaveExisted(streamId)) { if (logger.isDebugEnabled()) { logger.debug("{} Received a late RST_STREAM frame for a closed stream: {}", ctx.channel(), streamId); diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java index 2923f6e00a9..87e9f771ca0 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java @@ -1,7 +1,7 @@ /* - * Copyright 2025 LY Corporation + * Copyright 2017 LINE Corporation * - * LY Corporation licenses this file to you under the Apache License, + * LINE Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * @@ -60,6 +60,7 @@ */ public abstract class AbstractRetryingClient extends SimpleDecoratingClient { + private static final Logger logger = LoggerFactory.getLogger(AbstractRetryingClient.class); /** @@ -119,8 +120,7 @@ protected final RetryConfigMapping mapping() { * Invoked by {@link #execute(ClientRequestContext, Request)} * after the deadline for response timeout is set. */ - protected abstract O doExecute(ClientRequestContext ctx, I req) - throws Exception; + protected abstract O doExecute(ClientRequestContext ctx, I req) throws Exception; /** * todo(szymon): [doc]. @@ -373,7 +373,6 @@ protected static void scheduleNextRetry(ClientRequestContext ctx, */ @SuppressWarnings("MethodMayBeStatic") // Intentionally left non-static for better user experience. protected final boolean setResponseTimeout(ClientRequestContext ctx) { - // We do not need to acquire a lock on the state as this method body is thread-safe. requireNonNull(ctx, "ctx"); final long responseTimeoutMillis = state(ctx).responseTimeoutMillisForAttempt(); if (responseTimeoutMillis < 0) { @@ -392,7 +391,6 @@ protected final boolean setResponseTimeout(ClientRequestContext ctx) { * {@link ClientRequestContext}. */ protected static int getTotalAttempts(ClientRequestContext ctx) { - // We do not need to acquire a lock on the state as this method body is thread-safe. final State state = ctx.attr(STATE); if (state == null) { return 0; @@ -401,10 +399,10 @@ protected static int getTotalAttempts(ClientRequestContext ctx) { } /** - * Creates a new derived {@link ClientRequestContext} for a retrying attempt, replacing the requests. + * Creates a new derived {@link ClientRequestContext}, replacing the requests. * If {@link ClientRequestContext#endpointGroup()} exists, a new {@link Endpoint} will be selected. */ - protected static ClientRequestContext newAttemptContext(ClientRequestContext ctx, + protected static ClientRequestContext newDerivedContext(ClientRequestContext ctx, @Nullable HttpRequest req, @Nullable RpcRequest rpcReq, boolean initialAttempt) { @@ -604,7 +602,6 @@ boolean timeoutForWholeRetryEnabled() { return isTimeoutEnabled; } - // Is thread-safe. long responseTimeoutMillis() { assert isTimeoutEnabled; return Math.max(TimeUnit.NANOSECONDS.toMillis(deadlineNanos - System.nanoTime()), -1); @@ -755,7 +752,7 @@ private void completeIfNoPendingAttempts0() { complete0(); } - // takes ownership of current lock + // Takes ownership of the current retry lock acquisition private void complete0() { assert lock.isHeldByCurrentThread(); assert !isRetryingComplete(); @@ -763,7 +760,7 @@ private void complete0() { final boolean hasLastAttempt = lastAttempt != null; if (!hasLastAttempt) { - // takes ownership of current lock + // Takes ownership of the current retry lock acquisition completeExceptionally0( new IllegalStateException("Completed retrying without any attempts.")); } else { @@ -784,7 +781,7 @@ void completeExceptionally(Throwable cause) { return; } - // takes ownership of current lock + // Takes ownership of the current retry lock acquisition completeExceptionally0(cause); } @@ -813,7 +810,7 @@ private void notifyAllAttemptHandlers(@Nullable Attempt winningAttempt, @Null }); } - // takes ownership of current lock + // Takes ownership of the current retry lock acquisition private void completeExceptionally0(Throwable cause) { assert lock.isHeldByCurrentThread(); assert !isRetryingComplete(); diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/Backoff.java b/core/src/main/java/com/linecorp/armeria/client/retry/Backoff.java index dd5963e8774..5b8a289fd33 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/Backoff.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/Backoff.java @@ -1,7 +1,7 @@ /* - * Copyright 2025 LY Corporation + * Copyright 2017 LINE Corporation * - * LY Corporation licenses this file to you under the Apache License, + * LINE Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * @@ -187,7 +187,7 @@ static RandomBackoffBuilder builderForRandom() { } /** - * Returns the number of milliseconds to wait for before attempting a retry. This method is idempotent. + * Returns the number of milliseconds to wait for before attempting a retry. * * @param numAttemptsSoFar the number of attempts made by a client so far, including the first attempt and * its following retries. diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfig.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfig.java index 736056731ad..315e35fd660 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfig.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfig.java @@ -1,7 +1,7 @@ /* - * Copyright 2025 LY Corporation + * Copyright 2020 LINE Corporation * - * LY Corporation licenses this file to you under the Apache License, + * LINE Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfigBuilder.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfigBuilder.java index 05f9c81d9bf..1208fea8def 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfigBuilder.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryConfigBuilder.java @@ -1,7 +1,7 @@ /* - * Copyright 2025 LY Corporation + * Copyright 2020 LINE Corporation * - * LY Corporation licenses this file to you under the Apache License, + * LINE Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * @@ -94,7 +94,7 @@ public RetryConfigBuilder hedgingDelay(Duration hedgingDelay) { .toMillis(); checkArgument( millis >= 0, - "responseTimeoutForEachAttempt.toMillis(): %s (expected: >= 0)", + "hedgingDelay.toMillis(): %s (expected: >= 0)", millis); hedgingDelayMillis = millis; return this; @@ -175,7 +175,7 @@ ToStringHelper toStringHelper() { .add("retryRuleWithContent", retryRuleWithContent) .add("maxTotalAttempts", maxTotalAttempts) .add("responseTimeoutMillisForEachAttempt", responseTimeoutMillisForEachAttempt) - .add("hedgingDelayMillis", hedgingDelayMillis) - .add("maxContentLength", maxContentLength); + .add("maxContentLength", maxContentLength) + .add("hedgingDelayMillis", hedgingDelayMillis); } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java index 58f353b492c..412e428b575 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java @@ -105,9 +105,9 @@ ScheduledFuture getFuture() { private long retryTaskId; // The retry task that is about to be executed next. - // It is possible that the delay of this task is shorter than the `earliestNextRetryTimeNanos`, - // because of calls to `addEarliestNextRetryTimeNanos` with a pending call to `schedule` or - // `rescheduleCurrentRetryTaskIfTooEarly`. + // It is possible that the delay of this task is shorter than earliestNextRetryTimeNanos, + // because of calls to addEarliestNextRetryTimeNanos with a pending call to schedule or + // rescheduleCurrentRetryTaskIfTooEarly. private @Nullable RetryTaskHandle currentRetryTask; RetryScheduler(ReentrantLock retryLock, EventLoop eventLoop) { @@ -141,11 +141,11 @@ public void schedule(Consumer retryTask, long nextRetryTimeNanos, // currently scheduled retry task (even not by us in this method). if (currentRetryTask == null || nextRetryTimeNanos < Math.max(this.earliestNextRetryTimeNanos, currentRetryTask.retryTimeNanos())) { - // takes ownership of the acquired retry lock + // Takes ownership of the current retry lock acquisition scheduleNextRetryTask(retryTask, nextRetryTimeNanos, exceptionHandler, false); } else { // Make sure the current retry task is not scheduled too early. - // takes ownership of the acquired retry lock + // Takes ownership of the current retry lock acquisition rescheduleCurrentRetryTaskIfTooEarly0(); exceptionHandler.accept(new RetrySchedulingException( @@ -236,7 +236,7 @@ private void clearCurrentRetryTask() { } } - // takes ownership of the acquired retry lock + // Takes ownership of the current retry lock acquisition private void scheduleNextRetryTask(Consumer retryRunnable, long retryTimeNanos, Consumer exceptionHandler, boolean isReschedule) { @@ -287,7 +287,7 @@ private void scheduleNextRetryTask(Consumer retryRunnable, long r final long taskRunTimeNanos = System.nanoTime(); - // max to be robust against overflows. + // Math.max to be robust against overflows. if (Math.max(taskRunTimeNanos, taskRunTimeNanos + RESCHEDULING_OVERTAKING_TOLERANCE_NANOS) < earliestNextRetryTimeNanos) { @@ -297,9 +297,9 @@ private void scheduleNextRetryTask(Consumer retryRunnable, long r currentRetryTask.getExceptionHandler(); clearCurrentRetryTask(); - // do not invoke rescheduleCurrentRetryTaskIfTooEarly here as it will not be able + // Do not invoke rescheduleCurrentRetryTaskIfTooEarly here as it will not be able // to cancel us. - // takes ownership of the acquired retry lock + // Takes ownership of the current retry lock acquisition scheduleNextRetryTask(currentRetryRunnable, earliestNextRetryTimeNanos, currentExceptionHandler, false); @@ -350,14 +350,14 @@ public long addEarliestNextRetryTimeNanos(long earliestNextRetryTimeNanos) { } } - // takes ownership of the acquired retry lock (if any) + // Takes ownership of the current retry lock acquisition (if any) public void rescheduleCurrentRetryTaskIfTooEarly() { retryLock.lock(); - // takes ownership of the acquired retry lock + // Takes ownership of the current retry lock acquisition rescheduleCurrentRetryTaskIfTooEarly0(); } - // takes ownership of the acquired retry lock + // Takes ownership of the current retry lock acquisition private void rescheduleCurrentRetryTaskIfTooEarly0() { assert retryLock.isHeldByCurrentThread(); @@ -371,10 +371,10 @@ private void rescheduleCurrentRetryTaskIfTooEarly0() { currentRetryTask.retryTimeNanos() + RESCHEDULING_OVERTAKING_TOLERANCE_NANOS) < earliestNextRetryTimeNanos ) { - // Current retry task is going to be executed before the earliestNextRetryTimeNanos so + // The current retry task is going to be executed before earliestNextRetryTimeNanos so // we need to reschedule it. - // Takes ownership of the acquired lock. + // Takes ownership of the current retry lock acquisition scheduleNextRetryTask(currentRetryTask.getRetryTaskRunnable(), earliestNextRetryTimeNanos, currentRetryTask.getExceptionHandler(), true); diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetrySchedulingException.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetrySchedulingException.java index bb020446537..8dc0f0c6be8 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetrySchedulingException.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetrySchedulingException.java @@ -21,13 +21,15 @@ class RetrySchedulingException extends RuntimeException { private final Type type; enum Type { - RETRYING_ALREADY_COMPLETED("Retrying completed"), - NO_MORE_ATTEMPTS_IN_RETRY("No more attempts available in retry"), + RETRYING_ALREADY_COMPLETED("Retrying completed already"), + NO_MORE_ATTEMPTS_IN_RETRY("No more attempts with respect to `RetryConfig.maxAttempts`"), NO_MORE_ATTEMPTS_IN_BACKOFF("No more attempts available in backoff"), DELAY_FROM_BACKOFF_EXCEEDS_RESPONSE_TIMEOUT("Delay from backoff exceeds response timeout"), DELAY_FROM_SERVER_EXCEEDS_RESPONSE_TIMEOUT("Delay from server exceeds response timeout"), - RETRY_TASK_OVERTAKEN("Has earlier retry"), - RETRY_TASK_CANCELLED("Retry task cancelled unexpectedly."); + RETRY_TASK_OVERTAKEN( + "Retry task cannot run because of another retry task for the same attempt being" + + " scheduled earlier"), + RETRY_TASK_CANCELLED("A retry task was cancelled unexpectedly"); private final String message; diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index aa0c7eac157..7f0e7fb4255 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -1,7 +1,7 @@ /* - * Copyright 2025 LY Corporation + * Copyright 2017 LINE Corporation * - * LY Corporation licenses this file to you under the Apache License, + * LINE Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * @@ -220,7 +220,7 @@ public static Function newDecorator(RetryRul * requests. * * @param mapping the mapping that returns a {@link RetryConfig} for a given {@link ClientRequestContext} - * and {@link Request}. + * and {@link Request}. */ public static Function newDecoratorWithMapping(RetryConfigMapping mapping) { @@ -313,7 +313,7 @@ private void doExecute0(RetryingContext retryingContext, int attemptNo) { final ClientRequestContext attemptCtx; try { - attemptCtx = newAttemptContext(ctx, attemptReq, ctx.rpcRequest(), initialAttempt); + attemptCtx = newDerivedContext(ctx, attemptReq, ctx.rpcRequest(), initialAttempt); } catch (Throwable t) { completeRetryingExceptionally(retryingContext, t, initialAttempt); return; diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java index 0caf03c3c9d..8ce4622f8ed 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java @@ -1,7 +1,7 @@ /* - * Copyright 2025 LY Corporation + * Copyright 2017 LINE Corporation * - * LY Corporation licenses this file to you under the Apache License, + * LINE Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * @@ -165,7 +165,7 @@ private void doExecute0(ClientRequestContext ctx, RpcRequest req, return; } - final ClientRequestContext attemptCtx = newAttemptContext(ctx, null, req, initialAttempt); + final ClientRequestContext attemptCtx = newDerivedContext(ctx, null, req, initialAttempt); if (!initialAttempt) { attemptCtx.mutateAdditionalRequestHeaders( @@ -294,7 +294,6 @@ private static void completeRetryingExceptionally(ClientRequestContext ctx, if (endRequestLog) { ctx.logBuilder().endRequest(cause); } - ctx.logBuilder().endResponse(cause); completeRetryingExceptionally(ctx, cause); diff --git a/core/src/main/java/com/linecorp/armeria/common/logging/DefaultRequestLog.java b/core/src/main/java/com/linecorp/armeria/common/logging/DefaultRequestLog.java index cff87adfb62..0b12042fef3 100644 --- a/core/src/main/java/com/linecorp/armeria/common/logging/DefaultRequestLog.java +++ b/core/src/main/java/com/linecorp/armeria/common/logging/DefaultRequestLog.java @@ -1,7 +1,7 @@ /* - * Copyright 2025 LY Corporation + * Copyright 2016 LINE Corporation * - * LY Corporation licenses this file to you under the Apache License, + * LINE Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * @@ -401,7 +401,7 @@ private void updateFlags(RequestLogProperty property) { } private void updateFlags(int flags) { - for (;;) { + for (; ; ) { final int oldFlags = this.flags; final int newFlags = oldFlags | flags; if (oldFlags == newFlags) { @@ -517,7 +517,7 @@ private void defer(int flag) { flag |= RequestLogProperty.NAME.flag(); } - for (;;) { + for (; ; ) { final int oldFlags = deferredFlags; final int newFlags = oldFlags | flag; if (oldFlags == newFlags) { diff --git a/core/src/main/java/com/linecorp/armeria/common/logging/RequestLogBuilder.java b/core/src/main/java/com/linecorp/armeria/common/logging/RequestLogBuilder.java index 7b7a09fc817..2156e9da01d 100644 --- a/core/src/main/java/com/linecorp/armeria/common/logging/RequestLogBuilder.java +++ b/core/src/main/java/com/linecorp/armeria/common/logging/RequestLogBuilder.java @@ -1,7 +1,7 @@ /* - * Copyright 2025 LY Corporation + * Copyright 2016 LINE Corporation * - * LY Corporation licenses this file to you under the Apache License, + * LINE Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * diff --git a/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java b/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java index 7e7b3a95418..a4f65d598fc 100644 --- a/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java +++ b/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java @@ -1,7 +1,7 @@ /* - * Copyright 2025 LY Corporation + * Copyright 2018 LINE Corporation * - * LY Corporation licenses this file to you under the Apache License, + * LINE Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * @@ -235,10 +235,6 @@ public static ClientRequestContext newDerivedContext(ClientRequestContext ctx, derived = ctx.newDerivedContext(id, req, rpcReq, ctx.endpoint()); } - // We want to add the request log of the derived context as a child to the request log of the context - // we are deriving from. - // For that we copy over all log properties from the parent log to the derived log - // and add future actions to copy over content (previews). final RequestLogAccess parentLog = ctx.log(); final RequestLog partial = parentLog.partial(); final RequestLogBuilder logBuilder = derived.logBuilder(); @@ -281,10 +277,7 @@ public static ClientRequestContext newDerivedContext(ClientRequestContext ctx, .thenAccept(requestLog -> logBuilder.responseContentPreview( requestLog.responseContentPreview())); } - - // We finally add the derived log as a child of the parent log. ctx.logBuilder().addChild(derived.log()); - return derived; } diff --git a/core/src/main/java/com/linecorp/armeria/server/ServerHttp2ObjectEncoder.java b/core/src/main/java/com/linecorp/armeria/server/ServerHttp2ObjectEncoder.java index c9466dff548..3bceb2d7ca9 100644 --- a/core/src/main/java/com/linecorp/armeria/server/ServerHttp2ObjectEncoder.java +++ b/core/src/main/java/com/linecorp/armeria/server/ServerHttp2ObjectEncoder.java @@ -143,7 +143,7 @@ public void maybeResetStream(int streamId, Http2Error http2Error) { if (stream == null) { return; } - if (stream.state().localSideOpen() || stream.state().remoteSideOpen()) { + if (stream.state().remoteSideOpen()) { encoder().writeRstStream(ctx(), streamId, http2Error.code(), ctx().voidPromise()); ctx().flush(); } diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java index 97b151191a7..d4dd142cc38 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java @@ -1,7 +1,7 @@ /* - * Copyright 2025 LY Corporation + * Copyright 2017 LINE Corporation * - * LY Corporation licenses this file to you under the Apache License, + * LINE Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithContextAwareTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithContextAwareTest.java index 0e7654bd8ac..a33f0382322 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithContextAwareTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientWithContextAwareTest.java @@ -1,11 +1,11 @@ /* - * Copyright 2025 LY Corporation + * Copyright 2019 LINE Corporation * - * LY Corporation licenses this file to you under the Apache License, + * LINE Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT @@ -57,15 +57,7 @@ void contextAwareDoesNotThrowException() { final ServiceRequestContext dummyCtx = ServiceRequestContext.of(HttpRequest.of(HttpMethod.GET, "/")); try (SafeCloseable ignored = dummyCtx.push()) { final CompletableFuture future = client.get("/").aggregate(); - assertThatThrownBy(() -> { - try { - dummyCtx.makeContextAware(future).join(); - } catch (Exception e) { - throw e; - } - - System.out.println(future); - }).hasCauseInstanceOf( + assertThatThrownBy(() -> dummyCtx.makeContextAware(future).join()).hasCauseInstanceOf( ResponseTimeoutException.class); } } diff --git a/junit5/src/main/java/com/linecorp/armeria/testing/server/ServiceRequestContextCaptor.java b/junit5/src/main/java/com/linecorp/armeria/testing/server/ServiceRequestContextCaptor.java index a3f600e55ee..f2511fb01c6 100644 --- a/junit5/src/main/java/com/linecorp/armeria/testing/server/ServiceRequestContextCaptor.java +++ b/junit5/src/main/java/com/linecorp/armeria/testing/server/ServiceRequestContextCaptor.java @@ -1,7 +1,7 @@ /* - * Copyright 2025 LY Corporation + * Copyright 2021 LINE Corporation * - * LY Corporation licenses this file to you under the Apache License, + * LINE Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * From 9c1eccad4f3e8193242af9b147d5ef065b17bf4e Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Wed, 25 Jun 2025 12:29:15 +0200 Subject: [PATCH 36/36] refactor: revert updates to copyright headers and further cleanups --- .../linecorp/armeria/common/logging/DefaultRequestLog.java | 4 ++-- .../com/linecorp/armeria/client/retry/RetryingClientTest.java | 3 --- .../armeria/it/client/retry/RetryingRpcClientTest.java | 4 ++-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/common/logging/DefaultRequestLog.java b/core/src/main/java/com/linecorp/armeria/common/logging/DefaultRequestLog.java index 0b12042fef3..b9a8920a4a4 100644 --- a/core/src/main/java/com/linecorp/armeria/common/logging/DefaultRequestLog.java +++ b/core/src/main/java/com/linecorp/armeria/common/logging/DefaultRequestLog.java @@ -401,7 +401,7 @@ private void updateFlags(RequestLogProperty property) { } private void updateFlags(int flags) { - for (; ; ) { + for (;;) { final int oldFlags = this.flags; final int newFlags = oldFlags | flags; if (oldFlags == newFlags) { @@ -517,7 +517,7 @@ private void defer(int flag) { flag |= RequestLogProperty.NAME.flag(); } - for (; ; ) { + for (;;) { final int oldFlags = deferredFlags; final int newFlags = oldFlags | flag; if (oldFlags == newFlags) { diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java index d4dd142cc38..bd2c268f221 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java @@ -87,7 +87,6 @@ import com.linecorp.armeria.server.AbstractHttpService; import com.linecorp.armeria.server.ServerBuilder; import com.linecorp.armeria.server.ServiceRequestContext; -import com.linecorp.armeria.server.logging.LoggingService; import com.linecorp.armeria.testing.junit5.server.ServerExtension; import io.netty.channel.EventLoop; @@ -130,8 +129,6 @@ protected boolean runForEachTest() { @Override protected void configure(ServerBuilder sb) throws Exception { - sb.decorator(LoggingService.newDecorator()); - sb.service("/retry-content", new AbstractHttpService() { @Override protected HttpResponse doGet(ServiceRequestContext ctx, HttpRequest req) diff --git a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java index 4ff30f9f553..1e56c8b0472 100644 --- a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java +++ b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java @@ -1,7 +1,7 @@ /* - * Copyright 2025 LY Corporation + * Copyright 2017 LINE Corporation * - * LY Corporation licenses this file to you under the Apache License, + * LINE Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: *