@@ -6,8 +6,8 @@
  | spark.kubernetes.operator.api.retryMaxAttempts | Integer | 15 | false | Max attempts of retries on unhandled controller errors. Setting this to non-positive value means no retry. | 
  | spark.kubernetes.operator.api.secondaryResourceCreateBackoffJitterMillis | Long | 500 | false | Jitter (in milliseconds) added to backoff delay between retries when creating secondary resources for Spark application. | 
  | spark.kubernetes.operator.api.secondaryResourceCreateBackoffMultiplier | Double | 2.0 | false | Backoff multiplier applied to each retry interval when creating secondary resources for Spark application. | 
- | spark.kubernetes.operator.api.secondaryResourceCreateInitialBackoffMillis | Long | 1000 | false | Initial backoff (in milliseconds) between retries when creating secondary resources for Spark application. Backoff is applied whenever the API server response carries a Retry-After hint, or on 409 (Conflict) and 429 (Too Many Requests) responses. | 
- | spark.kubernetes.operator.api.secondaryResourceCreateMaxAttempts | Long | 3 | false | Maximal number of retry attempts of requesting secondary resource for Spark application. This would be performed on top of k8s client spark.kubernetes.operator.retry.maxAttempts to overcome potential conflicting reconcile on the same SparkApplication, as well as API server errors (408/500/502/503/504) and network-level timeouts. Exponential backoff with jitter is applied before retrying whenever the API server response carries a Retry-After hint, or on 409 (Conflict) and 429 (Too Many Requests) responses. Other retryable errors without a Retry-After hint are retried immediately. This should be positive number. | 
+ | spark.kubernetes.operator.api.secondaryResourceCreateInitialBackoffMillis | Long | 1000 | false | Initial backoff (in milliseconds) between retries when creating secondary resources for Spark application. Backoff is applied whenever the API server response carries a Retry-After hint, on 409 (Conflict) and 429 (Too Many Requests) responses, and on a request the API server never answered, which carries no response status and so can never supply a Retry-After hint. | 
+ | spark.kubernetes.operator.api.secondaryResourceCreateMaxAttempts | Long | 3 | false | Maximal number of retry attempts of requesting secondary resource for Spark application. This would be performed on top of k8s client spark.kubernetes.operator.retry.maxAttempts to overcome potential conflicting reconcile on the same SparkApplication, as well as API server errors (408/500/502/503/504) and requests the API server never answered. Exponential backoff with jitter is applied before retrying whenever the API server response carries a Retry-After hint, on 409 (Conflict) and 429 (Too Many Requests) responses, and on a request that went unanswered, which carries no response status and so can never supply a Retry-After hint. The remaining retryable errors are retried immediately. This should be positive number. | 
  | spark.kubernetes.operator.api.secondaryResourceCreateMaxBackoffMillis | Long | 40000 | false | Maximum backoff (in milliseconds) between retries when creating secondary resources for Spark application. | 
  | spark.kubernetes.operator.api.statusPatchMaxAttempts | Long | 3 | false | Maximal number of retry attempts of requests to k8s server for resource status update. This would be performed on top of k8s client spark.kubernetes.operator.retry.maxAttempts to overcome potential conflicting update on the same SparkApplication. This should be positive number. | 
  | spark.kubernetes.operator.dynamicConfig.enabled | Boolean | false | false | When enabled, operator would load config property overrides dynamically at runtime. The source of the overrides is controlled by spark.kubernetes.operator.dynamicConfig.source. | 
 
@@ -570,10 +570,11 @@ public final class SparkOperatorConf {
    * Maximal number of retry attempts of requesting secondary resource for Spark application. This
    * would be performed on top of k8s client spark.kubernetes.operator.retry.maxAttempts to overcome
    * potential conflicting reconcile on the same SparkApplication, as well as transient API server
-   * errors and network-level timeouts. Exponential backoff with jitter is applied before retrying
-   * whenever the API server response carries a Retry-After hint, or on 409 (Conflict) and 429 (Too
-   * Many Requests) responses. Other retryable errors (408/500/502/503/504 and network timeouts)
-   * without a Retry-After hint are retried immediately. This should be positive number
+   * errors and requests the API server never answered. Exponential backoff with jitter is applied
+   * before retrying whenever the API server response carries a Retry-After hint, on 409 (Conflict)
+   * and 429 (Too Many Requests) responses, and on a request that went unanswered, which carries no
+   * response status and so can never supply a Retry-After hint. The remaining retryable errors
+   * (408/500/502/503/504) are retried immediately. This should be positive number
    */
   public static final ConfigOption<Long> API_SECONDARY_RESOURCE_CREATE_MAX_ATTEMPTS =
       ConfigOption.<Long>builder()
@@ -584,11 +585,13 @@ public final class SparkOperatorConf {
                   + "application. This would be performed on top of k8s client "
                   + "spark.kubernetes.operator.retry.maxAttempts to overcome potential "
                   + "conflicting reconcile on the same SparkApplication, as well as API "
-                  + "server errors (408/500/502/503/504) and network-level timeouts. "
-                  + "Exponential backoff with jitter is applied before retrying whenever the "
-                  + "API server response carries a Retry-After hint, or on 409 (Conflict) and "
-                  + "429 (Too Many Requests) responses. Other retryable errors without a "
-                  + "Retry-After hint are retried immediately. This should be positive number.")
+                  + "server errors (408/500/502/503/504) and requests the API server never "
+                  + "answered. Exponential backoff with jitter is applied before retrying "
+                  + "whenever the API server response carries a Retry-After hint, on 409 "
+                  + "(Conflict) and 429 (Too Many Requests) responses, and on a request that "
+                  + "went unanswered, which carries no response status and so can never supply "
+                  + "a Retry-After hint. The remaining retryable errors are retried "
+                  + "immediately. This should be positive number.")
           .typeParameterClass(Long.class)
           .defaultValue(3L)
           .build();
@@ -807,8 +810,9 @@ public final class SparkOperatorConf {
 
   /**
    * Initial backoff (in milliseconds) between retries when creating secondary resources. Backoff
-   * is applied whenever the API server response carries a Retry-After hint, or on 409 (Conflict)
-   * and 429 (Too Many Requests) responses.
+   * is applied whenever the API server response carries a Retry-After hint, on 409 (Conflict) and
+   * 429 (Too Many Requests) responses, and on a request the API server never answered, which
+   * carries no response status and so can never supply a Retry-After hint.
    */
   public static final ConfigOption<Long> API_SECONDARY_RESOURCE_CREATE_INITIAL_BACKOFF_MILLIS =
       ConfigOption.<Long>builder()
@@ -818,8 +822,9 @@ public final class SparkOperatorConf {
               "Initial backoff (in milliseconds) between retries "
                   + "when creating secondary resources for Spark application. "
                   + "Backoff is applied whenever the API server response carries a "
-                  + "Retry-After hint, or on 409 (Conflict) and "
-                  + "429 (Too Many Requests) responses.")
+                  + "Retry-After hint, on 409 (Conflict) and 429 (Too Many Requests) "
+                  + "responses, and on a request the API server never answered, which "
+                  + "carries no response status and so can never supply a Retry-After hint.")
           .typeParameterClass(Long.class)
           .defaultValue(1000L)
           .build();
 
@@ -31,10 +31,15 @@
 import static org.apache.spark.k8s.operator.utils.ModelUtils.buildOwnerReferenceTo;
 import static org.apache.spark.k8s.operator.utils.SparkExceptionUtils.isConflictForExistingResource;
 
+import java.security.cert.CertPathBuilderException;
+import java.security.cert.CertPathValidatorException;
+import java.security.cert.CertificateException;
 import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 
+import javax.net.ssl.SSLPeerUnverifiedException;
+
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import io.fabric8.kubernetes.api.model.DeletionPropagation;
@@ -57,6 +62,20 @@
 @Slf4j
 public final class ReconcilerUtils {
 
+  /**
+   * The code fabric8 reports whenever no response status was received. It is the sentinel of
+   * {@link KubernetesClientException#getCode()}, not an HTTP status, and it covers three unrelated
+   * cases: a connection that broke on its way to the API server, which carries the failure that
+   * broke it; a rejection the client raised before sending the request, such as a missing name or
+   * resource version, which carries no cause at all; and a response that arrived but could not be
+   * parsed, which carries the parsing failure. Only the first heals on its own, see
+   * {@link #brokeOnTheWayToTheApiServer}.
+   */
+  private static final int NO_RESPONSE_CODE = -1;
+
+  /** Maximum number of links followed when looking for the cause that broke a request. */
+  private static final int MAX_CAUSE_DEPTH = 10;
+
   private ReconcilerUtils() {}
 
   /**
@@ -144,7 +163,7 @@ public static <T extends HasMetadata> Optional<T> getOrCreateSecondaryResource(
           } else if (e.getCode() == Constants.HTTP_TOO_MANY_REQUESTS) {
             log.debug("Server returned 429 Too Many Requests, will retry with backoff");
           } else if (isTransientError(e) || e.getCode() == HTTP_INTERNAL_ERROR) {
-            // GET to avoid duplicate create attempt for timeouts (0) and transient 5xx
+            // GET to avoid a duplicate create for transport failures, timeouts and transient 5xx
             current = getResource(client, resource);
             if (current.isPresent()) {
               return current;
@@ -279,7 +298,8 @@ private static boolean shouldBackoffBeforeRetry(KubernetesClientException e) {
       return true;
     }
     return switch (e.getCode()) {
-      case HTTP_CONFLICT, Constants.HTTP_TOO_MANY_REQUESTS -> true;
+      // A broken connection needs the delay most, and never carries a Retry-After to ask for one.
+      case HTTP_CONFLICT, Constants.HTTP_TOO_MANY_REQUESTS, NO_RESPONSE_CODE -> true;
       default -> false;
     };
   }
@@ -297,20 +317,59 @@ public static boolean isFirstAttempt(Context<?> context) {
   }
 
   /**
-   * Whether the given failure is transport level rather than a decision by the API server.
+   * Whether the given failure is expected to clear without anyone acting on it, so that a caller
+   * may wait it out rather than report it. A broken connection and an overloaded or proxied server
+   * qualify; a rejection by a reachable API server and a client side rejection do not.
    *
    * @param e The failure to classify.
-   * @return True if the request did not reach a healthy API server, false otherwise.
+   * @return True if the failure is expected to clear on its own, false otherwise.
    */
   public static boolean isTransientError(KubernetesClientException e) {
-    // code 0 is fabric8's sentinel for network-level failures (timeouts, connection resets, etc.)
     return switch (e.getCode()) {
-      case 0, HTTP_CLIENT_TIMEOUT, HTTP_BAD_GATEWAY,
-           HTTP_UNAVAILABLE, HTTP_GATEWAY_TIMEOUT -> true;
+      case NO_RESPONSE_CODE -> brokeOnTheWayToTheApiServer(e);
+      case HTTP_CLIENT_TIMEOUT, HTTP_BAD_GATEWAY, HTTP_UNAVAILABLE, HTTP_GATEWAY_TIMEOUT -> true;
       default -> false;
     };
   }
 
+  /**
+   * Whether the given failure left the request unanswered, so that asking again may yet work. It
+   * is defined as the complement of the status-less failures that repeating cannot change: a
+   * rejection the client raised before sending anything, which carries no cause at all; an answer
+   * that arrived but could not be parsed; a certificate the peer could not prove; and the caller
+   * being interrupted, which is a decision to stop rather than a failure. Anything else that
+   * carries a cause counts as unanswered.
+   *
+   * <p>Naming what cannot work, rather than what can, keeps this from tracking the exception types
+   * of whichever HTTP client is plugged in. A connection the peer closes mid-response is the case
+   * that matters: the client in use reports it with a type of its own which is not even an {@link
+   * java.io.IOException}, so any list of recognized connection failures would silently miss it.
+   *
+   * @param e The failure to inspect.
+   * @return True if the request went unanswered, false otherwise.
+   */
+  private static boolean brokeOnTheWayToTheApiServer(KubernetesClientException e) {
+    // Bounded walk: a cyclic cause chain must not hang the reconciler. The marker is nested, since
+    // fabric8 wraps the failure and its own HTTP client wraps it again.
+    Throwable cause = e.getCause();
+    for (int depth = 0; cause != null && depth < MAX_CAUSE_DEPTH; depth++) {
+      // The trust failures are the ones fabric8 itself refuses to retry. Keying on them rather
+      // than on the handshake that reported them keeps a handshake that merely timed out
+      // retriable. An interruption is matched on InterruptedException rather than on
+      // InterruptedIOException, which a plain read timeout also extends.
+      if (cause instanceof JsonProcessingException
+          || cause instanceof CertificateException
+          || cause instanceof CertPathValidatorException
+          || cause instanceof CertPathBuilderException
+          || cause instanceof SSLPeerUnverifiedException
+          || cause instanceof InterruptedException) {
+        return false;
+      }
+      cause = cause.getCause();
+    }
+    return e.getCause() != null;
+  }
+
   /**
    * Clones an object using JSON serialization and deserialization.
    *
 
@@ -32,6 +32,7 @@
 import static org.mockito.Mockito.verifyNoInteractions;
 import static org.mockito.Mockito.when;
 
+import java.net.SocketException;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.List;
@@ -989,6 +990,57 @@ void kueueTransientApiFailurePublishesNoEvent() {
     verifyNoInteractions(eventRecorder);
   }
 
+  @Test
+  void kueueUnansweredRequestPublishesNoEvent() {
+    AppInitStep appInitStep = new AppInitStep();
+    SparkAppContext mockContext = mock(SparkAppContext.class);
+    SparkAppStatusRecorder recorder = mock(SparkAppStatusRecorder.class);
+    SparkApplication application = new SparkApplication();
+    application.setMetadata(kueueApplicationMetadata);
+    // A request that never reached the API server carries no response code, so only its cause
+    // marks it as one to wait out rather than report.
+    KubernetesClient failingClient = mock(KubernetesClient.class, RETURNS_DEEP_STUBS);
+    when(failingClient.scheduling().v1().priorityClasses().list())
+        .thenReturn(new PriorityClassList());
+    when(failingClient.resource(any(Workload.class)).create())
+        .thenThrow(new KubernetesClientException("closed", new SocketException("reset")));
+    when(mockContext.getResource()).thenReturn(application);
+    when(mockContext.getClient()).thenReturn(failingClient);
+    when(mockContext.getEventRecorder()).thenReturn(eventRecorder);
+
+    ReconcileProgress progress = appInitStep.reconcile(mockContext, recorder);
+
+    Assertions.assertEquals(
+        ReconcileProgress.completeAndRequeueAfter(
+            KueueWorkloadUtils.STALE_WORKLOAD_REQUEUE_INTERVAL),
+        progress);
+    verifyNoInteractions(eventRecorder);
+  }
+
+  @Test
+  void kueueClientSideRejectionPublishesAnEvent() {
+    AppInitStep appInitStep = new AppInitStep();
+    SparkAppContext mockContext = mock(SparkAppContext.class);
+    SparkAppStatusRecorder recorder = mock(SparkAppStatusRecorder.class);
+    SparkApplication application = new SparkApplication();
+    application.setMetadata(kueueApplicationMetadata);
+    // A rejection raised before the request was sent shares the absent response code, but nothing
+    // clears it on its own, so it keeps its event and the default interval.
+    KubernetesClient failingClient = mock(KubernetesClient.class, RETURNS_DEEP_STUBS);
+    when(failingClient.scheduling().v1().priorityClasses().list())
+        .thenReturn(new PriorityClassList());
+    when(failingClient.resource(any(Workload.class)).create())
+        .thenThrow(new KubernetesClientException("resourceVersion cannot be null"));
+    when(mockContext.getResource()).thenReturn(application);
+    when(mockContext.getClient()).thenReturn(failingClient);
+    when(mockContext.getEventRecorder()).thenReturn(eventRecorder);
+
+    ReconcileProgress progress = appInitStep.reconcile(mockContext, recorder);
+
+    Assertions.assertEquals(ReconcileProgress.completeAndDefaultRequeue(), progress);
+    verify(eventRecorder).record(any(EventRecord.class));
+  }
+
   @Test
   void failedDriverLookupBeforeKueueAdmissionIsRetried() {
     // A failed verification is not an answer either before the admission: requesting quota for a