Merge pull request #2844 from hareldev/add-409-clusterresourcequotas

Broaden the 409 handling for OpenShift ClusterResourceQuota

Author: jglick

src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesLauncher.java

@@ -177,9 +177,9 @@ public synchronized void launch(SlaveComputer computer, TaskListener listener) {
                                             namespace,
                                             pod.getMetadata().getName(),
                                             e.getMessage());
-                        } else if (httpCode == 409
-                                && e.getMessage().contains("Operation cannot be fulfilled on resourcequotas")) {
+                        } else if (isResourceQuotaUpdateConflict(httpCode, e.getMessage())) {
                             // See: https://github.com/kubernetes/kubernetes/issues/67761 ; A retry usually works.
+                            // OpenShift ClusterResourceQuota conflicts use clusterresourcequotas.quota.openshift.io.
                             node.getRunListener()
                                     .getLogger()
                                     .printf(
@@ -391,6 +391,19 @@ private void logLastLines(
         }
     }
 
+    /**
+     * Detects transient quota update conflicts during pod creation.
+     * These occur when concurrent pod creations race to update ResourceQuota or OpenShift ClusterResourceQuota.
+     *
+     * @see <a href="https://github.com/kubernetes/kubernetes/issues/67761">kubernetes#67761</a>
+     */
+    static boolean isResourceQuotaUpdateConflict(int httpCode, @CheckForNull String message) {
+        return httpCode == 409
+                && message != null
+                && message.contains("Operation cannot be fulfilled on")
+                && message.contains("resourcequotas");
+    }
+
     /**
      * The last problem that occurred, if any.
      * @return

src/test/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesLauncherTest.java

@@ -0,0 +1,46 @@
+package org.csanchez.jenkins.plugins.kubernetes;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+import org.jvnet.hudson.test.WithoutJenkins;
+
+public class KubernetesLauncherTest {
+
+    private static final String K8S_RESOURCE_QUOTA_CONFLICT =
+            "Operation cannot be fulfilled on resourcequotas \"compute-resources\": the object has been modified; please apply your changes to the latest version and try again";
+
+    private static final String OPENSHIFT_CLUSTER_RESOURCE_QUOTA_CONFLICT =
+            "Operation cannot be fulfilled on clusterresourcequotas.quota.openshift.io \"dno--notterminating\": the object has been modified; please apply your changes to the latest version and try again";
+
+    @WithoutJenkins
+    @Test
+    public void isResourceQuotaUpdateConflict_kubernetesResourceQuota() {
+        assertTrue(KubernetesLauncher.isResourceQuotaUpdateConflict(409, K8S_RESOURCE_QUOTA_CONFLICT));
+    }
+
+    @WithoutJenkins
+    @Test
+    public void isResourceQuotaUpdateConflict_openshiftClusterResourceQuota() {
+        assertTrue(KubernetesLauncher.isResourceQuotaUpdateConflict(409, OPENSHIFT_CLUSTER_RESOURCE_QUOTA_CONFLICT));
+    }
+
+    @WithoutJenkins
+    @Test
+    public void isResourceQuotaUpdateConflict_not409() {
+        assertFalse(KubernetesLauncher.isResourceQuotaUpdateConflict(403, K8S_RESOURCE_QUOTA_CONFLICT));
+    }
+
+    @WithoutJenkins
+    @Test
+    public void isResourceQuotaUpdateConflict_nullMessage() {
+        assertFalse(KubernetesLauncher.isResourceQuotaUpdateConflict(409, null));
+    }
+
+    @WithoutJenkins
+    @Test
+    public void isResourceQuotaUpdateConflict_unrelated409() {
+        assertFalse(KubernetesLauncher.isResourceQuotaUpdateConflict(409, "pods \"my-pod\" already exists"));
+    }
+}