Wait for pod template on resume to avoid leaking agent pods after restart

README.md

@@ -814,6 +814,7 @@ Please read [Features controlled by system properties](https://www.jenkins.io/do
 * `io.jenkins.plugins.kubernetes.disableNoDelayProvisioning` (since 1.19.1) Whether to disable the no-delay provisioning strategy the plugin uses (defaults to `false`).
 * `io.jenkins.plugins.kubernetes.NoDelayProvisionerStrategy.disableCloudShuffle` Whether to disable the shuffling of clouds. When true clouds will be searched in order they are defined (defaults to `false`).
 * `jenkins.host.address` : (for unit tests) controls the host agents should use to contact Jenkins
+* `org.csanchez.jenkins.plugins.kubernetes.KubernetesLauncher.templateResolutionTimeout` : How long to wait for a dynamic pod template to be re-registered after a restart before giving up (defaults to `60sec`)
 * `org.csanchez.jenkins.plugins.kubernetes.PodTemplate.connectionTimeout` : The time in seconds to wait before considering the pod scheduling has failed (defaults to `1000`)
 * `org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator.stdinBufferSize` : stdin buffer size in bytes for commands sent to Kubernetes exec api. A low value will cause slowness in commands executed. A higher value will consume more memory (defaults to `16*1024`)
 * `org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator.websocketConnectionTimeout` : Time to wait for the websocket used by `container` step to connect (defaults to `30`)

pom.xml

@@ -41,7 +41,7 @@
 
   <properties>
     <changelist>999999-SNAPSHOT</changelist>
-    <jenkins.baseline>2.504</jenkins.baseline>
+    <jenkins.baseline>2.516</jenkins.baseline>
     <jenkins.version>${jenkins.baseline}.3</jenkins.version>
     <no-test-jar>false</no-test-jar>
     <useBeta>true</useBeta>
@@ -58,7 +58,7 @@
       <dependency>
         <groupId>io.jenkins.tools.bom</groupId>
         <artifactId>bom-${jenkins.baseline}.x</artifactId>
-        <version>5015.vb_52d36583443</version>
+        <version>6210.v69ea_fd8a_f010</version>
         <type>pom</type>
         <scope>import</scope>
       </dependency>

src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesLauncher.java

@@ -29,6 +29,7 @@
 import static java.util.logging.Level.WARNING;
 
 import edu.umd.cs.findbugs.annotations.CheckForNull;
+import edu.umd.cs.findbugs.annotations.NonNull;
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import hudson.Functions;
 import hudson.model.Descriptor;
@@ -43,6 +44,8 @@
 import io.fabric8.kubernetes.client.KubernetesClient;
 import io.fabric8.kubernetes.client.KubernetesClientException;
 import java.io.IOException;
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -72,6 +75,11 @@ public class KubernetesLauncher extends JNLPLauncher {
 
     private static final Logger LOGGER = Logger.getLogger(KubernetesLauncher.class.getName());
 
+    private static final Duration TEMPLATE_RESOLUTION_TIMEOUT = SystemProperties.getDuration(
+            KubernetesLauncher.class.getName() + ".templateResolutionTimeout",
+            ChronoUnit.SECONDS,
+            Duration.ofSeconds(60));
+
     private volatile boolean launched = false;
 
     private static final boolean DISABLE_DIAGNOSTIC_LOGS =
@@ -120,7 +128,7 @@ public synchronized void launch(SlaveComputer computer, TaskListener listener) {
         String cloudName = node.getCloudName();
 
         try {
-            PodTemplate template = node.getTemplate();
+            PodTemplate template = waitForTemplate(node);
             KubernetesCloud cloud = node.getKubernetesCloud();
             KubernetesClient client = cloud.connect();
             Pod pod;
@@ -334,6 +342,30 @@ public synchronized void launch(SlaveComputer computer, TaskListener listener) {
         }
     }
 
+    /**
+     * Resolve the nodes {@link PodTemplate}, waiting briefly for it to become available.
+     * <p>After a controller restart, {@link org.csanchez.jenkins.plugins.kubernetes.pipeline.PodTemplateStepExecution#onResume}
+     * re-registers the dynamic template asynchronously, and this launcher may run first. Rather than failing
+     * immediately (which would leak the pod), poll for the template for a short while. Only dynamic templates are
+     * waited for: a global template is configured on the cloud and so is already available, and if it is not then
+     * waiting will not help. Falls back to {@link KubernetesSlave#getTemplate()} (which throws a descriptive
+     * exception) if it never becomes available.
+     */
+    @NonNull
+    private static PodTemplate waitForTemplate(KubernetesSlave node) throws InterruptedException {
+        String id = node.getTemplateId();
+        boolean global = node.getKubernetesCloud().getTemplates().stream().anyMatch(t -> id.equals(t.getId()));
+        if (!global) {
+            long deadline = System.nanoTime() + TEMPLATE_RESOLUTION_TIMEOUT.toNanos();
+            while (node.getTemplateOrNull() == null && System.nanoTime() < deadline) {
+                LOGGER.fine(() -> "Pod template " + id + " not yet available for " + node.getNodeName()
+                        + " waiting for it to be (re-)registered");
+                Thread.sleep(1000);
+            }
+        }
+        return node.getTemplate();
+    }
+
     private static void terminateOrLog(KubernetesSlave node) {
         try {
             node.terminate();