diff --git a/README.md b/README.md index 80c6b3b34..e4aec45cc 100644 --- a/README.md +++ b/README.md @@ -814,6 +814,7 @@ Please read [Features controlled by system properties](https://www.jenkins.io/do * `io.jenkins.plugins.kubernetes.disableNoDelayProvisioning` (since 1.19.1) Whether to disable the no-delay provisioning strategy the plugin uses (defaults to `false`). * `io.jenkins.plugins.kubernetes.NoDelayProvisionerStrategy.disableCloudShuffle` Whether to disable the shuffling of clouds. When true clouds will be searched in order they are defined (defaults to `false`). * `jenkins.host.address` : (for unit tests) controls the host agents should use to contact Jenkins +* `org.csanchez.jenkins.plugins.kubernetes.KubernetesLauncher.templateResolutionTimeout` : How long to wait for a dynamic pod template to be re-registered after a restart before giving up (defaults to `60sec`) * `org.csanchez.jenkins.plugins.kubernetes.PodTemplate.connectionTimeout` : The time in seconds to wait before considering the pod scheduling has failed (defaults to `1000`) * `org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator.stdinBufferSize` : stdin buffer size in bytes for commands sent to Kubernetes exec api. A low value will cause slowness in commands executed. A higher value will consume more memory (defaults to `16*1024`) * `org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator.websocketConnectionTimeout` : Time to wait for the websocket used by `container` step to connect (defaults to `30`) diff --git a/pom.xml b/pom.xml index eadeea4f0..faeec1571 100644 --- a/pom.xml +++ b/pom.xml @@ -41,7 +41,7 @@ 999999-SNAPSHOT - 2.504 + 2.516 ${jenkins.baseline}.3 false true @@ -58,7 +58,7 @@ io.jenkins.tools.bom bom-${jenkins.baseline}.x - 5015.vb_52d36583443 + 6210.v69ea_fd8a_f010 pom import diff --git a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesLauncher.java b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesLauncher.java index 7326acd41..dd8421af9 100644 --- a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesLauncher.java +++ b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesLauncher.java @@ -29,6 +29,7 @@ import static java.util.logging.Level.WARNING; import edu.umd.cs.findbugs.annotations.CheckForNull; +import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import hudson.Functions; import hudson.model.Descriptor; @@ -43,6 +44,8 @@ import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.KubernetesClientException; import java.io.IOException; +import java.time.Duration; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -72,6 +75,11 @@ public class KubernetesLauncher extends JNLPLauncher { private static final Logger LOGGER = Logger.getLogger(KubernetesLauncher.class.getName()); + private static final Duration TEMPLATE_RESOLUTION_TIMEOUT = SystemProperties.getDuration( + KubernetesLauncher.class.getName() + ".templateResolutionTimeout", + ChronoUnit.SECONDS, + Duration.ofSeconds(60)); + private volatile boolean launched = false; private static final boolean DISABLE_DIAGNOSTIC_LOGS = @@ -120,7 +128,7 @@ public synchronized void launch(SlaveComputer computer, TaskListener listener) { String cloudName = node.getCloudName(); try { - PodTemplate template = node.getTemplate(); + PodTemplate template = waitForTemplate(node); KubernetesCloud cloud = node.getKubernetesCloud(); KubernetesClient client = cloud.connect(); Pod pod; @@ -334,6 +342,30 @@ public synchronized void launch(SlaveComputer computer, TaskListener listener) { } } + /** + * Resolve the nodes {@link PodTemplate}, waiting briefly for it to become available. + *

After a controller restart, {@link org.csanchez.jenkins.plugins.kubernetes.pipeline.PodTemplateStepExecution#onResume} + * re-registers the dynamic template asynchronously, and this launcher may run first. Rather than failing + * immediately (which would leak the pod), poll for the template for a short while. Only dynamic templates are + * waited for: a global template is configured on the cloud and so is already available, and if it is not then + * waiting will not help. Falls back to {@link KubernetesSlave#getTemplate()} (which throws a descriptive + * exception) if it never becomes available. + */ + @NonNull + private static PodTemplate waitForTemplate(KubernetesSlave node) throws InterruptedException { + String id = node.getTemplateId(); + boolean global = node.getKubernetesCloud().getTemplates().stream().anyMatch(t -> id.equals(t.getId())); + if (!global) { + long deadline = System.nanoTime() + TEMPLATE_RESOLUTION_TIMEOUT.toNanos(); + while (node.getTemplateOrNull() == null && System.nanoTime() < deadline) { + LOGGER.fine(() -> "Pod template " + id + " not yet available for " + node.getNodeName() + + " waiting for it to be (re-)registered"); + Thread.sleep(1000); + } + } + return node.getTemplate(); + } + private static void terminateOrLog(KubernetesSlave node) { try { node.terminate();