reconnect when agent is offline (#1142)

* reconnect when agent is offline

* address comments

* move offline check

---------

Co-authored-by: car-roll <[email protected]>
Co-authored-by: Raihaan Shouhell <[email protected]>

Author: 3 people

src/main/java/hudson/plugins/ec2/EC2RetentionStrategy.java

@@ -103,6 +103,7 @@ public long check(EC2Computer c) {
                 long currentTime = this.clock.millis();
 
                 if (currentTime > nextCheckAfter) {
+                    attemptReconnectIfOffline(c);
                     long intervalMins = internalCheck(c);
                     nextCheckAfter = currentTime + TimeUnit.MINUTES.toMillis(intervalMins);
                     return intervalMins;
@@ -248,6 +249,17 @@ private long internalCheck(EC2Computer computer) {
         return CHECK_INTERVAL_MINUTES;
     }
 
+    private void attemptReconnectIfOffline(EC2Computer computer) {
+        if (computer.isOffline()) {
+            LOGGER.warning("EC2Computer " + computer.getName() + " is offline");
+            if (!computer.isConnecting()) {
+                // Keep retrying connection to agent until the job times out
+                LOGGER.warning("Attempting to reconnect EC2Computer " + computer.getName());
+                computer.connect(false);
+            }
+        }
+    }
+
     /*
      * Checks if there are any items in the queue that are waiting for this node explicitly.
      * This prevents a node from being taken offline while there are Ivy/Maven Modules waiting to build.