Skip to content

Commit 4dc8759

Browse files
troymohltroymohl
committed
Stop/Restart of existing DumbSlave nodes that are EC2 backed
1 parent 57ed44e commit 4dc8759

File tree

9 files changed

+682
-0
lines changed

9 files changed

+682
-0
lines changed

‎src/main/java/hudson/plugins/ec2/AmazonEC2Cloud.java

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,27 +26,44 @@
2626
import com.amazonaws.SdkClientException;
2727
import com.amazonaws.auth.AWSCredentialsProvider;
2828
import com.amazonaws.services.ec2.AmazonEC2;
29+
import com.amazonaws.services.ec2.model.DescribeInstancesRequest;
30+
import com.amazonaws.services.ec2.model.DescribeInstancesResult;
2931
import com.amazonaws.services.ec2.model.DescribeRegionsResult;
32+
import com.amazonaws.services.ec2.model.Filter;
33+
import com.amazonaws.services.ec2.model.Instance;
34+
import com.amazonaws.services.ec2.model.InstanceStateName;
3035
import com.amazonaws.services.ec2.model.Region;
36+
import com.amazonaws.services.ec2.model.Reservation;
37+
import com.amazonaws.services.ec2.model.StartInstancesRequest;
38+
import com.amazonaws.services.ec2.model.StopInstancesRequest;
3139
import com.google.common.annotations.VisibleForTesting;
3240
import hudson.Extension;
3341
import hudson.Util;
42+
import hudson.model.Computer;
3443
import hudson.model.Failure;
44+
import hudson.model.Node;
45+
import hudson.model.labels.LabelAtom;
3546
import hudson.plugins.ec2.util.AmazonEC2Factory;
3647
import hudson.slaves.Cloud;
48+
import hudson.slaves.NodeProvisioner.PlannedNode;
3749
import hudson.util.FormValidation;
3850
import hudson.util.ListBoxModel;
3951
import java.io.IOException;
4052
import java.net.MalformedURLException;
4153
import java.net.URL;
54+
import java.util.Collections;
4255
import java.util.List;
4356
import java.util.Locale;
57+
import java.util.concurrent.TimeUnit;
58+
import java.util.logging.Level;
59+
import java.util.logging.Logger;
4460
import javax.annotation.Nullable;
4561
import javax.servlet.ServletException;
4662
import jenkins.model.Jenkins;
4763
import org.kohsuke.stapler.DataBoundConstructor;
4864
import org.kohsuke.stapler.DataBoundSetter;
4965
import org.kohsuke.stapler.QueryParameter;
66+
import org.kohsuke.stapler.StaplerResponse;
5067
import org.kohsuke.stapler.interceptor.RequirePOST;
5168

5269
/**
@@ -62,10 +79,26 @@ public class AmazonEC2Cloud extends EC2Cloud {
6279

6380
private String altEC2Endpoint;
6481

82+
private static final Logger LOGGER = Logger.getLogger(AmazonEC2Cloud.class.getName());
83+
6584
public static final String CLOUD_ID_PREFIX = "ec2-";
6685

86+
private static final int MAX_RESULTS = 1000;
87+
88+
private static final String INSTANCE_NAME_TAG = "Name";
89+
90+
private static final String TAG_PREFIX = "tag";
91+
6792
private boolean noDelayProvisioning;
6893

94+
private boolean startStopNodes;
95+
96+
private String instanceTagForJenkins;
97+
98+
private String nodeTagForEc2;
99+
100+
private String maxIdleMinutes;
101+
69102
@DataBoundConstructor
70103
public AmazonEC2Cloud(String cloudName, boolean useInstanceProfileForCredentials, String credentialsId, String region, String privateKey, String instanceCapStr, List<? extends SlaveTemplate> templates, String roleArn, String roleSessionName) {
71104
super(createCloudId(cloudName), useInstanceProfileForCredentials, credentialsId, privateKey, instanceCapStr, templates, roleArn, roleSessionName);
@@ -126,6 +159,24 @@ public void setNoDelayProvisioning(boolean noDelayProvisioning) {
126159
this.noDelayProvisioning = noDelayProvisioning;
127160
}
128161

162+
@DataBoundSetter
163+
public void setStartStopNodes(boolean startStopNodes) {
164+
this.startStopNodes = startStopNodes;
165+
}
166+
167+
public boolean isStartStopNodes() {
168+
return startStopNodes;
169+
}
170+
171+
public String getInstanceTagForJenkins() {
172+
return instanceTagForJenkins;
173+
}
174+
175+
@DataBoundSetter
176+
public void setInstanceTagForJenkins(String instanceTagForJenkins) {
177+
this.instanceTagForJenkins = instanceTagForJenkins;
178+
}
179+
129180
public String getAltEC2Endpoint() {
130181
return altEC2Endpoint;
131182
}
@@ -135,11 +186,159 @@ public void setAltEC2Endpoint(String altEC2Endpoint) {
135186
this.altEC2Endpoint = altEC2Endpoint;
136187
}
137188

189+
public String getNodeTagForEc2() {
190+
return nodeTagForEc2;
191+
}
192+
193+
@DataBoundSetter
194+
public void setNodeTagForEc2(String nodeTagForEc2) {
195+
this.nodeTagForEc2 = nodeTagForEc2;
196+
}
197+
198+
public boolean isEc2Node(Node node) {
199+
//If no label is specified then we check all nodes
200+
if (nodeTagForEc2 == null || nodeTagForEc2.trim().length() == 0) {
201+
return true;
202+
}
203+
204+
for (LabelAtom label : node.getAssignedLabels()) {
205+
if (label.getExpression().equalsIgnoreCase(nodeTagForEc2)) {
206+
return true;
207+
}
208+
}
209+
return false;
210+
}
211+
212+
public String getMaxIdleMinutes() {
213+
return maxIdleMinutes;
214+
}
215+
216+
@DataBoundSetter
217+
public void setMaxIdleMinutes(String maxIdleMinutes) {
218+
this.maxIdleMinutes = maxIdleMinutes;
219+
}
220+
221+
public PlannedNode startNode(Node node) {
222+
Instance nodeInstance = getInstanceByLabel(node.getSelfLabel().getExpression(), InstanceStateName.Stopped);
223+
if (nodeInstance == null) {
224+
nodeInstance = getInstanceByNodeName(node.getNodeName(), InstanceStateName.Stopped);
225+
}
226+
227+
if (nodeInstance == null) {
228+
return null;
229+
}
230+
231+
final String instanceId = nodeInstance.getInstanceId();
232+
233+
return new PlannedNode(node.getDisplayName(),
234+
Computer.threadPoolForRemoting.submit(() -> {
235+
try {
236+
while (true) {
237+
StartInstancesRequest startRequest = new StartInstancesRequest();
238+
startRequest.setInstanceIds(Collections.singletonList(instanceId));
239+
connect().startInstances(startRequest);
240+
241+
Instance instance = CloudHelper.getInstanceWithRetry(instanceId, this);
242+
if (instance == null) {
243+
LOGGER.log(Level.WARNING, "Can't find instance with instance id `{0}` in cloud {1}. Terminate provisioning ", new Object[] {
244+
instanceId, this.getCloudName() });
245+
return null;
246+
}
247+
248+
InstanceStateName state = InstanceStateName.fromValue(instance.getState().getName());
249+
if (state.equals(InstanceStateName.Running)) {
250+
long startTime = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - instance.getLaunchTime().getTime());
251+
LOGGER.log(Level.INFO, "{0} moved to RUNNING state in {1} seconds and is ready to be connected by Jenkins", new Object[] {
252+
instanceId, startTime });
253+
return node;
254+
}
255+
256+
if (!state.equals(InstanceStateName.Pending)) {
257+
LOGGER.log(Level.WARNING, "{0}. Node {1} is neither pending nor running, it's {2}. Terminate provisioning", new Object[] {
258+
instanceId, node.getNodeName(), state });
259+
return null;
260+
}
261+
262+
Thread.sleep(5000);
263+
}
264+
} catch (Exception e) {
265+
LOGGER.log(Level.WARNING, "Unable to start " + instanceId, e);
266+
return null;
267+
}
268+
})
269+
, node.getNumExecutors());
270+
}
271+
272+
public void stopNode(Node node) {
273+
Instance nodeInstance = getInstanceByLabel(node.getSelfLabel().getExpression(), InstanceStateName.Running);
274+
if (nodeInstance == null) {
275+
nodeInstance = getInstanceByNodeName(node.getNodeName(), InstanceStateName.Running);
276+
}
277+
278+
if (nodeInstance == null) {
279+
return;
280+
}
281+
282+
final String instanceId = nodeInstance.getInstanceId();
283+
284+
try {
285+
StopInstancesRequest request = new StopInstancesRequest();
286+
request.setInstanceIds(Collections.singletonList(instanceId));
287+
connect().stopInstances(request);
288+
LOGGER.log(Level.INFO, "Stopped instance: {0}", instanceId);
289+
} catch (Exception e) {
290+
LOGGER.log(Level.INFO, "Unable to stop instance: " + instanceId, e);
291+
}
292+
}
293+
138294
@Override
139295
protected AWSCredentialsProvider createCredentialsProvider() {
140296
return createCredentialsProvider(isUseInstanceProfileForCredentials(), getCredentialsId(), getRoleArn(), getRoleSessionName(), getRegion());
141297
}
142298

299+
private Instance getInstanceByLabel(String label, InstanceStateName desiredState) {
300+
String tag = getInstanceTagForJenkins();
301+
if (tag == null) {
302+
return null;
303+
}
304+
return getInstance(Collections.singletonList(getTagFilter(tag, label)), desiredState);
305+
}
306+
307+
private Instance getInstanceByNodeName(String name, InstanceStateName desiredState) {
308+
return getInstance(Collections.singletonList(getTagFilter(INSTANCE_NAME_TAG, name)), desiredState);
309+
}
310+
311+
private Filter getTagFilter(String name, String value) {
312+
Filter filter = new Filter();
313+
filter.setName(TAG_PREFIX + ":" + name.trim());
314+
filter.setValues(Collections.singletonList(value.trim()));
315+
LOGGER.log(Level.FINEST,"Created filter to query for instance: {0}", filter);
316+
return filter;
317+
}
318+
319+
private Instance getInstance(List<Filter> filters, InstanceStateName desiredState) {
320+
DescribeInstancesRequest request = new DescribeInstancesRequest();
321+
request.setFilters(filters);
322+
request.setMaxResults(MAX_RESULTS);
323+
request.setNextToken(null);
324+
DescribeInstancesResult response = connect().describeInstances( request );
325+
326+
if (!response.getReservations().isEmpty()) {
327+
for (Reservation reservation : response.getReservations()) {
328+
for (Instance instance : reservation.getInstances()) {
329+
com.amazonaws.services.ec2.model.InstanceState state = instance.getState();
330+
LOGGER.log(Level.FINEST,"Instance {0} state: {1}", new Object[] {instance.getInstanceId(), state.getName()});
331+
if (state.getName().equals(desiredState.toString())) {
332+
return instance;
333+
}
334+
}
335+
}
336+
} else {
337+
LOGGER.log(Level.FINEST,"No instances found that matched filter criteria");
338+
}
339+
return null;
340+
}
341+
143342
@Extension
144343
public static class DescriptorImpl extends EC2Cloud.DescriptorImpl {
145344

‎src/main/java/hudson/plugins/ec2/InstanceStopTimer.java

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
package hudson.plugins.ec2;
2+
3+
import com.google.common.annotations.VisibleForTesting;
4+
import hudson.Extension;
5+
import hudson.model.AsyncPeriodicWork;
6+
import hudson.model.Computer;
7+
import hudson.model.Executor;
8+
import hudson.model.Node;
9+
import hudson.model.TaskListener;
10+
import hudson.slaves.Cloud;
11+
import java.io.IOException;
12+
import java.util.concurrent.TimeUnit;
13+
import java.util.logging.Level;
14+
import java.util.logging.Logger;
15+
import jenkins.model.Jenkins;
16+
17+
@Extension
18+
public class InstanceStopTimer extends AsyncPeriodicWork {
19+
private static final Logger LOGGER = Logger.getLogger(InstanceStopTimer.class.getName());
20+
21+
private static final long STOP_DISABLED = -1;
22+
23+
public InstanceStopTimer() {
24+
super(InstanceStopTimer.class.getName());
25+
}
26+
27+
protected InstanceStopTimer(String name) {
28+
super(name);
29+
}
30+
31+
@Override protected void execute(TaskListener taskListener) throws IOException, InterruptedException {
32+
Jenkins jenkinsInstance = Jenkins.get();
33+
for (Node node : jenkinsInstance.getNodes()) {
34+
if (shouldStopNode(node)) {
35+
LOGGER.log(Level.FINEST, "{0} should be stopped", node.getNodeName());
36+
stopNode(node);
37+
}
38+
}
39+
}
40+
41+
@Override public long getRecurrencePeriod() {
42+
return TimeUnit.MINUTES.toMillis(1);
43+
}
44+
45+
private boolean shouldStopNode(Node node) {
46+
long maxIdleMillis = getMaxIdleMillis();
47+
48+
if (maxIdleMillis < 0) {
49+
return false;
50+
}
51+
boolean shouldStopNode = false;
52+
Computer computer = getComputer(node);
53+
if (computer != null && computer.isOnline() && !computer.isConnecting()) {
54+
boolean executorWasUsed = false;
55+
for (Executor executor : computer.getAllExecutors()) {
56+
if (executor.isIdle()) {
57+
long idleStart = executor.getIdleStartMilliseconds();
58+
long idleTime = System.currentTimeMillis() - idleStart;
59+
LOGGER.log(Level.FINEST, "{0} executor: {1} has been idle for: {2}", new Object[] {node.getNodeName() ,executor.getDisplayName(), idleTime});
60+
if (idleTime < maxIdleMillis) {
61+
executorWasUsed = true;
62+
break;
63+
}
64+
} else {
65+
executorWasUsed = true;
66+
break;
67+
}
68+
}
69+
shouldStopNode = !executorWasUsed;
70+
}
71+
return shouldStopNode;
72+
}
73+
74+
private void stopNode(Node node) {
75+
Jenkins jenkinsInstance = Jenkins.get();
76+
77+
for (Cloud cloud : jenkinsInstance.clouds) {
78+
if (!(cloud instanceof AmazonEC2Cloud))
79+
continue;
80+
AmazonEC2Cloud ec2 = (AmazonEC2Cloud) cloud;
81+
if (ec2.isStartStopNodes() && ec2.isEc2Node(node)) {
82+
LOGGER.log(Level.INFO, "Requesting stop on {0} of {1}", new Object[] {ec2.getCloudName(), node.getNodeName()});
83+
try {
84+
ec2.stopNode(node);
85+
} catch (Exception e) {
86+
LOGGER.log(Level.INFO, "Unable to start an EC2 Instance for node: " + node.getNodeName(), e);
87+
}
88+
}
89+
}
90+
}
91+
92+
private long getMaxIdleMillis() {
93+
long maxMinutes = STOP_DISABLED;
94+
Jenkins jenkinsInstance = Jenkins.get();
95+
for (Cloud cloud : jenkinsInstance.clouds) {
96+
if (!(cloud instanceof AmazonEC2Cloud))
97+
continue;
98+
AmazonEC2Cloud ec2 = (AmazonEC2Cloud) cloud;
99+
if (ec2.isStartStopNodes()) {
100+
Integer configuredMax = getInteger(ec2.getMaxIdleMinutes());
101+
if (configuredMax != null) {
102+
maxMinutes = Math.max(maxMinutes, configuredMax);
103+
}
104+
}
105+
}
106+
if (maxMinutes > 0) {
107+
return TimeUnit.MINUTES.toMillis(maxMinutes);
108+
}
109+
return maxMinutes;
110+
}
111+
112+
private Integer getInteger(String str) {
113+
if (str != null) {
114+
try {
115+
return Integer.parseInt(str);
116+
} catch (NumberFormatException nfe) {
117+
LOGGER.log(Level.INFO, "Couldn't get integer from string: {0}", str);
118+
return null;
119+
}
120+
}
121+
return null;
122+
}
123+
124+
@VisibleForTesting
125+
protected Computer getComputer(Node node) {
126+
return node.toComputer();
127+
}
128+
}

0 commit comments

Comments
 (0)