Skip to content

Commit fce8a1e

Browse files
zaldisDenys Zaluzhnyivladislav-sidorovich
authored
Feat: b/400932733 Add date range for cloudera manager tasks (#889)
* Feat: Add date range for cloudera manager tasks * fix tests * Add "to" HTTP parameter * Update tests * Extend tests to check custom dumped files * update dumped files * fix abstract builder * update task category * test zip files were generated by expected tasks * use single file for time range tasks --------- Co-authored-by: Denys Zaluzhnyi <dzaluzhnyi@google.com> Co-authored-by: Vladislav Sidorovich <vsidorovich@google.com>
1 parent 64035b5 commit fce8a1e

15 files changed

+345
-154
lines changed

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaTimeSeriesTask.java‎

Lines changed: 16 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,7 @@
1818

1919
import com.fasterxml.jackson.databind.JsonNode;
2020
import com.google.common.base.Preconditions;
21-
import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
2221
import java.net.URI;
23-
import java.time.LocalDateTime;
24-
import java.time.ZoneId;
2522
import java.time.ZonedDateTime;
2623
import java.time.format.DateTimeFormatter;
2724
import javax.annotation.Nonnull;
@@ -33,43 +30,38 @@
3330
abstract class AbstractClouderaTimeSeriesTask extends AbstractClouderaManagerTask {
3431
private static final DateTimeFormatter isoDateTimeFormatter =
3532
DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
36-
private final int includedLastDays;
33+
private final ZonedDateTime startDate;
34+
private final ZonedDateTime endDate;
3735
private final TimeSeriesAggregation tsAggregation;
38-
private final TaskCategory taskCategory;
3936

4037
public AbstractClouderaTimeSeriesTask(
41-
String targetPath,
42-
int includedLastDays,
43-
TimeSeriesAggregation tsAggregation,
44-
TaskCategory taskCategory) {
38+
@Nonnull String targetPath,
39+
@Nonnull ZonedDateTime startDate,
40+
@Nonnull ZonedDateTime endDate,
41+
@Nonnull TimeSeriesAggregation tsAggregation) {
4542
super(targetPath);
46-
Preconditions.checkNotNull(tsAggregation, "TimeSeriesAggregation must not be null.");
47-
Preconditions.checkArgument(
48-
includedLastDays >= 1,
49-
"The chart has to include at least one day. Received " + includedLastDays + " days.");
50-
Preconditions.checkNotNull(taskCategory, "TaskCategory must not be null.");
43+
Preconditions.checkNotNull(targetPath, "Target path must be not null.");
44+
Preconditions.checkState(!targetPath.isEmpty(), "Target file path must be not empty.");
45+
Preconditions.checkNotNull(tsAggregation, "TimeSeriesAggregation must be not null.");
46+
Preconditions.checkNotNull(startDate, "Start date must be not null.");
47+
Preconditions.checkNotNull(endDate, "End date must be not null.");
48+
Preconditions.checkState(startDate.isBefore(endDate), "Start Date has to be before End Date.");
5149

52-
this.includedLastDays = includedLastDays;
50+
this.startDate = startDate;
51+
this.endDate = endDate;
5352
this.tsAggregation = tsAggregation;
54-
this.taskCategory = taskCategory;
55-
}
56-
57-
@Nonnull
58-
@Override
59-
public final TaskCategory getCategory() {
60-
return taskCategory;
6153
}
6254

6355
protected JsonNode requestTimeSeriesChart(ClouderaManagerHandle handle, String query)
6456
throws Exception {
6557
String timeSeriesUrl = handle.getApiURI().toString() + "/timeseries";
66-
String fromDate = buildISODateTime(includedLastDays);
6758

6859
URIBuilder uriBuilder = new URIBuilder(timeSeriesUrl);
6960
uriBuilder.addParameter("query", query);
7061
uriBuilder.addParameter("desiredRollup", tsAggregation.toString());
7162
uriBuilder.addParameter("mustUseDesiredRollup", "true");
72-
uriBuilder.addParameter("from", fromDate);
63+
uriBuilder.addParameter("from", startDate.format(isoDateTimeFormatter));
64+
uriBuilder.addParameter("to", endDate.format(isoDateTimeFormatter));
7365
URI tsURI = uriBuilder.build();
7466

7567
CloseableHttpClient httpClient = handle.getHttpClient();
@@ -86,12 +78,6 @@ protected JsonNode requestTimeSeriesChart(ClouderaManagerHandle handle, String q
8678
return chartInJson;
8779
}
8880

89-
private String buildISODateTime(int deltaInDays) {
90-
ZonedDateTime dateTime =
91-
ZonedDateTime.of(LocalDateTime.now().minusDays(deltaInDays), ZoneId.of("UTC"));
92-
return dateTime.format(isoDateTimeFormatter);
93-
}
94-
9581
enum TimeSeriesAggregation {
9682
RAW,
9783
TEN_MINUTELY,

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaYarnApplicationTask.java‎

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,6 @@
2424
import java.io.IOException;
2525
import java.net.URI;
2626
import java.net.URISyntaxException;
27-
import java.time.LocalDateTime;
28-
import java.time.ZoneId;
2927
import java.time.ZonedDateTime;
3028
import java.time.format.DateTimeFormatter;
3129
import java.util.ArrayList;
@@ -40,16 +38,22 @@
4038

4139
public abstract class AbstractClouderaYarnApplicationTask extends AbstractClouderaManagerTask {
4240
private final ZonedDateTime fromDate;
41+
private final ZonedDateTime toDate;
4342
private final TaskCategory taskCategory;
4443

4544
public AbstractClouderaYarnApplicationTask(
46-
String fileName, int lastDaysToInclude, TaskCategory taskCategory) {
47-
super(String.format("%s-%dd.jsonl", fileName, lastDaysToInclude));
48-
Preconditions.checkArgument(
49-
lastDaysToInclude >= 1,
50-
String.format("Amount of days must be a positive number. Got %d.", lastDaysToInclude));
51-
52-
fromDate = ZonedDateTime.of(LocalDateTime.now().minusDays(lastDaysToInclude), ZoneId.of("UTC"));
45+
@Nonnull String targetPath,
46+
@Nonnull ZonedDateTime startDate,
47+
@Nonnull ZonedDateTime endDate,
48+
@Nonnull TaskCategory taskCategory) {
49+
super(targetPath);
50+
Preconditions.checkNotNull(startDate, "Start date must be not null.");
51+
Preconditions.checkNotNull(endDate, "End date must be not null.");
52+
Preconditions.checkNotNull(taskCategory, "Task category must be not null.");
53+
Preconditions.checkState(startDate.isBefore(endDate), "Start Date has to be before End Date.");
54+
55+
fromDate = startDate;
56+
toDate = endDate;
5357
this.taskCategory = taskCategory;
5458
}
5559

@@ -67,10 +71,7 @@ class PaginatedClouderaYarnApplicationsLoader {
6771
private final int limit;
6872
private int offset;
6973
private final String fromAppCreationDate;
70-
71-
public PaginatedClouderaYarnApplicationsLoader(ClouderaManagerHandle handle) {
72-
this(handle, 1000);
73-
}
74+
private final String toAppCreationDate;
7475

7576
public PaginatedClouderaYarnApplicationsLoader(ClouderaManagerHandle handle, int limit) {
7677
this.apiURI = handle.getApiURI();
@@ -79,6 +80,7 @@ public PaginatedClouderaYarnApplicationsLoader(ClouderaManagerHandle handle, int
7980

8081
final DateTimeFormatter dtFormatter = DateTimeFormatter.ofPattern(ISO_DATETIME_FORMAT);
8182
fromAppCreationDate = fromDate.format(dtFormatter);
83+
toAppCreationDate = toDate.format(dtFormatter);
8284
}
8385

8486
public int load(String clusterName, Consumer<List<ApiYARNApplicationDTO>> onPageLoad) {
@@ -144,7 +146,8 @@ private URI buildNextYARNApplicationPageURI(String clusterName, @Nullable String
144146
.setPathSegments("clusters", clusterName, "services", "yarn", "yarnApplications")
145147
.addParameter("limit", String.valueOf(limit))
146148
.addParameter("offset", String.valueOf(offset))
147-
.addParameter("from", fromAppCreationDate);
149+
.addParameter("from", fromAppCreationDate)
150+
.addParameter("to", toAppCreationDate);
148151
if (appType != null) {
149152
uriBuilder.addParameter("filter", String.format("applicationType=\"%s\"", appType));
150153
}

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTask.java‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,10 @@
2020
import com.google.common.io.ByteSink;
2121
import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
2222
import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaClusterDTO;
23-
import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
2423
import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;
2524
import java.io.Writer;
2625
import java.nio.charset.StandardCharsets;
26+
import java.time.ZonedDateTime;
2727
import java.util.ArrayList;
2828
import java.util.List;
2929
import javax.annotation.Nonnull;
@@ -45,8 +45,8 @@ public class ClouderaClusterCPUChartTask extends AbstractClouderaTimeSeriesTask
4545
"SELECT cpu_percent_across_hosts WHERE entityName = \"%s\" AND category = CLUSTER";
4646

4747
public ClouderaClusterCPUChartTask(
48-
int includedLastDays, TimeSeriesAggregation tsAggregation, TaskCategory taskCategory) {
49-
super(buildOutputFileName(includedLastDays), includedLastDays, tsAggregation, taskCategory);
48+
ZonedDateTime startDate, ZonedDateTime endDate, TimeSeriesAggregation tsAggregation) {
49+
super("cluster-cpu.jsonl", startDate, endDate, tsAggregation);
5050
}
5151

5252
@Override
@@ -88,8 +88,4 @@ private List<ClouderaClusterDTO> getClustersFromHandle(@Nonnull ClouderaManagerH
8888
}
8989
return cpuClusters;
9090
}
91-
92-
private static String buildOutputFileName(int includedLastDays) {
93-
return String.format("cluster-cpu-%sd.jsonl", includedLastDays);
94-
}
9591
}

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTask.java‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,10 @@
2020
import com.google.common.io.ByteSink;
2121
import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
2222
import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaHostDTO;
23-
import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
2423
import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;
2524
import java.io.Writer;
2625
import java.nio.charset.StandardCharsets;
26+
import java.time.ZonedDateTime;
2727
import java.util.List;
2828
import javax.annotation.Nonnull;
2929
import org.slf4j.Logger;
@@ -39,14 +39,15 @@
3939
* language.
4040
*/
4141
public class ClouderaHostRAMChartTask extends AbstractClouderaTimeSeriesTask {
42+
4243
private static final Logger logger = LoggerFactory.getLogger(ClouderaCMFHostsTask.class);
4344

4445
private static final String TS_RAM_QUERY_TEMPLATE =
4546
"select swap_used, physical_memory_used, physical_memory_total, physical_memory_cached, physical_memory_buffers where entityName = \"%s\"";
4647

4748
public ClouderaHostRAMChartTask(
48-
int includedLastDays, TimeSeriesAggregation tsAggregation, TaskCategory taskCategory) {
49-
super(buildOutputFileName(includedLastDays), includedLastDays, tsAggregation, taskCategory);
49+
ZonedDateTime startDate, ZonedDateTime endDate, TimeSeriesAggregation tsAggregation) {
50+
super("host-ram.jsonl", startDate, endDate, tsAggregation);
5051
}
5152

5253
@Override
@@ -72,8 +73,4 @@ protected void doRun(
7273
}
7374
}
7475
}
75-
76-
static String buildOutputFileName(int includedLastDays) {
77-
return String.format("host-ram-%sd.jsonl", includedLastDays);
78-
}
7976
}

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnector.java‎

Lines changed: 41 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,12 @@
1616
*/
1717
package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager;
1818

19-
import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.AbstractClouderaTimeSeriesTask.TimeSeriesAggregation.*;
20-
import static com.google.edwmigration.dumper.application.dumper.task.TaskCategory.*;
19+
import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.AbstractClouderaTimeSeriesTask.TimeSeriesAggregation.DAILY;
20+
import static com.google.edwmigration.dumper.application.dumper.task.TaskCategory.OPTIONAL;
2121

2222
import com.google.auto.service.AutoService;
23+
import com.google.common.base.Preconditions;
2324
import com.google.edwmigration.dumper.application.dumper.ConnectorArguments;
24-
import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
2525
import com.google.edwmigration.dumper.application.dumper.annotations.RespectsInput;
2626
import com.google.edwmigration.dumper.application.dumper.connector.AbstractConnector;
2727
import com.google.edwmigration.dumper.application.dumper.connector.Connector;
@@ -32,8 +32,9 @@
3232
import com.google.edwmigration.dumper.plugin.ext.jdk.annotation.Description;
3333
import java.net.URI;
3434
import java.time.Clock;
35-
import java.util.ArrayList;
35+
import java.time.ZonedDateTime;
3636
import java.util.List;
37+
import java.util.function.Supplier;
3738
import javax.annotation.Nonnull;
3839
import org.apache.http.conn.ssl.NoopHostnameVerifier;
3940
import org.apache.http.conn.ssl.TrustAllStrategy;
@@ -66,8 +67,18 @@ public class ClouderaManagerConnector extends AbstractConnector {
6667

6768
private static final String FORMAT_NAME = "cloudera-manager.dump.zip";
6869

70+
private static final int MAX_QUARTER_DAY = 93;
71+
72+
private final Supplier<ZonedDateTime> currentTimeProvider;
73+
74+
@SuppressWarnings("unused") // reflection call is expected
6975
public ClouderaManagerConnector() {
76+
this(ZonedDateTime::now);
77+
}
78+
79+
public ClouderaManagerConnector(Supplier<ZonedDateTime> currentTimeProvider) {
7080
super("cloudera-manager");
81+
this.currentTimeProvider = currentTimeProvider;
7182
}
7283

7384
@Nonnull
@@ -87,47 +98,48 @@ public void addTasksTo(@Nonnull List<? super Task<?>> out, @Nonnull ConnectorArg
8798
out.add(new ClouderaServicesTask());
8899
out.add(new ClouderaHostComponentsTask());
89100

90-
out.add(new ClouderaClusterCPUChartTask(1, HOURLY, REQUIRED));
91-
out.add(new ClouderaClusterCPUChartTask(7, DAILY, OPTIONAL));
92-
out.add(new ClouderaClusterCPUChartTask(30, DAILY, OPTIONAL));
93-
out.add(new ClouderaClusterCPUChartTask(90, DAILY, OPTIONAL));
94-
95-
out.add(new ClouderaHostRAMChartTask(1, HOURLY, REQUIRED));
96-
out.add(new ClouderaHostRAMChartTask(7, DAILY, OPTIONAL));
97-
out.add(new ClouderaHostRAMChartTask(30, DAILY, OPTIONAL));
98-
out.add(new ClouderaHostRAMChartTask(90, DAILY, OPTIONAL));
101+
ZonedDateTime startDate;
102+
ZonedDateTime endDate;
103+
boolean useDefaultDateRangeToFetch = arguments.getStartDate() == null;
104+
if (useDefaultDateRangeToFetch) {
105+
endDate = currentTimeProvider.get();
106+
startDate = endDate.minusDays(MAX_QUARTER_DAY);
107+
} else {
108+
startDate = arguments.getStartDate();
109+
endDate = arguments.getEndDate();
110+
}
99111

100-
out.add(new ClouderaYarnApplicationsTask(90, OPTIONAL));
101-
out.add(new ClouderaYarnApplicationTypeTask(90, OPTIONAL));
112+
out.add(new ClouderaClusterCPUChartTask(startDate, endDate, DAILY));
113+
out.add(new ClouderaHostRAMChartTask(startDate, endDate, DAILY));
114+
out.add(new ClouderaYarnApplicationsTask(startDate, endDate, OPTIONAL));
115+
out.add(new ClouderaYarnApplicationTypeTask(startDate, endDate, OPTIONAL));
102116
}
103117

104118
@Nonnull
105119
@Override
106120
public ClouderaManagerHandle open(@Nonnull ConnectorArguments arguments) throws Exception {
107-
List<String> errors = new ArrayList<>();
108-
if (arguments.getUri() == null) {
109-
errors.add("--url for Cloudera Manager API is required");
110-
}
111-
if (arguments.getUser() == null) {
112-
errors.add("--user is required for Cloudera Manager API connector");
113-
}
114-
115-
if (!errors.isEmpty()) {
116-
throw new MetadataDumperUsageException(
117-
"Missing arguments for connector generic-args : ", errors);
118-
}
119121
URI uri = new URI(arguments.getUri());
120-
121122
CloseableHttpClient httpClient = disableSSLVerification(HttpClients.custom()).build();
122123
ClouderaManagerHandle handle = new ClouderaManagerHandle(uri, httpClient);
123124

124125
String user = arguments.getUser();
125126
String password = arguments.getPasswordOrPrompt();
126127
doClouderaManagerLogin(handle.getBaseURI(), httpClient, user, password);
127-
128128
return handle;
129129
}
130130

131+
@Override
132+
public void validate(ConnectorArguments arguments) {
133+
String clouderaUri = arguments.getUri();
134+
Preconditions.checkNotNull(clouderaUri, "--url for Cloudera Manager API is required");
135+
136+
String clouderaUser = arguments.getUser();
137+
Preconditions.checkNotNull(
138+
clouderaUser, "--user is required for Cloudera Manager API connector");
139+
140+
validateDateRange(arguments);
141+
}
142+
131143
private void doClouderaManagerLogin(
132144
URI baseURI, CloseableHttpClient httpClient, String user, String password) throws Exception {
133145
ClouderaManagerLoginHelper.login(baseURI, httpClient, user, password);

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationTypeTask.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import java.io.IOException;
3030
import java.io.Writer;
3131
import java.nio.charset.StandardCharsets;
32+
import java.time.ZonedDateTime;
3233
import java.util.ArrayList;
3334
import java.util.HashSet;
3435
import java.util.List;
@@ -48,8 +49,9 @@ public class ClouderaYarnApplicationTypeTask extends AbstractClouderaYarnApplica
4849
private final ImmutableList<String> predefinedAppTypes =
4950
ImmutableList.of("MAPREDUCE", "SPARK", "Oozie Launcher");
5051

51-
public ClouderaYarnApplicationTypeTask(int days, TaskCategory taskCategory) {
52-
super("yarn-application-types", days, taskCategory);
52+
public ClouderaYarnApplicationTypeTask(
53+
ZonedDateTime startDate, ZonedDateTime endDate, TaskCategory taskCategory) {
54+
super("yarn-application-types.jsonl", startDate, endDate, taskCategory);
5355
}
5456

5557
@Override

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationsTask.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import java.io.IOException;
2727
import java.io.Writer;
2828
import java.nio.charset.StandardCharsets;
29+
import java.time.ZonedDateTime;
2930
import java.util.List;
3031
import javax.annotation.Nonnull;
3132
import org.slf4j.Logger;
@@ -35,8 +36,9 @@
3536
public class ClouderaYarnApplicationsTask extends AbstractClouderaYarnApplicationTask {
3637
private static final Logger logger = LoggerFactory.getLogger(ClouderaYarnApplicationsTask.class);
3738

38-
public ClouderaYarnApplicationsTask(int days, TaskCategory taskCategory) {
39-
super("yarn-applications", days, taskCategory);
39+
public ClouderaYarnApplicationsTask(
40+
ZonedDateTime startDate, ZonedDateTime endDate, TaskCategory taskCategory) {
41+
super("yarn-applications.jsonl", startDate, endDate, taskCategory);
4042
}
4143

4244
@Override

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/task/Task.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ public default String getName() {
118118
}
119119

120120
@Nonnull
121-
public default TaskCategory getCategory() {
121+
default TaskCategory getCategory() {
122122
return TaskCategory.REQUIRED;
123123
}
124124

0 commit comments

Comments
 (0)