[OPS] Mark some Cloudera tasks optional

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaTimeSeriesTask.java

@@ -17,15 +17,16 @@
 package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager;
 
 import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Preconditions;
+import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
+import javax.annotation.Nonnull;
 import org.apache.hc.core5.net.URIBuilder;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
@@ -34,19 +35,31 @@
 abstract class AbstractClouderaTimeSeriesTask extends AbstractClouderaManagerTask {
   private static final DateTimeFormatter isoDateTimeFormatter =
       DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
-  private final ObjectMapper objectMapper = new ObjectMapper();
   private final int includedLastDays;
   private final TimeSeriesAggregation tsAggregation;
+  private final TaskCategory taskCategory;
 
   public AbstractClouderaTimeSeriesTask(
-      String targetPath, int includedLastDays, TimeSeriesAggregation tsAggregation) {
+      String targetPath,
+      int includedLastDays,
+      TimeSeriesAggregation tsAggregation,
+      TaskCategory taskCategory) {
     super(targetPath);
-    Preconditions.checkNotNull(tsAggregation, "TimeSeriesAggregation has not to be a null.");
+    Preconditions.checkNotNull(tsAggregation, "TimeSeriesAggregation must not be null.");
     Preconditions.checkArgument(
         includedLastDays >= 1,
         "The chart has to include at least one day. Received " + includedLastDays + " days.");
+    Preconditions.checkNotNull(taskCategory, "TaskCategory must not be null.");
+
     this.includedLastDays = includedLastDays;
     this.tsAggregation = tsAggregation;
+    this.taskCategory = taskCategory;
+  }
+
+  @Nonnull
+  @Override
+  public final TaskCategory getCategory() {
+    return taskCategory;
   }
 
   protected JsonNode requestTimeSeriesChart(ClouderaManagerHandle handle, String query) {
@@ -61,21 +74,21 @@ protected JsonNode requestTimeSeriesChart(ClouderaManagerHandle handle, String q
       uriBuilder.addParameter("from", fromDate);
       tsURI = uriBuilder.build();
     } catch (URISyntaxException ex) {
-      throw new TimeSeriesException(ex.getMessage(), ex);
+      throw new RuntimeException(ex.getMessage(), ex);
     }
 
     CloseableHttpClient httpClient = handle.getHttpClient();
     JsonNode chartInJson;
     try (CloseableHttpResponse chart = httpClient.execute(new HttpGet(tsURI))) {
       int statusCode = chart.getStatusLine().getStatusCode();
       if (!isStatusCodeOK(statusCode)) {
-        throw new TimeSeriesException(
+        throw new RuntimeException(
             String.format(
                 "Cloudera Error: Response status code is %d but 2xx is expected.", statusCode));
       }
       chartInJson = readJsonTree(chart.getEntity().getContent());
     } catch (IOException ex) {
-      throw new TimeSeriesException(ex.getMessage(), ex);
+      throw new RuntimeException(ex.getMessage(), ex);
     }
     return chartInJson;
   }
@@ -86,23 +99,6 @@ private String buildISODateTime(int deltaInDays) {
     return dateTime.format(isoDateTimeFormatter);
   }
 
-  static class TimeSeriesException extends RuntimeException {
-    /* Exception which should be returned if something goes wrong with timeseries API.
-     *
-     * Includes:
-     * - unexpected HTTP status codes;
-     * - response with invalid JSON format.
-     */
-
-    public TimeSeriesException(String message) {
-      super(message);
-    }
-
-    public TimeSeriesException(String message, Throwable cause) {
-      super(message, cause);
-    }
-  }
-
   enum TimeSeriesAggregation {
     RAW,
     TEN_MINUTELY,

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaAPIHostsTask.java

@@ -16,7 +16,6 @@
  */
 package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager;
 
-import com.fasterxml.jackson.core.JsonParseException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.io.ByteSink;
 import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
@@ -66,13 +65,11 @@ protected void doRun(
             httpClient.execute(new HttpGet(hostPerClusterUrl))) {
           final int statusCode = hostsResponse.getStatusLine().getStatusCode();
           if (!isStatusCodeOK(statusCode)) {
-            throw new MetadataDumperUsageException(
+            throw new RuntimeException(
                 String.format(
                     "Cloudera Error: Response status code is %d but 2xx is expected.", statusCode));
           }
           jsonHosts = readJsonTree(hostsResponse.getEntity().getContent());
-        } catch (JsonParseException ex) {
-          throw new MetadataDumperUsageException("Cloudera Error:" + ex.getMessage());
         }
         String stringifiedHosts = jsonHosts.toString();
         writer.write(stringifiedHosts);

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaCMFHostsTask.java

@@ -19,11 +19,11 @@
 import com.fasterxml.jackson.core.JsonParseException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.io.ByteSink;
-import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaClusterDTO;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaHostDTO;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.CMFHostDTO;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.CMFHostListDTO;
+import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
 import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;
 import java.io.Writer;
 import java.net.URI;
@@ -49,14 +49,20 @@ public ClouderaCMFHostsTask() {
     super("cmf-hosts.jsonl");
   }
 
+  @Nonnull
+  @Override
+  public TaskCategory getCategory() {
+    return TaskCategory.OPTIONAL;
+  }
+
   @Override
   protected void doRun(
       TaskRunContext context, @Nonnull ByteSink sink, @Nonnull ClouderaManagerHandle handle)
       throws Exception {
     CloseableHttpClient httpClient = handle.getHttpClient();
     List<ClouderaClusterDTO> clusters = handle.getClusters();
     if (clusters == null) {
-      throw new MetadataDumperUsageException(
+      throw new IllegalStateException(
           "Cloudera clusters must be initialized before hosts dumping.");
     }
 
@@ -81,7 +87,7 @@ protected void doRun(
           try {
             hostsJson = readJsonTree(hostsResponse.getEntity().getContent());
           } catch (JsonParseException ex) {
-            LOG.warn("Cloudera Error: " + ex.getMessage());
+            LOG.warn("Could not parse json from cloudera hosts response: " + ex.getMessage(), ex);
             continue;
           }
         }

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTask.java

@@ -20,6 +20,7 @@
 import com.google.common.io.ByteSink;
 import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaClusterDTO;
+import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
 import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;
 import java.io.Writer;
 import java.nio.charset.StandardCharsets;
@@ -43,8 +44,9 @@ public class ClouderaClusterCPUChartTask extends AbstractClouderaTimeSeriesTask
   private static final String TS_CPU_QUERY_TEMPLATE =
       "SELECT cpu_percent_across_hosts WHERE entityName = \"%s\" AND category = CLUSTER";
 
-  public ClouderaClusterCPUChartTask(int includedLastDays, TimeSeriesAggregation tsAggregation) {
-    super(buildOutputFileName(includedLastDays), includedLastDays, tsAggregation);
+  public ClouderaClusterCPUChartTask(
+      int includedLastDays, TimeSeriesAggregation tsAggregation, TaskCategory taskCategory) {
+    super(buildOutputFileName(includedLastDays), includedLastDays, tsAggregation, taskCategory);
   }
 
   @Override
@@ -61,15 +63,7 @@ protected void doRun(
             cpuPerClusterQuery,
             cluster.getName());
 
-        JsonNode chartInJson;
-        try {
-          chartInJson = requestTimeSeriesChart(handle, cpuPerClusterQuery);
-        } catch (TimeSeriesException ex) {
-          MetadataDumperUsageException dumperException =
-              new MetadataDumperUsageException("Cloudera Error: " + ex.getMessage());
-          dumperException.initCause(ex);
-          throw dumperException;
-        }
+        JsonNode chartInJson = requestTimeSeriesChart(handle, cpuPerClusterQuery);
         writer.write(chartInJson.toString());
         writer.write('\n');
       }

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostComponentsTask.java

@@ -19,8 +19,8 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.io.ByteSink;
-import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaHostDTO;
+import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
 import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;
 import java.io.Writer;
 import java.nio.charset.StandardCharsets;
@@ -41,14 +41,20 @@ public ClouderaHostComponentsTask() {
     super("host-components.jsonl");
   }
 
+  @Nonnull
+  @Override
+  public TaskCategory getCategory() {
+    return TaskCategory.OPTIONAL;
+  }
+
   @Override
   protected void doRun(
       TaskRunContext context, @Nonnull ByteSink sink, @Nonnull ClouderaManagerHandle handle)
       throws Exception {
     CloseableHttpClient httpClient = handle.getHttpClient();
     List<ClouderaHostDTO> hosts = handle.getHosts();
     if (hosts == null) {
-      throw new MetadataDumperUsageException(
+      throw new IllegalStateException(
           "Cloudera hosts must be initialized before Host's components dumping.");
     }
 

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTask.java

@@ -20,6 +20,7 @@
 import com.google.common.io.ByteSink;
 import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaHostDTO;
+import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
 import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;
 import java.io.Writer;
 import java.nio.charset.StandardCharsets;
@@ -43,8 +44,9 @@ public class ClouderaHostRAMChartTask extends AbstractClouderaTimeSeriesTask {
   private static final String TS_RAM_QUERY_TEMPLATE =
       "select swap_used, physical_memory_used, physical_memory_total, physical_memory_cached, physical_memory_buffers where entityName = \"%s\"";
 
-  public ClouderaHostRAMChartTask(int includedLastDays, TimeSeriesAggregation tsAggregation) {
-    super(buildOutputFileName(includedLastDays), includedLastDays, tsAggregation);
+  public ClouderaHostRAMChartTask(
+      int includedLastDays, TimeSeriesAggregation tsAggregation, TaskCategory taskCategory) {
+    super(buildOutputFileName(includedLastDays), includedLastDays, tsAggregation, taskCategory);
   }
 
   @Override
@@ -63,15 +65,8 @@ protected void doRun(
         LOG.debug(
             "Execute RAM charts query: [{}] for the host: [{}].", ramPerHostQuery, host.getName());
 
-        JsonNode chartInJson;
-        try {
-          chartInJson = requestTimeSeriesChart(handle, ramPerHostQuery);
-        } catch (TimeSeriesException ex) {
-          MetadataDumperUsageException dumperException =
-              new MetadataDumperUsageException("Cloudera Error: " + ex.getMessage());
-          dumperException.initCause(ex);
-          throw dumperException;
-        }
+        JsonNode chartInJson = requestTimeSeriesChart(handle, ramPerHostQuery);
+
         writer.write(chartInJson.toString());
         writer.write('\n');
       }

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnector.java

@@ -16,13 +16,15 @@
  */
 package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager;
 
+import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.AbstractClouderaTimeSeriesTask.TimeSeriesAggregation.*;
+import static com.google.edwmigration.dumper.application.dumper.task.TaskCategory.*;
+
 import com.google.auto.service.AutoService;
 import com.google.edwmigration.dumper.application.dumper.ConnectorArguments;
 import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
 import com.google.edwmigration.dumper.application.dumper.annotations.RespectsInput;
 import com.google.edwmigration.dumper.application.dumper.connector.AbstractConnector;
 import com.google.edwmigration.dumper.application.dumper.connector.Connector;
-import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.AbstractClouderaTimeSeriesTask.TimeSeriesAggregation;
 import com.google.edwmigration.dumper.application.dumper.task.DumpMetadataTask;
 import com.google.edwmigration.dumper.application.dumper.task.FormatTask;
 import com.google.edwmigration.dumper.application.dumper.task.Task;
@@ -85,15 +87,15 @@ public void addTasksTo(@Nonnull List<? super Task<?>> out, @Nonnull ConnectorArg
     out.add(new ClouderaServicesTask());
     out.add(new ClouderaHostComponentsTask());
 
-    out.add(new ClouderaClusterCPUChartTask(1, TimeSeriesAggregation.HOURLY));
-    out.add(new ClouderaClusterCPUChartTask(7, TimeSeriesAggregation.DAILY));
-    out.add(new ClouderaClusterCPUChartTask(30, TimeSeriesAggregation.DAILY));
-    out.add(new ClouderaClusterCPUChartTask(90, TimeSeriesAggregation.DAILY));
+    out.add(new ClouderaClusterCPUChartTask(1, HOURLY, REQUIRED));
+    out.add(new ClouderaClusterCPUChartTask(7, DAILY, OPTIONAL));
+    out.add(new ClouderaClusterCPUChartTask(30, DAILY, OPTIONAL));
+    out.add(new ClouderaClusterCPUChartTask(90, DAILY, OPTIONAL));
 
-    out.add(new ClouderaHostRAMChartTask(1, TimeSeriesAggregation.HOURLY));
-    out.add(new ClouderaHostRAMChartTask(7, TimeSeriesAggregation.DAILY));
-    out.add(new ClouderaHostRAMChartTask(30, TimeSeriesAggregation.DAILY));
-    out.add(new ClouderaHostRAMChartTask(90, TimeSeriesAggregation.DAILY));
+    out.add(new ClouderaHostRAMChartTask(1, HOURLY, REQUIRED));
+    out.add(new ClouderaHostRAMChartTask(7, DAILY, OPTIONAL));
+    out.add(new ClouderaHostRAMChartTask(30, DAILY, OPTIONAL));
+    out.add(new ClouderaHostRAMChartTask(90, DAILY, OPTIONAL));
 
     out.add(new ClouderaYarnApplicationsTask(90));
     out.add(new ClouderaYarnApplicationTypeTask(90));

dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaTimeSeriesTaskTest.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2022-2025 Google LLC
+ * Copyright 2013-2021 CompilerWorks
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
+
+import com.google.common.io.ByteSink;
+import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.AbstractClouderaTimeSeriesTask.TimeSeriesAggregation;
+import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;
+import java.io.IOException;
+import javax.annotation.Nonnull;
+import org.junit.Test;
+
+public class AbstractClouderaTimeSeriesTaskTest {
+
+  @Test
+  public void doRun_missedAggregationParameter_throwsException() throws IOException {
+    // WHEN: CPU usage task is initiated with no aggregation parameter
+    NullPointerException exception =
+        assertThrows(
+            NullPointerException.class,
+            () ->
+                new AbstractClouderaTimeSeriesTask("some path", 5, null, null) {
+                  @Override
+                  protected void doRun(
+                      TaskRunContext context,
+                      @Nonnull ByteSink sink,
+                      @Nonnull ClouderaManagerHandle handle)
+                      throws Exception {}
+                });
+
+    // THEN: There is a relevant exception has been raised
+    assertEquals("TimeSeriesAggregation must not be null.", exception.getMessage());
+  }
+
+  @Test
+  public void doRun_missedCategoryParameter_throwsException() throws IOException {
+    // WHEN: CPU usage task is initiated with no aggregation parameter
+    NullPointerException exception =
+        assertThrows(
+            NullPointerException.class,
+            () ->
+                new AbstractClouderaTimeSeriesTask(
+                    "some path", 5, TimeSeriesAggregation.DAILY, null) {
+                  @Override
+                  protected void doRun(
+                      TaskRunContext context,
+                      @Nonnull ByteSink sink,
+                      @Nonnull ClouderaManagerHandle handle)
+                      throws Exception {}
+                });
+
+    // THEN: There is a relevant exception has been raised
+    assertEquals("TaskCategory must not be null.", exception.getMessage());
+  }
+}