[b455552691] Collect dynamic resource allocation of Cloudera services

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaTimeSeriesTask.java

@@ -18,6 +18,7 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.base.Preconditions;
+import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
 import java.net.URI;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
@@ -33,12 +34,14 @@ abstract class AbstractClouderaTimeSeriesTask extends AbstractClouderaManagerTas
   private final ZonedDateTime startDate;
   private final ZonedDateTime endDate;
   private final TimeSeriesAggregation tsAggregation;
+  private final TaskCategory taskCategory;
 
   public AbstractClouderaTimeSeriesTask(
       @Nonnull String targetPath,
       @Nonnull ZonedDateTime startDate,
       @Nonnull ZonedDateTime endDate,
-      @Nonnull TimeSeriesAggregation tsAggregation) {
+      @Nonnull TimeSeriesAggregation tsAggregation,
+      @Nonnull TaskCategory taskCategory) {
     super(targetPath);
     Preconditions.checkNotNull(targetPath, "Target path must be not null.");
     Preconditions.checkState(!targetPath.isEmpty(), "Target file path must be not empty.");
@@ -50,6 +53,13 @@ public AbstractClouderaTimeSeriesTask(
     this.startDate = startDate;
     this.endDate = endDate;
     this.tsAggregation = tsAggregation;
+    this.taskCategory = taskCategory;
+  }
+
+  @Nonnull
+  @Override
+  public TaskCategory getCategory() {
+    return taskCategory;
   }
 
   protected JsonNode requestTimeSeriesChart(ClouderaManagerHandle handle, String query)

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTask.java

@@ -20,6 +20,7 @@
 import com.google.common.io.ByteSink;
 import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaClusterDTO;
+import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
 import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;
 import java.io.Writer;
 import java.nio.charset.StandardCharsets;
@@ -40,13 +41,16 @@
  * language.
  */
 public class ClouderaClusterCPUChartTask extends AbstractClouderaTimeSeriesTask {
-  private static final Logger logger = LoggerFactory.getLogger(ClouderaCMFHostsTask.class);
+  private static final Logger logger = LoggerFactory.getLogger(ClouderaClusterCPUChartTask.class);
   private static final String TS_CPU_QUERY_TEMPLATE =
       "SELECT cpu_percent_across_hosts WHERE entityName = \"%s\" AND category = CLUSTER";
 
   public ClouderaClusterCPUChartTask(
-      ZonedDateTime startDate, ZonedDateTime endDate, TimeSeriesAggregation tsAggregation) {
-    super("cluster-cpu.jsonl", startDate, endDate, tsAggregation);
+      ZonedDateTime startDate,
+      ZonedDateTime endDate,
+      TimeSeriesAggregation tsAggregation,
+      TaskCategory taskCategory) {
+    super("cluster-cpu.jsonl", startDate, endDate, tsAggregation, taskCategory);
   }
 
   @Override

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTask.java

@@ -20,6 +20,7 @@
 import com.google.common.io.ByteSink;
 import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaHostDTO;
+import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
 import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;
 import java.io.Writer;
 import java.nio.charset.StandardCharsets;
@@ -40,14 +41,17 @@
  */
 public class ClouderaHostRAMChartTask extends AbstractClouderaTimeSeriesTask {
 
-  private static final Logger logger = LoggerFactory.getLogger(ClouderaCMFHostsTask.class);
+  private static final Logger logger = LoggerFactory.getLogger(ClouderaHostRAMChartTask.class);
 
   private static final String TS_RAM_QUERY_TEMPLATE =
       "select swap_used, physical_memory_used, physical_memory_total, physical_memory_cached, physical_memory_buffers where entityName = \"%s\"";
 
   public ClouderaHostRAMChartTask(
-      ZonedDateTime startDate, ZonedDateTime endDate, TimeSeriesAggregation tsAggregation) {
-    super("host-ram.jsonl", startDate, endDate, tsAggregation);
+      ZonedDateTime startDate,
+      ZonedDateTime endDate,
+      TimeSeriesAggregation tsAggregation,
+      TaskCategory taskCategory) {
+    super("host-ram.jsonl", startDate, endDate, tsAggregation, taskCategory);
   }
 
   @Override

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnector.java

@@ -18,7 +18,9 @@
 
 import static com.google.edwmigration.dumper.application.dumper.connector.Connector.validateDateRange;
 import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.AbstractClouderaTimeSeriesTask.TimeSeriesAggregation.DAILY;
+import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.AbstractClouderaTimeSeriesTask.TimeSeriesAggregation.HOURLY;
 import static com.google.edwmigration.dumper.application.dumper.task.TaskCategory.OPTIONAL;
+import static com.google.edwmigration.dumper.application.dumper.task.TaskCategory.REQUIRED;
 
 import com.google.auto.service.AutoService;
 import com.google.common.base.Preconditions;
@@ -110,8 +112,9 @@ public void addTasksTo(@Nonnull List<? super Task<?>> out, @Nonnull ConnectorArg
       endDate = arguments.getEndDate();
     }
 
-    out.add(new ClouderaClusterCPUChartTask(startDate, endDate, DAILY));
-    out.add(new ClouderaHostRAMChartTask(startDate, endDate, DAILY));
+    out.add(new ClouderaClusterCPUChartTask(startDate, endDate, DAILY, REQUIRED));
+    out.add(new ClouderaHostRAMChartTask(startDate, endDate, DAILY, REQUIRED));
+    out.add(new ClouderaServiceResourceAllocationChartTask(startDate, endDate, HOURLY, OPTIONAL));
     out.add(new ClouderaYarnApplicationsTask(startDate, endDate, OPTIONAL));
     out.add(new ClouderaYarnApplicationTypeTask(startDate, endDate, OPTIONAL));
   }

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaServiceResourceAllocationChartTask.java

@@ -0,0 +1,113 @@
+/*
+ * Copyright 2022-2025 Google LLC
+ * Copyright 2013-2021 CompilerWorks
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.io.ByteSink;
+import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
+import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaHostDTO;
+import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
+import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.time.ZonedDateTime;
+import java.util.ArrayList;
+import java.util.List;
+import javax.annotation.Nonnull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The task collects resource allocation metrics for <strong>all</strong> individual service roles
+ * present on a given host from the Cloudera Manager <a
+ * href="https://cldr2-aw-dl-gateway.cldr2-cd.svye-dcxb.a5.cloudera.site/static/apidocs/resource_TimeSeriesResource.html">TimeSeries
+ * API</a>.
+ *
+ * <p>This class provides a comprehensive, component-by-component breakdown of resource consumption,
+ * programmatically fetching the data that powers the stacked resource charts on a host's status
+ * page in the Cloudera Manager UI (found under the "Charts" tab for a specific host).
+ *
+ * <p>Queries are written in the <a
+ * href="https://docs.cloudera.com/documentation/enterprise/latest/topics/cm_dg_tsquery.html">tsquery</a>
+ * language and are filtered by a specific {@code hostId}.
+ *
+ * <p>Key metrics collected include:
+ *
+ * <ul>
+ *   <li><b>Memory:</b> {@code mem_rss}
+ *   <li><b>CPU:</b> {@code cpu_user_rate} and {@code cpu_system_rate}
+ *   <li><b>Disk Throughput:</b> {@code read_bytes_rate} and {@code write_bytes_rate}
+ * </ul>
+ */
+public class ClouderaServiceResourceAllocationChartTask extends AbstractClouderaTimeSeriesTask {
+
+  private static final Logger logger =
+      LoggerFactory.getLogger(ClouderaServiceResourceAllocationChartTask.class);
+
+  private static final String SERVICE_RESOURCE_ALLOCATION_QUERY_TEMPLATE =
+      "select mem_rss, cpu_user_rate, cpu_system_rate, read_bytes_rate, write_bytes_rate where category = \"ROLE\" AND hostId = \"%s\"";
+
+  public ClouderaServiceResourceAllocationChartTask(
+      ZonedDateTime startDate,
+      ZonedDateTime endDate,
+      TimeSeriesAggregation tsAggregation,
+      TaskCategory taskCategory) {
+    super("service-resource-allocation.jsonl", startDate, endDate, tsAggregation, taskCategory);
+  }
+
+  @Override
+  protected void doRun(
+      TaskRunContext context, @Nonnull ByteSink sink, @Nonnull ClouderaManagerHandle handle)
+      throws Exception {
+    List<ClouderaHostDTO> hosts = getHostsFromHandle(handle);
+
+    try (Writer writer = sink.asCharSink(StandardCharsets.UTF_8).openBufferedStream()) {
+      for (ClouderaHostDTO host : hosts) {
+        String resourceAllocationPerHostQuery =
+            String.format(SERVICE_RESOURCE_ALLOCATION_QUERY_TEMPLATE, host.getId());
+        logger.debug(
+            "Execute service resource allocation charts query: [{}] for the host: [{}].",
+            resourceAllocationPerHostQuery,
+            host.getName());
+
+        JsonNode chartInJson = requestTimeSeriesChart(handle, resourceAllocationPerHostQuery);
+
+        writer.write(chartInJson.toString());
+        writer.write('\n');
+      }
+    }
+  }
+
+  private List<ClouderaHostDTO> getHostsFromHandle(@Nonnull ClouderaManagerHandle handle) {
+    List<ClouderaHostDTO> hosts = handle.getHosts();
+    if (hosts == null) {
+      throw new MetadataDumperUsageException(
+          "Cloudera hosts must be initialized before service resource allocation charts dumping.");
+    }
+    List<ClouderaHostDTO> validHosts = new ArrayList<>();
+    for (ClouderaHostDTO host : hosts) {
+      if (host.getId() == null) {
+        logger.warn(
+            "Cloudera host id is null for host [{}]. Skip resource allocation metrics for services belonging to this host.",
+            host.getName());
+      } else {
+        validHosts.add(host);
+      }
+    }
+    return validHosts;
+  }
+}

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationTypeTask.java

@@ -44,7 +44,8 @@
 import org.slf4j.LoggerFactory;
 
 public class ClouderaYarnApplicationTypeTask extends AbstractClouderaYarnApplicationTask {
-  private static final Logger logger = LoggerFactory.getLogger(ClouderaYarnApplicationsTask.class);
+  private static final Logger logger =
+      LoggerFactory.getLogger(ClouderaYarnApplicationTypeTask.class);
 
   private final ImmutableList<String> predefinedAppTypes =
       ImmutableList.of("MAPREDUCE", "SPARK", "Oozie Launcher");

dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaTimeSeriesTaskTest.java

@@ -21,6 +21,7 @@
 
 import com.google.common.io.ByteSink;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.AbstractClouderaTimeSeriesTask.TimeSeriesAggregation;
+import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
 import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;
 import java.io.IOException;
 import java.time.LocalDateTime;
@@ -39,7 +40,11 @@ public void doRun_missedAggregationParameter_throwsException() {
             NullPointerException.class,
             () ->
                 new AbstractClouderaTimeSeriesTask(
-                    "some path", timeTravelDaysAgo(5), timeTravelDaysAgo(0), null) {
+                    "some path",
+                    timeTravelDaysAgo(5),
+                    timeTravelDaysAgo(0),
+                    null,
+                    TaskCategory.REQUIRED) {
                   @Override
                   protected void doRun(
                       TaskRunContext context,
@@ -63,7 +68,8 @@ public void doRun_emptyDateRange_throwsException() throws IOException {
                     "some path",
                     timeTravelDaysAgo(5),
                     timeTravelDaysAgo(8),
-                    TimeSeriesAggregation.DAILY) {
+                    TimeSeriesAggregation.DAILY,
+                    TaskCategory.REQUIRED) {
                   @Override
                   protected void doRun(
                       TaskRunContext context,

dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTaskTest.java

@@ -41,6 +41,7 @@
 import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.AbstractClouderaTimeSeriesTask.TimeSeriesAggregation;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaClusterDTO;
+import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
 import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;
 import java.io.IOException;
 import java.io.Writer;
@@ -69,7 +70,10 @@
 public class ClouderaClusterCPUChartTaskTest {
   private final ClouderaClusterCPUChartTask task =
       new ClouderaClusterCPUChartTask(
-          timeTravelDaysAgo(30), timeTravelDaysAgo(0), TimeSeriesAggregation.HOURLY);
+          timeTravelDaysAgo(30),
+          timeTravelDaysAgo(0),
+          TimeSeriesAggregation.HOURLY,
+          TaskCategory.REQUIRED);
   private ClouderaManagerHandle handle;
   private String servicesJson;
   private static WireMockServer server;

dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTaskTest.java

@@ -41,6 +41,7 @@
 import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.AbstractClouderaTimeSeriesTask.TimeSeriesAggregation;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaHostDTO;
+import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
 import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;
 import java.io.IOException;
 import java.io.Writer;
@@ -66,7 +67,10 @@
 public class ClouderaHostRAMChartTaskTest {
   private final ClouderaHostRAMChartTask task =
       new ClouderaHostRAMChartTask(
-          timeTravelDaysAgo(1), timeTravelDaysAgo(0), TimeSeriesAggregation.HOURLY);
+          timeTravelDaysAgo(1),
+          timeTravelDaysAgo(0),
+          TimeSeriesAggregation.HOURLY,
+          TaskCategory.REQUIRED);
   private ClouderaManagerHandle handle;
   private static WireMockServer server;
 

dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnectorTest.java

@@ -55,6 +55,7 @@ public void addTasksTo_checkFilesCategory() throws Exception {
                 ImmutableMap.of(
                     "cluster-cpu.jsonl", TaskCategory.REQUIRED,
                     "host-ram.jsonl", TaskCategory.REQUIRED,
+                    "service-resource-allocation.jsonl", TaskCategory.OPTIONAL,
                     "yarn-applications.jsonl", TaskCategory.OPTIONAL,
                     "yarn-application-types.jsonl", TaskCategory.OPTIONAL))
             .build();
@@ -87,6 +88,8 @@ public void addTasksTo_checkFileWasGeneratedByProperTask() throws Exception {
                     "host-components.jsonl", ClouderaHostComponentsTask.class,
                     "cluster-cpu.jsonl", ClouderaClusterCPUChartTask.class,
                     "host-ram.jsonl", ClouderaHostRAMChartTask.class,
+                    "service-resource-allocation.jsonl",
+                        ClouderaServiceResourceAllocationChartTask.class,
                     "yarn-applications.jsonl", ClouderaYarnApplicationsTask.class,
                     "yarn-application-types.jsonl", ClouderaYarnApplicationTypeTask.class))
             .build();
@@ -128,6 +131,7 @@ public void addTasksTo_checkFilesCategoryWithCustomDateRange() throws Exception
                 ImmutableMap.of(
                     "cluster-cpu.jsonl", TaskCategory.REQUIRED,
                     "host-ram.jsonl", TaskCategory.REQUIRED,
+                    "service-resource-allocation.jsonl", TaskCategory.OPTIONAL,
                     "yarn-applications.jsonl", TaskCategory.OPTIONAL,
                     "yarn-application-types.jsonl", TaskCategory.OPTIONAL))
             .build();
@@ -169,6 +173,8 @@ public void addTasksTo_checkFileWasGeneratedByProperTaskWithCustomDateRange() th
                 ImmutableMap.of(
                     "cluster-cpu.jsonl", ClouderaClusterCPUChartTask.class,
                     "host-ram.jsonl", ClouderaHostRAMChartTask.class,
+                    "service-resource-allocation.jsonl",
+                        ClouderaServiceResourceAllocationChartTask.class,
                     "yarn-applications.jsonl", ClouderaYarnApplicationsTask.class,
                     "yarn-application-types.jsonl", ClouderaYarnApplicationTypeTask.class))
             .build();