[b455552691] Collect dynamic resource allocation of Cloudera services (#1028)

dominik-burdzy · web-flow · commit 99809cbd8728 · 2026-01-05T10:26:58.000+01:00
Create a new task to iterate over Cloudera hosts and collect dynamic resource allocation of their services calling timeseries endpoint. It queries memory, cpu and disk throughput statistics in hourly manner and stores the results in `service-resource-allocation.jsonl` file.
diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaTimeSeriesTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaTimeSeriesTask.java
@@ -18,6 +18,7 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.base.Preconditions;
+import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
 import java.net.URI;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
@@ -33,12 +34,14 @@ abstract class AbstractClouderaTimeSeriesTask extends AbstractClouderaManagerTas
   private final ZonedDateTime startDate;
   private final ZonedDateTime endDate;
   private final TimeSeriesAggregation tsAggregation;
+  private final TaskCategory taskCategory;
 
   public AbstractClouderaTimeSeriesTask(
       @Nonnull String targetPath,
       @Nonnull ZonedDateTime startDate,
       @Nonnull ZonedDateTime endDate,
-      @Nonnull TimeSeriesAggregation tsAggregation) {
+      @Nonnull TimeSeriesAggregation tsAggregation,
+      @Nonnull TaskCategory taskCategory) {
     super(targetPath);
     Preconditions.checkNotNull(targetPath, "Target path must be not null.");
     Preconditions.checkState(!targetPath.isEmpty(), "Target file path must be not empty.");
@@ -50,6 +53,13 @@ public AbstractClouderaTimeSeriesTask(
     this.startDate = startDate;
     this.endDate = endDate;
     this.tsAggregation = tsAggregation;
+    this.taskCategory = taskCategory;
+  }
+
+  @Nonnull
+  @Override
+  public TaskCategory getCategory() {
+    return taskCategory;
   }
 
   protected JsonNode requestTimeSeriesChart(ClouderaManagerHandle handle, String query)
diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTask.java
@@ -20,6 +20,7 @@
 import com.google.common.io.ByteSink;
 import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaClusterDTO;
+import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
 import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;
 import java.io.Writer;
 import java.nio.charset.StandardCharsets;
@@ -40,13 +41,16 @@
  * language.
  */
 public class ClouderaClusterCPUChartTask extends AbstractClouderaTimeSeriesTask {
-  private static final Logger logger = LoggerFactory.getLogger(ClouderaCMFHostsTask.class);
+  private static final Logger logger = LoggerFactory.getLogger(ClouderaClusterCPUChartTask.class);
   private static final String TS_CPU_QUERY_TEMPLATE =
       "SELECT cpu_percent_across_hosts WHERE entityName = \"%s\" AND category = CLUSTER";
 
   public ClouderaClusterCPUChartTask(
-      ZonedDateTime startDate, ZonedDateTime endDate, TimeSeriesAggregation tsAggregation) {
-    super("cluster-cpu.jsonl", startDate, endDate, tsAggregation);
+      ZonedDateTime startDate,
+      ZonedDateTime endDate,
+      TimeSeriesAggregation tsAggregation,
+      TaskCategory taskCategory) {
+    super("cluster-cpu.jsonl", startDate, endDate, tsAggregation, taskCategory);
   }
 
   @Override
diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTask.java
@@ -20,6 +20,7 @@
 import com.google.common.io.ByteSink;
 import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaHostDTO;
+import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
 import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;
 import java.io.Writer;
 import java.nio.charset.StandardCharsets;
@@ -40,14 +41,17 @@
  */
 public class ClouderaHostRAMChartTask extends AbstractClouderaTimeSeriesTask {
 
-  private static final Logger logger = LoggerFactory.getLogger(ClouderaCMFHostsTask.class);
+  private static final Logger logger = LoggerFactory.getLogger(ClouderaHostRAMChartTask.class);
 
   private static final String TS_RAM_QUERY_TEMPLATE =
       "select swap_used, physical_memory_used, physical_memory_total, physical_memory_cached, physical_memory_buffers where entityName = \"%s\"";
 
   public ClouderaHostRAMChartTask(
-      ZonedDateTime startDate, ZonedDateTime endDate, TimeSeriesAggregation tsAggregation) {
-    super("host-ram.jsonl", startDate, endDate, tsAggregation);
+      ZonedDateTime startDate,
+      ZonedDateTime endDate,
+      TimeSeriesAggregation tsAggregation,
+      TaskCategory taskCategory) {
+    super("host-ram.jsonl", startDate, endDate, tsAggregation, taskCategory);
   }
 
   @Override
diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnector.java
@@ -18,7 +18,9 @@
 
 import static com.google.edwmigration.dumper.application.dumper.connector.Connector.validateDateRange;
 import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.AbstractClouderaTimeSeriesTask.TimeSeriesAggregation.DAILY;
+import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.AbstractClouderaTimeSeriesTask.TimeSeriesAggregation.HOURLY;
 import static com.google.edwmigration.dumper.application.dumper.task.TaskCategory.OPTIONAL;
+import static com.google.edwmigration.dumper.application.dumper.task.TaskCategory.REQUIRED;
 
 import com.google.auto.service.AutoService;
 import com.google.common.base.Preconditions;
@@ -110,8 +112,9 @@ public void addTasksTo(@Nonnull List<? super Task<?>> out, @Nonnull ConnectorArg
       endDate = arguments.getEndDate();
     }
 
-    out.add(new ClouderaClusterCPUChartTask(startDate, endDate, DAILY));
-    out.add(new ClouderaHostRAMChartTask(startDate, endDate, DAILY));
+    out.add(new ClouderaClusterCPUChartTask(startDate, endDate, DAILY, REQUIRED));
+    out.add(new ClouderaHostRAMChartTask(startDate, endDate, DAILY, REQUIRED));
+    out.add(new ClouderaServiceResourceAllocationChartTask(startDate, endDate, HOURLY, OPTIONAL));
     out.add(new ClouderaYarnApplicationsTask(startDate, endDate, OPTIONAL));
     out.add(new ClouderaYarnApplicationTypeTask(startDate, endDate, OPTIONAL));
   }
diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaServiceResourceAllocationChartTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaServiceResourceAllocationChartTask.java
@@ -0,0 +1,113 @@
+/*
+ * Copyright 2022-2025 Google LLC
+ * Copyright 2013-2021 CompilerWorks
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.io.ByteSink;
+import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
+import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaHostDTO;
+import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
+import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.time.ZonedDateTime;
+import java.util.ArrayList;
+import java.util.List;
+import javax.annotation.Nonnull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The task collects resource allocation metrics for <strong>all</strong> individual service roles
+ * present on a given host from the Cloudera Manager <a
+ * href="https://cldr2-aw-dl-gateway.cldr2-cd.svye-dcxb.a5.cloudera.site/static/apidocs/resource_TimeSeriesResource.html">TimeSeries
+ * API</a>.
+ *
+ * <p>This class provides a comprehensive, component-by-component breakdown of resource consumption,
+ * programmatically fetching the data that powers the stacked resource charts on a host's status
+ * page in the Cloudera Manager UI (found under the "Charts" tab for a specific host).
+ *
+ * <p>Queries are written in the <a
+ * href="https://docs.cloudera.com/documentation/enterprise/latest/topics/cm_dg_tsquery.html">tsquery</a>
+ * language and are filtered by a specific {@code hostId}.
+ *
+ * <p>Key metrics collected include:
+ *
+ * <ul>
+ *   <li><b>Memory:</b> {@code mem_rss}
+ *   <li><b>CPU:</b> {@code cpu_user_rate} and {@code cpu_system_rate}
+ *   <li><b>Disk Throughput:</b> {@code read_bytes_rate} and {@code write_bytes_rate}
+ * </ul>
+ */
+public class ClouderaServiceResourceAllocationChartTask extends AbstractClouderaTimeSeriesTask {
+
+  private static final Logger logger =
+      LoggerFactory.getLogger(ClouderaServiceResourceAllocationChartTask.class);
+
+  private static final String SERVICE_RESOURCE_ALLOCATION_QUERY_TEMPLATE =
+      "select mem_rss, cpu_user_rate, cpu_system_rate, read_bytes_rate, write_bytes_rate where category = \"ROLE\" AND hostId = \"%s\"";
+
+  public ClouderaServiceResourceAllocationChartTask(
+      ZonedDateTime startDate,
+      ZonedDateTime endDate,
+      TimeSeriesAggregation tsAggregation,
+      TaskCategory taskCategory) {
+    super("service-resource-allocation.jsonl", startDate, endDate, tsAggregation, taskCategory);
+  }
+
+  @Override
+  protected void doRun(
+      TaskRunContext context, @Nonnull ByteSink sink, @Nonnull ClouderaManagerHandle handle)
+      throws Exception {
+    List<ClouderaHostDTO> hosts = getHostsFromHandle(handle);
+
+    try (Writer writer = sink.asCharSink(StandardCharsets.UTF_8).openBufferedStream()) {
+      for (ClouderaHostDTO host : hosts) {
+        String resourceAllocationPerHostQuery =
+            String.format(SERVICE_RESOURCE_ALLOCATION_QUERY_TEMPLATE, host.getId());
+        logger.debug(
+            "Execute service resource allocation charts query: [{}] for the host: [{}].",
+            resourceAllocationPerHostQuery,
+            host.getName());
+
+        JsonNode chartInJson = requestTimeSeriesChart(handle, resourceAllocationPerHostQuery);
+
+        writer.write(chartInJson.toString());
+        writer.write('\n');
+      }
+    }
+  }
+
+  private List<ClouderaHostDTO> getHostsFromHandle(@Nonnull ClouderaManagerHandle handle) {
+    List<ClouderaHostDTO> hosts = handle.getHosts();
+    if (hosts == null) {
+      throw new MetadataDumperUsageException(
+          "Cloudera hosts must be initialized before service resource allocation charts dumping.");
+    }
+    List<ClouderaHostDTO> validHosts = new ArrayList<>();
+    for (ClouderaHostDTO host : hosts) {
+      if (host.getId() == null) {
+        logger.warn(
+            "Cloudera host id is null for host [{}]. Skip resource allocation metrics for services belonging to this host.",
+            host.getName());
+      } else {
+        validHosts.add(host);
+      }
+    }
+    return validHosts;
+  }
+}
diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationTypeTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationTypeTask.java
@@ -44,7 +44,8 @@
 import org.slf4j.LoggerFactory;
 
 public class ClouderaYarnApplicationTypeTask extends AbstractClouderaYarnApplicationTask {
-  private static final Logger logger = LoggerFactory.getLogger(ClouderaYarnApplicationsTask.class);
+  private static final Logger logger =
+      LoggerFactory.getLogger(ClouderaYarnApplicationTypeTask.class);
 
   private final ImmutableList<String> predefinedAppTypes =
       ImmutableList.of("MAPREDUCE", "SPARK", "Oozie Launcher");
diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaTimeSeriesTaskTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaTimeSeriesTaskTest.java
@@ -21,6 +21,7 @@
 
 import com.google.common.io.ByteSink;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.AbstractClouderaTimeSeriesTask.TimeSeriesAggregation;
+import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
 import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;
 import java.io.IOException;
 import java.time.LocalDateTime;
@@ -39,7 +40,11 @@ public void doRun_missedAggregationParameter_throwsException() {
             NullPointerException.class,
             () ->
                 new AbstractClouderaTimeSeriesTask(
-                    "some path", timeTravelDaysAgo(5), timeTravelDaysAgo(0), null) {
+                    "some path",
+                    timeTravelDaysAgo(5),
+                    timeTravelDaysAgo(0),
+                    null,
+                    TaskCategory.REQUIRED) {
                   @Override
                   protected void doRun(
                       TaskRunContext context,
@@ -63,7 +68,8 @@ public void doRun_emptyDateRange_throwsException() throws IOException {
                     "some path",
                     timeTravelDaysAgo(5),
                     timeTravelDaysAgo(8),
-                    TimeSeriesAggregation.DAILY) {
+                    TimeSeriesAggregation.DAILY,
+                    TaskCategory.REQUIRED) {
                   @Override
                   protected void doRun(
                       TaskRunContext context,
diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTaskTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTaskTest.java
@@ -41,6 +41,7 @@
 import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.AbstractClouderaTimeSeriesTask.TimeSeriesAggregation;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaClusterDTO;
+import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
 import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;
 import java.io.IOException;
 import java.io.Writer;
@@ -69,7 +70,10 @@
 public class ClouderaClusterCPUChartTaskTest {
   private final ClouderaClusterCPUChartTask task =
       new ClouderaClusterCPUChartTask(
-          timeTravelDaysAgo(30), timeTravelDaysAgo(0), TimeSeriesAggregation.HOURLY);
+          timeTravelDaysAgo(30),
+          timeTravelDaysAgo(0),
+          TimeSeriesAggregation.HOURLY,
+          TaskCategory.REQUIRED);
   private ClouderaManagerHandle handle;
   private String servicesJson;
   private static WireMockServer server;
diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTaskTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTaskTest.java
@@ -41,6 +41,7 @@
 import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.AbstractClouderaTimeSeriesTask.TimeSeriesAggregation;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaHostDTO;
+import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
 import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;
 import java.io.IOException;
 import java.io.Writer;
@@ -66,7 +67,10 @@
 public class ClouderaHostRAMChartTaskTest {
   private final ClouderaHostRAMChartTask task =
       new ClouderaHostRAMChartTask(
-          timeTravelDaysAgo(1), timeTravelDaysAgo(0), TimeSeriesAggregation.HOURLY);
+          timeTravelDaysAgo(1),
+          timeTravelDaysAgo(0),
+          TimeSeriesAggregation.HOURLY,
+          TaskCategory.REQUIRED);
   private ClouderaManagerHandle handle;
   private static WireMockServer server;
 
diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnectorTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnectorTest.java
@@ -55,6 +55,7 @@ public void addTasksTo_checkFilesCategory() throws Exception {
                 ImmutableMap.of(
                     "cluster-cpu.jsonl", TaskCategory.REQUIRED,
                     "host-ram.jsonl", TaskCategory.REQUIRED,
+                    "service-resource-allocation.jsonl", TaskCategory.OPTIONAL,
                     "yarn-applications.jsonl", TaskCategory.OPTIONAL,
                     "yarn-application-types.jsonl", TaskCategory.OPTIONAL))
             .build();
@@ -87,6 +88,8 @@ public void addTasksTo_checkFileWasGeneratedByProperTask() throws Exception {
                     "host-components.jsonl", ClouderaHostComponentsTask.class,
                     "cluster-cpu.jsonl", ClouderaClusterCPUChartTask.class,
                     "host-ram.jsonl", ClouderaHostRAMChartTask.class,
+                    "service-resource-allocation.jsonl",
+                        ClouderaServiceResourceAllocationChartTask.class,
                     "yarn-applications.jsonl", ClouderaYarnApplicationsTask.class,
                     "yarn-application-types.jsonl", ClouderaYarnApplicationTypeTask.class))
             .build();
@@ -128,6 +131,7 @@ public void addTasksTo_checkFilesCategoryWithCustomDateRange() throws Exception
                 ImmutableMap.of(
                     "cluster-cpu.jsonl", TaskCategory.REQUIRED,
                     "host-ram.jsonl", TaskCategory.REQUIRED,
+                    "service-resource-allocation.jsonl", TaskCategory.OPTIONAL,
                     "yarn-applications.jsonl", TaskCategory.OPTIONAL,
                     "yarn-application-types.jsonl", TaskCategory.OPTIONAL))
             .build();
@@ -169,6 +173,8 @@ public void addTasksTo_checkFileWasGeneratedByProperTaskWithCustomDateRange() th
                 ImmutableMap.of(
                     "cluster-cpu.jsonl", ClouderaClusterCPUChartTask.class,
                     "host-ram.jsonl", ClouderaHostRAMChartTask.class,
+                    "service-resource-allocation.jsonl",
+                        ClouderaServiceResourceAllocationChartTask.class,
                     "yarn-applications.jsonl", ClouderaYarnApplicationsTask.class,
                     "yarn-application-types.jsonl", ClouderaYarnApplicationTypeTask.class))
             .build();
diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaServiceResourceAllocationChartTaskTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaServiceResourceAllocationChartTaskTest.java
diff --git a/dumper/app/src/test/resources/cloudera/manager/host-services-resource-allocation-1.json b/dumper/app/src/test/resources/cloudera/manager/host-services-resource-allocation-1.json
diff --git a/dumper/app/src/test/resources/cloudera/manager/host-services-resource-allocation-2.json b/dumper/app/src/test/resources/cloudera/manager/host-services-resource-allocation-2.json
