diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaTimeSeriesTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaTimeSeriesTask.java index c8591e31a..830a486bb 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaTimeSeriesTask.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaTimeSeriesTask.java @@ -18,6 +18,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Preconditions; +import com.google.edwmigration.dumper.application.dumper.task.TaskCategory; import java.net.URI; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; @@ -33,12 +34,14 @@ abstract class AbstractClouderaTimeSeriesTask extends AbstractClouderaManagerTas private final ZonedDateTime startDate; private final ZonedDateTime endDate; private final TimeSeriesAggregation tsAggregation; + private final TaskCategory taskCategory; public AbstractClouderaTimeSeriesTask( @Nonnull String targetPath, @Nonnull ZonedDateTime startDate, @Nonnull ZonedDateTime endDate, - @Nonnull TimeSeriesAggregation tsAggregation) { + @Nonnull TimeSeriesAggregation tsAggregation, + @Nonnull TaskCategory taskCategory) { super(targetPath); Preconditions.checkNotNull(targetPath, "Target path must be not null."); Preconditions.checkState(!targetPath.isEmpty(), "Target file path must be not empty."); @@ -50,6 +53,13 @@ public AbstractClouderaTimeSeriesTask( this.startDate = startDate; this.endDate = endDate; this.tsAggregation = tsAggregation; + this.taskCategory = taskCategory; + } + + @Nonnull + @Override + public TaskCategory getCategory() { + return taskCategory; } protected JsonNode requestTimeSeriesChart(ClouderaManagerHandle handle, String query) diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTask.java index 7d5822766..2dddac4f3 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTask.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTask.java @@ -20,6 +20,7 @@ import com.google.common.io.ByteSink; import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException; import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaClusterDTO; +import com.google.edwmigration.dumper.application.dumper.task.TaskCategory; import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext; import java.io.Writer; import java.nio.charset.StandardCharsets; @@ -40,13 +41,16 @@ * language. */ public class ClouderaClusterCPUChartTask extends AbstractClouderaTimeSeriesTask { - private static final Logger logger = LoggerFactory.getLogger(ClouderaCMFHostsTask.class); + private static final Logger logger = LoggerFactory.getLogger(ClouderaClusterCPUChartTask.class); private static final String TS_CPU_QUERY_TEMPLATE = "SELECT cpu_percent_across_hosts WHERE entityName = \"%s\" AND category = CLUSTER"; public ClouderaClusterCPUChartTask( - ZonedDateTime startDate, ZonedDateTime endDate, TimeSeriesAggregation tsAggregation) { - super("cluster-cpu.jsonl", startDate, endDate, tsAggregation); + ZonedDateTime startDate, + ZonedDateTime endDate, + TimeSeriesAggregation tsAggregation, + TaskCategory taskCategory) { + super("cluster-cpu.jsonl", startDate, endDate, tsAggregation, taskCategory); } @Override diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTask.java index 702d96b56..e8d248118 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTask.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTask.java @@ -20,6 +20,7 @@ import com.google.common.io.ByteSink; import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException; import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaHostDTO; +import com.google.edwmigration.dumper.application.dumper.task.TaskCategory; import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext; import java.io.Writer; import java.nio.charset.StandardCharsets; @@ -40,14 +41,17 @@ */ public class ClouderaHostRAMChartTask extends AbstractClouderaTimeSeriesTask { - private static final Logger logger = LoggerFactory.getLogger(ClouderaCMFHostsTask.class); + private static final Logger logger = LoggerFactory.getLogger(ClouderaHostRAMChartTask.class); private static final String TS_RAM_QUERY_TEMPLATE = "select swap_used, physical_memory_used, physical_memory_total, physical_memory_cached, physical_memory_buffers where entityName = \"%s\""; public ClouderaHostRAMChartTask( - ZonedDateTime startDate, ZonedDateTime endDate, TimeSeriesAggregation tsAggregation) { - super("host-ram.jsonl", startDate, endDate, tsAggregation); + ZonedDateTime startDate, + ZonedDateTime endDate, + TimeSeriesAggregation tsAggregation, + TaskCategory taskCategory) { + super("host-ram.jsonl", startDate, endDate, tsAggregation, taskCategory); } @Override diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnector.java index 2c6535b1f..178c08e45 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnector.java @@ -18,7 +18,9 @@ import static com.google.edwmigration.dumper.application.dumper.connector.Connector.validateDateRange; import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.AbstractClouderaTimeSeriesTask.TimeSeriesAggregation.DAILY; +import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.AbstractClouderaTimeSeriesTask.TimeSeriesAggregation.HOURLY; import static com.google.edwmigration.dumper.application.dumper.task.TaskCategory.OPTIONAL; +import static com.google.edwmigration.dumper.application.dumper.task.TaskCategory.REQUIRED; import com.google.auto.service.AutoService; import com.google.common.base.Preconditions; @@ -110,8 +112,9 @@ public void addTasksTo(@Nonnull List> out, @Nonnull ConnectorArg endDate = arguments.getEndDate(); } - out.add(new ClouderaClusterCPUChartTask(startDate, endDate, DAILY)); - out.add(new ClouderaHostRAMChartTask(startDate, endDate, DAILY)); + out.add(new ClouderaClusterCPUChartTask(startDate, endDate, DAILY, REQUIRED)); + out.add(new ClouderaHostRAMChartTask(startDate, endDate, DAILY, REQUIRED)); + out.add(new ClouderaServiceResourceAllocationChartTask(startDate, endDate, HOURLY, OPTIONAL)); out.add(new ClouderaYarnApplicationsTask(startDate, endDate, OPTIONAL)); out.add(new ClouderaYarnApplicationTypeTask(startDate, endDate, OPTIONAL)); } diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaServiceResourceAllocationChartTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaServiceResourceAllocationChartTask.java new file mode 100644 index 000000000..b4e273e2a --- /dev/null +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaServiceResourceAllocationChartTask.java @@ -0,0 +1,113 @@ +/* + * Copyright 2022-2025 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.io.ByteSink; +import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException; +import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaHostDTO; +import com.google.edwmigration.dumper.application.dumper.task.TaskCategory; +import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.List; +import javax.annotation.Nonnull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The task collects resource allocation metrics for all individual service roles + * present on a given host from the Cloudera Manager TimeSeries + * API. + * + *

This class provides a comprehensive, component-by-component breakdown of resource consumption, + * programmatically fetching the data that powers the stacked resource charts on a host's status + * page in the Cloudera Manager UI (found under the "Charts" tab for a specific host). + * + *

Queries are written in the tsquery + * language and are filtered by a specific {@code hostId}. + * + *

Key metrics collected include: + * + *

+ */ +public class ClouderaServiceResourceAllocationChartTask extends AbstractClouderaTimeSeriesTask { + + private static final Logger logger = + LoggerFactory.getLogger(ClouderaServiceResourceAllocationChartTask.class); + + private static final String SERVICE_RESOURCE_ALLOCATION_QUERY_TEMPLATE = + "select mem_rss, cpu_user_rate, cpu_system_rate, read_bytes_rate, write_bytes_rate where category = \"ROLE\" AND hostId = \"%s\""; + + public ClouderaServiceResourceAllocationChartTask( + ZonedDateTime startDate, + ZonedDateTime endDate, + TimeSeriesAggregation tsAggregation, + TaskCategory taskCategory) { + super("service-resource-allocation.jsonl", startDate, endDate, tsAggregation, taskCategory); + } + + @Override + protected void doRun( + TaskRunContext context, @Nonnull ByteSink sink, @Nonnull ClouderaManagerHandle handle) + throws Exception { + List hosts = getHostsFromHandle(handle); + + try (Writer writer = sink.asCharSink(StandardCharsets.UTF_8).openBufferedStream()) { + for (ClouderaHostDTO host : hosts) { + String resourceAllocationPerHostQuery = + String.format(SERVICE_RESOURCE_ALLOCATION_QUERY_TEMPLATE, host.getId()); + logger.debug( + "Execute service resource allocation charts query: [{}] for the host: [{}].", + resourceAllocationPerHostQuery, + host.getName()); + + JsonNode chartInJson = requestTimeSeriesChart(handle, resourceAllocationPerHostQuery); + + writer.write(chartInJson.toString()); + writer.write('\n'); + } + } + } + + private List getHostsFromHandle(@Nonnull ClouderaManagerHandle handle) { + List hosts = handle.getHosts(); + if (hosts == null) { + throw new MetadataDumperUsageException( + "Cloudera hosts must be initialized before service resource allocation charts dumping."); + } + List validHosts = new ArrayList<>(); + for (ClouderaHostDTO host : hosts) { + if (host.getId() == null) { + logger.warn( + "Cloudera host id is null for host [{}]. Skip resource allocation metrics for services belonging to this host.", + host.getName()); + } else { + validHosts.add(host); + } + } + return validHosts; + } +} diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationTypeTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationTypeTask.java index 80b0736bd..cd03950cc 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationTypeTask.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationTypeTask.java @@ -44,7 +44,8 @@ import org.slf4j.LoggerFactory; public class ClouderaYarnApplicationTypeTask extends AbstractClouderaYarnApplicationTask { - private static final Logger logger = LoggerFactory.getLogger(ClouderaYarnApplicationsTask.class); + private static final Logger logger = + LoggerFactory.getLogger(ClouderaYarnApplicationTypeTask.class); private final ImmutableList predefinedAppTypes = ImmutableList.of("MAPREDUCE", "SPARK", "Oozie Launcher"); diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaTimeSeriesTaskTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaTimeSeriesTaskTest.java index b9629f1b4..c721a5f16 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaTimeSeriesTaskTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaTimeSeriesTaskTest.java @@ -21,6 +21,7 @@ import com.google.common.io.ByteSink; import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.AbstractClouderaTimeSeriesTask.TimeSeriesAggregation; +import com.google.edwmigration.dumper.application.dumper.task.TaskCategory; import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext; import java.io.IOException; import java.time.LocalDateTime; @@ -39,7 +40,11 @@ public void doRun_missedAggregationParameter_throwsException() { NullPointerException.class, () -> new AbstractClouderaTimeSeriesTask( - "some path", timeTravelDaysAgo(5), timeTravelDaysAgo(0), null) { + "some path", + timeTravelDaysAgo(5), + timeTravelDaysAgo(0), + null, + TaskCategory.REQUIRED) { @Override protected void doRun( TaskRunContext context, @@ -63,7 +68,8 @@ public void doRun_emptyDateRange_throwsException() throws IOException { "some path", timeTravelDaysAgo(5), timeTravelDaysAgo(8), - TimeSeriesAggregation.DAILY) { + TimeSeriesAggregation.DAILY, + TaskCategory.REQUIRED) { @Override protected void doRun( TaskRunContext context, diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTaskTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTaskTest.java index d66895147..6e8a0da30 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTaskTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTaskTest.java @@ -41,6 +41,7 @@ import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException; import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.AbstractClouderaTimeSeriesTask.TimeSeriesAggregation; import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaClusterDTO; +import com.google.edwmigration.dumper.application.dumper.task.TaskCategory; import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext; import java.io.IOException; import java.io.Writer; @@ -69,7 +70,10 @@ public class ClouderaClusterCPUChartTaskTest { private final ClouderaClusterCPUChartTask task = new ClouderaClusterCPUChartTask( - timeTravelDaysAgo(30), timeTravelDaysAgo(0), TimeSeriesAggregation.HOURLY); + timeTravelDaysAgo(30), + timeTravelDaysAgo(0), + TimeSeriesAggregation.HOURLY, + TaskCategory.REQUIRED); private ClouderaManagerHandle handle; private String servicesJson; private static WireMockServer server; diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTaskTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTaskTest.java index 3d0f87ffb..9e588372c 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTaskTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTaskTest.java @@ -41,6 +41,7 @@ import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException; import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.AbstractClouderaTimeSeriesTask.TimeSeriesAggregation; import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaHostDTO; +import com.google.edwmigration.dumper.application.dumper.task.TaskCategory; import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext; import java.io.IOException; import java.io.Writer; @@ -66,7 +67,10 @@ public class ClouderaHostRAMChartTaskTest { private final ClouderaHostRAMChartTask task = new ClouderaHostRAMChartTask( - timeTravelDaysAgo(1), timeTravelDaysAgo(0), TimeSeriesAggregation.HOURLY); + timeTravelDaysAgo(1), + timeTravelDaysAgo(0), + TimeSeriesAggregation.HOURLY, + TaskCategory.REQUIRED); private ClouderaManagerHandle handle; private static WireMockServer server; diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnectorTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnectorTest.java index ce9efee3b..c862339fc 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnectorTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnectorTest.java @@ -55,6 +55,7 @@ public void addTasksTo_checkFilesCategory() throws Exception { ImmutableMap.of( "cluster-cpu.jsonl", TaskCategory.REQUIRED, "host-ram.jsonl", TaskCategory.REQUIRED, + "service-resource-allocation.jsonl", TaskCategory.OPTIONAL, "yarn-applications.jsonl", TaskCategory.OPTIONAL, "yarn-application-types.jsonl", TaskCategory.OPTIONAL)) .build(); @@ -87,6 +88,8 @@ public void addTasksTo_checkFileWasGeneratedByProperTask() throws Exception { "host-components.jsonl", ClouderaHostComponentsTask.class, "cluster-cpu.jsonl", ClouderaClusterCPUChartTask.class, "host-ram.jsonl", ClouderaHostRAMChartTask.class, + "service-resource-allocation.jsonl", + ClouderaServiceResourceAllocationChartTask.class, "yarn-applications.jsonl", ClouderaYarnApplicationsTask.class, "yarn-application-types.jsonl", ClouderaYarnApplicationTypeTask.class)) .build(); @@ -128,6 +131,7 @@ public void addTasksTo_checkFilesCategoryWithCustomDateRange() throws Exception ImmutableMap.of( "cluster-cpu.jsonl", TaskCategory.REQUIRED, "host-ram.jsonl", TaskCategory.REQUIRED, + "service-resource-allocation.jsonl", TaskCategory.OPTIONAL, "yarn-applications.jsonl", TaskCategory.OPTIONAL, "yarn-application-types.jsonl", TaskCategory.OPTIONAL)) .build(); @@ -169,6 +173,8 @@ public void addTasksTo_checkFileWasGeneratedByProperTaskWithCustomDateRange() th ImmutableMap.of( "cluster-cpu.jsonl", ClouderaClusterCPUChartTask.class, "host-ram.jsonl", ClouderaHostRAMChartTask.class, + "service-resource-allocation.jsonl", + ClouderaServiceResourceAllocationChartTask.class, "yarn-applications.jsonl", ClouderaYarnApplicationsTask.class, "yarn-application-types.jsonl", ClouderaYarnApplicationTypeTask.class)) .build(); diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaServiceResourceAllocationChartTaskTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaServiceResourceAllocationChartTaskTest.java new file mode 100644 index 000000000..3af7c72e2 --- /dev/null +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaServiceResourceAllocationChartTaskTest.java @@ -0,0 +1,229 @@ +/* + * Copyright 2022-2025 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager; + +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.okJson; +import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyChar; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import com.google.common.collect.ImmutableSet; +import com.google.common.io.ByteSink; +import com.google.common.io.CharSink; +import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException; +import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.AbstractClouderaTimeSeriesTask.TimeSeriesAggregation; +import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaHostDTO; +import com.google.edwmigration.dumper.application.dumper.task.TaskCategory; +import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext; +import java.io.IOException; +import java.io.Writer; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import org.apache.http.HttpStatus; +import org.apache.http.impl.client.HttpClients; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class ClouderaServiceResourceAllocationChartTaskTest { + + private static WireMockServer server; + private final ClouderaServiceResourceAllocationChartTask task = + new ClouderaServiceResourceAllocationChartTask( + timeTravelDaysAgo(30), + timeTravelDaysAgo(0), + TimeSeriesAggregation.HOURLY, + TaskCategory.OPTIONAL); + private ClouderaManagerHandle handle; + + @Mock private TaskRunContext context; + @Mock private ByteSink sink; + @Mock private Writer writer; + @Mock private CharSink charSink; + + @BeforeClass + public static void beforeClass() throws Exception { + server = new WireMockServer(WireMockConfiguration.wireMockConfig().dynamicPort()); + server.start(); + } + + @AfterClass + public static void afterClass() throws Exception { + server.stop(); + } + + @Before + public void setUp() throws Exception { + server.resetAll(); + URI uri = URI.create(server.baseUrl() + "/api/vTest"); + handle = new ClouderaManagerHandle(uri, HttpClients.createDefault()); + + when(sink.asCharSink(eq(StandardCharsets.UTF_8))).thenReturn(charSink); + when(charSink.openBufferedStream()).thenReturn(writer); + } + + @Test + public void doRun_initiatedHostWithoutId_skipWrites() throws Exception { + initHosts(ClouderaHostDTO.create(null, "host1")); + + task.doRun(context, sink, handle); + + verifyNoWrites(); + } + + @Test + public void doRun_clouderaReturnsValidJson_writeJsonLines() throws Exception { + String firstHostId = "becfcf668a1861ab926151f1b11a726d"; + String secondHostId = "abcdef668a1861ab926151f1b11a726d"; + initHosts( + ClouderaHostDTO.create(firstHostId, "host1"), + ClouderaHostDTO.create(secondHostId, "host2")); + String firstHostServicesResourceAllocationJson = + readFileAsString("/cloudera/manager/host-services-resource-allocation-1.json"); + String secondHostServicesResourceAllocationJson = + readFileAsString("/cloudera/manager/host-services-resource-allocation-2.json"); + stubHttpRequestToFetchHostServicesResourceAllocationChart( + firstHostId, firstHostServicesResourceAllocationJson); + stubHttpRequestToFetchHostServicesResourceAllocationChart( + secondHostId, secondHostServicesResourceAllocationJson); + + task.doRun(context, sink, handle); + + Set fileLines = new HashSet<>(MockUtils.getWrittenJsonLines(writer, 2)); + assertEquals( + ImmutableSet.of( + tojsonl(firstHostServicesResourceAllocationJson), + tojsonl(secondHostServicesResourceAllocationJson)), + fileLines); + } + + @Test + public void doRun_hostsWereNotInitialized_throwsException() throws Exception { + assertNull(handle.getHosts()); + + MetadataDumperUsageException exception = + assertThrows(MetadataDumperUsageException.class, () -> task.doRun(context, sink, handle)); + + assertEquals( + "Cloudera hosts must be initialized before service resource allocation charts dumping.", + exception.getMessage()); + verifyNoWrites(); + } + + @Test + public void doRun_clouderaReturns4xx_throwsException() throws Exception { + initHosts(ClouderaHostDTO.create("id1", "host1")); + String hostServicesResourceAllocationJson = + readFileAsString("/cloudera/manager/host-services-resource-allocation-1.json"); + stubHttpRequestToFetchHostServicesResourceAllocationChart( + "id1", hostServicesResourceAllocationJson, HttpStatus.SC_BAD_REQUEST); + + assertThrows(RuntimeException.class, () -> task.doRun(context, sink, handle)); + + verifyNoWrites(); + } + + @Test + public void doRun_clouderaReturns5xx_throwsException() throws Exception { + initHosts(ClouderaHostDTO.create("id1", "host1")); + String hostServicesResourceAllocationJson = + readFileAsString("/cloudera/manager/host-services-resource-allocation-1.json"); + stubHttpRequestToFetchHostServicesResourceAllocationChart( + "id1", hostServicesResourceAllocationJson, HttpStatus.SC_INTERNAL_SERVER_ERROR); + + assertThrows(RuntimeException.class, () -> task.doRun(context, sink, handle)); + + verifyNoWrites(); + } + + @Test + public void doRun_clouderaReturnsInvalidJson_throwsException() throws Exception { + initHosts(ClouderaHostDTO.create("id1", "host1")); + String invalidJson = "{\"key\": []]"; + stubHttpRequestToFetchHostServicesResourceAllocationChart("id1", invalidJson); + + assertThrows(JsonParseException.class, () -> task.doRun(context, sink, handle)); + + verifyNoWrites(); + } + + private void initHosts(ClouderaHostDTO... hosts) { + handle.initHostsIfNull(Arrays.asList(hosts)); + } + + private void stubHttpRequestToFetchHostServicesResourceAllocationChart( + String hostId, String mockedContent) throws IOException { + stubHttpRequestToFetchHostServicesResourceAllocationChart( + hostId, mockedContent, HttpStatus.SC_OK); + } + + private void stubHttpRequestToFetchHostServicesResourceAllocationChart( + String hostId, String mockedContent, int statusCode) throws IOException { + server.stubFor( + get(urlMatching(String.format("/api/vTest/timeseries.*%s.*", hostId))) + .willReturn(okJson(mockedContent).withStatus(statusCode))); + } + + private void verifyNoWrites() throws IOException { + verify(writer, never()).write(anyChar()); + verify(writer, never()).write(anyString()); + verify(writer, never()).write(anyString(), anyInt(), anyInt()); + verify(writer, never()).write(any(char[].class)); + verify(writer, never()).write(any(char[].class), anyInt(), anyInt()); + } + + private String readFileAsString(String fileName) throws IOException, URISyntaxException { + return new String(Files.readAllBytes(Paths.get(this.getClass().getResource(fileName).toURI()))); + } + + private String tojsonl(String json) throws Exception { + return new ObjectMapper().readTree(json).toString(); + } + + private ZonedDateTime timeTravelDaysAgo(int days) { + ZonedDateTime today = ZonedDateTime.of(LocalDateTime.now(), ZoneId.of("UTC")); + return today.minusDays(days); + } +} diff --git a/dumper/app/src/test/resources/cloudera/manager/host-services-resource-allocation-1.json b/dumper/app/src/test/resources/cloudera/manager/host-services-resource-allocation-1.json new file mode 100644 index 000000000..96f85d16e --- /dev/null +++ b/dumper/app/src/test/resources/cloudera/manager/host-services-resource-allocation-1.json @@ -0,0 +1,1192 @@ +{ + "items": [ + { + "timeSeries": [ + { + "metadata": { + "metricName": "mem_rss", + "entityName": "Impala Catalog Server (cldr4-data-hub-master0.cldr4-cd.svye-dcxb.a5.cloudera.site)", + "startTime": "2025-10-30T08:21:00.000Z", + "endTime": "2025-10-30T09:21:00.000Z", + "attributes": { + "serviceType": "IMPALA", + "clusterDisplayName": "cldr4-data-hub", + "hostId": "becfcf668a1861ab926151f1b11a726d", + "active": "true", + "serviceDisplayName": "Impala", + "serviceName": "impala", + "roleType": "CATALOGSERVER", + "version": "CDH 7.3.1", + "hostname": "cldr4-data-hub-master0.cldr4-cd.svye-dcxb.a5.cloudera.site", + "entityName": "impala-CATALOGSERVER-9391f56d8aaa03e550a59fa59439aa09", + "clusterName": "cldr4-data-hub", + "roleName": "impala-CATALOGSERVER-9391f56d8aaa03e550a59fa59439aa09", + "roleConfigGroup": "Impala Catalog Server Default Group", + "category": "ROLE", + "rackId": "/eu-west-3b" + }, + "unitNumerators": [ + "bytes" + ], + "unitDenominators": [], + "expression": "SELECT mem_rss WHERE entityName = \"impala-CATALOGSERVER-9391f56d8aaa03e550a59fa59439aa09\" AND category = ROLE", + "metricCollectionFrequencyMs": 60000, + "rollupUsed": "TEN_MINUTELY" + }, + "data": [ + { + "timestamp": "2025-10-30T08:30:00.000Z", + "value": 583757824, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:29:33.000Z", + "sampleValue": 583757824, + "count": 10, + "min": 583757824, + "minTime": "2025-10-30T08:20:32.000Z", + "max": 583757824, + "maxTime": "2025-10-30T08:20:32.000Z", + "mean": 583757824, + "stdDev": 0.0 + } + }, + { + "timestamp": "2025-10-30T08:40:00.000Z", + "value": 583757824, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:39:33.000Z", + "sampleValue": 583757824, + "count": 10, + "min": 583757824, + "minTime": "2025-10-30T08:30:33.000Z", + "max": 583757824, + "maxTime": "2025-10-30T08:30:33.000Z", + "mean": 583757824, + "stdDev": 0.0 + } + }, + { + "timestamp": "2025-10-30T08:50:00.000Z", + "value": 583757824, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:49:32.000Z", + "sampleValue": 583757824, + "count": 10, + "min": 583757824, + "minTime": "2025-10-30T08:40:33.000Z", + "max": 583757824, + "maxTime": "2025-10-30T08:40:33.000Z", + "mean": 583757824, + "stdDev": 0.0 + } + }, + { + "timestamp": "2025-10-30T09:00:00.000Z", + "value": 583757824, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:59:32.000Z", + "sampleValue": 583757824, + "count": 10, + "min": 583757824, + "minTime": "2025-10-30T08:50:32.000Z", + "max": 583757824, + "maxTime": "2025-10-30T08:50:32.000Z", + "mean": 583757824, + "stdDev": 0.0 + } + }, + { + "timestamp": "2025-10-30T09:10:00.000Z", + "value": 583757824, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T09:09:32.000Z", + "sampleValue": 583757824, + "count": 10, + "min": 583757824, + "minTime": "2025-10-30T09:00:32.000Z", + "max": 583757824, + "maxTime": "2025-10-30T09:00:32.000Z", + "mean": 583757824, + "stdDev": 0.0 + } + }, + { + "timestamp": "2025-10-30T09:20:00.000Z", + "value": 583757824, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T09:19:31.000Z", + "sampleValue": 583757824, + "count": 10, + "min": 583757824, + "minTime": "2025-10-30T09:10:32.000Z", + "max": 583757824, + "maxTime": "2025-10-30T09:10:32.000Z", + "mean": 583757824, + "stdDev": 0.0 + } + } + ] + }, + { + "metadata": { + "metricName": "mem_rss", + "entityName": "Impala StateStore (cldr4-data-hub-master0.cldr4-cd.svye-dcxb.a5.cloudera.site)", + "startTime": "2025-10-30T08:21:00.000Z", + "endTime": "2025-10-30T09:21:00.000Z", + "attributes": { + "serviceType": "IMPALA", + "clusterDisplayName": "cldr4-data-hub", + "hostId": "becfcf668a1861ab926151f1b11a726d", + "active": "true", + "serviceDisplayName": "Impala", + "serviceName": "impala", + "roleType": "STATESTORE", + "version": "CDH 7.3.1", + "hostname": "cldr4-data-hub-master0.cldr4-cd.svye-dcxb.a5.cloudera.site", + "entityName": "impala-STATESTORE-9391f56d8aaa03e550a59fa59439aa09", + "clusterName": "cldr4-data-hub", + "roleName": "impala-STATESTORE-9391f56d8aaa03e550a59fa59439aa09", + "roleConfigGroup": "Impala StateStore Default Group", + "category": "ROLE", + "rackId": "/eu-west-3b" + }, + "unitNumerators": [ + "bytes" + ], + "unitDenominators": [], + "expression": "SELECT mem_rss WHERE entityName = \"impala-STATESTORE-9391f56d8aaa03e550a59fa59439aa09\" AND category = ROLE", + "metricCollectionFrequencyMs": 60000, + "rollupUsed": "TEN_MINUTELY" + }, + "data": [ + { + "timestamp": "2025-10-30T08:30:00.000Z", + "value": 3.633152E+7, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:29:53.000Z", + "sampleValue": 3.633152E+7, + "count": 10, + "min": 3.633152E+7, + "minTime": "2025-10-30T08:20:53.000Z", + "max": 3.633152E+7, + "maxTime": "2025-10-30T08:20:53.000Z", + "mean": 3.633152E+7, + "stdDev": 0.0 + } + }, + { + "timestamp": "2025-10-30T08:40:00.000Z", + "value": 3.633152E+7, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:39:53.000Z", + "sampleValue": 3.633152E+7, + "count": 10, + "min": 3.633152E+7, + "minTime": "2025-10-30T08:30:53.000Z", + "max": 3.633152E+7, + "maxTime": "2025-10-30T08:30:53.000Z", + "mean": 3.633152E+7, + "stdDev": 0.0 + } + }, + { + "timestamp": "2025-10-30T08:50:00.000Z", + "value": 3.633152E+7, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:49:53.000Z", + "sampleValue": 3.633152E+7, + "count": 10, + "min": 3.633152E+7, + "minTime": "2025-10-30T08:40:53.000Z", + "max": 3.633152E+7, + "maxTime": "2025-10-30T08:40:53.000Z", + "mean": 3.633152E+7, + "stdDev": 0.0 + } + }, + { + "timestamp": "2025-10-30T09:00:00.000Z", + "value": 3.633152E+7, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:59:53.000Z", + "sampleValue": 3.633152E+7, + "count": 10, + "min": 3.633152E+7, + "minTime": "2025-10-30T08:50:53.000Z", + "max": 3.633152E+7, + "maxTime": "2025-10-30T08:50:53.000Z", + "mean": 3.633152E+7, + "stdDev": 0.0 + } + }, + { + "timestamp": "2025-10-30T09:10:00.000Z", + "value": 3.633152E+7, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T09:09:53.000Z", + "sampleValue": 3.633152E+7, + "count": 10, + "min": 3.633152E+7, + "minTime": "2025-10-30T09:00:53.000Z", + "max": 3.633152E+7, + "maxTime": "2025-10-30T09:00:53.000Z", + "mean": 3.633152E+7, + "stdDev": 0.0 + } + }, + { + "timestamp": "2025-10-30T09:20:00.000Z", + "value": 3.633152E+7, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T09:19:53.000Z", + "sampleValue": 3.633152E+7, + "count": 10, + "min": 3.633152E+7, + "minTime": "2025-10-30T09:10:53.000Z", + "max": 3.633152E+7, + "maxTime": "2025-10-30T09:10:53.000Z", + "mean": 3.633152E+7, + "stdDev": 0.0 + } + } + ] + }, + { + "metadata": { + "metricName": "mem_rss", + "entityName": "LIVY_SERVER_FOR_SPARK3 (cldr4-data-hub-master0.cldr4-cd.svye-dcxb.a5.cloudera.site)", + "startTime": "2025-10-30T08:21:00.000Z", + "endTime": "2025-10-30T09:21:00.000Z", + "attributes": { + "serviceType": "LIVY_FOR_SPARK3", + "clusterDisplayName": "cldr4-data-hub", + "hostId": "becfcf668a1861ab926151f1b11a726d", + "active": "true", + "serviceDisplayName": "livy_for_spark3", + "serviceName": "livy_for_spark3", + "roleType": "LIVY_SERVER_FOR_SPARK3", + "version": "CDH 7.3.1", + "hostname": "cldr4-data-hub-master0.cldr4-cd.svye-dcxb.a5.cloudera.site", + "entityName": "livy446c4191-LIVY_SERVER_FOR_SPARK3-9391f56d8aaa03e550a59fa59439", + "clusterName": "cldr4-data-hub", + "roleName": "livy446c4191-LIVY_SERVER_FOR_SPARK3-9391f56d8aaa03e550a59fa59439", + "roleConfigGroup": "Livy Server for Spark 3 Default Group", + "category": "ROLE", + "rackId": "/eu-west-3b" + }, + "unitNumerators": [ + "bytes" + ], + "unitDenominators": [], + "expression": "SELECT mem_rss WHERE entityName = \"livy446c4191-LIVY_SERVER_FOR_SPARK3-9391f56d8aaa03e550a59fa59439\" AND category = ROLE", + "metricCollectionFrequencyMs": 60000, + "rollupUsed": "TEN_MINUTELY" + }, + "data": [ + { + "timestamp": "2025-10-30T08:30:00.000Z", + "value": 2.338816E+8, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:29:22.000Z", + "sampleValue": 2.338816E+8, + "count": 10, + "min": 2.338816E+8, + "minTime": "2025-10-30T08:20:22.000Z", + "max": 2.338816E+8, + "maxTime": "2025-10-30T08:20:22.000Z", + "mean": 2.338816E+8, + "stdDev": 0.0 + } + }, + { + "timestamp": "2025-10-30T08:40:00.000Z", + "value": 2.338816E+8, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:39:23.000Z", + "sampleValue": 2.338816E+8, + "count": 10, + "min": 2.338816E+8, + "minTime": "2025-10-30T08:30:22.000Z", + "max": 2.338816E+8, + "maxTime": "2025-10-30T08:30:22.000Z", + "mean": 2.338816E+8, + "stdDev": 0.0 + } + }, + { + "timestamp": "2025-10-30T08:50:00.000Z", + "value": 2.338816E+8, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:49:22.000Z", + "sampleValue": 2.338816E+8, + "count": 10, + "min": 2.338816E+8, + "minTime": "2025-10-30T08:40:22.000Z", + "max": 2.338816E+8, + "maxTime": "2025-10-30T08:40:22.000Z", + "mean": 2.338816E+8, + "stdDev": 0.0 + } + }, + { + "timestamp": "2025-10-30T09:00:00.000Z", + "value": 2.338816E+8, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:59:22.000Z", + "sampleValue": 2.338816E+8, + "count": 10, + "min": 2.338816E+8, + "minTime": "2025-10-30T08:50:22.000Z", + "max": 2.338816E+8, + "maxTime": "2025-10-30T08:50:22.000Z", + "mean": 2.338816E+8, + "stdDev": 0.0 + } + }, + { + "timestamp": "2025-10-30T09:10:00.000Z", + "value": 2.338816E+8, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T09:09:22.000Z", + "sampleValue": 2.338816E+8, + "count": 10, + "min": 2.338816E+8, + "minTime": "2025-10-30T09:00:22.000Z", + "max": 2.338816E+8, + "maxTime": "2025-10-30T09:00:22.000Z", + "mean": 2.338816E+8, + "stdDev": 0.0 + } + }, + { + "timestamp": "2025-10-30T09:20:00.000Z", + "value": 2.338816E+8, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T09:19:21.000Z", + "sampleValue": 2.338816E+8, + "count": 10, + "min": 2.338816E+8, + "minTime": "2025-10-30T09:10:22.000Z", + "max": 2.338816E+8, + "maxTime": "2025-10-30T09:10:22.000Z", + "mean": 2.338816E+8, + "stdDev": 0.0 + } + } + ] + }, + { + "metadata": { + "metricName": "cpu_user_rate", + "entityName": "Impala Catalog Server (cldr4-data-hub-master0.cldr4-cd.svye-dcxb.a5.cloudera.site)", + "startTime": "2025-10-30T08:21:00.000Z", + "endTime": "2025-10-30T09:21:00.000Z", + "attributes": { + "serviceType": "IMPALA", + "clusterDisplayName": "cldr4-data-hub", + "hostId": "becfcf668a1861ab926151f1b11a726d", + "active": "true", + "serviceDisplayName": "Impala", + "serviceName": "impala", + "roleType": "CATALOGSERVER", + "version": "CDH 7.3.1", + "hostname": "cldr4-data-hub-master0.cldr4-cd.svye-dcxb.a5.cloudera.site", + "entityName": "impala-CATALOGSERVER-9391f56d8aaa03e550a59fa59439aa09", + "clusterName": "cldr4-data-hub", + "roleName": "impala-CATALOGSERVER-9391f56d8aaa03e550a59fa59439aa09", + "roleConfigGroup": "Impala Catalog Server Default Group", + "category": "ROLE", + "rackId": "/eu-west-3b" + }, + "unitNumerators": [ + "seconds" + ], + "unitDenominators": [ + "seconds" + ], + "expression": "SELECT cpu_user_rate WHERE entityName = \"impala-CATALOGSERVER-9391f56d8aaa03e550a59fa59439aa09\" AND category = ROLE", + "metricCollectionFrequencyMs": 60000, + "rollupUsed": "TEN_MINUTELY" + }, + "data": [ + { + "timestamp": "2025-10-30T08:30:00.000Z", + "value": 0.000682378020178791, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:29:33.000Z", + "sampleValue": 0.0004838709677414954, + "count": 10, + "min": 0.0004838709677414954, + "minTime": "2025-10-30T08:29:33.000Z", + "max": 0.0008474576271188367, + "maxTime": "2025-10-30T08:26:31.000Z", + "mean": 0.000682378020178791, + "stdDev": 0.0001227485829983224 + } + }, + { + "timestamp": "2025-10-30T08:40:00.000Z", + "value": 0.0007165528100079016, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:39:33.000Z", + "sampleValue": 0.0008196721311477274, + "count": 10, + "min": 0.0006557377049174364, + "minTime": "2025-10-30T08:37:33.000Z", + "max": 0.0008474576271188367, + "maxTime": "2025-10-30T08:38:32.000Z", + "mean": 0.0007165528100079016, + "stdDev": 0.00007836407647083641 + } + }, + { + "timestamp": "2025-10-30T08:50:00.000Z", + "value": 0.0007183986292488786, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:49:32.000Z", + "sampleValue": 0.0006666666666670077, + "count": 10, + "min": 0.0004999999999995453, + "minTime": "2025-10-30T08:40:33.000Z", + "max": 0.0008474576271188367, + "maxTime": "2025-10-30T08:42:32.000Z", + "mean": 0.0007183986292488786, + "stdDev": 0.00011645769798271054 + } + }, + { + "timestamp": "2025-10-30T09:00:00.000Z", + "value": 0.0006830273683528945, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:59:32.000Z", + "sampleValue": 0.0004999999999995453, + "count": 10, + "min": 0.0004999999999995453, + "minTime": "2025-10-30T08:50:32.000Z", + "max": 0.0008474576271188367, + "maxTime": "2025-10-30T08:58:32.000Z", + "mean": 0.0006830273683528945, + "stdDev": 0.00014331234384495252 + } + }, + { + "timestamp": "2025-10-30T09:10:00.000Z", + "value": 0.0007167870704825545, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T09:09:32.000Z", + "sampleValue": 0.0006779661016942986, + "count": 10, + "min": 0.0005000000000004927, + "minTime": "2025-10-30T09:01:32.000Z", + "max": 0.0008474576271188367, + "maxTime": "2025-10-30T09:03:32.000Z", + "mean": 0.0007167870704825545, + "stdDev": 0.00011294200427276238 + } + }, + { + "timestamp": "2025-10-30T09:20:00.000Z", + "value": 0.0007012503473187268, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T09:19:31.000Z", + "sampleValue": 0.0006779661016952621, + "count": 10, + "min": 0.0006557377049174364, + "minTime": "2025-10-30T09:15:32.000Z", + "max": 0.0008474576271188367, + "maxTime": "2025-10-30T09:14:31.000Z", + "mean": 0.0007012503473187268, + "stdDev": 0.00007052001289904478 + } + } + ] + }, + { + "metadata": { + "metricName": "cpu_user_rate", + "entityName": "Impala StateStore (cldr4-data-hub-master0.cldr4-cd.svye-dcxb.a5.cloudera.site)", + "startTime": "2025-10-30T08:21:00.000Z", + "endTime": "2025-10-30T09:21:00.000Z", + "attributes": { + "serviceType": "IMPALA", + "clusterDisplayName": "cldr4-data-hub", + "hostId": "becfcf668a1861ab926151f1b11a726d", + "active": "true", + "serviceDisplayName": "Impala", + "serviceName": "impala", + "roleType": "STATESTORE", + "version": "CDH 7.3.1", + "hostname": "cldr4-data-hub-master0.cldr4-cd.svye-dcxb.a5.cloudera.site", + "entityName": "impala-STATESTORE-9391f56d8aaa03e550a59fa59439aa09", + "clusterName": "cldr4-data-hub", + "roleName": "impala-STATESTORE-9391f56d8aaa03e550a59fa59439aa09", + "roleConfigGroup": "Impala StateStore Default Group", + "category": "ROLE", + "rackId": "/eu-west-3b" + }, + "unitNumerators": [ + "seconds" + ], + "unitDenominators": [ + "seconds" + ], + "expression": "SELECT cpu_user_rate WHERE entityName = \"impala-STATESTORE-9391f56d8aaa03e550a59fa59439aa09\" AND category = ROLE", + "metricCollectionFrequencyMs": 60000, + "rollupUsed": "TEN_MINUTELY" + }, + "data": [ + { + "timestamp": "2025-10-30T08:30:00.000Z", + "value": 0.0006166666666666742, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:29:53.000Z", + "sampleValue": 0.0006666666666670077, + "count": 10, + "min": 0.0004999999999995453, + "minTime": "2025-10-30T08:25:53.000Z", + "max": 0.0006666666666670077, + "maxTime": "2025-10-30T08:22:53.000Z", + "mean": 0.0006166666666666742, + "stdDev": 0.00008050764858982454 + } + }, + { + "timestamp": "2025-10-30T08:40:00.000Z", + "value": 0.0006166666666666743, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:39:53.000Z", + "sampleValue": 0.0006666666666670077, + "count": 10, + "min": 0.0004999999999995453, + "minTime": "2025-10-30T08:31:53.000Z", + "max": 0.0008333333333335228, + "maxTime": "2025-10-30T08:30:53.000Z", + "mean": 0.0006166666666666743, + "stdDev": 0.00011249142628511458 + } + }, + { + "timestamp": "2025-10-30T08:50:00.000Z", + "value": 0.0006166666666666743, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:49:53.000Z", + "sampleValue": 0.0008333333333335228, + "count": 10, + "min": 0.0003333333333330302, + "minTime": "2025-10-30T08:45:53.000Z", + "max": 0.0008333333333335228, + "maxTime": "2025-10-30T08:41:53.000Z", + "mean": 0.0006166666666666743, + "stdDev": 0.00019325030145454037 + } + }, + { + "timestamp": "2025-10-30T09:00:00.000Z", + "value": 0.0005999999999999281, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:59:53.000Z", + "sampleValue": 0.0006666666666660604, + "count": 10, + "min": 0.0004999999999995453, + "minTime": "2025-10-30T08:50:53.000Z", + "max": 0.0006666666666670077, + "maxTime": "2025-10-30T08:53:53.000Z", + "mean": 0.0005999999999999281, + "stdDev": 0.00008606629658230875 + } + }, + { + "timestamp": "2025-10-30T09:10:00.000Z", + "value": 0.0006166666666666742, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T09:09:53.000Z", + "sampleValue": 0.0006666666666660604, + "count": 10, + "min": 0.0005000000000004927, + "minTime": "2025-10-30T09:00:53.000Z", + "max": 0.0006666666666670077, + "maxTime": "2025-10-30T09:02:53.000Z", + "mean": 0.0006166666666666742, + "stdDev": 0.00008050764858960659 + } + }, + { + "timestamp": "2025-10-30T09:20:00.000Z", + "value": 0.0006333333333333258, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T09:19:53.000Z", + "sampleValue": 0.0006666666666660604, + "count": 10, + "min": 0.0004999999999995453, + "minTime": "2025-10-30T09:12:53.000Z", + "max": 0.0008333333333335228, + "maxTime": "2025-10-30T09:13:53.000Z", + "mean": 0.0006333333333333258, + "stdDev": 0.00010540925533921625 + } + } + ] + }, + { + "metadata": { + "metricName": "cpu_user_rate", + "entityName": "LIVY_SERVER_FOR_SPARK3 (cldr4-data-hub-master0.cldr4-cd.svye-dcxb.a5.cloudera.site)", + "startTime": "2025-10-30T08:21:00.000Z", + "endTime": "2025-10-30T09:21:00.000Z", + "attributes": { + "serviceType": "LIVY_FOR_SPARK3", + "clusterDisplayName": "cldr4-data-hub", + "hostId": "becfcf668a1861ab926151f1b11a726d", + "active": "true", + "serviceDisplayName": "livy_for_spark3", + "serviceName": "livy_for_spark3", + "roleType": "LIVY_SERVER_FOR_SPARK3", + "version": "CDH 7.3.1", + "hostname": "cldr4-data-hub-master0.cldr4-cd.svye-dcxb.a5.cloudera.site", + "entityName": "livy446c4191-LIVY_SERVER_FOR_SPARK3-9391f56d8aaa03e550a59fa59439", + "clusterName": "cldr4-data-hub", + "roleName": "livy446c4191-LIVY_SERVER_FOR_SPARK3-9391f56d8aaa03e550a59fa59439", + "roleConfigGroup": "Livy Server for Spark 3 Default Group", + "category": "ROLE", + "rackId": "/eu-west-3b" + }, + "unitNumerators": [ + "seconds" + ], + "unitDenominators": [ + "seconds" + ], + "expression": "SELECT cpu_user_rate WHERE entityName = \"livy446c4191-LIVY_SERVER_FOR_SPARK3-9391f56d8aaa03e550a59fa59439\" AND category = ROLE", + "metricCollectionFrequencyMs": 60000, + "rollupUsed": "TEN_MINUTELY" + }, + "data": [ + { + "timestamp": "2025-10-30T08:30:00.000Z", + "value": 0.00013336993250813114, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:29:22.000Z", + "sampleValue": 0.00016666666666675193, + "count": 10, + "min": 0.0, + "minTime": "2025-10-30T08:22:23.000Z", + "max": 0.00016949152542381553, + "maxTime": "2025-10-30T08:21:21.000Z", + "mean": 0.00013336993250813114, + "stdDev": 0.00007033976289240952 + } + }, + { + "timestamp": "2025-10-30T08:40:00.000Z", + "value": 0.00014948088925538397, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:39:23.000Z", + "sampleValue": 0.00016129032258072767, + "count": 10, + "min": 0.0, + "minTime": "2025-10-30T08:35:22.000Z", + "max": 0.000333333333333267, + "maxTime": "2025-10-30T08:36:22.000Z", + "mean": 0.00014948088925538397, + "stdDev": 0.00009453959716339184 + } + }, + { + "timestamp": "2025-10-30T08:50:00.000Z", + "value": 0.00015028248587568795, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:49:22.000Z", + "sampleValue": 0.0001666666666665151, + "count": 10, + "min": 0.0, + "minTime": "2025-10-30T08:46:22.000Z", + "max": 0.00016949152542357465, + "maxTime": "2025-10-30T08:40:22.000Z", + "mean": 0.00015028248587568795, + "stdDev": 0.000052811345375001784 + } + }, + { + "timestamp": "2025-10-30T09:00:00.000Z", + "value": 0.00014973603778827964, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:59:22.000Z", + "sampleValue": 0.0001666666666665151, + "count": 10, + "min": 0.0, + "minTime": "2025-10-30T08:52:21.000Z", + "max": 0.00016949152542357465, + "maxTime": "2025-10-30T08:56:22.000Z", + "mean": 0.00014973603778827964, + "stdDev": 0.00005263525005820667 + } + }, + { + "timestamp": "2025-10-30T09:10:00.000Z", + "value": 0.00013334259516532083, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T09:09:22.000Z", + "sampleValue": 0.0, + "count": 10, + "min": 0.0, + "minTime": "2025-10-30T09:03:22.000Z", + "max": 0.00016949152542381553, + "maxTime": "2025-10-30T09:07:22.000Z", + "mean": 0.00013334259516532083, + "stdDev": 0.00007028992586747114 + } + }, + { + "timestamp": "2025-10-30T09:20:00.000Z", + "value": 0.00015029174770770284, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T09:19:21.000Z", + "sampleValue": 0.00016949152542381553, + "count": 10, + "min": 0.0, + "minTime": "2025-10-30T09:18:22.000Z", + "max": 0.00016949152542381553, + "maxTime": "2025-10-30T09:14:21.000Z", + "mean": 0.00015029174770770284, + "stdDev": 0.00005283078107447756 + } + } + ] + }, + { + "metadata": { + "metricName": "cpu_system_rate", + "entityName": "Impala Catalog Server (cldr4-data-hub-master0.cldr4-cd.svye-dcxb.a5.cloudera.site)", + "startTime": "2025-10-30T08:21:00.000Z", + "endTime": "2025-10-30T09:21:00.000Z", + "attributes": { + "serviceType": "IMPALA", + "clusterDisplayName": "cldr4-data-hub", + "hostId": "becfcf668a1861ab926151f1b11a726d", + "active": "true", + "serviceDisplayName": "Impala", + "serviceName": "impala", + "roleType": "CATALOGSERVER", + "version": "CDH 7.3.1", + "hostname": "cldr4-data-hub-master0.cldr4-cd.svye-dcxb.a5.cloudera.site", + "entityName": "impala-CATALOGSERVER-9391f56d8aaa03e550a59fa59439aa09", + "clusterName": "cldr4-data-hub", + "roleName": "impala-CATALOGSERVER-9391f56d8aaa03e550a59fa59439aa09", + "roleConfigGroup": "Impala Catalog Server Default Group", + "category": "ROLE", + "rackId": "/eu-west-3b" + }, + "unitNumerators": [ + "seconds" + ], + "unitDenominators": [ + "seconds" + ], + "expression": "SELECT cpu_system_rate WHERE entityName = \"impala-CATALOGSERVER-9391f56d8aaa03e550a59fa59439aa09\" AND category = ROLE", + "metricCollectionFrequencyMs": 60000, + "rollupUsed": "TEN_MINUTELY" + }, + "data": [ + { + "timestamp": "2025-10-30T08:30:00.000Z", + "value": 0.000532085824317977, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:29:33.000Z", + "sampleValue": 0.0006451612903219939, + "count": 10, + "min": 0.0004918032786880772, + "minTime": "2025-10-30T08:27:32.000Z", + "max": 0.0006666666666670077, + "maxTime": "2025-10-30T08:24:33.000Z", + "mean": 0.000532085824317977, + "stdDev": 0.00006574733165151588 + } + }, + { + "timestamp": "2025-10-30T08:40:00.000Z", + "value": 0.0004995568692453127, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:39:33.000Z", + "sampleValue": 0.0004918032786880772, + "count": 10, + "min": 0.0003389830508481128, + "minTime": "2025-10-30T08:38:32.000Z", + "max": 0.0006557377049174364, + "maxTime": "2025-10-30T08:37:33.000Z", + "mean": 0.0004995568692453127, + "stdDev": 0.00007498915820862658 + } + }, + { + "timestamp": "2025-10-30T08:50:00.000Z", + "value": 0.0005169861998703364, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:49:32.000Z", + "sampleValue": 0.0006666666666670077, + "count": 10, + "min": 0.0003333333333330302, + "minTime": "2025-10-30T08:41:33.000Z", + "max": 0.0006666666666670077, + "maxTime": "2025-10-30T08:40:33.000Z", + "mean": 0.0005169861998703364, + "stdDev": 0.00012050842873426478 + } + }, + { + "timestamp": "2025-10-30T09:00:00.000Z", + "value": 0.0005498935816534313, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:59:32.000Z", + "sampleValue": 0.0004999999999995453, + "count": 10, + "min": 0.00048387096774241226, + "minTime": "2025-10-30T08:54:33.000Z", + "max": 0.0006666666666670077, + "maxTime": "2025-10-30T08:50:32.000Z", + "mean": 0.0005498935816534313, + "stdDev": 0.00007865842284112788 + } + }, + { + "timestamp": "2025-10-30T09:10:00.000Z", + "value": 0.0005503658423636874, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T09:09:32.000Z", + "sampleValue": 0.0006779661016952621, + "count": 10, + "min": 0.0003333333333330302, + "minTime": "2025-10-30T09:04:32.000Z", + "max": 0.0006779661016952621, + "maxTime": "2025-10-30T09:09:32.000Z", + "mean": 0.0005503658423636874, + "stdDev": 0.00011438777095696416 + } + }, + { + "timestamp": "2025-10-30T09:20:00.000Z", + "value": 0.0005173242567379876, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T09:19:31.000Z", + "sampleValue": 0.0005084745762716875, + "count": 10, + "min": 0.0004918032786880772, + "minTime": "2025-10-30T09:15:32.000Z", + "max": 0.0006557377049183682, + "maxTime": "2025-10-30T09:16:33.000Z", + "mean": 0.0005173242567379876, + "stdDev": 0.000049074297720045846 + } + } + ] + }, + { + "metadata": { + "metricName": "cpu_system_rate", + "entityName": "Impala StateStore (cldr4-data-hub-master0.cldr4-cd.svye-dcxb.a5.cloudera.site)", + "startTime": "2025-10-30T08:21:00.000Z", + "endTime": "2025-10-30T09:21:00.000Z", + "attributes": { + "serviceType": "IMPALA", + "clusterDisplayName": "cldr4-data-hub", + "hostId": "becfcf668a1861ab926151f1b11a726d", + "active": "true", + "serviceDisplayName": "Impala", + "serviceName": "impala", + "roleType": "STATESTORE", + "version": "CDH 7.3.1", + "hostname": "cldr4-data-hub-master0.cldr4-cd.svye-dcxb.a5.cloudera.site", + "entityName": "impala-STATESTORE-9391f56d8aaa03e550a59fa59439aa09", + "clusterName": "cldr4-data-hub", + "roleName": "impala-STATESTORE-9391f56d8aaa03e550a59fa59439aa09", + "roleConfigGroup": "Impala StateStore Default Group", + "category": "ROLE", + "rackId": "/eu-west-3b" + }, + "unitNumerators": [ + "seconds" + ], + "unitDenominators": [ + "seconds" + ], + "expression": "SELECT cpu_system_rate WHERE entityName = \"impala-STATESTORE-9391f56d8aaa03e550a59fa59439aa09\" AND category = ROLE", + "metricCollectionFrequencyMs": 60000, + "rollupUsed": "TEN_MINUTELY" + }, + "data": [ + { + "timestamp": "2025-10-30T08:30:00.000Z", + "value": 0.0006166666666666742, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:29:53.000Z", + "sampleValue": 0.0005000000000004927, + "count": 10, + "min": 0.0004999999999995453, + "minTime": "2025-10-30T08:26:53.000Z", + "max": 0.0008333333333335228, + "maxTime": "2025-10-30T08:25:53.000Z", + "mean": 0.0006166666666666742, + "stdDev": 0.0001124914262851146 + } + }, + { + "timestamp": "2025-10-30T08:40:00.000Z", + "value": 0.0006333333333333257, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:39:53.000Z", + "sampleValue": 0.0006666666666670077, + "count": 10, + "min": 0.0004999999999995453, + "minTime": "2025-10-30T08:30:53.000Z", + "max": 0.0008333333333335228, + "maxTime": "2025-10-30T08:31:53.000Z", + "mean": 0.0006333333333333257, + "stdDev": 0.00013146843962434301 + } + }, + { + "timestamp": "2025-10-30T08:50:00.000Z", + "value": 0.0006666666666666288, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:49:53.000Z", + "sampleValue": 0.0006666666666660604, + "count": 10, + "min": 0.0004999999999995453, + "minTime": "2025-10-30T08:41:53.000Z", + "max": 0.0008333333333335228, + "maxTime": "2025-10-30T08:44:53.000Z", + "mean": 0.0006666666666666288, + "stdDev": 0.00015713484026386924 + } + }, + { + "timestamp": "2025-10-30T09:00:00.000Z", + "value": 0.0006500000000000721, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:59:53.000Z", + "sampleValue": 0.0006666666666670077, + "count": 10, + "min": 0.0005000000000004927, + "minTime": "2025-10-30T08:56:53.000Z", + "max": 0.0006666666666670077, + "maxTime": "2025-10-30T08:50:53.000Z", + "mean": 0.0006500000000000721, + "stdDev": 0.00005270462766932517 + } + }, + { + "timestamp": "2025-10-30T09:10:00.000Z", + "value": 0.0006333333333333258, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T09:09:53.000Z", + "sampleValue": 0.0006666666666670077, + "count": 10, + "min": 0.0004999999999995453, + "minTime": "2025-10-30T09:05:53.000Z", + "max": 0.0006666666666670077, + "maxTime": "2025-10-30T09:01:53.000Z", + "mean": 0.0006333333333333258, + "stdDev": 0.00007027283689261668 + } + }, + { + "timestamp": "2025-10-30T09:20:00.000Z", + "value": 0.0006166666666666743, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T09:19:53.000Z", + "sampleValue": 0.0006666666666670077, + "count": 10, + "min": 0.0004999999999995453, + "minTime": "2025-10-30T09:10:53.000Z", + "max": 0.0006666666666670077, + "maxTime": "2025-10-30T09:11:53.000Z", + "mean": 0.0006166666666666743, + "stdDev": 0.00008050764859026035 + } + } + ] + }, + { + "metadata": { + "metricName": "cpu_system_rate", + "entityName": "LIVY_SERVER_FOR_SPARK3 (cldr4-data-hub-master0.cldr4-cd.svye-dcxb.a5.cloudera.site)", + "startTime": "2025-10-30T08:21:00.000Z", + "endTime": "2025-10-30T09:21:00.000Z", + "attributes": { + "serviceType": "LIVY_FOR_SPARK3", + "clusterDisplayName": "cldr4-data-hub", + "hostId": "becfcf668a1861ab926151f1b11a726d", + "active": "true", + "serviceDisplayName": "livy_for_spark3", + "serviceName": "livy_for_spark3", + "roleType": "LIVY_SERVER_FOR_SPARK3", + "version": "CDH 7.3.1", + "hostname": "cldr4-data-hub-master0.cldr4-cd.svye-dcxb.a5.cloudera.site", + "entityName": "livy446c4191-LIVY_SERVER_FOR_SPARK3-9391f56d8aaa03e550a59fa59439", + "clusterName": "cldr4-data-hub", + "roleName": "livy446c4191-LIVY_SERVER_FOR_SPARK3-9391f56d8aaa03e550a59fa59439", + "roleConfigGroup": "Livy Server for Spark 3 Default Group", + "category": "ROLE", + "rackId": "/eu-west-3b" + }, + "unitNumerators": [ + "seconds" + ], + "unitDenominators": [ + "seconds" + ], + "expression": "SELECT cpu_system_rate WHERE entityName = \"livy446c4191-LIVY_SERVER_FOR_SPARK3-9391f56d8aaa03e550a59fa59439\" AND category = ROLE", + "metricCollectionFrequencyMs": 60000, + "rollupUsed": "TEN_MINUTELY" + }, + "data": [ + { + "timestamp": "2025-10-30T08:30:00.000Z", + "value": 0.0001503190850505131, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:29:22.000Z", + "sampleValue": 0.0001666666666665151, + "count": 10, + "min": 0.0, + "minTime": "2025-10-30T08:27:23.000Z", + "max": 0.00016949152542381553, + "maxTime": "2025-10-30T08:21:21.000Z", + "mean": 0.0001503190850505131, + "stdDev": 0.000052887336283355536 + } + }, + { + "timestamp": "2025-10-30T08:40:00.000Z", + "value": 0.00011559184600263572, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:39:23.000Z", + "sampleValue": 0.00016129032258072767, + "count": 10, + "min": 0.0, + "minTime": "2025-10-30T08:32:22.000Z", + "max": 0.00016949152542381553, + "maxTime": "2025-10-30T08:34:21.000Z", + "mean": 0.00011559184600263572, + "stdDev": 0.00007979573964598811 + } + }, + { + "timestamp": "2025-10-30T08:50:00.000Z", + "value": 0.00013361581920903645, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:49:22.000Z", + "sampleValue": 0.0, + "count": 10, + "min": 0.0, + "minTime": "2025-10-30T08:43:22.000Z", + "max": 0.00016949152542357465, + "maxTime": "2025-10-30T08:40:22.000Z", + "mean": 0.00013361581920903645, + "stdDev": 0.00007042722816723391 + } + }, + { + "timestamp": "2025-10-30T09:00:00.000Z", + "value": 0.00015001852366398603, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:59:22.000Z", + "sampleValue": 0.00016666666666675193, + "count": 10, + "min": 0.0, + "minTime": "2025-10-30T08:55:23.000Z", + "max": 0.0003278688524591841, + "maxTime": "2025-10-30T08:54:22.000Z", + "mean": 0.00015001852366398603, + "stdDev": 0.00009356284475002657 + } + }, + { + "timestamp": "2025-10-30T09:10:00.000Z", + "value": 0.00014972677595626602, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T09:09:22.000Z", + "sampleValue": 0.0001666666666665151, + "count": 10, + "min": 0.0, + "minTime": "2025-10-30T09:07:22.000Z", + "max": 0.00016666666666675193, + "maxTime": "2025-10-30T09:01:22.000Z", + "mean": 0.00014972677595626602, + "stdDev": 0.00005261563344226861 + } + }, + { + "timestamp": "2025-10-30T09:20:00.000Z", + "value": 0.0001502917477077028, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T09:19:21.000Z", + "sampleValue": 0.00016949152542381553, + "count": 10, + "min": 0.0, + "minTime": "2025-10-30T09:13:22.000Z", + "max": 0.00016949152542381553, + "maxTime": "2025-10-30T09:14:21.000Z", + "mean": 0.0001502917477077028, + "stdDev": 0.00005283078107447755 + } + } + ] + } + ], + "warnings": [], + "timeSeriesQuery": "SELECT mem_rss, cpu_user_rate, cpu_system_rate, read_bytes_rate, write_bytes_rate WHERE category = \"ROLE\" AND hostId = \"becfcf668a1861ab926151f1b11a726d\"" + } + ] +} \ No newline at end of file diff --git a/dumper/app/src/test/resources/cloudera/manager/host-services-resource-allocation-2.json b/dumper/app/src/test/resources/cloudera/manager/host-services-resource-allocation-2.json new file mode 100644 index 000000000..c8323af7f --- /dev/null +++ b/dumper/app/src/test/resources/cloudera/manager/host-services-resource-allocation-2.json @@ -0,0 +1,404 @@ +{ + "items": [ + { + "timeSeries": [ + { + "metadata": { + "metricName": "mem_rss", + "entityName": "HiveServer2 (cldr4-data-hub-master0.cldr4-cd.svye-dcxb.a5.cloudera.site)", + "startTime": "2025-10-30T08:21:00.000Z", + "endTime": "2025-10-30T09:21:00.000Z", + "attributes": { + "serviceType": "HIVE_ON_TEZ", + "clusterDisplayName": "cldr4-data-hub", + "hostId": "abcdef668a1861ab926151f1b11a726d", + "active": "true", + "serviceDisplayName": "Hive", + "serviceName": "hive_on_tez", + "roleType": "HIVESERVER2", + "version": "CDH 7.3.1", + "hostname": "cldr4-data-hub-master0.cldr4-cd.svye-dcxb.a5.cloudera.site", + "entityName": "hive_on_tez-HIVESERVER2-9391f56d8aaa03e550a59fa59439aa09", + "clusterName": "cldr4-data-hub", + "roleName": "hive_on_tez-HIVESERVER2-9391f56d8aaa03e550a59fa59439aa09", + "roleConfigGroup": "HiveServer2 Default Group", + "category": "ROLE", + "rackId": "/eu-west-3b" + }, + "unitNumerators": [ + "bytes" + ], + "unitDenominators": [], + "expression": "SELECT mem_rss WHERE entityName = \"hive_on_tez-HIVESERVER2-9391f56d8aaa03e550a59fa59439aa09\" AND category = ROLE", + "metricCollectionFrequencyMs": 60000, + "rollupUsed": "TEN_MINUTELY" + }, + "data": [ + { + "timestamp": "2025-10-30T08:30:00.000Z", + "value": 11792621568, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:29:53.000Z", + "sampleValue": 11792621568, + "count": 10, + "min": 11792621568, + "minTime": "2025-10-30T08:20:52.000Z", + "max": 11792621568, + "maxTime": "2025-10-30T08:20:52.000Z", + "mean": 11792621568, + "stdDev": 0.0 + } + }, + { + "timestamp": "2025-10-30T08:40:00.000Z", + "value": 11792621568, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:39:53.000Z", + "sampleValue": 11792621568, + "count": 10, + "min": 11792621568, + "minTime": "2025-10-30T08:30:53.000Z", + "max": 11792621568, + "maxTime": "2025-10-30T08:30:53.000Z", + "mean": 11792621568, + "stdDev": 0.0 + } + }, + { + "timestamp": "2025-10-30T08:50:00.000Z", + "value": 11792621568, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:49:53.000Z", + "sampleValue": 11792621568, + "count": 10, + "min": 11792621568, + "minTime": "2025-10-30T08:40:53.000Z", + "max": 11792621568, + "maxTime": "2025-10-30T08:40:53.000Z", + "mean": 11792621568, + "stdDev": 0.0 + } + }, + { + "timestamp": "2025-10-30T09:00:00.000Z", + "value": 11792621568, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:59:53.000Z", + "sampleValue": 11792621568, + "count": 10, + "min": 11792621568, + "minTime": "2025-10-30T08:50:53.000Z", + "max": 11792621568, + "maxTime": "2025-10-30T08:50:53.000Z", + "mean": 11792621568, + "stdDev": 0.0 + } + }, + { + "timestamp": "2025-10-30T09:10:00.000Z", + "value": 11792621568, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T09:09:53.000Z", + "sampleValue": 11792621568, + "count": 10, + "min": 11792621568, + "minTime": "2025-10-30T09:00:53.000Z", + "max": 11792621568, + "maxTime": "2025-10-30T09:00:53.000Z", + "mean": 11792621568, + "stdDev": 0.0 + } + }, + { + "timestamp": "2025-10-30T09:20:00.000Z", + "value": 11792621568, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T09:19:53.000Z", + "sampleValue": 11792621568, + "count": 10, + "min": 11792621568, + "minTime": "2025-10-30T09:10:53.000Z", + "max": 11792621568, + "maxTime": "2025-10-30T09:10:53.000Z", + "mean": 11792621568, + "stdDev": 0.0 + } + } + ] + }, + { + "metadata": { + "metricName": "cpu_user_rate", + "entityName": "HiveServer2 (cldr4-data-hub-master0.cldr4-cd.svye-dcxb.a5.cloudera.site)", + "startTime": "2025-10-30T08:21:00.000Z", + "endTime": "2025-10-30T09:21:00.000Z", + "attributes": { + "serviceType": "HIVE_ON_TEZ", + "clusterDisplayName": "cldr4-data-hub", + "hostId": "abcdef668a1861ab926151f1b11a726d", + "active": "true", + "serviceDisplayName": "Hive", + "serviceName": "hive_on_tez", + "roleType": "HIVESERVER2", + "version": "CDH 7.3.1", + "hostname": "cldr4-data-hub-master0.cldr4-cd.svye-dcxb.a5.cloudera.site", + "entityName": "hive_on_tez-HIVESERVER2-9391f56d8aaa03e550a59fa59439aa09", + "clusterName": "cldr4-data-hub", + "roleName": "hive_on_tez-HIVESERVER2-9391f56d8aaa03e550a59fa59439aa09", + "roleConfigGroup": "HiveServer2 Default Group", + "category": "ROLE", + "rackId": "/eu-west-3b" + }, + "unitNumerators": [ + "seconds" + ], + "unitDenominators": [ + "seconds" + ], + "expression": "SELECT cpu_user_rate WHERE entityName = \"hive_on_tez-HIVESERVER2-9391f56d8aaa03e550a59fa59439aa09\" AND category = ROLE", + "metricCollectionFrequencyMs": 60000, + "rollupUsed": "TEN_MINUTELY" + }, + "data": [ + { + "timestamp": "2025-10-30T08:30:00.000Z", + "value": 0.0012328424562378596, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:29:53.000Z", + "sampleValue": 0.0013333333333340155, + "count": 10, + "min": 0.001016949152543375, + "minTime": "2025-10-30T08:20:52.000Z", + "max": 0.0013333333333340155, + "maxTime": "2025-10-30T08:24:53.000Z", + "mean": 0.0012328424562378596, + "stdDev": 0.00011090720144477281 + } + }, + { + "timestamp": "2025-10-30T08:40:00.000Z", + "value": 0.001283333333333303, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:39:53.000Z", + "sampleValue": 0.0013333333333321207, + "count": 10, + "min": 0.0011666666666656055, + "minTime": "2025-10-30T08:31:53.000Z", + "max": 0.0015000000000005306, + "maxTime": "2025-10-30T08:33:53.000Z", + "mean": 0.001283333333333303, + "stdDev": 0.00011249142628523944 + } + }, + { + "timestamp": "2025-10-30T08:50:00.000Z", + "value": 0.001333333333333447, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:49:53.000Z", + "sampleValue": 0.0015000000000005306, + "count": 10, + "min": 0.0011666666666675003, + "minTime": "2025-10-30T08:42:53.000Z", + "max": 0.0015000000000005306, + "maxTime": "2025-10-30T08:43:53.000Z", + "mean": 0.001333333333333447, + "stdDev": 0.00013608276348757277 + } + }, + { + "timestamp": "2025-10-30T09:00:00.000Z", + "value": 0.001349999999999909, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:59:53.000Z", + "sampleValue": 0.0011666666666675003, + "count": 10, + "min": 0.0011666666666656055, + "minTime": "2025-10-30T08:52:53.000Z", + "max": 0.0015000000000005306, + "maxTime": "2025-10-30T08:53:53.000Z", + "mean": 0.001349999999999909, + "stdDev": 0.00014593250596154188 + } + }, + { + "timestamp": "2025-10-30T09:10:00.000Z", + "value": 0.0014000000000000531, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T09:09:53.000Z", + "sampleValue": 0.0013333333333340155, + "count": 10, + "min": 0.0013333333333321207, + "minTime": "2025-10-30T09:04:53.000Z", + "max": 0.0015000000000005306, + "maxTime": "2025-10-30T09:00:53.000Z", + "mean": 0.0014000000000000531, + "stdDev": 0.00008606629658239033 + } + }, + { + "timestamp": "2025-10-30T09:20:00.000Z", + "value": 0.0014500000000000077, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T09:19:53.000Z", + "sampleValue": 0.0013333333333340155, + "count": 10, + "min": 0.0013333333333321207, + "minTime": "2025-10-30T09:10:53.000Z", + "max": 0.0016666666666670456, + "maxTime": "2025-10-30T09:13:53.000Z", + "mean": 0.0014500000000000077, + "stdDev": 0.00011249142628502104 + } + } + ] + }, + { + "metadata": { + "metricName": "cpu_system_rate", + "entityName": "HiveServer2 (cldr4-data-hub-master0.cldr4-cd.svye-dcxb.a5.cloudera.site)", + "startTime": "2025-10-30T08:21:00.000Z", + "endTime": "2025-10-30T09:21:00.000Z", + "attributes": { + "serviceType": "HIVE_ON_TEZ", + "clusterDisplayName": "cldr4-data-hub", + "hostId": "abcdef668a1861ab926151f1b11a726d", + "active": "true", + "serviceDisplayName": "Hive", + "serviceName": "hive_on_tez", + "roleType": "HIVESERVER2", + "version": "CDH 7.3.1", + "hostname": "cldr4-data-hub-master0.cldr4-cd.svye-dcxb.a5.cloudera.site", + "entityName": "hive_on_tez-HIVESERVER2-9391f56d8aaa03e550a59fa59439aa09", + "clusterName": "cldr4-data-hub", + "roleName": "hive_on_tez-HIVESERVER2-9391f56d8aaa03e550a59fa59439aa09", + "roleConfigGroup": "HiveServer2 Default Group", + "category": "ROLE", + "rackId": "/eu-west-3b" + }, + "unitNumerators": [ + "seconds" + ], + "unitDenominators": [ + "seconds" + ], + "expression": "SELECT cpu_system_rate WHERE entityName = \"hive_on_tez-HIVESERVER2-9391f56d8aaa03e550a59fa59439aa09\" AND category = ROLE", + "metricCollectionFrequencyMs": 60000, + "rollupUsed": "TEN_MINUTELY" + }, + "data": [ + { + "timestamp": "2025-10-30T08:30:00.000Z", + "value": 0.00023362508104100694, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:29:53.000Z", + "sampleValue": 0.0001666666666665151, + "count": 10, + "min": 0.00016393442622982503, + "minTime": "2025-10-30T08:22:53.000Z", + "max": 0.00033898305084763107, + "maxTime": "2025-10-30T08:20:52.000Z", + "mean": 0.00023362508104100694, + "stdDev": 0.00008705007637557179 + } + }, + { + "timestamp": "2025-10-30T08:40:00.000Z", + "value": 0.0002500000000000095, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:39:53.000Z", + "sampleValue": 0.0001666666666665151, + "count": 10, + "min": 0.0001666666666665151, + "minTime": "2025-10-30T08:33:53.000Z", + "max": 0.00033333333333350387, + "maxTime": "2025-10-30T08:32:53.000Z", + "mean": 0.0002500000000000095, + "stdDev": 0.00008784104611575835 + } + }, + { + "timestamp": "2025-10-30T08:50:00.000Z", + "value": 0.0002500000000000095, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:49:53.000Z", + "sampleValue": 0.00033333333333350387, + "count": 10, + "min": 0.0001666666666665151, + "minTime": "2025-10-30T08:41:53.000Z", + "max": 0.00033333333333350387, + "maxTime": "2025-10-30T08:42:53.000Z", + "mean": 0.0002500000000000095, + "stdDev": 0.00008784104611585822 + } + }, + { + "timestamp": "2025-10-30T09:00:00.000Z", + "value": 0.0002500000000000095, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T08:59:53.000Z", + "sampleValue": 0.00033333333333350387, + "count": 10, + "min": 0.0001666666666665151, + "minTime": "2025-10-30T08:50:53.000Z", + "max": 0.00033333333333350387, + "maxTime": "2025-10-30T08:51:53.000Z", + "mean": 0.0002500000000000095, + "stdDev": 0.00008784104611595809 + } + }, + { + "timestamp": "2025-10-30T09:10:00.000Z", + "value": 0.0002333333333333106, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T09:09:53.000Z", + "sampleValue": 0.0001666666666665151, + "count": 10, + "min": 0.0001666666666665151, + "minTime": "2025-10-30T09:00:53.000Z", + "max": 0.00033333333333350387, + "maxTime": "2025-10-30T09:08:53.000Z", + "mean": 0.0002333333333333106, + "stdDev": 0.00008606629658224762 + } + }, + { + "timestamp": "2025-10-30T09:20:00.000Z", + "value": 0.0002166666666666591, + "type": "SAMPLE", + "aggregateStatistics": { + "sampleTime": "2025-10-30T09:19:53.000Z", + "sampleValue": 0.0003333333333330302, + "count": 10, + "min": 0.0, + "minTime": "2025-10-30T09:14:53.000Z", + "max": 0.00033333333333350387, + "maxTime": "2025-10-30T09:10:53.000Z", + "mean": 0.0002166666666666591, + "stdDev": 0.00011249142628497426 + } + } + ] + } + ], + "warnings": [], + "timeSeriesQuery": "SELECT mem_rss, cpu_user_rate, cpu_system_rate, read_bytes_rate, write_bytes_rate WHERE category = \"ROLE\" AND hostId = \"abcdef668a1861ab926151f1b11a726d\"" + } + ] +} \ No newline at end of file