diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaYarnApplicationTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaYarnApplicationTask.java index c69a9651e..dd30ac583 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaYarnApplicationTask.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaYarnApplicationTask.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.google.common.base.Preconditions; -import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiYARNApplicationDTO; +import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiYarnApplicationDto; import com.google.edwmigration.dumper.application.dumper.task.TaskCategory; import java.io.IOException; import java.net.URI; @@ -83,20 +83,20 @@ public PaginatedClouderaYarnApplicationsLoader(ClouderaManagerHandle handle, int toAppCreationDate = toDate.format(dtFormatter); } - public int load(String clusterName, Consumer> onPageLoad) { + public int load(String clusterName, Consumer> onPageLoad) { return load(clusterName, null, onPageLoad); } public int load( String clusterName, @Nullable String appType, - Consumer> onPageLoad) { + Consumer> onPageLoad) { offset = 0; boolean fetchedNewApps; do { fetchedNewApps = false; URI yarnAppsURI = buildNextYARNApplicationPageURI(clusterName, appType); - List newLoad = load(yarnAppsURI); + List newLoad = load(yarnAppsURI); if (!newLoad.isEmpty()) { onPageLoad.accept(newLoad); offset += newLoad.size(); @@ -106,7 +106,7 @@ public int load( return offset; } - private List load(URI yarnAppURI) { + private List load(URI yarnAppURI) { try (CloseableHttpResponse resp = httpClient.execute(new HttpGet(yarnAppURI))) { int statusCode = resp.getStatusLine().getStatusCode(); if (!isStatusCodeOK(statusCode)) { @@ -130,10 +130,10 @@ private List load(URI yarnAppURI) { } } - private List toDTOs(ArrayNode applicationsArray) { - List yarnApplicationDTOs = new ArrayList<>(); + private List toDTOs(ArrayNode applicationsArray) { + List yarnApplicationDTOs = new ArrayList<>(); for (JsonNode application : applicationsArray) { - yarnApplicationDTOs.add(new ApiYARNApplicationDTO(application)); + yarnApplicationDTOs.add(new ApiYarnApplicationDto(application)); } return yarnApplicationDTOs; diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaAPIHostsTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaApiHostsTask.java similarity index 91% rename from dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaAPIHostsTask.java rename to dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaApiHostsTask.java index 796e9dcb2..6305fce50 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaAPIHostsTask.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaApiHostsTask.java @@ -21,8 +21,8 @@ import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException; import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaClusterDTO; import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaHostDTO; -import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiHostDTO; -import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiHostListDTO; +import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiHostDto; +import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiHostListDto; import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext; import java.io.Writer; import java.nio.charset.StandardCharsets; @@ -38,9 +38,9 @@ * href="https://archive.cloudera.com/cm7/7.11.3.0/generic/jar/cm_api/apidocs/json_ApiHostList.html">Hosts * API which doesn't contain usage and disk data and collected as a fallback. */ -public class ClouderaAPIHostsTask extends AbstractClouderaManagerTask { +public class ClouderaApiHostsTask extends AbstractClouderaManagerTask { - public ClouderaAPIHostsTask() { + public ClouderaApiHostsTask() { super("api-hosts.jsonl"); } @@ -75,8 +75,8 @@ protected void doRun( writer.write(stringifiedHosts); writer.write('\n'); - ApiHostListDTO apiHosts = parseJsonStringToObject(stringifiedHosts, ApiHostListDTO.class); - for (ApiHostDTO apiHost : apiHosts.getHosts()) { + ApiHostListDto apiHosts = parseJsonStringToObject(stringifiedHosts, ApiHostListDto.class); + for (ApiHostDto apiHost : apiHosts.getHosts()) { hosts.add(ClouderaHostDTO.create(apiHost.getId(), apiHost.getName())); } } diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCpuChartTask.java similarity index 96% rename from dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTask.java rename to dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCpuChartTask.java index 2dddac4f3..5bfaaddfc 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTask.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCpuChartTask.java @@ -40,12 +40,12 @@ * href="https://docs.cloudera.com/documentation/enterprise/latest/topics/cm_dg_tsquery.html">tsquery * language. */ -public class ClouderaClusterCPUChartTask extends AbstractClouderaTimeSeriesTask { - private static final Logger logger = LoggerFactory.getLogger(ClouderaClusterCPUChartTask.class); +public class ClouderaClusterCpuChartTask extends AbstractClouderaTimeSeriesTask { + private static final Logger logger = LoggerFactory.getLogger(ClouderaClusterCpuChartTask.class); private static final String TS_CPU_QUERY_TEMPLATE = "SELECT cpu_percent_across_hosts WHERE entityName = \"%s\" AND category = CLUSTER"; - public ClouderaClusterCPUChartTask( + public ClouderaClusterCpuChartTask( ZonedDateTime startDate, ZonedDateTime endDate, TimeSeriesAggregation tsAggregation, diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClustersTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClustersTask.java index ff48eaf8a..e8697ca30 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClustersTask.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClustersTask.java @@ -20,8 +20,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.io.ByteSink; import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaClusterDTO; -import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiClusterDTO; -import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiClusterListDTO; +import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiClusterDto; +import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiClusterListDto; import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext; import java.io.Writer; import java.net.URI; @@ -52,18 +52,18 @@ protected void doRun( throws Exception { CloseableHttpClient httpClient = handle.getClouderaManagerHttpClient(); - ApiClusterListDTO clusterList; + ApiClusterListDto clusterList; if (context.getArguments().getCluster() != null) { final String clusterName = context.getArguments().getCluster(); try (CloseableHttpResponse clusterResponse = httpClient.execute(new HttpGet(handle.getApiURI() + "/clusters/" + clusterName))) { - ApiClusterDTO cluster = + ApiClusterDto cluster = parseJsonStringToObject( - EntityUtils.toString(clusterResponse.getEntity()), ApiClusterDTO.class); + EntityUtils.toString(clusterResponse.getEntity()), ApiClusterDto.class); - clusterList = new ApiClusterListDTO(); + clusterList = new ApiClusterListDto(); clusterList.setClusters(ImmutableList.of(cluster)); } } else { @@ -74,7 +74,7 @@ protected void doRun( try (CloseableHttpResponse clustersResponse = httpClient.execute(new HttpGet(handle.getApiURI() + "/clusters?clusterType=ANY"))) { String clustersJson = EntityUtils.toString(clustersResponse.getEntity()); - clusterList = parseJsonStringToObject(clustersJson, ApiClusterListDTO.class); + clusterList = parseJsonStringToObject(clustersJson, ApiClusterListDto.class); } } @@ -83,7 +83,7 @@ protected void doRun( } List clusters = new ArrayList<>(); - for (ApiClusterDTO item : clusterList.getClusters()) { + for (ApiClusterDto item : clusterList.getClusters()) { String clusterId = requestClusterIdByName(httpClient, handle.getBaseURI(), item.getName()); clusters.add(ClouderaClusterDTO.create(clusterId, item.getName())); } diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaCMFHostsTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaCmfHostsTask.java similarity index 96% rename from dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaCMFHostsTask.java rename to dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaCmfHostsTask.java index a3075c91f..592a293e0 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaCMFHostsTask.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaCmfHostsTask.java @@ -37,11 +37,11 @@ * The task collects hosts from Cloudera Manager {@code /cmf/} urls. These API contains * well-structured data but is not well documented. */ -public class ClouderaCMFHostsTask extends AbstractClouderaManagerTask { +public class ClouderaCmfHostsTask extends AbstractClouderaManagerTask { - private static final Logger logger = LoggerFactory.getLogger(ClouderaCMFHostsTask.class); + private static final Logger logger = LoggerFactory.getLogger(ClouderaCmfHostsTask.class); - public ClouderaCMFHostsTask() { + public ClouderaCmfHostsTask() { super("cmf-hosts.jsonl"); } diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRamChartTask.java similarity index 96% rename from dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTask.java rename to dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRamChartTask.java index e8d248118..824afec7a 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTask.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRamChartTask.java @@ -39,14 +39,14 @@ * href="https://docs.cloudera.com/documentation/enterprise/latest/topics/cm_dg_tsquery.html">tsquery * language. */ -public class ClouderaHostRAMChartTask extends AbstractClouderaTimeSeriesTask { +public class ClouderaHostRamChartTask extends AbstractClouderaTimeSeriesTask { - private static final Logger logger = LoggerFactory.getLogger(ClouderaHostRAMChartTask.class); + private static final Logger logger = LoggerFactory.getLogger(ClouderaHostRamChartTask.class); private static final String TS_RAM_QUERY_TEMPLATE = "select swap_used, physical_memory_used, physical_memory_total, physical_memory_cached, physical_memory_buffers where entityName = \"%s\""; - public ClouderaHostRAMChartTask( + public ClouderaHostRamChartTask( ZonedDateTime startDate, ZonedDateTime endDate, TimeSeriesAggregation tsAggregation, diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnector.java index d1e791095..de94ca470 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnector.java @@ -91,8 +91,8 @@ public void addTasksTo(@Nonnull List> out, @Nonnull ConnectorArg out.add(new DumpMetadataTask(arguments, FORMAT_NAME)); out.add(new FormatTask(FORMAT_NAME)); out.add(new ClouderaClustersTask()); - out.add(new ClouderaCMFHostsTask()); - out.add(new ClouderaAPIHostsTask()); + out.add(new ClouderaCmfHostsTask()); + out.add(new ClouderaApiHostsTask()); out.add(new ClouderaServicesTask()); out.add(new ClouderaHostComponentsTask()); @@ -107,8 +107,8 @@ public void addTasksTo(@Nonnull List> out, @Nonnull ConnectorArg endDate = arguments.getEndDate(); } - out.add(new ClouderaClusterCPUChartTask(startDate, endDate, DAILY, REQUIRED)); - out.add(new ClouderaHostRAMChartTask(startDate, endDate, DAILY, REQUIRED)); + out.add(new ClouderaClusterCpuChartTask(startDate, endDate, DAILY, REQUIRED)); + out.add(new ClouderaHostRamChartTask(startDate, endDate, DAILY, REQUIRED)); out.add(new ClouderaServiceResourceAllocationChartTask(startDate, endDate, HOURLY, OPTIONAL)); out.add(new ClouderaYarnApplicationsTask(startDate, endDate, OPTIONAL)); out.add(new ClouderaYarnApplicationTypeTask(startDate, endDate, OPTIONAL)); diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerHandle.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerHandle.java index 1df806bef..4f3620021 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerHandle.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerHandle.java @@ -19,6 +19,7 @@ import com.google.auto.value.AutoValue; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.io.Closer; import com.google.edwmigration.dumper.application.dumper.handle.Handle; import java.io.IOException; import java.net.URI; @@ -35,6 +36,7 @@ public class ClouderaManagerHandle implements Handle { private final URI apiURI; private final CloseableHttpClient clouderaManagerHttpClient; private final CloseableHttpClient basicAuthHttpClient; + private final Closer closer; private ImmutableList clusters; private ImmutableList hosts; @@ -52,6 +54,9 @@ public ClouderaManagerHandle( this.apiURI = unify(apiURI); this.clouderaManagerHttpClient = clouderaManagerHttpClient; this.basicAuthHttpClient = basicAuthHttpClient; + this.closer = Closer.create(); + closer.register(this.clouderaManagerHttpClient); + closer.register(this.basicAuthHttpClient); } /** 1. Remove query params and url fragments 2. Add trailing slash for safety */ @@ -130,10 +135,9 @@ public synchronized void initSparkYarnApplications( } @Override - public void close() throws IOException { + public synchronized void close() throws IOException { try { - clouderaManagerHttpClient.close(); - basicAuthHttpClient.close(); + closer.close(); } catch (IOException ignore) { // The intention is to do graceful shutdown and try to release the resource. // In case of errors we do not need to interrupt the execution flow diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationTypeTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationTypeTask.java index ac3c5a5ea..4a294ae7f 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationTypeTask.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationTypeTask.java @@ -27,7 +27,7 @@ import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException; import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaClusterDTO; import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaYarnApplicationDTO; -import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiYARNApplicationDTO; +import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiYarnApplicationDto; import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.model.YarnApplicationType; import com.google.edwmigration.dumper.application.dumper.task.TaskCategory; import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext; @@ -108,9 +108,9 @@ protected void doRun( } private void writeYarnAppTypes( - Writer writer, List yarnApps, String appType, String clusterName) { + Writer writer, List yarnApps, String appType, String clusterName) { List yarnAppTypeMappings = new ArrayList<>(); - for (ApiYARNApplicationDTO yarnApp : yarnApps) { + for (ApiYarnApplicationDto yarnApp : yarnApps) { yarnAppTypeMappings.add( new ApplicationTypeToYarnApplication(yarnApp.getApplicationId(), appType, clusterName)); } diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationsTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationsTask.java index ca9afe40e..9bb09b4a4 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationsTask.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationsTask.java @@ -20,7 +20,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteSink; import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaClusterDTO; -import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiYARNApplicationDTO; +import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiYarnApplicationDto; import com.google.edwmigration.dumper.application.dumper.task.TaskCategory; import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext; import java.io.IOException; @@ -67,8 +67,8 @@ protected void doRun( } private void writeYarnApplications( - Writer writer, List yarnApps, String clusterName) { - for (ApiYARNApplicationDTO yarnApp : yarnApps) { + Writer writer, List yarnApps, String clusterName) { + for (ApiYarnApplicationDto yarnApp : yarnApps) { yarnApp.setClusterName(clusterName); } try { diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/SparkHistoryDiscoveryService.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/SparkHistoryDiscoveryService.java new file mode 100644 index 000000000..2a5daf28d --- /dev/null +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/SparkHistoryDiscoveryService.java @@ -0,0 +1,225 @@ +/* + * Copyright 2022-2025 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; +import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiConfigDto; +import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiConfigListDTO; +import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiRoleDto; +import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiRoleListDto; +import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiServiceDto; +import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiServiceListDto; +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SparkHistoryDiscoveryService { + + private static final Logger logger = LoggerFactory.getLogger(SparkHistoryDiscoveryService.class); + + private static final List DEFAULT_CANDIDATE_PATHS = + ImmutableList.of("spark3history", "sparkhistory"); + + private final ObjectMapper objectMapper; + private final CloseableHttpClient clouderaManagerHttpClient; + private final URI apiURI; + + public SparkHistoryDiscoveryService( + ObjectMapper objectMapper, CloseableHttpClient clouderaManagerHttpClient, URI apiURI) { + this.objectMapper = objectMapper; + this.clouderaManagerHttpClient = clouderaManagerHttpClient; + this.apiURI = apiURI; + } + + /** + * Discovers the active Spark History Server URLs. Probes Spark 3 and Spark 2 endpoints, including + * any custom paths, against the discovered Knox Gateway instance. + */ + public List resolveUrl( + String clusterName, CloseableHttpClient knoxClient, List customCandidatePaths) { + Set reachableUrls = new LinkedHashSet<>(); + try { + Optional knoxInfo = getKnoxGatewayInfo(clusterName); + + if (!knoxInfo.isPresent()) { + logger.warn("Could not find Knox Service or Role for cluster: {}", clusterName); + return ImmutableList.of(); + } + + KnoxGatewayInfo info = knoxInfo.get(); + + Set allCandidates = new LinkedHashSet<>(customCandidatePaths); + allCandidates.addAll(DEFAULT_CANDIDATE_PATHS); + + for (String context : allCandidates) { + String candidateUrl = + String.format( + "https://%s/%s/%s/%s/api/v1", + info.hostname, info.gatewayPath, info.topologyName, context); + + if (isReachable(candidateUrl, knoxClient)) { + logger.info("Found active Spark History Server at: {}", candidateUrl); + reachableUrls.add(candidateUrl); + } + } + + if (reachableUrls.isEmpty()) { + logger.warn( + "Knox is active, but no Spark history endpoints are reachable for candidates: {}", + allCandidates); + } + } catch (IOException e) { + logger.warn( + "Cluster '{}': Failed to discover Spark History URL due to error: {}", + clusterName, + e.getMessage()); + } + return new ArrayList<>(reachableUrls); + } + + private boolean isReachable(String url, CloseableHttpClient knoxHttpClient) { + String probeUrl = url + "/applications?limit=1"; + + try (CloseableHttpResponse response = knoxHttpClient.execute(new HttpGet(probeUrl))) { + int status = response.getStatusLine().getStatusCode(); + return status == 200; + } catch (IOException e) { + logger.debug("Probe failed for URL: {}", url); + return false; + } + } + + private Optional getKnoxGatewayInfo(String clusterName) throws IOException { + Optional knoxServiceName = getKnoxServiceName(clusterName); + if (!knoxServiceName.isPresent()) { + return Optional.empty(); + } + + String serviceName = knoxServiceName.get(); + Optional knoxRole = getKnoxRole(clusterName, serviceName); + if (!knoxRole.isPresent()) { + return Optional.empty(); + } + + ApiRoleDto role = knoxRole.get(); + if (role.getHostRef() == null || role.getRoleConfigGroupRef() == null) { + logger.warn("Knox role is missing hostRef or roleConfigGroupRef."); + return Optional.empty(); + } + + String hostname = role.getHostRef().getHostname(); + String roleConfigGroup = role.getRoleConfigGroupRef().getRoleConfigGroupName(); + if (hostname == null || roleConfigGroup == null) { + logger.warn("Knox role has null hostname or roleConfigGroupName."); + return Optional.empty(); + } + + String gatewayPath = getGatewayPath(clusterName, serviceName, roleConfigGroup); + String topologyName = getTopologyName(clusterName, serviceName, roleConfigGroup); + + return Optional.of(new KnoxGatewayInfo(hostname, gatewayPath, topologyName)); + } + + private Optional getKnoxServiceName(String clusterName) throws IOException { + String path = String.format("clusters/%s/services", clusterName); + Optional serviceList = get(path, ApiServiceListDto.class); + return serviceList.flatMap( + list -> + list.getItems().stream() + .filter(service -> "KNOX".equals(service.getType())) + .map(ApiServiceDto::getName) + .findFirst()); + } + + private Optional getKnoxRole(String clusterName, String knoxServiceName) + throws IOException { + String path = String.format("clusters/%s/services/%s/roles", clusterName, knoxServiceName); + Optional roleList = get(path, ApiRoleListDto.class); + return roleList.flatMap(list -> list.getItems().stream().findFirst()); + } + + private String getGatewayPath( + String clusterName, String knoxServiceName, String roleConfigGroupName) throws IOException { + return getConfigValue(clusterName, knoxServiceName, roleConfigGroupName, "gateway_path") + .orElse(clusterName); + } + + private String getTopologyName( + String clusterName, String knoxServiceName, String roleConfigGroupName) throws IOException { + return getConfigValue( + clusterName, knoxServiceName, roleConfigGroupName, "gateway_default_api_topology_name") + .orElse("cdp-proxy-api"); + } + + private Optional getConfigValue( + String clusterName, String knoxServiceName, String roleConfigGroupName, String configName) + throws IOException { + String path = + String.format( + "clusters/%s/services/%s/roleConfigGroups/%s/config", + clusterName, knoxServiceName, roleConfigGroupName); + Optional configList = get(path, ApiConfigListDTO.class); + return configList.flatMap( + list -> + list.getItems().stream() + .filter(config -> configName.equals(config.getName())) + .map(ApiConfigDto::getValue) + .findFirst()); + } + + private Optional get(String path, Class responseType) throws IOException { + URI requestUri = apiURI.resolve(path); + try (CloseableHttpResponse response = + clouderaManagerHttpClient.execute(new HttpGet(requestUri))) { + if (response.getStatusLine().getStatusCode() != 200) { + throw new IOException( + "Unexpected status code " + + response.getStatusLine().getStatusCode() + + " from " + + requestUri); + } + HttpEntity entity = response.getEntity(); + if (entity == null) { + return Optional.empty(); + } + return Optional.of(objectMapper.readValue(entity.getContent(), responseType)); + } + } + + private static class KnoxGatewayInfo { + String hostname; + String gatewayPath; + String topologyName; + + KnoxGatewayInfo(String hostname, String gatewayPath, String topologyName) { + this.hostname = hostname; + this.gatewayPath = gatewayPath; + this.topologyName = topologyName; + } + } +} diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiClusterDTO.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiClusterDto.java similarity index 98% rename from dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiClusterDTO.java rename to dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiClusterDto.java index cef381e15..4f2dc09a6 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiClusterDTO.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiClusterDto.java @@ -29,7 +29,7 @@ * code is unclear, the own model for public schema used instead of it. */ @JsonIgnoreProperties(ignoreUnknown = true) -public final class ApiClusterDTO { +public final class ApiClusterDto { @JsonProperty(required = true) private String name; diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiClusterListDTO.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiClusterListDto.java similarity index 88% rename from dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiClusterListDTO.java rename to dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiClusterListDto.java index c01f6bd96..625e6bb82 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiClusterListDTO.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiClusterListDto.java @@ -29,16 +29,16 @@ * code is unclear, the own model for public schema used instead of it. */ @JsonIgnoreProperties(ignoreUnknown = true) -public final class ApiClusterListDTO { +public final class ApiClusterListDto { @JsonProperty(required = true) - private List items; + private List items; - public List getClusters() { + public List getClusters() { return items; } - public void setClusters(List items) { + public void setClusters(List items) { this.items = items; } } diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiConfigDto.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiConfigDto.java new file mode 100644 index 000000000..5fb11dec0 --- /dev/null +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiConfigDto.java @@ -0,0 +1,38 @@ +/* + * Copyright 2022-2025 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class ApiConfigDto { + + @JsonProperty("name") + private String name; + + @JsonProperty("value") + private String value; + + public String getName() { + return name; + } + + public String getValue() { + return value; + } +} diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiConfigListDTO.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiConfigListDTO.java new file mode 100644 index 000000000..30f5aae9c --- /dev/null +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiConfigListDTO.java @@ -0,0 +1,32 @@ +/* + * Copyright 2022-2025 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class ApiConfigListDTO { + + @JsonProperty("items") + private List items; + + public List getItems() { + return items; + } +} diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiHostDTO.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiHostDto.java similarity index 98% rename from dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiHostDTO.java rename to dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiHostDto.java index 910806c68..49221b5a1 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiHostDTO.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiHostDto.java @@ -28,7 +28,7 @@ * code is unclear, the own model for public schema used instead of it. */ @JsonIgnoreProperties(ignoreUnknown = true) -public class ApiHostDTO { +public class ApiHostDto { @JsonProperty(required = true) private String hostname; diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiHostListDTO.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiHostListDto.java similarity index 87% rename from dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiHostListDTO.java rename to dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiHostListDto.java index ad38c2a00..6db7d3405 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiHostListDTO.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiHostListDto.java @@ -30,15 +30,15 @@ * code is unclear, the own model for public schema used instead of it. */ @JsonIgnoreProperties(ignoreUnknown = true) -public class ApiHostListDTO { - private final List hosts; +public class ApiHostListDto { + private final List hosts; @JsonCreator - public ApiHostListDTO(@JsonProperty(value = "items", required = true) List hosts) { + public ApiHostListDto(@JsonProperty(value = "items", required = true) List hosts) { this.hosts = hosts; } - public List getHosts() { + public List getHosts() { return hosts; } } diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiHostRefDto.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiHostRefDto.java new file mode 100644 index 000000000..17d668f99 --- /dev/null +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiHostRefDto.java @@ -0,0 +1,31 @@ +/* + * Copyright 2022-2025 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class ApiHostRefDto { + + @JsonProperty("hostname") + private String hostname; + + public String getHostname() { + return hostname; + } +} diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiRoleConfigGroupRefDto.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiRoleConfigGroupRefDto.java new file mode 100644 index 000000000..76a9cf009 --- /dev/null +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiRoleConfigGroupRefDto.java @@ -0,0 +1,31 @@ +/* + * Copyright 2022-2025 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class ApiRoleConfigGroupRefDto { + + @JsonProperty("roleConfigGroupName") + private String roleConfigGroupName; + + public String getRoleConfigGroupName() { + return roleConfigGroupName; + } +} diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiRoleDto.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiRoleDto.java new file mode 100644 index 000000000..13bb8dcf5 --- /dev/null +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiRoleDto.java @@ -0,0 +1,38 @@ +/* + * Copyright 2022-2025 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class ApiRoleDto { + + @JsonProperty("hostRef") + private ApiHostRefDto hostRef; + + @JsonProperty("roleConfigGroupRef") + private ApiRoleConfigGroupRefDto roleConfigGroupRef; + + public ApiHostRefDto getHostRef() { + return hostRef; + } + + public ApiRoleConfigGroupRefDto getRoleConfigGroupRef() { + return roleConfigGroupRef; + } +} diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiRoleListDto.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiRoleListDto.java new file mode 100644 index 000000000..2cb871d29 --- /dev/null +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiRoleListDto.java @@ -0,0 +1,32 @@ +/* + * Copyright 2022-2025 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class ApiRoleListDto { + + @JsonProperty("items") + private List items; + + public List getItems() { + return items; + } +} diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiServiceDto.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiServiceDto.java new file mode 100644 index 000000000..3a9c0a219 --- /dev/null +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiServiceDto.java @@ -0,0 +1,38 @@ +/* + * Copyright 2022-2025 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class ApiServiceDto { + + @JsonProperty("name") + private String name; + + @JsonProperty("type") + private String type; + + public String getName() { + return name; + } + + public String getType() { + return type; + } +} diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiServiceListDto.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiServiceListDto.java new file mode 100644 index 000000000..99a9bfb84 --- /dev/null +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiServiceListDto.java @@ -0,0 +1,32 @@ +/* + * Copyright 2022-2025 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class ApiServiceListDto { + + @JsonProperty("items") + private List items; + + public List getItems() { + return items; + } +} diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiYARNApplicationDTO.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiYarnApplicationDto.java similarity index 95% rename from dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiYARNApplicationDTO.java rename to dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiYarnApplicationDto.java index ed56b840e..7a98ab3aa 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiYARNApplicationDTO.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiYarnApplicationDto.java @@ -28,7 +28,7 @@ * with additional information about cluster. */ @JsonIgnoreProperties(ignoreUnknown = true) -public class ApiYARNApplicationDTO { +public class ApiYarnApplicationDto { @JsonProperty(required = true) private String clusterName; @@ -40,7 +40,7 @@ public class ApiYARNApplicationDTO { */ private final JsonNode apiYarnApplication; - public ApiYARNApplicationDTO(JsonNode apiYarnApplication) { + public ApiYarnApplicationDto(JsonNode apiYarnApplication) { this.apiYarnApplication = apiYarnApplication; } diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaYarnApplicationTaskTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaYarnApplicationTaskTest.java index efecd31d4..698dfeb18 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaYarnApplicationTaskTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaYarnApplicationTaskTest.java @@ -26,7 +26,7 @@ import com.github.tomakehurst.wiremock.core.WireMockConfiguration; import com.github.tomakehurst.wiremock.matching.StringValuePattern; import com.google.common.io.ByteSink; -import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiYARNApplicationDTO; +import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiYarnApplicationDto; import com.google.edwmigration.dumper.application.dumper.task.TaskCategory; import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext; import java.io.IOException; @@ -54,7 +54,7 @@ public class AbstractClouderaYarnApplicationTaskTest { private static WireMockServer server; private MockedYarnApplicationTask task; private ClouderaManagerHandle handle; - private List loadResponse; + private List loadResponse; @BeforeClass public static void beforeClass() { diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaAPIHostsTaskTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaApiHostsTaskTest.java similarity index 98% rename from dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaAPIHostsTaskTest.java rename to dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaApiHostsTaskTest.java index 084319961..4feeaa07f 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaAPIHostsTaskTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaApiHostsTaskTest.java @@ -59,10 +59,10 @@ import org.mockito.junit.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) -public class ClouderaAPIHostsTaskTest { +public class ClouderaApiHostsTaskTest { private static WireMockServer server; - private final ClouderaAPIHostsTask task = new ClouderaAPIHostsTask(); + private final ClouderaApiHostsTask task = new ClouderaApiHostsTask(); private ClouderaManagerHandle handle; @Mock private TaskRunContext context; diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTaskTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCpuChartTaskTest.java similarity index 98% rename from dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTaskTest.java rename to dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCpuChartTaskTest.java index ecaeda4f1..e8f5fd637 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTaskTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCpuChartTaskTest.java @@ -60,9 +60,9 @@ import org.mockito.junit.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) -public class ClouderaClusterCPUChartTaskTest { - private final ClouderaClusterCPUChartTask task = - new ClouderaClusterCPUChartTask( +public class ClouderaClusterCpuChartTaskTest { + private final ClouderaClusterCpuChartTask task = + new ClouderaClusterCpuChartTask( timeTravelDaysAgo(30), timeTravelDaysAgo(0), TimeSeriesAggregation.HOURLY, diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClustersTaskTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClustersTaskTest.java index 56a89453f..1e99d8f14 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClustersTaskTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClustersTaskTest.java @@ -39,7 +39,7 @@ import com.google.common.io.CharSink; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaClusterDTO; -import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiClusterListDTO; +import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiClusterListDto; import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext; import java.io.Writer; import java.net.URI; @@ -128,8 +128,8 @@ public void doRun_clusterNotProvided_fetchAllClusters() throws Exception { argThat( content -> { try { - ApiClusterListDTO listDto = - objectMapper.readValue((String) content, ApiClusterListDTO.class); + ApiClusterListDto listDto = + objectMapper.readValue((String) content, ApiClusterListDto.class); assertNotNull(listDto.getClusters()); } catch (JsonProcessingException e) { throw new RuntimeException(e); @@ -170,8 +170,8 @@ public void doRun_clusterProvided_fetchOnlyProvidedCluster() throws Exception { argThat( content -> { try { - ApiClusterListDTO listDto = - objectMapper.readValue((String) content, ApiClusterListDTO.class); + ApiClusterListDto listDto = + objectMapper.readValue((String) content, ApiClusterListDto.class); assertNotNull(listDto.getClusters()); } catch (JsonProcessingException e) { throw new RuntimeException(e); diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaCMFHostsTaskTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaCmfHostsTaskTest.java similarity index 98% rename from dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaCMFHostsTaskTest.java rename to dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaCmfHostsTaskTest.java index 9be755398..6aa2a8595 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaCMFHostsTaskTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaCmfHostsTaskTest.java @@ -53,10 +53,10 @@ import org.mockito.junit.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) -public class ClouderaCMFHostsTaskTest { +public class ClouderaCmfHostsTaskTest { private static WireMockServer server; - private final ClouderaCMFHostsTask task = new ClouderaCMFHostsTask(); + private final ClouderaCmfHostsTask task = new ClouderaCmfHostsTask(); private ClouderaManagerHandle handle; @Mock private TaskRunContext context; diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTaskTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRamChartTaskTest.java similarity index 98% rename from dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTaskTest.java rename to dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRamChartTaskTest.java index 9abf1835e..4164a895d 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTaskTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRamChartTaskTest.java @@ -60,9 +60,9 @@ import org.mockito.junit.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) -public class ClouderaHostRAMChartTaskTest { - private final ClouderaHostRAMChartTask task = - new ClouderaHostRAMChartTask( +public class ClouderaHostRamChartTaskTest { + private final ClouderaHostRamChartTask task = + new ClouderaHostRamChartTask( timeTravelDaysAgo(1), timeTravelDaysAgo(0), TimeSeriesAggregation.HOURLY, diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnectorTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnectorTest.java index 70b755421..e6d843f1f 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnectorTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnectorTest.java @@ -87,14 +87,14 @@ public void addTasksTo_checkFileWasGeneratedByProperTask() throws Exception { "compilerworks-metadata.yaml", DumpMetadataTask.class, "compilerworks-format.txt", FormatTask.class, "clusters.json", ClouderaClustersTask.class, - "cmf-hosts.jsonl", ClouderaCMFHostsTask.class, - "api-hosts.jsonl", ClouderaAPIHostsTask.class, + "cmf-hosts.jsonl", ClouderaCmfHostsTask.class, + "api-hosts.jsonl", ClouderaApiHostsTask.class, "services.jsonl", ClouderaServicesTask.class)) .putAll( ImmutableMap.of( "host-components.jsonl", ClouderaHostComponentsTask.class, - "cluster-cpu.jsonl", ClouderaClusterCPUChartTask.class, - "host-ram.jsonl", ClouderaHostRAMChartTask.class, + "cluster-cpu.jsonl", ClouderaClusterCpuChartTask.class, + "host-ram.jsonl", ClouderaHostRamChartTask.class, "service-resource-allocation.jsonl", ClouderaServiceResourceAllocationChartTask.class, "yarn-applications.jsonl", ClouderaYarnApplicationsTask.class, @@ -172,14 +172,14 @@ public void addTasksTo_checkFileWasGeneratedByProperTaskWithCustomDateRange() th "compilerworks-metadata.yaml", DumpMetadataTask.class, "compilerworks-format.txt", FormatTask.class, "clusters.json", ClouderaClustersTask.class, - "cmf-hosts.jsonl", ClouderaCMFHostsTask.class, - "api-hosts.jsonl", ClouderaAPIHostsTask.class, + "cmf-hosts.jsonl", ClouderaCmfHostsTask.class, + "api-hosts.jsonl", ClouderaApiHostsTask.class, "services.jsonl", ClouderaServicesTask.class, "host-components.jsonl", ClouderaHostComponentsTask.class)) .putAll( ImmutableMap.of( - "cluster-cpu.jsonl", ClouderaClusterCPUChartTask.class, - "host-ram.jsonl", ClouderaHostRAMChartTask.class, + "cluster-cpu.jsonl", ClouderaClusterCpuChartTask.class, + "host-ram.jsonl", ClouderaHostRamChartTask.class, "service-resource-allocation.jsonl", ClouderaServiceResourceAllocationChartTask.class, "yarn-applications.jsonl", ClouderaYarnApplicationsTask.class, diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/SparkHistoryDiscoveryServiceTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/SparkHistoryDiscoveryServiceTest.java new file mode 100644 index 000000000..3a7332c66 --- /dev/null +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/SparkHistoryDiscoveryServiceTest.java @@ -0,0 +1,374 @@ +/* + * Copyright 2022-2025 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager; + +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.okJson; +import static com.github.tomakehurst.wiremock.client.WireMock.serverError; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import java.net.URI; +import java.util.List; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class SparkHistoryDiscoveryServiceTest { + + private static WireMockServer server; + private SparkHistoryDiscoveryService service; + private CloseableHttpClient cmClient; + private ObjectMapper objectMapper = new ObjectMapper(); + + @Mock private CloseableHttpClient knoxClient; + + @BeforeClass + public static void beforeClass() { + server = new WireMockServer(WireMockConfiguration.wireMockConfig().dynamicPort()); + server.start(); + } + + @AfterClass + public static void afterClass() { + server.stop(); + } + + @Before + public void setUp() { + server.resetAll(); + cmClient = HttpClients.createDefault(); + URI apiUri = URI.create(server.baseUrl() + "/api/v41/"); + service = new SparkHistoryDiscoveryService(objectMapper, cmClient, apiUri); + } + + @Test + public void resolveUrl_spark3_success() throws Exception { + // 1. Mock services list + server.stubFor( + get("/api/v41/clusters/my-cluster/services") + .willReturn(okJson("{\"items\": [{\"name\": \"knox-1\", \"type\": \"KNOX\"}]}"))); + + // 2. Mock roles list + server.stubFor( + get("/api/v41/clusters/my-cluster/services/knox-1/roles") + .willReturn( + okJson( + "{\"items\": [{\"hostRef\": {\"hostname\": \"knox-host\"}, \"roleConfigGroupRef\": {\"roleConfigGroupName\": \"knox-BASE\"}}]}"))); + + // 3. Mock config + server.stubFor( + get("/api/v41/clusters/my-cluster/services/knox-1/roleConfigGroups/knox-BASE/config") + .willReturn( + okJson( + "{\"items\": [{\"name\": \"gateway_path\", \"value\": \"gateway\"}, {\"name\": \"gateway_default_api_topology_name\", \"value\": \"cdp-proxy-api\"}]}"))); + + // 4. Mock Knox probe for Spark 3 + CloseableHttpResponse mockResponse = mock(CloseableHttpResponse.class); + StatusLine mockStatusLine = mock(StatusLine.class); + when(mockStatusLine.getStatusCode()).thenReturn(200); + when(mockResponse.getStatusLine()).thenReturn(mockStatusLine); + + when(knoxClient.execute(any(HttpGet.class))) + .thenAnswer( + invocation -> { + HttpGet get = invocation.getArgument(0); + if (get.getURI().toString().contains("spark3history")) { + return mockResponse; + } + CloseableHttpResponse failResponse = mock(CloseableHttpResponse.class); + StatusLine failStatus = mock(StatusLine.class); + when(failStatus.getStatusCode()).thenReturn(404); + when(failResponse.getStatusLine()).thenReturn(failStatus); + return failResponse; + }); + + // Act + List result = + service.resolveUrl("my-cluster", knoxClient, com.google.common.collect.ImmutableList.of()); + + // Assert + assertEquals(1, result.size()); + assertEquals("https://knox-host/gateway/cdp-proxy-api/spark3history/api/v1", result.get(0)); + } + + @Test + public void resolveUrl_spark2_success() throws Exception { + // 1. Mock services list + server.stubFor( + get("/api/v41/clusters/my-cluster/services") + .willReturn(okJson("{\"items\": [{\"name\": \"knox-1\", \"type\": \"KNOX\"}]}"))); + + // 2. Mock roles list + server.stubFor( + get("/api/v41/clusters/my-cluster/services/knox-1/roles") + .willReturn( + okJson( + "{\"items\": [{\"hostRef\": {\"hostname\": \"knox-host\"}, \"roleConfigGroupRef\": {\"roleConfigGroupName\": \"knox-BASE\"}}]}"))); + + // 3. Mock config + server.stubFor( + get("/api/v41/clusters/my-cluster/services/knox-1/roleConfigGroups/knox-BASE/config") + .willReturn( + okJson( + "{\"items\": [{\"name\": \"gateway_path\", \"value\": \"gateway\"}, {\"name\": \"gateway_default_api_topology_name\", \"value\": \"cdp-proxy-api\"}]}"))); + + // 4. Mock Knox probe for Spark 2 (Spark 3 fails) + CloseableHttpResponse mockResponse = mock(CloseableHttpResponse.class); + StatusLine mockStatusLine = mock(StatusLine.class); + when(mockStatusLine.getStatusCode()).thenReturn(200); + when(mockResponse.getStatusLine()).thenReturn(mockStatusLine); + + when(knoxClient.execute(any(HttpGet.class))) + .thenAnswer( + invocation -> { + HttpGet get = invocation.getArgument(0); + if (get.getURI().toString().contains("sparkhistory")) { + return mockResponse; + } + CloseableHttpResponse failResponse = mock(CloseableHttpResponse.class); + StatusLine failStatus = mock(StatusLine.class); + when(failStatus.getStatusCode()).thenReturn(404); + when(failResponse.getStatusLine()).thenReturn(failStatus); + return failResponse; + }); + + // Act + List result = + service.resolveUrl("my-cluster", knoxClient, com.google.common.collect.ImmutableList.of()); + + // Assert + assertEquals(1, result.size()); + assertEquals("https://knox-host/gateway/cdp-proxy-api/sparkhistory/api/v1", result.get(0)); + } + + @Test + public void resolveUrl_multipleReachable_success() throws Exception { + // 1. Mock services list + server.stubFor( + get("/api/v41/clusters/my-cluster/services") + .willReturn(okJson("{\"items\": [{\"name\": \"knox-1\", \"type\": \"KNOX\"}]}"))); + + // 2. Mock roles list + server.stubFor( + get("/api/v41/clusters/my-cluster/services/knox-1/roles") + .willReturn( + okJson( + "{\"items\": [{\"hostRef\": {\"hostname\": \"knox-host\"}, \"roleConfigGroupRef\": {\"roleConfigGroupName\": \"knox-BASE\"}}]}"))); + + // 3. Mock config + server.stubFor( + get("/api/v41/clusters/my-cluster/services/knox-1/roleConfigGroups/knox-BASE/config") + .willReturn( + okJson( + "{\"items\": [{\"name\": \"gateway_path\", \"value\": \"gateway\"}, {\"name\": \"gateway_default_api_topology_name\", \"value\": \"cdp-proxy-api\"}]}"))); + + // 4. Mock Knox probe for both Spark 2 and Spark 3 + CloseableHttpResponse mockResponse = mock(CloseableHttpResponse.class); + StatusLine mockStatusLine = mock(StatusLine.class); + when(mockStatusLine.getStatusCode()).thenReturn(200); + when(mockResponse.getStatusLine()).thenReturn(mockStatusLine); + + when(knoxClient.execute(any(HttpGet.class))).thenReturn(mockResponse); + + // Act + List result = + service.resolveUrl("my-cluster", knoxClient, com.google.common.collect.ImmutableList.of()); + + // Assert + assertEquals(2, result.size()); + assertTrue(result.contains("https://knox-host/gateway/cdp-proxy-api/spark3history/api/v1")); + assertTrue(result.contains("https://knox-host/gateway/cdp-proxy-api/sparkhistory/api/v1")); + } + + @Test + public void resolveUrl_customCandidate_success() throws Exception { + // 1. Mock services list + server.stubFor( + get("/api/v41/clusters/my-cluster/services") + .willReturn(okJson("{\"items\": [{\"name\": \"knox-1\", \"type\": \"KNOX\"}]}"))); + + // 2. Mock roles list + server.stubFor( + get("/api/v41/clusters/my-cluster/services/knox-1/roles") + .willReturn( + okJson( + "{\"items\": [{\"hostRef\": {\"hostname\": \"knox-host\"}, \"roleConfigGroupRef\": {\"roleConfigGroupName\": \"knox-BASE\"}}]}"))); + + // 3. Mock config + server.stubFor( + get("/api/v41/clusters/my-cluster/services/knox-1/roleConfigGroups/knox-BASE/config") + .willReturn( + okJson( + "{\"items\": [{\"name\": \"gateway_path\", \"value\": \"gateway\"}, {\"name\": \"gateway_default_api_topology_name\", \"value\": \"cdp-proxy-api\"}]}"))); + + // 4. Mock Knox probe for custom path + CloseableHttpResponse mockResponse = mock(CloseableHttpResponse.class); + StatusLine mockStatusLine = mock(StatusLine.class); + when(mockStatusLine.getStatusCode()).thenReturn(200); + when(mockResponse.getStatusLine()).thenReturn(mockStatusLine); + + when(knoxClient.execute(any(HttpGet.class))) + .thenAnswer( + invocation -> { + HttpGet get = invocation.getArgument(0); + if (get.getURI().toString().contains("custom-spark")) { + return mockResponse; + } + CloseableHttpResponse failResponse = mock(CloseableHttpResponse.class); + StatusLine failStatus = mock(StatusLine.class); + when(failStatus.getStatusCode()).thenReturn(404); + when(failResponse.getStatusLine()).thenReturn(failStatus); + return failResponse; + }); + + // Act + List result = + service.resolveUrl( + "my-cluster", knoxClient, com.google.common.collect.ImmutableList.of("custom-spark")); + + // Assert + assertEquals(1, result.size()); + assertEquals("https://knox-host/gateway/cdp-proxy-api/custom-spark/api/v1", result.get(0)); + } + + @Test + public void resolveUrl_neitherSparkVersionReachable_returnsEmpty() throws Exception { + // 1. Mock services list + server.stubFor( + get("/api/v41/clusters/my-cluster/services") + .willReturn(okJson("{\"items\": [{\"name\": \"knox-1\", \"type\": \"KNOX\"}]}"))); + + // 2. Mock roles list + server.stubFor( + get("/api/v41/clusters/my-cluster/services/knox-1/roles") + .willReturn( + okJson( + "{\"items\": [{\"hostRef\": {\"hostname\": \"knox-host\"}, \"roleConfigGroupRef\": {\"roleConfigGroupName\": \"knox-BASE\"}}]}"))); + + // 3. Mock config + server.stubFor( + get("/api/v41/clusters/my-cluster/services/knox-1/roleConfigGroups/knox-BASE/config") + .willReturn( + okJson( + "{\"items\": [{\"name\": \"gateway_path\", \"value\": \"gateway\"}, {\"name\": \"gateway_default_api_topology_name\", \"value\": \"cdp-proxy-api\"}]}"))); + + // 4. Mock Knox probe always returning 404 + CloseableHttpResponse failResponse = mock(CloseableHttpResponse.class); + StatusLine failStatus = mock(StatusLine.class); + when(failStatus.getStatusCode()).thenReturn(404); + when(failResponse.getStatusLine()).thenReturn(failStatus); + when(knoxClient.execute(any(HttpGet.class))).thenReturn(failResponse); + + List result = + service.resolveUrl("my-cluster", knoxClient, com.google.common.collect.ImmutableList.of()); + + assertTrue(result.isEmpty()); + } + + @Test + public void resolveUrl_missingHostRef_returnsEmpty() throws Exception { + // 1. Mock services list + server.stubFor( + get("/api/v41/clusters/my-cluster/services") + .willReturn(okJson("{\"items\": [{\"name\": \"knox-1\", \"type\": \"KNOX\"}]}"))); + + // 2. Mock roles list with missing hostRef + server.stubFor( + get("/api/v41/clusters/my-cluster/services/knox-1/roles") + .willReturn( + okJson( + "{\"items\": [{\"roleConfigGroupRef\": {\"roleConfigGroupName\": \"knox-BASE\"}}]}"))); + + // Act + List result = + service.resolveUrl("my-cluster", knoxClient, com.google.common.collect.ImmutableList.of()); + + // Assert + assertTrue(result.isEmpty()); + } + + @Test + public void resolveUrl_noKnoxService_returnsEmpty() throws Exception { + // Mock empty services list + server.stubFor( + get("/api/v41/clusters/my-cluster/services").willReturn(okJson("{\"items\": []}"))); + + List result = + service.resolveUrl("my-cluster", knoxClient, com.google.common.collect.ImmutableList.of()); + + assertTrue(result.isEmpty()); + } + + @Test + public void resolveUrl_cmApiError_returnsEmpty() throws Exception { + // Mock CM API returning 500 + server.stubFor(get("/api/v41/clusters/my-cluster/services").willReturn(serverError())); + + List result = + service.resolveUrl("my-cluster", knoxClient, com.google.common.collect.ImmutableList.of()); + + assertTrue(result.isEmpty()); + } + + @Test + public void resolveUrl_knoxNotReachable_returnsEmpty() throws Exception { + // 1. Mock services list + server.stubFor( + get("/api/v41/clusters/my-cluster/services") + .willReturn(okJson("{\"items\": [{\"name\": \"knox-1\", \"type\": \"KNOX\"}]}"))); + + // 2. Mock roles list + server.stubFor( + get("/api/v41/clusters/my-cluster/services/knox-1/roles") + .willReturn( + okJson( + "{\"items\": [{\"hostRef\": {\"hostname\": \"knox-host\"}, \"roleConfigGroupRef\": {\"roleConfigGroupName\": \"knox-BASE\"}}]}"))); + + // 3. Mock config + server.stubFor( + get("/api/v41/clusters/my-cluster/services/knox-1/roleConfigGroups/knox-BASE/config") + .willReturn( + okJson( + "{\"items\": [{\"name\": \"gateway_path\", \"value\": \"gateway\"}, {\"name\": \"gateway_default_api_topology_name\", \"value\": \"cdp-proxy-api\"}]}"))); + + // 4. Mock Knox probe always failing + when(knoxClient.execute(any(HttpGet.class))) + .thenThrow(new java.io.IOException("Connection refused")); + + List result = + service.resolveUrl("my-cluster", knoxClient, com.google.common.collect.ImmutableList.of()); + + assertTrue(result.isEmpty()); + } +} diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiYARNApplicationDTOTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiYarnApplicationDtoTest.java similarity index 88% rename from dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiYARNApplicationDTOTest.java rename to dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiYarnApplicationDtoTest.java index 248161a22..2aa35a396 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiYARNApplicationDTOTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiYarnApplicationDtoTest.java @@ -27,14 +27,14 @@ import java.nio.file.Paths; import org.junit.Test; -public class ApiYARNApplicationDTOTest { +public class ApiYarnApplicationDtoTest { private static final ObjectMapper objectMapper = new ObjectMapper(); @Test public void applicationIdExists() throws Exception { JsonNode jsonNode = objectMapper.readTree(readString("/cloudera/manager/dto/ApiYARNApplicationDTO.json")); - ApiYARNApplicationDTO dto = new ApiYARNApplicationDTO(jsonNode); + ApiYarnApplicationDto dto = new ApiYarnApplicationDto(jsonNode); assertEquals("job_1741847944157_0018", dto.getApplicationId()); } @@ -42,14 +42,14 @@ public void applicationIdExists() throws Exception { @Test public void applicationIdDoesNotExist() throws Exception { JsonNode jsonNode = objectMapper.readTree("{ \"prop\": \"val\"}"); - ApiYARNApplicationDTO dto = new ApiYARNApplicationDTO(jsonNode); + ApiYarnApplicationDto dto = new ApiYarnApplicationDto(jsonNode); assertNull("ApplicationId must be null if doesn't exist", dto.getApplicationId()); } @Test public void nullJsonFieldHandled() { - ApiYARNApplicationDTO dto = new ApiYARNApplicationDTO(null); + ApiYarnApplicationDto dto = new ApiYarnApplicationDto(null); assertNull("ApplicationId must be null for null json", dto.getApplicationId()); } @@ -57,7 +57,7 @@ public void nullJsonFieldHandled() { @Test public void jsonSerialization() throws Exception { JsonNode yarnApp = objectMapper.readTree("{ \"prop\": \"val\"}"); - ApiYARNApplicationDTO dto = new ApiYARNApplicationDTO(yarnApp); + ApiYarnApplicationDto dto = new ApiYarnApplicationDto(yarnApp); JsonNode outputJsonl = objectMapper.convertValue(dto, JsonNode.class);