[b456377401] Implement Spark History discovery

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaYarnApplicationTask.java

@@ -19,7 +19,7 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.google.common.base.Preconditions;
-import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiYARNApplicationDTO;
+import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiYarnApplicationDto;
 import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
 import java.io.IOException;
 import java.net.URI;
@@ -83,20 +83,20 @@ public PaginatedClouderaYarnApplicationsLoader(ClouderaManagerHandle handle, int
       toAppCreationDate = toDate.format(dtFormatter);
     }
 
-    public int load(String clusterName, Consumer<List<ApiYARNApplicationDTO>> onPageLoad) {
+    public int load(String clusterName, Consumer<List<ApiYarnApplicationDto>> onPageLoad) {
       return load(clusterName, null, onPageLoad);
     }
 
     public int load(
         String clusterName,
         @Nullable String appType,
-        Consumer<List<ApiYARNApplicationDTO>> onPageLoad) {
+        Consumer<List<ApiYarnApplicationDto>> onPageLoad) {
       offset = 0;
       boolean fetchedNewApps;
       do {
         fetchedNewApps = false;
         URI yarnAppsURI = buildNextYARNApplicationPageURI(clusterName, appType);
-        List<ApiYARNApplicationDTO> newLoad = load(yarnAppsURI);
+        List<ApiYarnApplicationDto> newLoad = load(yarnAppsURI);
         if (!newLoad.isEmpty()) {
           onPageLoad.accept(newLoad);
           offset += newLoad.size();
@@ -106,7 +106,7 @@ public int load(
       return offset;
     }
 
-    private List<ApiYARNApplicationDTO> load(URI yarnAppURI) {
+    private List<ApiYarnApplicationDto> load(URI yarnAppURI) {
       try (CloseableHttpResponse resp = httpClient.execute(new HttpGet(yarnAppURI))) {
         int statusCode = resp.getStatusLine().getStatusCode();
         if (!isStatusCodeOK(statusCode)) {
@@ -130,10 +130,10 @@ private List<ApiYARNApplicationDTO> load(URI yarnAppURI) {
       }
     }
 
-    private List<ApiYARNApplicationDTO> toDTOs(ArrayNode applicationsArray) {
-      List<ApiYARNApplicationDTO> yarnApplicationDTOs = new ArrayList<>();
+    private List<ApiYarnApplicationDto> toDTOs(ArrayNode applicationsArray) {
+      List<ApiYarnApplicationDto> yarnApplicationDTOs = new ArrayList<>();
       for (JsonNode application : applicationsArray) {
-        yarnApplicationDTOs.add(new ApiYARNApplicationDTO(application));
+        yarnApplicationDTOs.add(new ApiYarnApplicationDto(application));
       }
 
       return yarnApplicationDTOs;

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaAPIHostsTask.java → dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaApiHostsTask.java

@@ -21,8 +21,8 @@
 import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaClusterDTO;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaHostDTO;
-import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiHostDTO;
-import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiHostListDTO;
+import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiHostDto;
+import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiHostListDto;
 import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;
 import java.io.Writer;
 import java.nio.charset.StandardCharsets;
@@ -38,9 +38,9 @@
  * href="https://archive.cloudera.com/cm7/7.11.3.0/generic/jar/cm_api/apidocs/json_ApiHostList.html">Hosts
  * API</a> which doesn't contain usage and disk data and collected as a fallback.
  */
-public class ClouderaAPIHostsTask extends AbstractClouderaManagerTask {
+public class ClouderaApiHostsTask extends AbstractClouderaManagerTask {
 
-  public ClouderaAPIHostsTask() {
+  public ClouderaApiHostsTask() {
     super("api-hosts.jsonl");
   }
 
@@ -75,8 +75,8 @@ protected void doRun(
         writer.write(stringifiedHosts);
         writer.write('\n');
 
-        ApiHostListDTO apiHosts = parseJsonStringToObject(stringifiedHosts, ApiHostListDTO.class);
-        for (ApiHostDTO apiHost : apiHosts.getHosts()) {
+        ApiHostListDto apiHosts = parseJsonStringToObject(stringifiedHosts, ApiHostListDto.class);
+        for (ApiHostDto apiHost : apiHosts.getHosts()) {
           hosts.add(ClouderaHostDTO.create(apiHost.getId(), apiHost.getName()));
         }
       }

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTask.java → dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCpuChartTask.java

@@ -40,12 +40,12 @@
  * href="https://docs.cloudera.com/documentation/enterprise/latest/topics/cm_dg_tsquery.html">tsquery</a>
  * language.
  */
-public class ClouderaClusterCPUChartTask extends AbstractClouderaTimeSeriesTask {
-  private static final Logger logger = LoggerFactory.getLogger(ClouderaClusterCPUChartTask.class);
+public class ClouderaClusterCpuChartTask extends AbstractClouderaTimeSeriesTask {
+  private static final Logger logger = LoggerFactory.getLogger(ClouderaClusterCpuChartTask.class);
   private static final String TS_CPU_QUERY_TEMPLATE =
       "SELECT cpu_percent_across_hosts WHERE entityName = \"%s\" AND category = CLUSTER";
 
-  public ClouderaClusterCPUChartTask(
+  public ClouderaClusterCpuChartTask(
       ZonedDateTime startDate,
       ZonedDateTime endDate,
       TimeSeriesAggregation tsAggregation,

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClustersTask.java

@@ -20,8 +20,8 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.io.ByteSink;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaClusterDTO;
-import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiClusterDTO;
-import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiClusterListDTO;
+import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiClusterDto;
+import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiClusterListDto;
 import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;
 import java.io.Writer;
 import java.net.URI;
@@ -52,18 +52,18 @@ protected void doRun(
       throws Exception {
     CloseableHttpClient httpClient = handle.getClouderaManagerHttpClient();
 
-    ApiClusterListDTO clusterList;
+    ApiClusterListDto clusterList;
 
     if (context.getArguments().getCluster() != null) {
       final String clusterName = context.getArguments().getCluster();
       try (CloseableHttpResponse clusterResponse =
           httpClient.execute(new HttpGet(handle.getApiURI() + "/clusters/" + clusterName))) {
 
-        ApiClusterDTO cluster =
+        ApiClusterDto cluster =
             parseJsonStringToObject(
-                EntityUtils.toString(clusterResponse.getEntity()), ApiClusterDTO.class);
+                EntityUtils.toString(clusterResponse.getEntity()), ApiClusterDto.class);
 
-        clusterList = new ApiClusterListDTO();
+        clusterList = new ApiClusterListDto();
         clusterList.setClusters(ImmutableList.of(cluster));
       }
     } else {
@@ -74,7 +74,7 @@ protected void doRun(
       try (CloseableHttpResponse clustersResponse =
           httpClient.execute(new HttpGet(handle.getApiURI() + "/clusters?clusterType=ANY"))) {
         String clustersJson = EntityUtils.toString(clustersResponse.getEntity());
-        clusterList = parseJsonStringToObject(clustersJson, ApiClusterListDTO.class);
+        clusterList = parseJsonStringToObject(clustersJson, ApiClusterListDto.class);
       }
     }
 
@@ -83,7 +83,7 @@ protected void doRun(
     }
 
     List<ClouderaClusterDTO> clusters = new ArrayList<>();
-    for (ApiClusterDTO item : clusterList.getClusters()) {
+    for (ApiClusterDto item : clusterList.getClusters()) {
       String clusterId = requestClusterIdByName(httpClient, handle.getBaseURI(), item.getName());
       clusters.add(ClouderaClusterDTO.create(clusterId, item.getName()));
     }

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaCMFHostsTask.java → dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaCmfHostsTask.java

@@ -37,11 +37,11 @@
  * The task collects hosts from Cloudera Manager {@code /cmf/} urls. These API contains
  * well-structured data but is not well documented.
  */
-public class ClouderaCMFHostsTask extends AbstractClouderaManagerTask {
+public class ClouderaCmfHostsTask extends AbstractClouderaManagerTask {
 
-  private static final Logger logger = LoggerFactory.getLogger(ClouderaCMFHostsTask.class);
+  private static final Logger logger = LoggerFactory.getLogger(ClouderaCmfHostsTask.class);
 
-  public ClouderaCMFHostsTask() {
+  public ClouderaCmfHostsTask() {
     super("cmf-hosts.jsonl");
   }
 

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTask.java → dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRamChartTask.java

@@ -39,14 +39,14 @@
  * href="https://docs.cloudera.com/documentation/enterprise/latest/topics/cm_dg_tsquery.html">tsquery</a>
  * language.
  */
-public class ClouderaHostRAMChartTask extends AbstractClouderaTimeSeriesTask {
+public class ClouderaHostRamChartTask extends AbstractClouderaTimeSeriesTask {
 
-  private static final Logger logger = LoggerFactory.getLogger(ClouderaHostRAMChartTask.class);
+  private static final Logger logger = LoggerFactory.getLogger(ClouderaHostRamChartTask.class);
 
   private static final String TS_RAM_QUERY_TEMPLATE =
       "select swap_used, physical_memory_used, physical_memory_total, physical_memory_cached, physical_memory_buffers where entityName = \"%s\"";
 
-  public ClouderaHostRAMChartTask(
+  public ClouderaHostRamChartTask(
       ZonedDateTime startDate,
       ZonedDateTime endDate,
       TimeSeriesAggregation tsAggregation,

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnector.java

@@ -91,8 +91,8 @@ public void addTasksTo(@Nonnull List<? super Task<?>> out, @Nonnull ConnectorArg
     out.add(new DumpMetadataTask(arguments, FORMAT_NAME));
     out.add(new FormatTask(FORMAT_NAME));
     out.add(new ClouderaClustersTask());
-    out.add(new ClouderaCMFHostsTask());
-    out.add(new ClouderaAPIHostsTask());
+    out.add(new ClouderaCmfHostsTask());
+    out.add(new ClouderaApiHostsTask());
     out.add(new ClouderaServicesTask());
     out.add(new ClouderaHostComponentsTask());
 
@@ -107,8 +107,8 @@ public void addTasksTo(@Nonnull List<? super Task<?>> out, @Nonnull ConnectorArg
       endDate = arguments.getEndDate();
     }
 
-    out.add(new ClouderaClusterCPUChartTask(startDate, endDate, DAILY, REQUIRED));
-    out.add(new ClouderaHostRAMChartTask(startDate, endDate, DAILY, REQUIRED));
+    out.add(new ClouderaClusterCpuChartTask(startDate, endDate, DAILY, REQUIRED));
+    out.add(new ClouderaHostRamChartTask(startDate, endDate, DAILY, REQUIRED));
     out.add(new ClouderaServiceResourceAllocationChartTask(startDate, endDate, HOURLY, OPTIONAL));
     out.add(new ClouderaYarnApplicationsTask(startDate, endDate, OPTIONAL));
     out.add(new ClouderaYarnApplicationTypeTask(startDate, endDate, OPTIONAL));

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerHandle.java

@@ -19,6 +19,7 @@
 import com.google.auto.value.AutoValue;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
+import com.google.common.io.Closer;
 import com.google.edwmigration.dumper.application.dumper.handle.Handle;
 import java.io.IOException;
 import java.net.URI;
@@ -35,6 +36,7 @@ public class ClouderaManagerHandle implements Handle {
   private final URI apiURI;
   private final CloseableHttpClient clouderaManagerHttpClient;
   private final CloseableHttpClient basicAuthHttpClient;
+  private final Closer closer;
 
   private ImmutableList<ClouderaClusterDTO> clusters;
   private ImmutableList<ClouderaHostDTO> hosts;
@@ -52,6 +54,9 @@ public ClouderaManagerHandle(
     this.apiURI = unify(apiURI);
     this.clouderaManagerHttpClient = clouderaManagerHttpClient;
     this.basicAuthHttpClient = basicAuthHttpClient;
+    this.closer = Closer.create();
+    closer.register(this.clouderaManagerHttpClient);
+    closer.register(this.basicAuthHttpClient);
   }
 
   /** 1. Remove query params and url fragments 2. Add trailing slash for safety */
@@ -130,10 +135,9 @@ public synchronized void initSparkYarnApplications(
   }
 
   @Override
-  public void close() throws IOException {
+  public synchronized void close() throws IOException {
     try {
-      clouderaManagerHttpClient.close();
-      basicAuthHttpClient.close();
+      closer.close();
     } catch (IOException ignore) {
       // The intention is to do graceful shutdown and try to release the resource.
       // In case of errors we do not need to interrupt the execution flow

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationTypeTask.java

@@ -27,7 +27,7 @@
 import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaClusterDTO;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaYarnApplicationDTO;
-import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiYARNApplicationDTO;
+import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiYarnApplicationDto;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.model.YarnApplicationType;
 import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
 import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;
@@ -108,9 +108,9 @@ protected void doRun(
   }
 
   private void writeYarnAppTypes(
-      Writer writer, List<ApiYARNApplicationDTO> yarnApps, String appType, String clusterName) {
+      Writer writer, List<ApiYarnApplicationDto> yarnApps, String appType, String clusterName) {
     List<ApplicationTypeToYarnApplication> yarnAppTypeMappings = new ArrayList<>();
-    for (ApiYARNApplicationDTO yarnApp : yarnApps) {
+    for (ApiYarnApplicationDto yarnApp : yarnApps) {
       yarnAppTypeMappings.add(
           new ApplicationTypeToYarnApplication(yarnApp.getApplicationId(), appType, clusterName));
     }

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationsTask.java

@@ -20,7 +20,7 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.io.ByteSink;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaClusterDTO;
-import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiYARNApplicationDTO;
+import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiYarnApplicationDto;
 import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
 import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;
 import java.io.IOException;
@@ -67,8 +67,8 @@ protected void doRun(
   }
 
   private void writeYarnApplications(
-      Writer writer, List<ApiYARNApplicationDTO> yarnApps, String clusterName) {
-    for (ApiYARNApplicationDTO yarnApp : yarnApps) {
+      Writer writer, List<ApiYarnApplicationDto> yarnApps, String clusterName) {
+    for (ApiYarnApplicationDto yarnApp : yarnApps) {
       yarnApp.setClusterName(clusterName);
     }
     try {