[b/413037987] Add more fields to Yarn applications response (#880)

* add more fields to Cloudera YARN API

* change Cloudera YARN application output

* rename variable

* remove unused code

* rename ouput json YARN to Yarn

* add unit test for json field name

Author: vladislav-sidorovich

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaYarnApplicationTask.java

@@ -16,9 +16,10 @@
  */
 package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager;
 
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.google.common.base.Preconditions;
 import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiYARNApplicationDTO;
-import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiYARNApplicationListDTO;
 import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
 import java.io.IOException;
 import java.net.URI;
@@ -27,6 +28,7 @@
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Consumer;
 import javax.annotation.Nonnull;
@@ -111,14 +113,30 @@ private List<ApiYARNApplicationDTO> load(URI yarnAppURI) {
                   "YARN application API returned HTTP status %d. Message: %s",
                   statusCode, readFromStream(resp.getEntity().getContent())));
         }
-        ApiYARNApplicationListDTO yarnAppListDto =
-            parseJsonStreamToObject(resp.getEntity().getContent(), ApiYARNApplicationListDTO.class);
-        return yarnAppListDto.getApplications();
+
+        JsonNode applicationsResponse = readJsonTree(resp.getEntity().getContent());
+        JsonNode applicationsArray = applicationsResponse.at("/applications");
+        if (!applicationsArray.isArray()) {
+          throw new IllegalArgumentException(
+              "Unexpected JSON response without `applications`. "
+                  + "Response: "
+                  + applicationsResponse);
+        }
+        return toDTOs((ArrayNode) applicationsArray);
       } catch (IOException ex) {
         throw new ClouderaConnectorException(ex.getMessage(), ex);
       }
     }
 
+    private List<ApiYARNApplicationDTO> toDTOs(ArrayNode applicationsArray) {
+      List<ApiYARNApplicationDTO> yarnApplicationDTOs = new ArrayList<>();
+      for (JsonNode application : applicationsArray) {
+        yarnApplicationDTOs.add(new ApiYARNApplicationDTO(application));
+      }
+
+      return yarnApplicationDTOs;
+    }
+
     private URI buildNextYARNApplicationPageURI(String clusterName, @Nullable String appType) {
       try {
         URIBuilder uriBuilder =

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiYARNApplicationDTO.java

@@ -16,146 +16,52 @@
  */
 package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import java.util.List;
-import java.util.Map;
+import com.fasterxml.jackson.databind.JsonNode;
+import org.springframework.util.StringUtils;
 
 /**
  * DTO class for the official Cloudera API <a
  * href="https://archive.cloudera.com/cm7/7.0.3/generic/jar/cm_api/apidocs/json_ApiYarnApplication.html">json</a>
+ * with additional information about cluster.
  */
 @JsonIgnoreProperties(ignoreUnknown = true)
 public class ApiYARNApplicationDTO {
-  @JsonProperty(required = true)
-  private String clusterName;
-
-  @JsonProperty(required = true)
-  private String applicationId;
-
-  @JsonProperty(required = true)
-  private String name;
-
-  @JsonProperty(required = true)
-  private String state;
-
-  @JsonProperty(required = true)
-  private String startTime;
-
-  @JsonProperty(required = true)
-  private String endTime;
-
-  @JsonProperty(required = true)
-  private String user;
-
-  @JsonProperty(required = true)
-  private String pool;
 
   @JsonProperty(required = true)
-  private List<String> applicationTags;
-
-  @JsonProperty(required = true)
-  private long allocatedMemorySeconds;
-
-  @JsonProperty(required = true)
-  private long allocatedVcoreSeconds;
+  private String clusterName;
 
-  @JsonProperty(required = true)
-  private Map<String, String> attributes;
+  /**
+   * DTO class for the official Cloudera API <a
+   * href="https://archive.cloudera.com/cm7/7.0.3/generic/jar/cm_api/apidocs/json_ApiYarnApplication.html">json</a>.
+   * JsonNode used to support schema evaluation.
+   */
+  private final JsonNode apiYarnApplication;
 
-  public String getClusterName() {
-    return clusterName;
+  public ApiYARNApplicationDTO(JsonNode apiYarnApplication) {
+    this.apiYarnApplication = apiYarnApplication;
   }
 
+  @JsonIgnore
   public String getApplicationId() {
-    return applicationId;
-  }
-
-  public String getName() {
-    return name;
-  }
-
-  public String getState() {
-    return state;
-  }
-
-  public String getStartTime() {
-    return startTime;
-  }
-
-  public String getEndTime() {
-    return endTime;
+    if (apiYarnApplication == null) {
+      return null;
+    }
+    String applicationId = apiYarnApplication.at("/applicationId").asText();
+    return StringUtils.hasText(applicationId) ? applicationId : null;
   }
 
-  public String getUser() {
-    return user;
-  }
-
-  public String getPool() {
-    return pool;
-  }
-
-  public List<String> getApplicationTags() {
-    return applicationTags;
-  }
-
-  public long getAllocatedMemorySeconds() {
-    return allocatedMemorySeconds;
-  }
-
-  public long getAllocatedVcoreSeconds() {
-    return allocatedVcoreSeconds;
-  }
-
-  public Map<String, String> getAttributes() {
-    return attributes;
+  public String getClusterName() {
+    return clusterName;
   }
 
   public void setClusterName(String clusterName) {
     this.clusterName = clusterName;
   }
 
-  public void setApplicationId(String applicationId) {
-    this.applicationId = applicationId;
-  }
-
-  public void setName(String name) {
-    this.name = name;
-  }
-
-  public void setState(String state) {
-    this.state = state;
-  }
-
-  public void setStartTime(String startTime) {
-    this.startTime = startTime;
-  }
-
-  public void setEndTime(String endTime) {
-    this.endTime = endTime;
-  }
-
-  public void setUser(String user) {
-    this.user = user;
-  }
-
-  public void setPool(String pool) {
-    this.pool = pool;
-  }
-
-  public void setApplicationTags(List<String> applicationTags) {
-    this.applicationTags = applicationTags;
-  }
-
-  public void setAllocatedMemorySeconds(long allocatedMemorySeconds) {
-    this.allocatedMemorySeconds = allocatedMemorySeconds;
-  }
-
-  public void setAllocatedVcoreSeconds(long allocatedVcoreSeconds) {
-    this.allocatedVcoreSeconds = allocatedVcoreSeconds;
-  }
-
-  public void setAttributes(Map<String, String> attributes) {
-    this.attributes = attributes;
+  public JsonNode getApiYarnApplication() {
+    return apiYarnApplication;
   }
 }

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/ApiYARNApplicationListDTO.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2022-2025 Google LLC
+ * Copyright 2013-2021 CompilerWorks
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import org.junit.Test;
+
+public class ApiYARNApplicationDTOTest {
+  private static final ObjectMapper objectMapper = new ObjectMapper();
+
+  @Test
+  public void applicationIdExists() throws Exception {
+    JsonNode jsonNode =
+        objectMapper.readTree(readString("/cloudera/manager/dto/ApiYARNApplicationDTO.json"));
+    ApiYARNApplicationDTO dto = new ApiYARNApplicationDTO(jsonNode);
+
+    assertEquals("job_1741847944157_0018", dto.getApplicationId());
+  }
+
+  @Test
+  public void applicationIdDoesNotExist() throws Exception {
+    JsonNode jsonNode = objectMapper.readTree("{ \"prop\": \"val\"}");
+    ApiYARNApplicationDTO dto = new ApiYARNApplicationDTO(jsonNode);
+
+    assertNull("ApplicationId must be null if doesn't exist", dto.getApplicationId());
+  }
+
+  @Test
+  public void nullJsonFieldHandled() {
+    ApiYARNApplicationDTO dto = new ApiYARNApplicationDTO(null);
+
+    assertNull("ApplicationId must be null for null json", dto.getApplicationId());
+  }
+
+  @Test
+  public void jsonSerialization() throws Exception {
+    JsonNode yarnApp = objectMapper.readTree("{ \"prop\": \"val\"}");
+    ApiYARNApplicationDTO dto = new ApiYARNApplicationDTO(yarnApp);
+
+    JsonNode outputJsonl = objectMapper.convertValue(dto, JsonNode.class);
+
+    // json filed apiYarnApplication is expected on server side
+    assertEquals(yarnApp, outputJsonl.at("/apiYarnApplication"));
+  }
+
+  private String readString(String name) throws IOException, URISyntaxException {
+    return new String(Files.readAllBytes(Paths.get(this.getClass().getResource(name).toURI())));
+  }
+}