diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaAPIHostsTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaAPIHostsTask.java index 31a0d4186..75ed4db75 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaAPIHostsTask.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaAPIHostsTask.java @@ -80,7 +80,11 @@ protected void doRun( hosts.add(ClouderaHostDTO.create(apiHost.getId(), apiHost.getName())); } } - handle.initHostsIfNull(hosts); + if (hosts.isEmpty()) { + throw new MetadataDumperUsageException( + "No hosts were found in any of the initialized Cloudera clusters."); + } + handle.initHosts(hosts); } } } diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaCMFHostsTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaCMFHostsTask.java index aaaaa7867..f1aa14931 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaCMFHostsTask.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaCMFHostsTask.java @@ -20,15 +20,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.io.ByteSink; import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaClusterDTO; -import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaHostDTO; -import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.CMFHostDTO; -import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.CMFHostListDTO; import com.google.edwmigration.dumper.application.dumper.task.TaskCategory; import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext; import java.io.Writer; import java.net.URI; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.List; import javax.annotation.Nonnull; import org.apache.http.client.methods.CloseableHttpResponse; @@ -67,7 +63,6 @@ protected void doRun( } final URI baseURI = handle.getBaseURI(); - List hosts = new ArrayList<>(); try (Writer writer = sink.asCharSink(StandardCharsets.UTF_8).openBufferedStream()) { for (ClouderaClusterDTO cluster : clusters) { if (cluster.getId() == null) { @@ -95,14 +90,7 @@ protected void doRun( String stringifiedHosts = hostsJson.toString(); writer.write(stringifiedHosts); writer.write('\n'); - - CMFHostListDTO apiHosts = parseJsonStringToObject(stringifiedHosts, CMFHostListDTO.class); - for (CMFHostDTO apiHost : apiHosts.getHosts()) { - hosts.add(ClouderaHostDTO.create(apiHost.getId(), apiHost.getName())); - } } } - - handle.initHostsIfNull(hosts); } } diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerHandle.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerHandle.java index 0db73465b..b4eba2bfe 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerHandle.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerHandle.java @@ -37,6 +37,7 @@ public class ClouderaManagerHandle implements Handle { private ImmutableList clusters; private ImmutableList hosts; + private ImmutableList sparkYarnApplications; public ClouderaManagerHandle(URI apiURI, CloseableHttpClient httpClient) { Preconditions.checkNotNull(apiURI, "Cloudera's apiURI can't be null."); @@ -85,10 +86,8 @@ public synchronized void initClusters(List clusters) { Preconditions.checkNotNull(clusters, "Clusters can't be initialised to null list."); Preconditions.checkArgument( !clusters.isEmpty(), "Clusters can't be initialised to empty list."); + Preconditions.checkState(this.clusters == null, "The cluster already initialized."); - if (this.clusters != null) { - throw new IllegalStateException("The cluster already initialized!"); - } this.clusters = ImmutableList.copyOf(clusters); } @@ -97,14 +96,27 @@ public synchronized ImmutableList getHosts() { return hosts; } - public synchronized void initHostsIfNull(List hosts) { - // Todo - // Preconditions.checkNotNull(hosts, "Hosts can't be initialised to null list."); - // Preconditions.checkArgument(!hosts.isEmpty(), "Hosts can't be initialised to empty list."); + public synchronized void initHosts(List hosts) { + Preconditions.checkNotNull(hosts, "Hosts can't be initialised to null list."); + Preconditions.checkArgument(!hosts.isEmpty(), "Hosts can't be initialised to empty list."); + Preconditions.checkState(this.hosts == null, "Hosts already initialized."); - if (this.hosts == null) { - this.hosts = ImmutableList.copyOf(hosts); - } + this.hosts = ImmutableList.copyOf(hosts); + } + + @CheckForNull + public synchronized ImmutableList getSparkYarnApplications() { + return sparkYarnApplications; + } + + public synchronized void initSparkYarnApplications( + List sparkYarnApplications) { + Preconditions.checkNotNull( + sparkYarnApplications, "Spark YARN applications can't be initialised to null list."); + Preconditions.checkState( + this.sparkYarnApplications == null, "Spark YARN applications already initialized."); + + this.sparkYarnApplications = ImmutableList.copyOf(sparkYarnApplications); } @Override @@ -145,4 +157,17 @@ public static ClouderaHostDTO create(String id, String name) { abstract String getName(); } + + @AutoValue + public abstract static class ClouderaYarnApplicationDTO { + public static ClouderaYarnApplicationDTO create(String id, String clusterName) { + return new AutoValue_ClouderaManagerHandle_ClouderaYarnApplicationDTO(id, clusterName); + } + + @CheckForNull + @Nullable + abstract String getId(); + + abstract String getClusterName(); + } } diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationTypeTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationTypeTask.java index cd03950cc..4df12239e 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationTypeTask.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationTypeTask.java @@ -16,6 +16,9 @@ */ package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Arrays.stream; + import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -23,7 +26,9 @@ import com.google.common.io.ByteSink; import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException; import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaClusterDTO; +import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaYarnApplicationDTO; import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiYARNApplicationDTO; +import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.model.YarnApplicationType; import com.google.edwmigration.dumper.application.dumper.task.TaskCategory; import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext; import java.io.IOException; @@ -34,7 +39,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; import java.util.stream.StreamSupport; import javax.annotation.Nonnull; import org.apache.http.client.methods.CloseableHttpResponse; @@ -47,9 +51,6 @@ public class ClouderaYarnApplicationTypeTask extends AbstractClouderaYarnApplica private static final Logger logger = LoggerFactory.getLogger(ClouderaYarnApplicationTypeTask.class); - private final ImmutableList predefinedAppTypes = - ImmutableList.of("MAPREDUCE", "SPARK", "Oozie Launcher"); - public ClouderaYarnApplicationTypeTask( ZonedDateTime startDate, ZonedDateTime endDate, TaskCategory taskCategory) { super("yarn-application-types.jsonl", startDate, endDate, taskCategory); @@ -73,21 +74,28 @@ protected void doRun( new PaginatedClouderaYarnApplicationsLoader( handle, context.getArguments().getPaginationPageSize()); + List sparkYarnApplications = new ArrayList<>(); try (Writer writer = sink.asCharSink(StandardCharsets.UTF_8).openBufferedStream()) { for (ClouderaClusterDTO cluster : clusters) { final String clusterName = cluster.getName(); - Set yarnAppTypes = new HashSet<>(fetchYARNApplicationTypes(handle, clusterName)); - yarnAppTypes.addAll(predefinedAppTypes); - yarnAppTypes.addAll(context.getArguments().getYarnApplicationTypes()); - for (String yarnAppType : yarnAppTypes) { + for (String yarnAppType : collectYarnApplicationTypes(context, handle, clusterName)) { logger.info( "Dump YARN applications with {} type from {} cluster", yarnAppType, clusterName); int loadedAppsCount = appLoader.load( clusterName, yarnAppType, - yarnAppsPage -> - writeYarnAppTypes(writer, yarnAppsPage, yarnAppType, clusterName)); + yarnAppsPage -> { + writeYarnAppTypes(writer, yarnAppsPage, yarnAppType, clusterName); + if (yarnAppType.equals(YarnApplicationType.SPARK.getValue())) { + yarnAppsPage.stream() + .map( + yarnApp -> + ClouderaYarnApplicationDTO.create( + yarnApp.getApplicationId(), clusterName)) + .forEach(sparkYarnApplications::add); + } + }); logger.info( "Dumped {} YARN applications with {} type from {} cluster", loadedAppsCount, @@ -96,6 +104,7 @@ protected void doRun( } } } + handle.initSparkYarnApplications(sparkYarnApplications); } private void writeYarnAppTypes( @@ -115,22 +124,37 @@ private void writeYarnAppTypes( } } - private List fetchYARNApplicationTypes(ClouderaManagerHandle handle, String clusterName) { - String yarnAppTypesUrl = + private Set collectYarnApplicationTypes( + TaskRunContext context, ClouderaManagerHandle handle, String clusterName) { + Set yarnApplicationTypes = new HashSet<>(); + ImmutableList predefinedYarnAppTypes = + stream(YarnApplicationType.values()) + .map(YarnApplicationType::getValue) + .collect(toImmutableList()); + yarnApplicationTypes.addAll(predefinedYarnAppTypes); + yarnApplicationTypes.addAll(fetchClusterServiceTypes(handle, clusterName)); + yarnApplicationTypes.addAll(context.getArguments().getYarnApplicationTypes()); + return yarnApplicationTypes; + } + + private ImmutableList fetchClusterServiceTypes( + ClouderaManagerHandle handle, String clusterName) { + String serviceTypesUrl = handle.getApiURI().toString() + "clusters/" + clusterName + "/serviceTypes"; CloseableHttpClient httpClient = handle.getHttpClient(); - try (CloseableHttpResponse appTypesResp = httpClient.execute(new HttpGet(yarnAppTypesUrl))) { - int statusCode = appTypesResp.getStatusLine().getStatusCode(); + try (CloseableHttpResponse serviceTypesResp = + httpClient.execute(new HttpGet(serviceTypesUrl))) { + int statusCode = serviceTypesResp.getStatusLine().getStatusCode(); if (!isStatusCodeOK(statusCode)) { throw new ClouderaConnectorException( String.format( "Cloudera API returned bad http status: %d. Message: %s", - statusCode, readFromStream(appTypesResp.getEntity().getContent()))); + statusCode, readFromStream(serviceTypesResp.getEntity().getContent()))); } - JsonNode appTypesJson = readJsonTree(appTypesResp.getEntity().getContent()); - return StreamSupport.stream(appTypesJson.get("items").spliterator(), false) + JsonNode serviceTypesJson = readJsonTree(serviceTypesResp.getEntity().getContent()); + return StreamSupport.stream(serviceTypesJson.get("items").spliterator(), false) .map(JsonNode::asText) - .collect(Collectors.toList()); + .collect(toImmutableList()); } catch (IOException ex) { throw new ClouderaConnectorException(ex.getMessage(), ex); } diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/CMFHostDTO.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/model/YarnApplicationType.java similarity index 56% rename from dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/CMFHostDTO.java rename to dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/model/YarnApplicationType.java index 97a11d569..ff9ff60c6 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/CMFHostDTO.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/model/YarnApplicationType.java @@ -14,28 +14,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto; +package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.model; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; +public enum YarnApplicationType { + MAPREDUCE("MAPREDUCE"), + SPARK("SPARK"), + OOZIE_LAUNCHER("Oozie Launcher"); -/** - * DTO class for the unofficial UI part of Cloudera Management. Display the host from a Memory Usage - * chart. - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class CMFHostDTO { - @JsonProperty(required = true) - private String hostName; - - @JsonProperty(required = true) - private String hostId; + private final String value; - public String getName() { - return hostName; + YarnApplicationType(String value) { + this.value = value; } - public String getId() { - return hostId; + public String getValue() { + return value; } } diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaAPIHostsTaskTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaAPIHostsTaskTest.java index e959d5b52..74a54c4fa 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaAPIHostsTaskTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaAPIHostsTaskTest.java @@ -19,18 +19,15 @@ import static com.github.tomakehurst.wiremock.client.WireMock.get; import static com.github.tomakehurst.wiremock.client.WireMock.okJson; import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching; +import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.MockUtils.getWrittenJsonLines; +import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.MockUtils.verifyNoWrites; +import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.TestUtils.readFileAsString; +import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.TestUtils.toJsonl; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyChar; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -38,19 +35,19 @@ import com.fasterxml.jackson.core.JsonParseException; import com.github.tomakehurst.wiremock.WireMockServer; import com.github.tomakehurst.wiremock.core.WireMockConfiguration; -import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableList; import com.google.common.io.ByteSink; import com.google.common.io.CharSink; import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException; import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaClusterDTO; +import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaHostDTO; import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext; import java.io.IOException; import java.io.Writer; import java.net.URI; import java.nio.charset.StandardCharsets; import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; +import java.util.List; import org.apache.http.HttpStatus; import org.apache.http.impl.client.HttpClients; import org.junit.AfterClass; @@ -64,9 +61,9 @@ @RunWith(MockitoJUnitRunner.class) public class ClouderaAPIHostsTaskTest { + private static WireMockServer server; private final ClouderaAPIHostsTask task = new ClouderaAPIHostsTask(); private ClouderaManagerHandle handle; - private static WireMockServer server; @Mock private TaskRunContext context; @Mock private ByteSink sink; @@ -100,21 +97,19 @@ public void doRun_clouderaReturnsValidJsonLines_writeJsonLines() throws Exceptio initClusters( ClouderaClusterDTO.create("id1", "first-cluster"), ClouderaClusterDTO.create("id125", "second-cluster")); - - stubClouderaClusterAPIResponse("first-cluster", "[]"); - stubClouderaClusterAPIResponse("second-cluster", "[]\r\n"); + String hostsJson = readFileAsString("/cloudera/manager/api-hosts.json"); + stubClouderaClusterAPIResponse("first-cluster", hostsJson); + stubClouderaClusterAPIResponse("second-cluster", "{\"items\":[]}\r\n"); task.doRun(context, sink, handle); - // write jsonl. https://jsonlines.org/ - Set fileLines = getWrittenJsonLines(); - verify(writer, times(2)).write('\n'); + List fileLines = getWrittenJsonLines(writer, 2); + assertEquals(ImmutableList.of(toJsonl(hostsJson), "{\"items\":[]}"), fileLines); assertEquals( - ImmutableSet.of( - "{\"clusterName\":\"first-cluster\",\"items\":[]}", - "{\"clusterName\":\"second-cluster\",\"items\":[]}"), - fileLines); - + ImmutableList.of( + ClouderaHostDTO.create("1", "first-host"), ClouderaHostDTO.create("2", "second-host")), + handle.getHosts()); + verify(writer, times(2)).write('\n'); verify(writer).close(); } @@ -128,7 +123,7 @@ public void doRun_clustersWereNotInitialized_throwsCriticalException() throws Ex assertEquals( "Cloudera clusters must be initialized before hosts dumping.", exception.getMessage()); - verifyNoWrites(); + verifyNoWrites(writer); } @Test @@ -167,46 +162,31 @@ public void doRun_clouderaReturnsInvalidJsonFormat_throwsException() throws Exce assertThrows(JsonParseException.class, () -> task.doRun(context, sink, handle)); } + @Test + public void doRun_clouderaReturnsNoHosts_throwsException() throws Exception { + initClusters(ClouderaClusterDTO.create("id1", "first-cluster")); + stubClouderaClusterAPIResponse("first-cluster", "{\"items\":[]}"); + + MetadataDumperUsageException exception = + assertThrows(MetadataDumperUsageException.class, () -> task.doRun(context, sink, handle)); + + assertEquals( + "No hosts were found in any of the initialized Cloudera clusters.", exception.getMessage()); + } + private void initClusters(ClouderaClusterDTO... clusters) { handle.initClusters(Arrays.asList(clusters)); } - private void stubClouderaClusterAPIResponse(String clusterName, String itemsJsonLine) + private void stubClouderaClusterAPIResponse(String clusterName, String responseJson) throws IOException { - stubClouderaClusterAPIResponse(clusterName, itemsJsonLine, HttpStatus.SC_OK); + stubClouderaClusterAPIResponse(clusterName, responseJson, HttpStatus.SC_OK); } private void stubClouderaClusterAPIResponse( - String clusterName, String itemsJsonLine, int statusCode) throws IOException { - String clusterAPIResponse = - String.format("{\"clusterName\":\"%s\",\"items\":%s}", clusterName, itemsJsonLine); + String clusterName, String responseJson, int statusCode) { server.stubFor( get(urlMatching(String.format(".*/%s/hosts.*", clusterName))) - .willReturn(okJson(clusterAPIResponse).withStatus(statusCode))); - } - - private Set getWrittenJsonLines() throws IOException { - // https://jsonlines.org/ - Set fileLines = new HashSet<>(); - verify(writer, times(2)) - .write( - (String) - argThat( - content -> { - String str = (String) content; - assertFalse(str.contains("\n")); - assertFalse(str.contains("\r")); - fileLines.add(str); - return true; - })); - return fileLines; - } - - private void verifyNoWrites() throws IOException { - verify(writer, never()).write(anyChar()); - verify(writer, never()).write(anyString()); - verify(writer, never()).write(anyString(), anyInt(), anyInt()); - verify(writer, never()).write(any(char[].class)); - verify(writer, never()).write(any(char[].class), anyInt(), anyInt()); + .willReturn(okJson(responseJson).withStatus(statusCode))); } } diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaCMFHostsTaskTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaCMFHostsTaskTest.java index c629e950e..343a7f9fe 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaCMFHostsTaskTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaCMFHostsTaskTest.java @@ -19,38 +19,30 @@ import static com.github.tomakehurst.wiremock.client.WireMock.get; import static com.github.tomakehurst.wiremock.client.WireMock.okJson; import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching; +import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.MockUtils.getWrittenJsonLines; +import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.MockUtils.verifyNoWrites; +import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.TestUtils.readFileAsString; +import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.TestUtils.toJsonl; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyChar; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import com.fasterxml.jackson.databind.exc.MismatchedInputException; import com.github.tomakehurst.wiremock.WireMockServer; import com.github.tomakehurst.wiremock.core.WireMockConfiguration; -import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableList; import com.google.common.io.ByteSink; import com.google.common.io.CharSink; import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaClusterDTO; import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext; -import java.io.IOException; import java.io.Writer; import java.net.URI; import java.nio.charset.StandardCharsets; import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; -import org.apache.http.HttpStatus; +import java.util.List; import org.apache.http.impl.client.HttpClients; import org.junit.AfterClass; import org.junit.Before; @@ -63,10 +55,9 @@ @RunWith(MockitoJUnitRunner.class) public class ClouderaCMFHostsTaskTest { + private static WireMockServer server; private final ClouderaCMFHostsTask task = new ClouderaCMFHostsTask(); - private ClouderaManagerHandle handle; - private static WireMockServer server; @Mock private TaskRunContext context; @Mock private ByteSink sink; @@ -99,47 +90,25 @@ public void doRun_clouderaReturnsValidJson_writeJsonLines() throws Exception { initClusters( ClouderaClusterDTO.create("id1", "first-cluster"), ClouderaClusterDTO.create("id34", "next-cluster")); - - stubCMFHostResponse("id1", "first-cluster", "[]"); - stubCMFHostResponse("id34", "next-cluster", "[]\n\r"); + String hostsJson = readFileAsString("/cloudera/manager/cmf-hosts.json"); + stubCMFHostResponse("id1", hostsJson); + stubCMFHostResponse("id34", "{\"hosts\":[]}\n\r"); task.doRun(context, sink, handle); - // write jsonl. https://jsonlines.org/ - Set fileLines = getWrittenJsonLines(); + List fileLines = getWrittenJsonLines(writer, 2); + assertEquals(ImmutableList.of(toJsonl(hostsJson), "{\"hosts\":[]}"), fileLines); verify(writer, times(2)).write('\n'); - assertEquals( - ImmutableSet.of( - "{\"clusterName\":\"first-cluster\",\"hosts\":[]}", - "{\"clusterName\":\"next-cluster\",\"hosts\":[]}"), - fileLines); verify(writer).close(); } - @Test - public void doRun_clouderaReturnsNoHostForCluster_throwsWarningException() throws Exception { - // GIVEN: The cluster which has no host - initClusters(ClouderaClusterDTO.create("id1", "first-cluster")); - String mockedResponse = String.format("{\"clusterName\" :\"%s\"}", "first-cluster"); - server.stubFor( - get(urlMatching("/cmf/hardware/hosts/hostsOverview.json\\?clusterId=id1.*")) - .willReturn(okJson(mockedResponse).withStatus(HttpStatus.SC_OK))); - - // WHEN: Hosts are requested from the API and no one has been returned - MismatchedInputException exception = - assertThrows(MismatchedInputException.class, () -> task.doRun(context, sink, handle)); - - // THEN: The exception has to be raised - assertTrue(exception.getMessage().contains("hosts")); - } - @Test public void doRun_initClusterWithoutId_skipWrites() throws Exception { initClusters(ClouderaClusterDTO.create(null, "single cluster")); task.doRun(context, sink, handle); - verifyNoWrites(); + verifyNoWrites(writer); } @Test @@ -151,52 +120,28 @@ public void doRun_clustersWereNotInitialized_throwsException() throws Exception assertEquals( "Cloudera clusters must be initialized before hosts dumping.", exception.getMessage()); - verifyNoWrites(); + verifyNoWrites(writer); } @Test public void doRun_clouderaReturnsInvalidJson_continueTaskWithoutWriting() throws Exception { initClusters(ClouderaClusterDTO.create("id1", "first-cluster")); - stubCMFHostResponse("id1", "first-cluster", "[}"); - verifyNoWrites(); + stubCMFHostResponse("id1", "[}"); + + task.doRun(context, sink, handle); + + verifyNoWrites(writer); } private void initClusters(ClouderaClusterDTO... clusters) { handle.initClusters(Arrays.asList(clusters)); } - private void stubCMFHostResponse(String clusterId, String clusterName, String jsonHosts) { - String cmfResponse = - String.format("{\"clusterName\" :\"%s\", \"hosts\": %s}", clusterName, jsonHosts); + private void stubCMFHostResponse(String clusterId, String responseJson) { server.stubFor( get(urlMatching( String.format( "/cmf/hardware/hosts/hostsOverview\\.json\\?clusterId=%s.*", clusterId))) - .willReturn(okJson(cmfResponse))); - } - - private Set getWrittenJsonLines() throws IOException { - // https://jsonlines.org/ - Set fileLines = new HashSet<>(); - verify(writer, times(2)) - .write( - (String) - argThat( - content -> { - String str = (String) content; - assertFalse(str.contains("\n")); - assertFalse(str.contains("\r")); - fileLines.add(str); - return true; - })); - return fileLines; - } - - private void verifyNoWrites() throws IOException { - verify(writer, never()).write(anyChar()); - verify(writer, never()).write(anyString()); - verify(writer, never()).write(anyString(), anyInt(), anyInt()); - verify(writer, never()).write(any(char[].class)); - verify(writer, never()).write(any(char[].class), anyInt(), anyInt()); + .willReturn(okJson(responseJson))); } } diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTaskTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTaskTest.java index 6e8a0da30..39a5f73fb 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTaskTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClusterCPUChartTaskTest.java @@ -19,20 +19,16 @@ import static com.github.tomakehurst.wiremock.client.WireMock.get; import static com.github.tomakehurst.wiremock.client.WireMock.okJson; import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching; +import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.MockUtils.verifyNoWrites; +import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.TestUtils.readFileAsString; +import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.TestUtils.toJsonl; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyChar; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.ObjectMapper; import com.github.tomakehurst.wiremock.WireMockServer; import com.github.tomakehurst.wiremock.core.WireMockConfiguration; import com.google.common.collect.ImmutableSet; @@ -46,10 +42,7 @@ import java.io.IOException; import java.io.Writer; import java.net.URI; -import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.ZonedDateTime; @@ -114,7 +107,7 @@ public void doRun_initiatedClusterWithoutId_skipWrites() throws Exception { task.doRun(context, sink, handle); // THEN: Task for such clusters should be skipped - verifyNoWrites(); + verifyNoWrites(writer); } @Test @@ -134,7 +127,7 @@ public void doRun_clouderaReturnsValidJson_writeJsonLines() throws Exception { // THEN: the output should be dumped into the jsonl format for both clusters Set fileLines = new HashSet<>(MockUtils.getWrittenJsonLines(writer, 2)); assertEquals( - ImmutableSet.of(tojsonl(firstClusterServicesJson), tojsonl(secondClusterServicesJson)), + ImmutableSet.of(toJsonl(firstClusterServicesJson), toJsonl(secondClusterServicesJson)), fileLines); } @@ -150,7 +143,7 @@ public void doRun_clustersWereNotInitialized_throwsException() throws Exception // THEN: There is a relevant exception has been raised assertEquals( "Cloudera clusters must be initialized before CPU charts dumping.", exception.getMessage()); - verifyNoWrites(); + verifyNoWrites(writer); } @Test @@ -164,7 +157,7 @@ public void doRun_clouderaReturns4xx_throwsException() throws Exception { // WHEN: Cloudera returns 4xx http status code assertThrows(RuntimeException.class, () -> task.doRun(context, sink, handle)); - verifyNoWrites(); + verifyNoWrites(writer); } @Test @@ -179,7 +172,7 @@ public void doRun_clouderaReturns5xx_throwsException() throws Exception { assertThrows(RuntimeException.class, () -> task.doRun(context, sink, handle)); // THEN: There is a relevant exception has been raised - verifyNoWrites(); + verifyNoWrites(writer); } @Test @@ -192,7 +185,7 @@ public void doRun_clouderaReturnsInvalidJson_throwsException() throws Exception // WHEN: Cloudera returns 4xx http status code assertThrows(JsonParseException.class, () -> task.doRun(context, sink, handle)); - verifyNoWrites(); + verifyNoWrites(writer); } private void initClusters(ClouderaClusterDTO... clusters) { @@ -211,22 +204,6 @@ private void stubHttpRequestToFetchClusterCPUChart( .willReturn(okJson(mockedContent).withStatus(statusCode))); } - private void verifyNoWrites() throws IOException { - verify(writer, never()).write(anyChar()); - verify(writer, never()).write(anyString()); - verify(writer, never()).write(anyString(), anyInt(), anyInt()); - verify(writer, never()).write(any(char[].class)); - verify(writer, never()).write(any(char[].class), anyInt(), anyInt()); - } - - private String readFileAsString(String fileName) throws IOException, URISyntaxException { - return new String(Files.readAllBytes(Paths.get(this.getClass().getResource(fileName).toURI()))); - } - - private String tojsonl(String json) throws Exception { - return new ObjectMapper().readTree(json).toString(); - } - private ZonedDateTime timeTravelDaysAgo(int days) { ZonedDateTime today = ZonedDateTime.of(LocalDateTime.now(), ZoneId.of("UTC")); return today.minusDays(days); diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClustersTaskTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClustersTaskTest.java index b4719a913..8f1bfc679 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClustersTaskTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClustersTaskTest.java @@ -21,6 +21,7 @@ import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; import static com.github.tomakehurst.wiremock.client.WireMock.okJson; import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.TestUtils.readFileAsString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -40,13 +41,9 @@ import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaClusterDTO; import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto.ApiClusterListDTO; import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext; -import java.io.IOException; import java.io.Writer; import java.net.URI; -import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.junit.AfterClass; @@ -98,9 +95,9 @@ public void setUp() throws Exception { when(charSink.openBufferedStream()).thenReturn(writer); when(context.getArguments()).thenReturn(arguments); - apiClusterListJson = readString("/cloudera/manager/dto/ApiClusterList.json"); - apiClusterJson = readString("/cloudera/manager/dto/ApiCluster.json"); - clusterStatusJson = readString("/cloudera/manager/cluster-status.json"); + apiClusterListJson = readFileAsString("/cloudera/manager/dto/ApiClusterList.json"); + apiClusterJson = readFileAsString("/cloudera/manager/dto/ApiCluster.json"); + clusterStatusJson = readFileAsString("/cloudera/manager/cluster-status.json"); } @Test @@ -190,8 +187,4 @@ public void doRun_clusterProvided_fetchOnlyProvidedCluster() throws Exception { private String clusterStatusJsonWithId(String clusterId) { return clusterStatusJson.replaceAll("" + clusterStatusId, clusterId); } - - private String readString(String name) throws IOException, URISyntaxException { - return new String(Files.readAllBytes(Paths.get(this.getClass().getResource(name).toURI()))); - } } diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostComponentsTaskTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostComponentsTaskTest.java index e76656881..c773987d8 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostComponentsTaskTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostComponentsTaskTest.java @@ -18,17 +18,13 @@ import static com.github.tomakehurst.wiremock.client.WireMock.get; import static com.github.tomakehurst.wiremock.client.WireMock.okJson; +import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.MockUtils.verifyNoWrites; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyChar; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -41,7 +37,6 @@ import com.google.common.io.CharSink; import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaHostDTO; import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext; -import java.io.IOException; import java.io.Writer; import java.net.URI; import java.nio.charset.StandardCharsets; @@ -92,7 +87,7 @@ public void setUp() throws Exception { @Test public void hostsWereInitialized_writesSuccess() throws Exception { - handle.initHostsIfNull( + handle.initHosts( ImmutableList.of( ClouderaHostDTO.create("id1", "host one"), ClouderaHostDTO.create("id2", "host two"))); server.stubFor(get("/hosts/id1/components").willReturn(okJson("{\"valid\":\n \"json1\"}"))); @@ -136,14 +131,6 @@ public void hostsWereNotInitialized_throwsException() throws Exception { "Cloudera hosts must be initialized before Host's components dumping.", exception.getMessage()); - verifyNoWrites(); - } - - private void verifyNoWrites() throws IOException { - verify(writer, never()).write(anyChar()); - verify(writer, never()).write(anyString()); - verify(writer, never()).write(anyString(), anyInt(), anyInt()); - verify(writer, never()).write(any(char[].class)); - verify(writer, never()).write(any(char[].class), anyInt(), anyInt()); + verifyNoWrites(writer); } } diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTaskTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTaskTest.java index 9e588372c..7ef8e94af 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTaskTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostRAMChartTaskTest.java @@ -19,15 +19,11 @@ import static com.github.tomakehurst.wiremock.client.WireMock.get; import static com.github.tomakehurst.wiremock.client.WireMock.okJson; import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching; +import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.MockUtils.verifyNoWrites; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyChar; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -126,7 +122,7 @@ public void doRun_clustersWereNotInitialized_throwsCriticalException() throws Ex assertEquals( "Cloudera hosts must be initialized before RAM charts dumping.", exception.getMessage()); - verifyNoWrites(); + verifyNoWrites(writer); } @Test @@ -136,7 +132,7 @@ public void doRun_clouderaServerReturnsInvalidJson_throwsException() throws Exce assertThrows(JsonParseException.class, () -> task.doRun(context, sink, handle)); - verifyNoWrites(); + verifyNoWrites(writer); } @Test @@ -146,7 +142,7 @@ public void doRun_clouderaServerReturns4xx_throwsException() throws Exception { assertThrows(RuntimeException.class, () -> task.doRun(context, sink, handle)); - verifyNoWrites(); + verifyNoWrites(writer); } @Test @@ -156,11 +152,11 @@ public void doRun_clouderaServerReturns5xx_throwsException() throws Exception { assertThrows(RuntimeException.class, () -> task.doRun(context, sink, handle)); - verifyNoWrites(); + verifyNoWrites(writer); } private void initHosts(ClouderaHostDTO... hosts) { - handle.initHostsIfNull(Arrays.asList(hosts)); + handle.initHosts(Arrays.asList(hosts)); } private void stubHostAPIResponse(String hostId, int statusCode, String responseContent) @@ -170,14 +166,6 @@ private void stubHostAPIResponse(String hostId, int statusCode, String responseC .willReturn(okJson(responseContent).withStatus(statusCode))); } - private void verifyNoWrites() throws IOException { - verify(writer, never()).write(anyChar()); - verify(writer, never()).write(anyString()); - verify(writer, never()).write(anyString(), anyInt(), anyInt()); - verify(writer, never()).write(any(char[].class)); - verify(writer, never()).write(any(char[].class), anyInt(), anyInt()); - } - private ZonedDateTime timeTravelDaysAgo(int days) { ZonedDateTime today = ZonedDateTime.of(LocalDateTime.now(), ZoneId.of("UTC")); return today.minusDays(days); diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerHandleTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerHandleTest.java index a659fd4e1..b661d2abf 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerHandleTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerHandleTest.java @@ -21,6 +21,8 @@ import com.google.common.collect.ImmutableList; import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaClusterDTO; +import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaHostDTO; +import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaYarnApplicationDTO; import java.net.URI; import java.util.ArrayList; import java.util.List; @@ -128,22 +130,98 @@ public void initClustersTwice_throwsException() { assertThrows(IllegalStateException.class, () -> handle.initClusters(second)); assertEquals(first, handle.getClusters()); - assertEquals("The cluster already initialized!", exception.getMessage()); + assertEquals("The cluster already initialized.", exception.getMessage()); } - // @Test todo - public void initHostsWithNullOrEmpty_throwsException() { + @Test + public void initHosts_success() { + ClouderaManagerHandle handle = new ClouderaManagerHandle(localhost, httpClient); + List dtos = new ArrayList<>(); + dtos.add(ClouderaHostDTO.create("1", "first")); + dtos.add(ClouderaHostDTO.create("2", "second")); + + handle.initHosts(dtos); + + assertEquals(dtos, handle.getHosts()); + } + + @Test + public void initHostsWithNull_throwsException() { + ClouderaManagerHandle handle = new ClouderaManagerHandle(localhost, httpClient); + + NullPointerException npe = + assertThrows(NullPointerException.class, () -> handle.initHosts(null)); + + assertEquals("Hosts can't be initialised to null list.", npe.getMessage()); + } + + @Test + public void initHostsWithEmptyList_throwsException() { ClouderaManagerHandle handle = new ClouderaManagerHandle(localhost, httpClient); IllegalArgumentException exception = - assertThrows( - IllegalArgumentException.class, () -> handle.initHostsIfNull(ImmutableList.of())); + assertThrows(IllegalArgumentException.class, () -> handle.initHosts(ImmutableList.of())); assertEquals("Hosts can't be initialised to empty list.", exception.getMessage()); + } + + @Test + public void initHostsTwice_throwsException() { + ClouderaManagerHandle handle = new ClouderaManagerHandle(localhost, httpClient); + List first = new ArrayList<>(); + first.add(ClouderaHostDTO.create("1", "first")); + first.add(ClouderaHostDTO.create("2", "second")); + List second = new ArrayList<>(); + second.add(ClouderaHostDTO.create("3", "third")); + second.add(ClouderaHostDTO.create("4", "fourth")); + + handle.initHosts(first); + + IllegalStateException exception = + assertThrows(IllegalStateException.class, () -> handle.initHosts(second)); + + assertEquals(first, handle.getHosts()); + assertEquals("Hosts already initialized.", exception.getMessage()); + } + + @Test + public void initSparkYarnApplications_success() { + ClouderaManagerHandle handle = new ClouderaManagerHandle(localhost, httpClient); + List dtos = new ArrayList<>(); + dtos.add(ClouderaYarnApplicationDTO.create("1", "clusterOne")); + dtos.add(ClouderaYarnApplicationDTO.create("2", "clusterTwo")); + + handle.initSparkYarnApplications(dtos); + + assertEquals(dtos, handle.getSparkYarnApplications()); + } + + @Test + public void initSparkYarnApplicationsWithNull_throwsException() { + ClouderaManagerHandle handle = new ClouderaManagerHandle(localhost, httpClient); NullPointerException npe = - assertThrows(NullPointerException.class, () -> handle.initHostsIfNull(null)); + assertThrows(NullPointerException.class, () -> handle.initSparkYarnApplications(null)); - assertEquals("Hosts can't be initialised to null list.", npe.getMessage()); + assertEquals("Spark YARN applications can't be initialised to null list.", npe.getMessage()); + } + + @Test + public void initSparkYarnApplicationsTwice_throwsException() { + ClouderaManagerHandle handle = new ClouderaManagerHandle(localhost, httpClient); + List first = new ArrayList<>(); + first.add(ClouderaYarnApplicationDTO.create("1", "clusterOne")); + first.add(ClouderaYarnApplicationDTO.create("2", "clusterTwo")); + List second = new ArrayList<>(); + second.add(ClouderaYarnApplicationDTO.create("3", "clusterOne")); + second.add(ClouderaYarnApplicationDTO.create("4", "clusterTwo")); + + handle.initSparkYarnApplications(first); + + IllegalStateException exception = + assertThrows(IllegalStateException.class, () -> handle.initSparkYarnApplications(second)); + + assertEquals(first, handle.getSparkYarnApplications()); + assertEquals("Spark YARN applications already initialized.", exception.getMessage()); } } diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaServiceResourceAllocationChartTaskTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaServiceResourceAllocationChartTaskTest.java index 3af7c72e2..14267f320 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaServiceResourceAllocationChartTaskTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaServiceResourceAllocationChartTaskTest.java @@ -190,7 +190,7 @@ public void doRun_clouderaReturnsInvalidJson_throwsException() throws Exception } private void initHosts(ClouderaHostDTO... hosts) { - handle.initHostsIfNull(Arrays.asList(hosts)); + handle.initHosts(Arrays.asList(hosts)); } private void stubHttpRequestToFetchHostServicesResourceAllocationChart( diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaServicesTaskTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaServicesTaskTest.java index dc2a8d0f0..3d17eb174 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaServicesTaskTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaServicesTaskTest.java @@ -20,20 +20,17 @@ import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; import static com.github.tomakehurst.wiremock.client.WireMock.okJson; import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.MockUtils.verifyNoWrites; +import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.TestUtils.readFileAsString; +import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.TestUtils.toJsonl; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyChar; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import com.fasterxml.jackson.databind.ObjectMapper; import com.github.tomakehurst.wiremock.WireMockServer; import com.github.tomakehurst.wiremock.core.WireMockConfiguration; import com.google.common.collect.ImmutableList; @@ -42,13 +39,9 @@ import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException; import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaClusterDTO; import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext; -import java.io.IOException; import java.io.Writer; import java.net.URI; -import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; import org.apache.http.impl.client.HttpClients; import org.junit.AfterClass; import org.junit.Before; @@ -115,7 +108,7 @@ public void clustersExists_do_writes_success() throws Exception { server.verify(getRequestedFor(urlEqualTo("/api/vTest/clusters/first-cluster/services"))); server.verify(getRequestedFor(urlEqualTo("/api/vTest/clusters/second-cluster/services"))); - verify(writer).write(tojsonl(servicesJson)); + verify(writer).write(toJsonl(servicesJson)); verify(writer).write("{\"some\":\"json\"}"); verify(writer, times(2)).write('\n'); verify(writer).close(); @@ -131,22 +124,6 @@ public void clustersWereNotInitialized_throwsException() throws Exception { assertEquals( "Cloudera clusters must be initialized before services dumping.", exception.getMessage()); - verifyNoWrites(); - } - - private void verifyNoWrites() throws IOException { - verify(writer, never()).write(anyChar()); - verify(writer, never()).write(anyString()); - verify(writer, never()).write(anyString(), anyInt(), anyInt()); - verify(writer, never()).write(any(char[].class)); - verify(writer, never()).write(any(char[].class), anyInt(), anyInt()); - } - - private String tojsonl(String json) throws Exception { - return new ObjectMapper().readTree(json).toString(); - } - - private String readFileAsString(String fileName) throws IOException, URISyntaxException { - return new String(Files.readAllBytes(Paths.get(this.getClass().getResource(fileName).toURI()))); + verifyNoWrites(writer); } } diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationTypeTaskTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationTypeTaskTest.java index f825882f5..24b44a119 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationTypeTaskTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaYarnApplicationTypeTaskTest.java @@ -31,11 +31,13 @@ import com.github.tomakehurst.wiremock.WireMockServer; import com.github.tomakehurst.wiremock.core.WireMockConfiguration; import com.github.tomakehurst.wiremock.matching.StringValuePattern; +import com.google.common.collect.ImmutableList; import com.google.common.io.ByteSink; import com.google.common.io.CharSink; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException; import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaClusterDTO; +import com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.ClouderaManagerHandle.ClouderaYarnApplicationDTO; import com.google.edwmigration.dumper.application.dumper.task.TaskCategory; import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext; import java.io.Writer; @@ -211,6 +213,25 @@ public void doRun_applicationTypeContainsSpace_queryParameterIsEncoded() throws "{\"yarnAppTypes\":[{\"applicationId\":\"oozie\",\"applicationType\":\"Oozie Launcher\",\"clusterName\":\"test-cluster\"}]}")); } + @Test + public void doRun_initializedClusters_cachesSparkYarnApplications() throws Exception { + initClusters( + ClouderaClusterDTO.create("cluster-1", "first-cluster"), + ClouderaClusterDTO.create("cluster-2", "second-cluster")); + stubYARNApplicationTypesAPI("first-cluster", "{\"items\":[]}"); + stubYARNApplicationTypesAPI("second-cluster", "{\"items\":[]}"); + stubPredefinedApplicationTypes("first-cluster"); + stubPredefinedApplicationTypes("second-cluster"); + + task.doRun(context, sink, handle); + + assertEquals( + ImmutableList.of( + ClouderaYarnApplicationDTO.create("spark-app", "first-cluster"), + ClouderaYarnApplicationDTO.create("spark-app", "second-cluster")), + handle.getSparkYarnApplications()); + } + private void initClusters(ClouderaClusterDTO... clusters) { handle.initClusters(Arrays.asList(clusters)); } diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/MockUtils.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/MockUtils.java index 3e9cb2908..c5b2c18b5 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/MockUtils.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/MockUtils.java @@ -17,6 +17,11 @@ package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager; import static org.junit.Assert.assertFalse; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyChar; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -36,4 +41,12 @@ public static List getWrittenJsonLines(Writer writer, int callsCount) th } return fileLines; } + + public static void verifyNoWrites(Writer writer) throws IOException { + verify(writer, never()).write(anyChar()); + verify(writer, never()).write(anyString()); + verify(writer, never()).write(anyString(), anyInt(), anyInt()); + verify(writer, never()).write(any(char[].class)); + verify(writer, never()).write(any(char[].class), anyInt(), anyInt()); + } } diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/CMFHostListDTO.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/TestUtils.java similarity index 51% rename from dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/CMFHostListDTO.java rename to dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/TestUtils.java index 1b3dd25f9..bce2e1ee2 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/dto/CMFHostListDTO.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/TestUtils.java @@ -14,27 +14,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.dto; +package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Objects; -/** - * DTO class for the unofficial UI part of Cloudera Management. Display the host list from a Memory - * Usage chart. - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class CMFHostListDTO { - private final List hosts; +public class TestUtils { + + private static final ObjectMapper MAPPER = new ObjectMapper(); - @JsonCreator - public CMFHostListDTO(@JsonProperty(value = "hosts", required = true) List hosts) { - this.hosts = hosts; + public static String toJsonl(String json) throws Exception { + return MAPPER.readTree(json).toString(); } - public List getHosts() { - return hosts; + public static String readFileAsString(String fileName) throws IOException, URISyntaxException { + URI uri = Objects.requireNonNull(TestUtils.class.getResource(fileName)).toURI(); + return new String(Files.readAllBytes(Paths.get(uri))); } } diff --git a/dumper/app/src/test/resources/cloudera/manager/api-hosts.json b/dumper/app/src/test/resources/cloudera/manager/api-hosts.json new file mode 100644 index 000000000..5ff63cfa2 --- /dev/null +++ b/dumper/app/src/test/resources/cloudera/manager/api-hosts.json @@ -0,0 +1,60 @@ +{ + "items": [ + { + "maintenanceOwners": [], + "hostId": "1", + "ipAddress": "10.10.1.155", + "hostname": "first-host", + "rackId": "/us-west-2b", + "hostUrl": "https://cldr-e2e-data-hub-master0.cldr-e2e.svye-dcxb.a5.cloudera.site/cmf/hostRedirect/855220dcd69cd2da77eed368e9d4df0f", + "maintenanceMode": false, + "commissionState": "COMMISSIONED", + "numCores": 4, + "numPhysicalCores": 2, + "totalPhysMemBytes": 33046466560, + "clusterRef": { + "clusterName": "first-cluster", + "displayName": "first-cluster" + }, + "distribution": { + "distributionType": "RHEL8", + "name": "redhat", + "version": "8.10" + }, + "tags": [ + { + "name": "_cldr_cm_host_template_name", + "value": "compute" + } + ] + }, + { + "maintenanceOwners": [], + "hostId": "2", + "ipAddress": "10.10.1.233", + "hostname": "second-host", + "rackId": "/us-west-2b", + "hostUrl": "https://cldr-e2e-data-hub-master0.cldr-e2e.svye-dcxb.a5.cloudera.site/cmf/hostRedirect/f04fa33e3af5ec7dcbe16c394adeb17c", + "maintenanceMode": false, + "commissionState": "COMMISSIONED", + "numCores": 16, + "numPhysicalCores": 8, + "totalPhysMemBytes": 65809149952, + "clusterRef": { + "clusterName": "first-cluster", + "displayName": "first-cluster" + }, + "distribution": { + "distributionType": "RHEL8", + "name": "redhat", + "version": "8.10" + }, + "tags": [ + { + "name": "_cldr_cm_host_template_name", + "value": "master" + } + ] + } + ] +} diff --git a/dumper/app/src/test/resources/cloudera/manager/cmf-hosts.json b/dumper/app/src/test/resources/cloudera/manager/cmf-hosts.json new file mode 100644 index 000000000..40eb5a126 --- /dev/null +++ b/dumper/app/src/test/resources/cloudera/manager/cmf-hosts.json @@ -0,0 +1,172 @@ +{ + "healthTestDisplayNames": {}, + "healthTestFilters": {}, + "hosts": [ + { + "cdhVersion": "CDH 7.2.18", + "clusterId": 1546337345, + "clusterName": "first-cluster", + "commissionStateTag": "COMMISSIONED", + "commissionStateText": "Commissioned", + "diskPercentageUsed": 0.16187594620605952, + "diskTotal": 361243934720, + "diskUsed": 58476703744, + "displayStatusTag": "GOOD_HEALTH", + "displayStatusText": "Good Health", + "heartbeatHealth": "good", + "hostId": "855220dcd69cd2da77eed368e9d4df0f", + "hostName": "cldr-e2e-data-hub-compute0.cldr-e2e.svye-dcxb.a5.cloudera.site", + "id": 1546336928, + "ipAddress": "10.10.1.155", + "loadAverage15Min": "0.00", + "loadAverage1Min": "0.01", + "loadAverage5Min": "0.00", + "maintenanceMode": "no", + "msSinceLastSeen": 2460, + "numCores": 4, + "physicalMemoryPercentageUsed": 0.055655904169380585, + "physicalMemoryTotal": 33046466560, + "physicalMemoryUsed": 1839230976, + "rackId": "/us-west-2b", + "roleCount": 9, + "tagCount": 1 + }, + { + "cdhVersion": "CDH 7.2.18", + "clusterId": 1546337345, + "clusterName": "first-cluster", + "commissionStateTag": "COMMISSIONED", + "commissionStateText": "Commissioned", + "diskPercentageUsed": 0.18000125199076944, + "diskTotal": 424914059264, + "diskUsed": 76485062656, + "displayStatusTag": "GOOD_HEALTH", + "displayStatusText": "Good Health", + "heartbeatHealth": "good", + "hostId": "f04fa33e3af5ec7dcbe16c394adeb17c", + "hostName": "cldr-e2e-data-hub-master0.cldr-e2e.svye-dcxb.a5.cloudera.site", + "id": 1546336918, + "ipAddress": "10.10.1.233", + "loadAverage15Min": "0.99", + "loadAverage1Min": "0.55", + "loadAverage5Min": "0.59", + "maintenanceMode": "no", + "msSinceLastSeen": 2174, + "numCores": 16, + "physicalMemoryPercentageUsed": 0.46078678199183193, + "physicalMemoryTotal": 65809149952, + "physicalMemoryUsed": 30323986432, + "rackId": "/us-west-2b", + "roleCount": 34, + "tagCount": 1 + } + ], + "humanizedValues": { + "no": "No", + "yes": "Yes" + }, + "roleFilters": { + "YARN": [ + 1546336921, + 1546336925, + 1546336931, + 1546336928, + 1546336918 + ], + "TEZ": [ + 1546336921, + 1546336925, + 1546336931, + 1546336928, + 1546336918 + ], + "HUE": [ + 1546336918 + ], + "ZOOKEEPER": [ + 1546336918 + ], + "SPARK3_ON_YARN": [ + 1546336921, + 1546336925, + 1546336931, + 1546336928, + 1546336918 + ], + "LIVY_FOR_SPARK3": [ + 1546336921, + 1546336925, + 1546336931, + 1546336928, + 1546336918 + ], + "HIVE_ON_TEZ": [ + 1546336921, + 1546336925, + 1546336931, + 1546336928, + 1546336918 + ], + "HDFS": [ + 1546336921, + 1546336925, + 1546336931, + 1546336928, + 1546336918 + ], + "SQOOP_CLIENT": [ + 1546336918 + ], + "DATA_CONTEXT_CONNECTOR": [ + 1546336921, + 1546336925, + 1546336931, + 1546336928, + 1546336918 + ], + "HIVE": [ + 1546336921, + 1546336925, + 1546336931, + 1546336928, + 1546336918 + ], + "QUERY_PROCESSOR": [ + 1546336918 + ], + "ZEPPELIN": [ + 1546336918 + ], + "RANGER_RAZ": [ + 1546336918 + ], + "OOZIE": [ + 1546336918 + ], + "QUEUEMANAGER": [ + 1546336918 + ], + "KNOX": [ + 1546336918 + ] + }, + "serviceDisplayNames": { + "YARN": "YARN", + "TEZ": "Tez", + "HUE": "Hue", + "ZOOKEEPER": "ZooKeeper", + "SPARK3_ON_YARN": "Spark 3", + "LIVY_FOR_SPARK3": "Livy for Spark 3", + "HIVE_ON_TEZ": "Hive on Tez", + "HDFS": "HDFS", + "SQOOP_CLIENT": "SQOOP_CLIENT", + "DATA_CONTEXT_CONNECTOR": "Data Context Connector", + "HIVE": "Hive", + "QUERY_PROCESSOR": "Query Processor", + "ZEPPELIN": "Zeppelin", + "RANGER_RAZ": "Ranger Raz", + "OOZIE": "Oozie", + "QUEUEMANAGER": "YARN Queue Manager", + "KNOX": "Knox" + } +} \ No newline at end of file