diff --git a/examples/local-testing-project/gradle.properties b/examples/local-testing-project/gradle.properties index a3e4f280..cbb8308f 100644 --- a/examples/local-testing-project/gradle.properties +++ b/examples/local-testing-project/gradle.properties @@ -1,6 +1,6 @@ # Set this to the version you used when running # "gradle -Pversion=(something) publishToMavenLocal" on your local ml-gradle repo -mlGradleVersion=6.0-SNAPSHOT +mlGradleVersion=6.2-SNAPSHOT mlHost=localhost mlAppName=example diff --git a/ml-app-deployer/src/main/java/com/marklogic/appdeployer/command/pdc/DeployMarkLogicEndpointsCommand.java b/ml-app-deployer/src/main/java/com/marklogic/appdeployer/command/pdc/DeployMarkLogicEndpointsCommand.java index 7967143e..ab8a805a 100644 --- a/ml-app-deployer/src/main/java/com/marklogic/appdeployer/command/pdc/DeployMarkLogicEndpointsCommand.java +++ b/ml-app-deployer/src/main/java/com/marklogic/appdeployer/command/pdc/DeployMarkLogicEndpointsCommand.java @@ -39,18 +39,19 @@ public void execute(CommandContext context) { return; } - final List endpoints = readEndpointDefinitionsFromFiles(context, pdcConfigPaths); - if (!endpoints.isEmpty()) { + final Map> endpointsByDnsName = readEndpointDefinitionsFromFiles(context, pdcConfigPaths); + if (!endpointsByDnsName.isEmpty()) { if (!StringUtils.hasText(context.getAppConfig().getCloudApiKey())) { - logger.warn("Found configuration for {} MarkLogic endpoint(s), but not deploying them because no cloud API key has been specified.", endpoints.size()); + int totalEndpoints = endpointsByDnsName.values().stream().mapToInt(List::size).sum(); + logger.warn("Found configuration for {} MarkLogic endpoint(s), but not deploying them because no cloud API key has been specified.", totalEndpoints); } else { - deployEndpoints(context, endpoints); + deployEndpointsByService(context, endpointsByDnsName); } } } - private List readEndpointDefinitionsFromFiles(CommandContext context, List pdcConfigPaths) { - List endpoints = new ArrayList<>(); + private Map> readEndpointDefinitionsFromFiles(CommandContext context, List pdcConfigPaths) { + Map> endpointsByDnsName = new HashMap<>(); for (String pdcConfigPath : pdcConfigPaths) { File serviceDir = new File(pdcConfigPath, "service"); @@ -62,17 +63,33 @@ private List readEndpointDefinitionsFromFiles(CommandCont logger.info("Reading MarkLogic endpoints from: {}", endpointsDir.getAbsolutePath()); - try (Stream paths = Files.walk(endpointsDir.toPath())) { - paths.filter(Files::isRegularFile) - .filter(path -> path.toString().endsWith(".json")) - .forEach(path -> endpoints.add(buildEndpointFromFile(context, path.toFile()))); - } catch (IOException e) { - throw new RuntimeException("Failed to read MarkLogic endpoint configuration files from: " + - endpointsDir.getAbsolutePath(), e); + File[] dnsNameDirs = endpointsDir.listFiles(File::isDirectory); + if (dnsNameDirs == null || dnsNameDirs.length == 0) { + logger.warn("No dnsName directories found under: {}. Endpoints should be organized under mlendpoints//*.json", endpointsDir.getAbsolutePath()); + continue; + } + + for (File dnsNameDir : dnsNameDirs) { + String dnsName = dnsNameDir.getName(); + List endpoints = new ArrayList<>(); + + try (Stream paths = Files.walk(dnsNameDir.toPath())) { + paths.filter(Files::isRegularFile) + .filter(path -> path.toString().endsWith(".json")) + .forEach(path -> endpoints.add(buildEndpointFromFile(context, path.toFile()))); + } catch (IOException e) { + throw new RuntimeException("Failed to read MarkLogic endpoint configuration files from: " + + dnsNameDir.getAbsolutePath(), e); + } + + if (!endpoints.isEmpty()) { + endpointsByDnsName.put(dnsName, endpoints); + logger.info("Found {} endpoint(s) for MarkLogic service: {}", endpoints.size(), dnsName); + } } } - return endpoints; + return endpointsByDnsName; } private MarkLogicHttpEndpoint buildEndpointFromFile(CommandContext context, File endpointFile) { @@ -91,55 +108,69 @@ private MarkLogicHttpEndpoint buildEndpointFromFile(CommandContext context, File } } - private void deployEndpoints(CommandContext context, List endpoints) { + private void deployEndpointsByService(CommandContext context, Map> endpointsByDnsName) { final String host = context.getManageClient().getManageConfig().getHost(); try (PdcClient pdcClient = new PdcClient(host, context.getAppConfig().getCloudApiKey())) { - final UUID markLogicServiceId = getFirstMarkLogicServiceId(pdcClient); - final ServiceApi serviceApi = new ServiceApi(pdcClient.getApiClient()); - try { - MarkLogicEndpointMappingData existingEndpoints = serviceApi.apiServiceMlendpointsIdGet(markLogicServiceId); - List endpointsToDeploy = filterOutExistingEndpoints(endpoints, existingEndpoints); - if (endpointsToDeploy.isEmpty()) { - logger.info("All {} endpoint(s) are up to date; nothing to deploy.", endpoints.size()); - } else { - logger.info("Deploying {} new or updated endpoint(s) out of {} total.", endpointsToDeploy.size(), endpoints.size()); - serviceApi.apiServiceMlendpointsIdHttpPut(markLogicServiceId, endpointsToDeploy); - logger.info("Successfully deployed {} endpoint(s).", endpointsToDeploy.size()); - } - } catch (ApiException e) { - throw new RuntimeException("Unable to create MarkLogic endpoints in PDC; cause: %s".formatted(e.getMessage()), e); + for (Map.Entry> entry : endpointsByDnsName.entrySet()) { + String dnsName = entry.getKey(); + List endpoints = entry.getValue(); + logger.info("Processing {} endpoint(s) for MarkLogic service: {}", endpoints.size(), dnsName); + deployEndpoints(pdcClient, dnsName, endpoints); + } + } + } + + private void deployEndpoints(PdcClient pdcClient, String dnsName, List endpoints) { + final UUID markLogicServiceId = getMarkLogicServiceIdByDnsName(pdcClient, dnsName); + final ServiceApi serviceApi = new ServiceApi(pdcClient.getApiClient()); + try { + MarkLogicEndpointMappingData existingEndpointData = serviceApi.apiServiceMlendpointsIdGet(markLogicServiceId); + if (allEndpointsMatch(endpoints, existingEndpointData)) { + logger.info("All {} endpoint(s) for '{}' are up to date; nothing to deploy.", endpoints.size(), dnsName); + } else { + logger.info("Deploying all {} endpoint(s) for '{}' due to changes or count mismatch.", endpoints.size(), dnsName); + serviceApi.apiServiceMlendpointsIdHttpPut(markLogicServiceId, endpoints); + logger.info("Successfully deployed {} endpoint(s) for '{}'.", endpoints.size(), dnsName); } + } catch (ApiException e) { + throw new RuntimeException("Unable to create MarkLogic endpoints for '%s' in PDC; cause: %s".formatted(dnsName, e.getMessage()), e); } } /** - * Filters out endpoints that don't need to be deployed. An endpoint needs to be deployed if: - * 1. It doesn't exist in PDC (based on name, which is unique), OR - * 2. It exists but has different properties (needs to be updated) + * Checks if all endpoints match the existing endpoints in PDC. Returns true only if: + * 1. The count of endpoints matches, AND + * 2. Every endpoint exists with identical properties *

- * An endpoint can take a surprisingly long time to create, so we only want to deploy ones that - * are new or have changed. + * If any endpoint is new, modified, or removed, this returns false and all endpoints + * will be deployed. The PDC API requires sending the complete list of endpoints. * * @param endpoints the list of endpoints to potentially deploy * @param existingEndpointData the existing endpoint data from PDC - * @return a list of endpoints that need to be deployed (new or updated) + * @return true if all endpoints are up to date, false if deployment is needed */ - private List filterOutExistingEndpoints( + private boolean allEndpointsMatch( List endpoints, MarkLogicEndpointMappingData existingEndpointData ) { if (existingEndpointData == null || existingEndpointData.getEndpoints() == null || existingEndpointData.getEndpoints().getHttpEndpoints() == null) { - return endpoints; + return false; } - Map existingEndpointsByName = existingEndpointData.getEndpoints() - .getHttpEndpoints().stream() + List existingEndpoints = existingEndpointData.getEndpoints().getHttpEndpoints(); + + // If counts don't match, something changed + if (endpoints.size() != existingEndpoints.size()) { + return false; + } + + Map existingEndpointsByName = existingEndpoints.stream() .collect(Collectors.toMap(MarkLogicHttpEndpoint::getName, Function.identity())); + // Check if all endpoints exist and match return endpoints.stream() - .filter(endpoint -> needsDeployment(endpoint, existingEndpointsByName.get(endpoint.getName()))) - .collect(Collectors.toList()); + .allMatch(endpoint -> !needsDeployment(endpoint, existingEndpointsByName.get(endpoint.getName()))); } /** @@ -162,14 +193,24 @@ private boolean needsDeployment(MarkLogicHttpEndpoint endpoint, MarkLogicHttpEnd || !Objects.equals(endpoint.getSupportedByCloud(), existingEndpoint.getSupportedByCloud()); } - private UUID getFirstMarkLogicServiceId(PdcClient pdcClient) { + private UUID getMarkLogicServiceIdByDnsName(PdcClient pdcClient, String dnsName) { try { final UUID environmentId = pdcClient.getEnvironmentId(); List apps = new ServiceApi(pdcClient.getApiClient()).apiServiceAppsGet(environmentId).getMarkLogic(); if (apps == null || apps.isEmpty()) { throw new RuntimeException("No instances of MarkLogic found in PDC tenancy; host: %s".formatted(pdcClient.getHost())); } - return apps.get(0).getId(); + + return apps.stream() + .filter(app -> dnsName.equals(app.getDnsName())) + .findFirst() + .map(MarkLogicApp::getId) + .orElseThrow(() -> new RuntimeException( + "No MarkLogic service found with dnsName '%s'. Available services: %s".formatted( + dnsName, + apps.stream().map(MarkLogicApp::getDnsName).collect(Collectors.joining(", ")) + ) + )); } catch (ApiException e) { throw new RuntimeException("Unable to lookup instances of MarkLogic in PDC; cause: %s".formatted(e), e); } diff --git a/ml-app-deployer/src/test/resources/cloud-project/src/main/pdc-config/service/mlendpoints/endpoint1.json b/ml-app-deployer/src/test/resources/cloud-project/src/main/pdc-config/service/mlendpoints/exampledns/endpoint1.json similarity index 100% rename from ml-app-deployer/src/test/resources/cloud-project/src/main/pdc-config/service/mlendpoints/endpoint1.json rename to ml-app-deployer/src/test/resources/cloud-project/src/main/pdc-config/service/mlendpoints/exampledns/endpoint1.json diff --git a/ml-app-deployer/src/test/resources/cloud-project/src/main/pdc-config/service/mlendpoints/endpoint2.json b/ml-app-deployer/src/test/resources/cloud-project/src/main/pdc-config/service/mlendpoints/exampledns/endpoint2.json similarity index 100% rename from ml-app-deployer/src/test/resources/cloud-project/src/main/pdc-config/service/mlendpoints/endpoint2.json rename to ml-app-deployer/src/test/resources/cloud-project/src/main/pdc-config/service/mlendpoints/exampledns/endpoint2.json diff --git a/ml-gradle/src/main/groovy/com/marklogic/gradle/MarkLogicPlugin.groovy b/ml-gradle/src/main/groovy/com/marklogic/gradle/MarkLogicPlugin.groovy index 4daddacb..a6a8a280 100644 --- a/ml-gradle/src/main/groovy/com/marklogic/gradle/MarkLogicPlugin.groovy +++ b/ml-gradle/src/main/groovy/com/marklogic/gradle/MarkLogicPlugin.groovy @@ -38,6 +38,7 @@ import com.marklogic.gradle.task.groups.SetTraceEventsTask import com.marklogic.gradle.task.hosts.AssignHostsToGroupsTask import com.marklogic.gradle.task.mimetypes.DeployMimetypesTask import com.marklogic.gradle.task.mimetypes.UndeployMimetypesTask +import com.marklogic.gradle.task.pdc.DeployMarkLogicEndpointsTask import com.marklogic.gradle.task.plugins.InstallPluginsTask import com.marklogic.gradle.task.plugins.UninstallPluginsTask import com.marklogic.gradle.task.qconsole.ExportWorkspacesTask @@ -341,7 +342,12 @@ class MarkLogicPlugin implements Plugin { "Use -PrunCodeCoverage to enable code coverage support when running the tests. " + "Use -PrunTeardown and -PrunSuiteTeardown to control whether teardown and suite teardown scripts are run; these default to 'true' and can be set to 'false' instead. ") - // Any granular task that deploys/undeploys resources may need to do so for a resource in a bundle, so these + String pdcGroup = "PDC" + project.tasks.register("pdcDeployMarkLogicEndpoints", DeployMarkLogicEndpointsTask.class) { + group = pdcGroup + description = "Deploy MarkLogic endpoints to a PDC instance. Endpoints default to being defined at src/main/pdc-config/service/mlendpoints/(dnsName)." + } + // tasks must all depend on mlPrepareBundles project.tasks.each { task -> if (task.name.startsWith("mlDeploy") || task.name.startsWith("mlUndeploy")) { diff --git a/ml-gradle/src/main/groovy/com/marklogic/gradle/task/pdc/DeployMarkLogicEndpointsTask.groovy b/ml-gradle/src/main/groovy/com/marklogic/gradle/task/pdc/DeployMarkLogicEndpointsTask.groovy new file mode 100644 index 00000000..d1d98a74 --- /dev/null +++ b/ml-gradle/src/main/groovy/com/marklogic/gradle/task/pdc/DeployMarkLogicEndpointsTask.groovy @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2015-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.gradle.task.pdc + +import com.marklogic.appdeployer.command.pdc.DeployMarkLogicEndpointsCommand +import com.marklogic.gradle.task.MarkLogicTask +import org.gradle.api.tasks.TaskAction + +class DeployMarkLogicEndpointsTask extends MarkLogicTask { + + @TaskAction + void deployMarkLogicEndpoints() { + invokeDeployerCommandWithClassName(DeployMarkLogicEndpointsCommand.class.getSimpleName()) + } +}