Skip to content

Commit bb4dc6a

Browse files
rjrudinrjrudin
committed
MLE-26325 Can now deploy endpoints to multiple MarkLogic services
1 parent 0e4e758 commit bb4dc6a

File tree

6 files changed

+108
-45
lines changed

6 files changed

+108
-45
lines changed

‎examples/local-testing-project/gradle.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Set this to the version you used when running
22
# "gradle -Pversion=(something) publishToMavenLocal" on your local ml-gradle repo
3-
mlGradleVersion=6.0-SNAPSHOT
3+
mlGradleVersion=6.2-SNAPSHOT
44

55
mlHost=localhost
66
mlAppName=example

‎ml-app-deployer/src/main/java/com/marklogic/appdeployer/command/pdc/DeployMarkLogicEndpointsCommand.java‎

Lines changed: 84 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -39,18 +39,19 @@ public void execute(CommandContext context) {
3939
return;
4040
}
4141

42-
final List<MarkLogicHttpEndpoint> endpoints = readEndpointDefinitionsFromFiles(context, pdcConfigPaths);
43-
if (!endpoints.isEmpty()) {
42+
final Map<String, List<MarkLogicHttpEndpoint>> endpointsByDnsName = readEndpointDefinitionsFromFiles(context, pdcConfigPaths);
43+
if (!endpointsByDnsName.isEmpty()) {
4444
if (!StringUtils.hasText(context.getAppConfig().getCloudApiKey())) {
45-
logger.warn("Found configuration for {} MarkLogic endpoint(s), but not deploying them because no cloud API key has been specified.", endpoints.size());
45+
int totalEndpoints = endpointsByDnsName.values().stream().mapToInt(List::size).sum();
46+
logger.warn("Found configuration for {} MarkLogic endpoint(s), but not deploying them because no cloud API key has been specified.", totalEndpoints);
4647
} else {
47-
deployEndpoints(context, endpoints);
48+
deployEndpointsByService(context, endpointsByDnsName);
4849
}
4950
}
5051
}
5152

52-
private List<MarkLogicHttpEndpoint> readEndpointDefinitionsFromFiles(CommandContext context, List<String> pdcConfigPaths) {
53-
List<MarkLogicHttpEndpoint> endpoints = new ArrayList<>();
53+
private Map<String, List<MarkLogicHttpEndpoint>> readEndpointDefinitionsFromFiles(CommandContext context, List<String> pdcConfigPaths) {
54+
Map<String, List<MarkLogicHttpEndpoint>> endpointsByDnsName = new HashMap<>();
5455

5556
for (String pdcConfigPath : pdcConfigPaths) {
5657
File serviceDir = new File(pdcConfigPath, "service");
@@ -62,17 +63,33 @@ private List<MarkLogicHttpEndpoint> readEndpointDefinitionsFromFiles(CommandCont
6263

6364
logger.info("Reading MarkLogic endpoints from: {}", endpointsDir.getAbsolutePath());
6465

65-
try (Stream<Path> paths = Files.walk(endpointsDir.toPath())) {
66-
paths.filter(Files::isRegularFile)
67-
.filter(path -> path.toString().endsWith(".json"))
68-
.forEach(path -> endpoints.add(buildEndpointFromFile(context, path.toFile())));
69-
} catch (IOException e) {
70-
throw new RuntimeException("Failed to read MarkLogic endpoint configuration files from: " +
71-
endpointsDir.getAbsolutePath(), e);
66+
File[] dnsNameDirs = endpointsDir.listFiles(File::isDirectory);
67+
if (dnsNameDirs == null || dnsNameDirs.length == 0) {
68+
logger.warn("No dnsName directories found under: {}. Endpoints should be organized under mlendpoints/<dnsName>/*.json", endpointsDir.getAbsolutePath());
69+
continue;
70+
}
71+
72+
for (File dnsNameDir : dnsNameDirs) {
73+
String dnsName = dnsNameDir.getName();
74+
List<MarkLogicHttpEndpoint> endpoints = new ArrayList<>();
75+
76+
try (Stream<Path> paths = Files.walk(dnsNameDir.toPath())) {
77+
paths.filter(Files::isRegularFile)
78+
.filter(path -> path.toString().endsWith(".json"))
79+
.forEach(path -> endpoints.add(buildEndpointFromFile(context, path.toFile())));
80+
} catch (IOException e) {
81+
throw new RuntimeException("Failed to read MarkLogic endpoint configuration files from: " +
82+
dnsNameDir.getAbsolutePath(), e);
83+
}
84+
85+
if (!endpoints.isEmpty()) {
86+
endpointsByDnsName.put(dnsName, endpoints);
87+
logger.info("Found {} endpoint(s) for MarkLogic service: {}", endpoints.size(), dnsName);
88+
}
7289
}
7390
}
7491

75-
return endpoints;
92+
return endpointsByDnsName;
7693
}
7794

7895
private MarkLogicHttpEndpoint buildEndpointFromFile(CommandContext context, File endpointFile) {
@@ -91,55 +108,69 @@ private MarkLogicHttpEndpoint buildEndpointFromFile(CommandContext context, File
91108
}
92109
}
93110

94-
private void deployEndpoints(CommandContext context, List<MarkLogicHttpEndpoint> endpoints) {
111+
private void deployEndpointsByService(CommandContext context, Map<String, List<MarkLogicHttpEndpoint>> endpointsByDnsName) {
95112
final String host = context.getManageClient().getManageConfig().getHost();
96113
try (PdcClient pdcClient = new PdcClient(host, context.getAppConfig().getCloudApiKey())) {
97-
final UUID markLogicServiceId = getFirstMarkLogicServiceId(pdcClient);
98-
final ServiceApi serviceApi = new ServiceApi(pdcClient.getApiClient());
99-
try {
100-
MarkLogicEndpointMappingData existingEndpoints = serviceApi.apiServiceMlendpointsIdGet(markLogicServiceId);
101-
List<MarkLogicHttpEndpoint> endpointsToDeploy = filterOutExistingEndpoints(endpoints, existingEndpoints);
102-
if (endpointsToDeploy.isEmpty()) {
103-
logger.info("All {} endpoint(s) are up to date; nothing to deploy.", endpoints.size());
104-
} else {
105-
logger.info("Deploying {} new or updated endpoint(s) out of {} total.", endpointsToDeploy.size(), endpoints.size());
106-
serviceApi.apiServiceMlendpointsIdHttpPut(markLogicServiceId, endpointsToDeploy);
107-
logger.info("Successfully deployed {} endpoint(s).", endpointsToDeploy.size());
108-
}
109-
} catch (ApiException e) {
110-
throw new RuntimeException("Unable to create MarkLogic endpoints in PDC; cause: %s".formatted(e.getMessage()), e);
114+
for (Map.Entry<String, List<MarkLogicHttpEndpoint>> entry : endpointsByDnsName.entrySet()) {
115+
String dnsName = entry.getKey();
116+
List<MarkLogicHttpEndpoint> endpoints = entry.getValue();
117+
logger.info("Processing {} endpoint(s) for MarkLogic service: {}", endpoints.size(), dnsName);
118+
deployEndpoints(pdcClient, dnsName, endpoints);
119+
}
120+
}
121+
}
122+
123+
private void deployEndpoints(PdcClient pdcClient, String dnsName, List<MarkLogicHttpEndpoint> endpoints) {
124+
final UUID markLogicServiceId = getMarkLogicServiceIdByDnsName(pdcClient, dnsName);
125+
final ServiceApi serviceApi = new ServiceApi(pdcClient.getApiClient());
126+
try {
127+
MarkLogicEndpointMappingData existingEndpointData = serviceApi.apiServiceMlendpointsIdGet(markLogicServiceId);
128+
if (allEndpointsMatch(endpoints, existingEndpointData)) {
129+
logger.info("All {} endpoint(s) for '{}' are up to date; nothing to deploy.", endpoints.size(), dnsName);
130+
} else {
131+
logger.info("Deploying all {} endpoint(s) for '{}' due to changes or count mismatch.", endpoints.size(), dnsName);
132+
serviceApi.apiServiceMlendpointsIdHttpPut(markLogicServiceId, endpoints);
133+
logger.info("Successfully deployed {} endpoint(s) for '{}'.", endpoints.size(), dnsName);
111134
}
135+
} catch (ApiException e) {
136+
throw new RuntimeException("Unable to create MarkLogic endpoints for '%s' in PDC; cause: %s".formatted(dnsName, e.getMessage()), e);
112137
}
113138
}
114139

115140
/**
116-
* Filters out endpoints that don't need to be deployed. An endpoint needs to be deployed if:
117-
* 1. It doesn't exist in PDC (based on name, which is unique), OR
118-
* 2. It exists but has different properties (needs to be updated)
141+
* Checks if all endpoints match the existing endpoints in PDC. Returns true only if:
142+
* 1. The count of endpoints matches, AND
143+
* 2. Every endpoint exists with identical properties
119144
* <p>
120-
* An endpoint can take a surprisingly long time to create, so we only want to deploy ones that
121-
* are new or have changed.
145+
* If any endpoint is new, modified, or removed, this returns false and all endpoints
146+
* will be deployed. The PDC API requires sending the complete list of endpoints.
122147
*
123148
* @param endpoints the list of endpoints to potentially deploy
124149
* @param existingEndpointData the existing endpoint data from PDC
125-
* @return a list of endpoints that need to be deployed (new or updated)
150+
* @return true if all endpoints are up to date, false if deployment is needed
126151
*/
127-
private List<MarkLogicHttpEndpoint> filterOutExistingEndpoints(
152+
private boolean allEndpointsMatch(
128153
List<MarkLogicHttpEndpoint> endpoints,
129154
MarkLogicEndpointMappingData existingEndpointData
130155
) {
131156
if (existingEndpointData == null || existingEndpointData.getEndpoints() == null
132157
|| existingEndpointData.getEndpoints().getHttpEndpoints() == null) {
133-
return endpoints;
158+
return false;
134159
}
135160

136-
Map<String, MarkLogicHttpEndpoint> existingEndpointsByName = existingEndpointData.getEndpoints()
137-
.getHttpEndpoints().stream()
161+
List<MarkLogicHttpEndpoint> existingEndpoints = existingEndpointData.getEndpoints().getHttpEndpoints();
162+
163+
// If counts don't match, something changed
164+
if (endpoints.size() != existingEndpoints.size()) {
165+
return false;
166+
}
167+
168+
Map<String, MarkLogicHttpEndpoint> existingEndpointsByName = existingEndpoints.stream()
138169
.collect(Collectors.toMap(MarkLogicHttpEndpoint::getName, Function.identity()));
139170

171+
// Check if all endpoints exist and match
140172
return endpoints.stream()
141-
.filter(endpoint -> needsDeployment(endpoint, existingEndpointsByName.get(endpoint.getName())))
142-
.collect(Collectors.toList());
173+
.allMatch(endpoint -> !needsDeployment(endpoint, existingEndpointsByName.get(endpoint.getName())));
143174
}
144175

145176
/**
@@ -162,14 +193,24 @@ private boolean needsDeployment(MarkLogicHttpEndpoint endpoint, MarkLogicHttpEnd
162193
|| !Objects.equals(endpoint.getSupportedByCloud(), existingEndpoint.getSupportedByCloud());
163194
}
164195

165-
private UUID getFirstMarkLogicServiceId(PdcClient pdcClient) {
196+
private UUID getMarkLogicServiceIdByDnsName(PdcClient pdcClient, String dnsName) {
166197
try {
167198
final UUID environmentId = pdcClient.getEnvironmentId();
168199
List<MarkLogicApp> apps = new ServiceApi(pdcClient.getApiClient()).apiServiceAppsGet(environmentId).getMarkLogic();
169200
if (apps == null || apps.isEmpty()) {
170201
throw new RuntimeException("No instances of MarkLogic found in PDC tenancy; host: %s".formatted(pdcClient.getHost()));
171202
}
172-
return apps.get(0).getId();
203+
204+
return apps.stream()
205+
.filter(app -> dnsName.equals(app.getDnsName()))
206+
.findFirst()
207+
.map(MarkLogicApp::getId)
208+
.orElseThrow(() -> new RuntimeException(
209+
"No MarkLogic service found with dnsName '%s'. Available services: %s".formatted(
210+
dnsName,
211+
apps.stream().map(MarkLogicApp::getDnsName).collect(Collectors.joining(", "))
212+
)
213+
));
173214
} catch (ApiException e) {
174215
throw new RuntimeException("Unable to lookup instances of MarkLogic in PDC; cause: %s".formatted(e), e);
175216
}

ml-app-deployer/src/test/resources/cloud-project/src/main/pdc-config/service/mlendpoints/endpoint1.json renamed to ml-app-deployer/src/test/resources/cloud-project/src/main/pdc-config/service/mlendpoints/exampledns/endpoint1.json

File renamed without changes.

ml-app-deployer/src/test/resources/cloud-project/src/main/pdc-config/service/mlendpoints/endpoint2.json renamed to ml-app-deployer/src/test/resources/cloud-project/src/main/pdc-config/service/mlendpoints/exampledns/endpoint2.json

File renamed without changes.

‎ml-gradle/src/main/groovy/com/marklogic/gradle/MarkLogicPlugin.groovy‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ import com.marklogic.gradle.task.groups.SetTraceEventsTask
3838
import com.marklogic.gradle.task.hosts.AssignHostsToGroupsTask
3939
import com.marklogic.gradle.task.mimetypes.DeployMimetypesTask
4040
import com.marklogic.gradle.task.mimetypes.UndeployMimetypesTask
41+
import com.marklogic.gradle.task.pdc.DeployMarkLogicEndpointsTask
4142
import com.marklogic.gradle.task.plugins.InstallPluginsTask
4243
import com.marklogic.gradle.task.plugins.UninstallPluginsTask
4344
import com.marklogic.gradle.task.qconsole.ExportWorkspacesTask
@@ -341,7 +342,12 @@ class MarkLogicPlugin implements Plugin<Project> {
341342
"Use -PrunCodeCoverage to enable code coverage support when running the tests. " +
342343
"Use -PrunTeardown and -PrunSuiteTeardown to control whether teardown and suite teardown scripts are run; these default to 'true' and can be set to 'false' instead. ")
343344

344-
// Any granular task that deploys/undeploys resources may need to do so for a resource in a bundle, so these
345+
String pdcGroup = "PDC"
346+
project.tasks.register("pdcDeployMarkLogicEndpoints", DeployMarkLogicEndpointsTask.class) {
347+
group = pdcGroup
348+
description = "Deploy MarkLogic endpoints to a PDC instance. Endpoints default to being defined at src/main/pdc-config/service/mlendpoints/(dnsName)."
349+
}
350+
345351
// tasks must all depend on mlPrepareBundles
346352
project.tasks.each { task ->
347353
if (task.name.startsWith("mlDeploy") || task.name.startsWith("mlUndeploy")) {

‎ml-gradle/src/main/groovy/com/marklogic/gradle/task/pdc/DeployMarkLogicEndpointsTask.groovy‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
/*
2+
* Copyright (c) 2015-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved.
3+
*/
4+
package com.marklogic.gradle.task.pdc
5+
6+
import com.marklogic.appdeployer.command.pdc.DeployMarkLogicEndpointsCommand
7+
import com.marklogic.gradle.task.MarkLogicTask
8+
import org.gradle.api.tasks.TaskAction
9+
10+
class DeployMarkLogicEndpointsTask extends MarkLogicTask {
11+
12+
@TaskAction
13+
void deployMarkLogicEndpoints() {
14+
invokeDeployerCommandWithClassName(DeployMarkLogicEndpointsCommand.class.getSimpleName())
15+
}
16+
}

0 commit comments

Comments
 (0)