MLE-25666 Deferring construction of client when loading data

Author: rjrudin

ml-app-deployer/src/main/java/com/marklogic/appdeployer/command/data/LoadDataCommand.java

@@ -11,6 +11,7 @@
 import com.marklogic.client.DatabaseClient;
 import com.marklogic.client.ext.file.FileLoader;
 import com.marklogic.client.ext.file.GenericFileLoader;
+import com.marklogic.client.ext.helper.DatabaseClientSupplier;
 import org.springframework.util.StringUtils;
 
 import java.util.List;
@@ -42,21 +43,16 @@ public void execute(CommandContext context) {
 			return;
 		}
 
-		final FileLoader fileLoader = buildFileLoader(context.getAppConfig());
-		for (String dataPath : dataPaths) {
-			fileLoader.loadFiles(dataPath);
+		try (DatabaseClientSupplier clientSupplier = new DatabaseClientSupplier(() -> determineDatabaseClient(context.getAppConfig()))) {
+			final FileLoader fileLoader = buildFileLoader(context.getAppConfig(), clientSupplier);
+			for (String dataPath : dataPaths) {
+				fileLoader.loadFiles(dataPath);
+			}
 		}
 	}
 
-	/**
-	 * Build a FileLoader based on the configuration in the given AppConfig object.
-	 *
-	 * @param appConfig
-	 * @return
-	 */
-	protected FileLoader buildFileLoader(AppConfig appConfig) {
-		final DatabaseClient client = determineDatabaseClient(appConfig);
-		final GenericFileLoader loader = new GenericFileLoader(client);
+	protected final FileLoader buildFileLoader(AppConfig appConfig, DatabaseClientSupplier supplier) {
+		final GenericFileLoader loader = new GenericFileLoader(supplier);
 
 		loader.setCascadeCollections(appConfig.isCascadeCollections());
 		loader.setCascadePermissions(appConfig.isCascadePermissions());
@@ -97,11 +93,11 @@ protected FileLoader buildFileLoader(AppConfig appConfig) {
 	 * @param appConfig
 	 * @return
 	 */
-	protected DatabaseClient determineDatabaseClient(AppConfig appConfig) {
+	protected final DatabaseClient determineDatabaseClient(AppConfig appConfig) {
 		DataConfig dataConfig = appConfig.getDataConfig();
 		final String databaseName = dataConfig.getDatabaseName();
 		if (StringUtils.hasText(databaseName)) {
-			logger.info("Will load data via App-Services port into database: " + databaseName);
+			logger.info("Will load data via App-Services port into database: {}", databaseName);
 			return appConfig.newAppServicesDatabaseClient(databaseName);
 		}
 		return appConfig.newDatabaseClient();

ml-app-deployer/src/main/java/com/marklogic/appdeployer/command/pdc/DeployMarkLogicEndpointsCommand.java

@@ -56,15 +56,11 @@ private List<MarkLogicHttpEndpoint> readEndpointDefinitionsFromFiles(CommandCont
 			File serviceDir = new File(pdcConfigPath, "service");
 			File endpointsDir = new File(serviceDir, "mlendpoints");
 			if (!endpointsDir.exists()) {
-				if (logger.isDebugEnabled()) {
-					logger.debug("MarkLogic endpoints directory does not exist: {}", endpointsDir.getAbsolutePath());
-				}
+				logger.info("MarkLogic endpoints directory does not exist: {}", endpointsDir.getAbsolutePath());
 				continue;
 			}
 
-			if (logger.isInfoEnabled()) {
-				logger.info("Reading MarkLogic endpoints from: {}", endpointsDir.getAbsolutePath());
-			}
+			logger.info("Reading MarkLogic endpoints from: {}", endpointsDir.getAbsolutePath());
 
 			try (Stream<Path> paths = Files.walk(endpointsDir.toPath())) {
 				paths.filter(Files::isRegularFile)
@@ -106,9 +102,7 @@ private void deployEndpoints(CommandContext context, List<MarkLogicHttpEndpoint>
 				if (endpointsToDeploy.isEmpty()) {
 					logger.info("All {} endpoint(s) are up to date; nothing to deploy.", endpoints.size());
 				} else {
-					if (logger.isInfoEnabled()) {
-						logger.info("Deploying {} new or updated endpoint(s) out of {} total.", endpointsToDeploy.size(), endpoints.size());
-					}
+					logger.info("Deploying {} new or updated endpoint(s) out of {} total.", endpointsToDeploy.size(), endpoints.size());
 					serviceApi.apiServiceMlendpointsIdHttpPut(markLogicServiceId, endpointsToDeploy);
 					logger.info("Successfully deployed {} endpoint(s).", endpointsToDeploy.size());
 				}

ml-app-deployer/src/test/java/com/marklogic/appdeployer/command/data/LoadDataTest.java

@@ -16,16 +16,12 @@
 import java.io.File;
 import java.util.Set;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.*;
 
-public class LoadDataTest extends AbstractAppDeployerTest {
+class LoadDataTest extends AbstractAppDeployerTest {
 
 	@AfterEach
-	public void teardown() {
+	void teardown() {
 		undeploySampleApp();
 	}
 
@@ -34,7 +30,7 @@ public void teardown() {
 	 * just does some basic assertions to make sure all is well.
 	 */
 	@Test
-	public void multiplePaths() {
+	void multiplePaths() {
 		DataConfig dataConfig = appConfig.getDataConfig();
 		dataConfig.getDataPaths().add(new File(dataConfig.getProjectDir(), "src/main/more-data").getAbsolutePath());
 
@@ -72,7 +68,7 @@ public void multiplePaths() {
 	}
 
 	@Test
-	public void databaseNameIsSet() {
+	void databaseNameIsSet() {
 		LoadDataCommand command = new LoadDataCommand();
 		DatabaseClient client = command.determineDatabaseClient(appConfig);
 		assertNull(client.getDatabase(), "The database property isn't set on a DatabaseClient when no value is provided when the " +
@@ -94,7 +90,7 @@ void cascadeCollectionsAndPermissions() {
 		appConfig.setCascadePermissions(true);
 		appConfig.setCascadeCollections(true);
 
-		GenericFileLoader loader = (GenericFileLoader) new LoadDataCommand().buildFileLoader(appConfig);
+		GenericFileLoader loader = (GenericFileLoader) new LoadDataCommand().buildFileLoader(appConfig, null);
 
 		assertTrue(loader.isCascadeCollections());
 		assertTrue(loader.isCascadePermissions());

ml-javaclient-util/src/main/java/com/marklogic/client/ext/file/GenericFileLoader.java

@@ -7,12 +7,15 @@
 import com.marklogic.client.document.DocumentWriteOperation;
 import com.marklogic.client.ext.batch.BatchWriter;
 import com.marklogic.client.ext.batch.RestBatchWriter;
+import com.marklogic.client.ext.helper.DatabaseClientSupplier;
 import com.marklogic.client.ext.helper.LoggingObject;
 import com.marklogic.client.ext.tokenreplacer.TokenReplacer;
 
 import java.io.FileFilter;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 /**
@@ -28,6 +31,7 @@ public class GenericFileLoader extends LoggingObject implements FileLoader {
 
 	private DocumentFileReader documentFileReader;
 	private BatchWriter batchWriter;
+	private Supplier<DatabaseClient> databaseClientSupplier;
 	private boolean waitForCompletion = true;
 	private boolean logFileUris = true;
 	private Integer batchSize;
@@ -44,6 +48,20 @@ public class GenericFileLoader extends LoggingObject implements FileLoader {
 	private boolean cascadeCollections;
 	private boolean cascadePermissions;
 
+	/**
+	 * The given DatabaseClientSupplier is used to lazily construct a BatchWriter that writes to MarkLogic via the REST API.
+	 * The DatabaseClient will only be created when files are actually written, avoiding unnecessary client creation.
+	 * <p>
+	 * The expectation is that the client will then either call setDocumentFileReader or will rely on the default one
+	 * that's created by this class if one has not yet been set.
+	 *
+	 * @param databaseClientSupplier supplier for lazy client creation
+	 * @since 6.2.0
+	 */
+	public GenericFileLoader(Supplier<DatabaseClient> databaseClientSupplier) {
+		this.databaseClientSupplier = databaseClientSupplier;
+	}
+
 	/**
 	 * The given DatabaseClient is used to construct a BatchWriter that writes to MarkLogic via the REST API. The
 	 * expectation is that the client will then either call setDocumentFileReader or will rely on the default one
@@ -52,9 +70,7 @@ public class GenericFileLoader extends LoggingObject implements FileLoader {
 	 * @param client
 	 */
 	public GenericFileLoader(DatabaseClient client) {
-		RestBatchWriter restBatchWriter = new RestBatchWriter(client);
-		restBatchWriter.setReleaseDatabaseClients(false);
-		this.batchWriter = restBatchWriter;
+		this(new DatabaseClientSupplier(() -> client));
 	}
 
 	/**
@@ -87,7 +103,7 @@ protected final List<DocumentFile> getDocumentFiles(String... paths) {
 
 	protected final void writeDocumentFiles(List<DocumentFile> documentFiles) {
 		if (documentFiles != null && !documentFiles.isEmpty()) {
-			batchWriter.initialize();
+			initializeBatchWriter();
 			writeBatchOfDocuments(documentFiles, 0);
 			if (waitForCompletion) {
 				batchWriter.waitForCompletion();
@@ -140,6 +156,21 @@ protected void writeBatchOfDocuments(List<DocumentFile> documentFiles, final int
 		}
 	}
 
+	/**
+	 * Lazily initializes the BatchWriter from the DatabaseClientSupplier if needed.
+	 * This ensures the DatabaseClient is only created when files actually need to be written.
+	 */
+	private void initializeBatchWriter() {
+		if (batchWriter == null) {
+			Objects.requireNonNull(databaseClientSupplier, "A DatabaseClient supplier is required to lazily create a BatchWriter.");
+			RestBatchWriter restBatchWriter = new RestBatchWriter(databaseClientSupplier.get());
+			restBatchWriter.setReleaseDatabaseClients(false);
+			this.batchWriter = restBatchWriter;
+		}
+
+		batchWriter.initialize();
+	}
+
 	/**
 	 * If no DocumentFileReader is set, this will construct a DefaultDocumentFileReader, which is then configured based
 	 * on several properties of this class.

ml-javaclient-util/src/main/java/com/marklogic/client/ext/helper/DatabaseClientSupplier.java

@@ -10,7 +10,7 @@
 
 /**
  * Preferred mechanism for lazy instantiation of a DatabaseClient. Will eventually deprecate DatabaseClientProvider in
- * favor of this.
+ * favor of this. Wraps a {@code Supplier<DatabaseClient>} and handles releasing the client if it was created.
  *
  * @since 6.2.0
  */
@@ -27,6 +27,10 @@ public DatabaseClientSupplier(Supplier<DatabaseClient> databaseClientSupplier) {
 		this.databaseClientSupplier = databaseClientSupplier;
 	}
 
+	/**
+	 * @return the DatabaseClient, creating it via the supplier if it hasn't already been created. May return null if
+	 * the supplier is not able to create a client.
+	 */
 	public DatabaseClient get() {
 		if (databaseClient == null) {
 			databaseClient = databaseClientSupplier.get();

ml-javaclient-util/src/main/java/com/marklogic/client/ext/modulesloader/impl/AssetFileLoader.java

@@ -21,7 +21,7 @@ public AssetFileLoader(DatabaseClient modulesDatabaseClient) {
 	}
 
 	public AssetFileLoader(DatabaseClient modulesDatabaseClient, ModulesManager modulesManager) {
-		super(modulesDatabaseClient);
+		super(() -> modulesDatabaseClient);
 		initializeAssetFileLoader(modulesManager);
 	}
 

ml-javaclient-util/src/test/java/com/marklogic/client/ext/file/SetAdditionalBinaryExtensionsTest.java

@@ -10,14 +10,14 @@
 import java.nio.file.Paths;
 import java.util.List;
 
-import static org.junit.jupiter.api.Assertions.*;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
-public class SetAdditionalBinaryExtensionsTest extends AbstractIntegrationTest {
+class SetAdditionalBinaryExtensionsTest extends AbstractIntegrationTest {
 
 	@Test
-	public void test() {
+	void test() {
 		client = newClient(CONTENT_DATABASE);
-		client.newServerEval().xquery("cts:uris((), (), cts:not-query(cts:collection-query('test-data'))) ! xdmp:document-delete(.)").eval();
+		client.newServerEval().xquery("cts:uris((), (), cts:not-query(cts:collection-query('test-data'))) ! xdmp:document-delete(.)").evalAs(String.class);
 
 		GenericFileLoader loader = new GenericFileLoader(client);
 		loader.setAdditionalBinaryExtensions("test1", "test2");