[test] Harden DeadlockIT store/remove coordination

Share document naming between store and remove tasks, wait for store
completion before removals, use per-test failure tracking, and retry
when concurrent removes race on the same resource.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: duncdrum

exist-core/src/test/java/org/exist/storage/lock/DeadlockIT.java

@@ -26,6 +26,7 @@
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.*;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
@@ -58,6 +59,7 @@
 import org.xml.sax.SAXException;
 import org.xmldb.api.DatabaseManager;
 import org.xmldb.api.base.Database;
+import org.xmldb.api.base.ErrorCodes;
 import org.xmldb.api.base.ResourceSet;
 import org.xmldb.api.base.XMLDBException;
 import org.xmldb.api.base.Resource;
@@ -95,12 +97,35 @@ public class DeadlockIT {
     /** Max attempts to find and remove an existing document before failing. */
     private static final int MAX_REMOVE_ATTEMPTS = 100;
 
-    private static final AtomicReference<Throwable> taskFailure = new AtomicReference<>();
+    private final AtomicReference<Throwable> taskFailure = new AtomicReference<>();
 
-    private static void recordTaskFailure(final Throwable t) {
+    private void recordTaskFailure(final Throwable t) {
         taskFailure.compareAndSet(null, t);
     }
 
+    private void rethrowTaskFailure() {
+        final Throwable failure = taskFailure.get();
+        if (failure != null) {
+            if (failure instanceof RuntimeException re) {
+                throw re;
+            }
+            if (failure instanceof Error err) {
+                throw err;
+            }
+            throw new AssertionError(failure.getMessage(), failure);
+        }
+    }
+
+    /** Matches {@link StoreTask} global document numbering. */
+    private static String documentName(final int collectionId, final int indexInCollection) {
+        return "test" + (collectionId * DOC_COUNT + indexInCollection) + ".xml";
+    }
+
+    private static boolean isConcurrentRemoveRace(final XMLDBException e) {
+        return e.errorCode == ErrorCodes.INVALID_RESOURCE
+                || e.errorCode == ErrorCodes.NO_SUCH_RESOURCE;
+    }
+
     /** Use 4 test runs, querying different collections */
     @Parameters(name = "{0}")
     public static java.util.Collection<Object[]> data() {
@@ -197,7 +222,8 @@ public void clearDB() throws XMLDBException {
 	public void runTasks() {
 		taskFailure.set(null);
 		final ExecutorService executor = Executors.newFixedThreadPool(N_THREADS);
-        executor.submit(new StoreTask("store", COLL_COUNT, DOC_COUNT));
+        final CountDownLatch storeComplete = new CountDownLatch(1);
+        executor.submit(new StoreTask("store", COLL_COUNT, DOC_COUNT, storeComplete));
         synchronized (this) {
             try {
                 wait(DELAY);
@@ -211,6 +237,15 @@ public void runTasks() {
 			executor.submit(new QueryTask(COLL_COUNT));
 		}
         if (mode == TEST_REMOVE) {
+            try {
+                assertTrue("Store task did not finish before document removals started",
+                        storeComplete.await(AWAIT_TERMINATION_MINUTES, TimeUnit.MINUTES));
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                LOG.error(e.getMessage(), e);
+                fail(e.getMessage());
+            }
+            rethrowTaskFailure();
             for (int i = 0; i < REMOVE_COUNT; i++) {
                 executor.submit(new RemoveDocumentTask(COLL_COUNT, DOC_COUNT));
             }
@@ -228,29 +263,23 @@ public void runTasks() {
 			executor.shutdownNow();
 			assertTrue("Executor did not terminate within " + AWAIT_TERMINATION_MINUTES + " minutes; possible deadlock or hang", terminated);
 		}
-		final Throwable failure = taskFailure.get();
-		if (failure != null) {
-			if (failure instanceof RuntimeException re) {
-				throw re;
-			}
-			if (failure instanceof Error err) {
-				throw err;
-			}
-			throw new AssertionError(failure.getMessage(), failure);
-		}
+		rethrowTaskFailure();
 	}
 
-	private static class StoreTask implements Runnable {
+	private class StoreTask implements Runnable {
 
 		@SuppressWarnings("unused")
 		private final String id;
 		private final int docCount;
 		private final int collectionCount;
+        private final CountDownLatch storeComplete;
 
-		public StoreTask(final String id, final int collectionCount, final int docCount) {
+		public StoreTask(final String id, final int collectionCount, final int docCount,
+                final CountDownLatch storeComplete) {
 			this.id = id;
 			this.collectionCount = collectionCount;
 			this.docCount = docCount;
+            this.storeComplete = storeComplete;
 		}
 
 		@Override
@@ -261,7 +290,6 @@ public void run() {
 
 				final TestDataGenerator generator = new TestDataGenerator("xdb", docCount);
 				Collection coll;
-				int fileCount = 0;
 				for (int i = 0; i < collectionCount; i++) {
                     try(final Txn transaction = transact.beginTransaction()) {
                         coll = broker.getOrCreateCollection(transaction,
@@ -273,12 +301,12 @@ public void run() {
                     }
 
                     final Path[] files = generator.generate(broker, coll, generateXQ);
-                    for (int j = 0; j < files.length; j++, fileCount++) {
+                    for (int j = 0; j < files.length; j++) {
                         try(final Txn transaction = transact.beginTransaction()) {
                             final InputSource is = new InputSource(files[j].toUri()
                                     .toASCIIString());
 
-							broker.storeDocument(transaction, XmldbURI.create("test" + fileCount + ".xml"), is, MimeType.XML_TYPE, coll);
+							broker.storeDocument(transaction, XmldbURI.create(documentName(i, j)), is, MimeType.XML_TYPE, coll);
                             transact.commit(transaction);
                         }
                     }
@@ -287,7 +315,9 @@ public void run() {
 			} catch (Exception e) {
 				LOG.error(e.getMessage(), e);
 				recordTaskFailure(e);
-			}
+			} finally {
+                storeComplete.countDown();
+            }
 		}
 	}
 
@@ -372,8 +402,7 @@ public void run() {
             for (int attempt = 0; !removed && attempt < MAX_REMOVE_ATTEMPTS; attempt++) {
                 final int collectionId = random.nextInt(collectionCount);
                 final String collection = "/db/test/" + collectionId;
-                final int docId = collectionId * documentCount + random.nextInt(documentCount);
-                final String document = "test" + docId + ".xml";
+                final String document = documentName(collectionId, random.nextInt(documentCount));
                 try {
                     final org.xmldb.api.base.Collection testCollection = DatabaseManager.getCollection("xmldb:exist://" + collection, "admin", "");
                     final Resource resource = testCollection.getResource(document);
@@ -382,6 +411,9 @@ public void run() {
                         removed = true;
                     }
                 } catch (final XMLDBException e) {
+                    if (isConcurrentRemoveRace(e)) {
+                        continue;
+                    }
 					LOG.error(e.getMessage(), e);
                     recordTaskFailure(e);
                     return;