[refactor] index worker priorities, Jetty listener API, test server builder

- IndexWorker.getChainPriority() replaces IndexController substring ranking
- JettyStartListener replaces deprecated Observable for Jetty lifecycle signals
- Event-driven webapp readiness wait with CountDownLatch
- org.exist.jetty.startup.timeout.ms override; EXIST/PORTAL context constants
- ExistWebServer.builder() for readable test configuration

Author: duncdrum

exist-core/src/main/java/org/exist/indexing/IndexController.java

@@ -42,9 +42,9 @@
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.TreeMap;
 import org.exist.security.PermissionDeniedException;
 
 /**
@@ -60,14 +60,12 @@ public enum CollectionIndexRemovalMode {
 
     /**
      * Stable iteration order for listener chains and {@link #flush()}.
-     * Alphabetical {@link TreeMap} put Lucene first and broke store-time indexing
-     * (see {@code LuceneTests} facet/field-type tests). Core indexes run before Lucene.
      */
-    private static final Comparator<String> INDEX_WORKER_ORDER = Comparator
-            .comparingInt(IndexController::indexWorkerChainRank)
-            .thenComparing(Comparator.naturalOrder());
+    private static final Comparator<IndexWorker> INDEX_WORKER_ORDER = Comparator
+            .comparingInt(IndexWorker::getChainPriority)
+            .thenComparing(IndexWorker::getIndexId);
 
-    private final Map<String, IndexWorker> indexWorkers = new TreeMap<>(INDEX_WORKER_ORDER);
+    private final Map<String, IndexWorker> indexWorkers = new LinkedHashMap<>();
 
     private final DBBroker broker;
     private StreamListener listener = null;
@@ -79,26 +77,9 @@ public enum CollectionIndexRemovalMode {
     public IndexController(final DBBroker broker) {
         this.broker = broker;
         final List<IndexWorker> workers = broker.getBrokerPool().getIndexManager().getWorkers(broker);
-        for (final IndexWorker worker : workers) {
-            indexWorkers.put(worker.getIndexId(), worker);
-        }
-    }
-
-    /**
-     * Preferred {@link StreamListener} chain order: structural → statistics → Lucene → others.
-     * Uses index-id substrings so exist-core does not depend on extension modules.
-     */
-    private static int indexWorkerChainRank(final String indexId) {
-        if (indexId.contains("NativeStructuralIndex")) {
-            return 0;
-        }
-        if (indexId.contains("IndexStatistics")) {
-            return 1;
-        }
-        if (indexId.contains("lucene.LuceneIndex")) {
-            return 2;
-        }
-        return 3;
+        workers.stream()
+                .sorted(INDEX_WORKER_ORDER)
+                .forEach(worker -> indexWorkers.put(worker.getIndexId(), worker));
     }
 
     /**

exist-core/src/main/java/org/exist/indexing/IndexWorker.java

@@ -53,6 +53,21 @@ public interface IndexWorker {
      */	
     public static final String VALUE_COUNT = "value_count";
 
+    /**
+     * Lower values run earlier in {@link IndexController} listener chains and {@link #flush()}.
+     */
+    int CHAIN_PRIORITY_STRUCTURAL = 0;
+    int CHAIN_PRIORITY_STATISTICS = 100;
+    int CHAIN_PRIORITY_LUCENE = 1000;
+
+    /**
+     * Chain-order priority for {@link IndexController}. Default {@link Integer#MAX_VALUE} preserves
+     * legacy ordering among workers that do not override.
+     */
+    default int getChainPriority() {
+        return Integer.MAX_VALUE;
+    }
+
     /**
      * Returns an ID which uniquely identifies this worker's index.
      * @return a unique name identifying this worker's index.

exist-core/src/main/java/org/exist/jetty/JettyStart.java

@@ -57,6 +57,10 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.*;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
 import static org.exist.util.ThreadUtils.newGlobalThread;
@@ -69,10 +73,14 @@
  *
  * @author wolf
  */
-public class JettyStart extends Observable implements LifeCycle.Listener {
+public class JettyStart implements LifeCycle.Listener {
 
     public static final String JETTY_HOME_PROP = "jetty.home";
     public static final String JETTY_BASE_PROP = "jetty.base";
+    public static final String STARTUP_TIMEOUT_MS_PROPERTY = "org.exist.jetty.startup.timeout.ms";
+
+    private static final String EXIST_CONTEXT_PATH = "/exist";
+    private static final String PORTAL_CONTEXT_PATH = "/";
 
     private static final String JETTY_PROPETIES_FILENAME = "jetty.properties";
     private static final Logger logger = LogManager.getLogger(JettyStart.class);
@@ -101,6 +109,8 @@ public class JettyStart extends Observable implements LifeCycle.Listener {
     @GuardedBy("this") private boolean webAppStartedSuccessfully = false;
     @GuardedBy("this") private String webAppStartupFailureDetail = null;
 
+    private final CopyOnWriteArrayList<JettyStartListener> jettyStartListeners = new CopyOnWriteArrayList<>();
+
 
     public static void main(final String[] args) {
         try {
@@ -168,7 +178,25 @@ public synchronized void run(final boolean standalone) {
         run(new String[] { jettyConfig.toAbsolutePath().toString() }, null);
     }
 
-    public synchronized void run(final String[] args, final Observer observer) {
+    public void addJettyStartListener(final JettyStartListener listener) {
+        if (listener != null) {
+            jettyStartListeners.addIfAbsent(listener);
+        }
+    }
+
+    public void removeJettyStartListener(final JettyStartListener listener) {
+        if (listener != null) {
+            jettyStartListeners.remove(listener);
+        }
+    }
+
+    private void notifyJettyStartListeners(final String signal) {
+        for (final JettyStartListener listener : jettyStartListeners) {
+            listener.onJettyStartEvent(signal);
+        }
+    }
+
+    public synchronized void run(final String[] args, final JettyStartListener listener) {
         if (args.length == 0) {
             logger.error("No configuration file specified!");
             return;
@@ -209,8 +237,8 @@ public synchronized void run(final String[] args, final Observer observer) {
                 configProperties.put(JETTY_BASE_PROP, jettyClasspathHome);
             }
 
-            if (observer != null) {
-                addObserver(observer);
+            if (listener != null) {
+                addJettyStartListener(listener);
             }
 
             logger.info("Running with Java {} [{} ({}) in {}]",
@@ -249,7 +277,10 @@ public synchronized void run(final String[] args, final Observer observer) {
                         .map(Path::normalize).map(Path::toAbsolutePath).map(Path::toString)
                         .orElse("<UNKNOWN>"));
 
-            BrokerPool.configure(1, 5, config, Optional.ofNullable(observer));
+            final Optional<Observer> brokerPoolObserver = listener instanceof Observer observer
+                    ? Optional.of(observer)
+                    : Optional.empty();
+            BrokerPool.configure(1, 5, config, brokerPoolObserver);
 
             // register the XMLDB driver
             final Database xmldb = new DatabaseImpl();
@@ -362,13 +393,11 @@ public synchronized void run(final String[] args, final Observer observer) {
             webAppStartedSuccessfully = true;
             webAppStartupFailureDetail = null;
 
-            setChanged();
-            notifyObservers(SIGNAL_STARTED);
+            notifyJettyStartListeners(SIGNAL_STARTED);
 
         } catch (final SocketException e) {
             recordStartupFailure("Could not bind to port: " + e.getMessage(), e);
-            setChanged();
-            notifyObservers(SIGNAL_ERROR);
+            notifyJettyStartListeners(SIGNAL_ERROR);
 
         } catch (final Exception e) {
             if (webAppStartupFailureDetail == null) {
@@ -377,8 +406,7 @@ public synchronized void run(final String[] args, final Observer observer) {
             } else {
                 recordStartupFailure(webAppStartupFailureDetail, e);
             }
-            setChanged();
-            notifyObservers(SIGNAL_ERROR);
+            notifyJettyStartListeners(SIGNAL_ERROR);
         }
     }
 
@@ -529,9 +557,9 @@ private Optional<Server> startJetty(final List<Object> configuredObjects) throws
     /**
      * Block until deployed webapps reach the readiness level required for tests.
      * <p>
-     * Every context except the distribution portal at {@code /} must be
+     * Every context except the distribution portal at {@link #PORTAL_CONTEXT_PATH} must be
      * {@link org.eclipse.jetty.server.handler.ContextHandler#isAvailable()} — Jetty returns
-     * {@code 503} on all paths while unavailable. The portal coexists with {@code /exist} and is
+     * {@code 503} on all paths while unavailable. The portal coexists with {@link #EXIST_CONTEXT_PATH} and is
      * non-gating.
      */
     private void awaitWebAppContextsStarted(final List<Handler> handlers) throws InterruptedException {
@@ -547,29 +575,87 @@ private void awaitWebAppContextsStarted(final List<Handler> handlers) throws Int
 
         final boolean distributionLayout = isDistributionLayout(webApps);
         final long timeoutMs = slowEnvironmentStartupDeadlineMs();
-        final long deadline = System.currentTimeMillis() + timeoutMs;
-        while (System.currentTimeMillis() < deadline) {
-            boolean allReady = true;
+        final CountDownLatch readyLatch = new CountDownLatch(1);
+        final AtomicReference<IllegalStateException> failure = new AtomicReference<>();
+
+        final LifeCycle.Listener readinessListener = new LifeCycle.Listener() {
+            @Override
+            public void lifeCycleStarted(final LifeCycle event) {
+                evaluateReadiness();
+            }
+
+            @Override
+            public void lifeCycleFailure(final LifeCycle event, final Throwable cause) {
+                if (event instanceof WebAppContext webApp) {
+                    failure.compareAndSet(null, new IllegalStateException(
+                            "Web application failed to start: " + webApp.getContextPath(), cause));
+                } else {
+                    failure.compareAndSet(null, new IllegalStateException("Web application failed to start", cause));
+                }
+                readyLatch.countDown();
+            }
+
+            private void evaluateReadiness() {
+                for (final WebAppContext webApp : webApps) {
+                    if (webApp.isFailed()) {
+                        failure.compareAndSet(null, new IllegalStateException(
+                                "Web application failed to start: " + webApp.getContextPath()));
+                        readyLatch.countDown();
+                        return;
+                    }
+                }
+                if (allWebAppsReady(webApps, distributionLayout)) {
+                    readyLatch.countDown();
+                }
+            }
+        };
+
+        for (final WebAppContext webApp : webApps) {
+            webApp.addEventListener(readinessListener);
+        }
+
+        try {
+            if (allWebAppsReady(webApps, distributionLayout)) {
+                logger.info("All required web application contexts are ready.");
+                return;
+            }
             for (final WebAppContext webApp : webApps) {
                 if (webApp.isFailed()) {
                     throw new IllegalStateException(
                             "Web application failed to start: " + webApp.getContextPath());
                 }
-                if (!isWebAppContextReady(webApp, distributionLayout)) {
-                    allReady = false;
-                    break;
-                }
             }
-            if (allReady) {
-                logger.info("All required web application contexts are ready.");
-                return;
+            if (!readyLatch.await(timeoutMs, TimeUnit.MILLISECONDS)) {
+                throw new IllegalStateException(
+                        "Web application context did not become ready within " + timeoutMs + "ms: "
+                                + describePendingWebApps(webApps, distributionLayout),
+                        firstUnavailableCause(webApps));
+            }
+            final IllegalStateException startupFailure = failure.get();
+            if (startupFailure != null) {
+                throw startupFailure;
+            }
+            if (!allWebAppsReady(webApps, distributionLayout)) {
+                throw new IllegalStateException(
+                        "Web application context did not become ready: "
+                                + describePendingWebApps(webApps, distributionLayout),
+                        firstUnavailableCause(webApps));
+            }
+            logger.info("All required web application contexts are ready.");
+        } finally {
+            for (final WebAppContext webApp : webApps) {
+                webApp.removeEventListener(readinessListener);
+            }
+        }
+    }
+
+    private static boolean allWebAppsReady(final List<WebAppContext> webApps, final boolean distributionLayout) {
+        for (final WebAppContext webApp : webApps) {
+            if (!isWebAppContextReady(webApp, distributionLayout)) {
+                return false;
             }
-            Thread.sleep(200);
         }
-        throw new IllegalStateException(
-                "Web application context did not become ready within " + timeoutMs + "ms: "
-                        + describePendingWebApps(webApps, distributionLayout),
-                firstUnavailableCause(webApps));
+        return true;
     }
 
     private static Throwable firstUnavailableCause(final List<WebAppContext> webApps) {
@@ -583,18 +669,18 @@ private static Throwable firstUnavailableCause(final List<WebAppContext> webApps
     }
 
     private static boolean isDistributionLayout(final List<WebAppContext> webApps) {
-        return webApps.stream().anyMatch(webApp -> "/exist".equals(webApp.getContextPath()));
+        return webApps.stream().anyMatch(webApp -> EXIST_CONTEXT_PATH.equals(webApp.getContextPath()));
     }
 
     /**
-     * Distribution portal {@code /} only needs {@code isStarted()}. Standalone {@code /} and
-     * {@code /exist} must be {@code isAvailable()} or HTTP clients see {@code 503}.
+     * Distribution portal {@link #PORTAL_CONTEXT_PATH} only needs {@code isStarted()}. Standalone
+     * {@link #PORTAL_CONTEXT_PATH} and {@link #EXIST_CONTEXT_PATH} must be {@code isAvailable()} or HTTP clients see {@code 503}.
      */
     private static boolean isWebAppContextReady(final WebAppContext webApp, final boolean distributionLayout) {
         if (!webApp.isStarted()) {
             return false;
         }
-        if (distributionLayout && "/".equals(webApp.getContextPath())) {
+        if (distributionLayout && PORTAL_CONTEXT_PATH.equals(webApp.getContextPath())) {
             return true;
         }
         return webApp.isAvailable();
@@ -639,10 +725,14 @@ private static String describeWebAppWar(final WebAppContext webApp) {
     }
 
     private static boolean requiresAvailability(final WebAppContext webApp, final boolean distributionLayout) {
-        return !(distributionLayout && "/".equals(webApp.getContextPath()));
+        return !(distributionLayout && PORTAL_CONTEXT_PATH.equals(webApp.getContextPath()));
     }
 
     private static long slowEnvironmentStartupDeadlineMs() {
+        final String override = System.getProperty(STARTUP_TIMEOUT_MS_PROPERTY);
+        if (override != null && !override.isBlank()) {
+            return Long.parseLong(override);
+        }
         if (System.getenv("CI") != null) {
             return 180_000L;
         }
@@ -799,17 +889,15 @@ public synchronized boolean isStarted() {
     @Override
     public synchronized void lifeCycleStarting(final LifeCycle lifeCycle) {
         logger.info("Jetty server starting...");
-        setChanged();
-        notifyObservers(SIGNAL_STARTING);
+        notifyJettyStartListeners(SIGNAL_STARTING);
         status = STATUS_STARTING;
         notifyAll();
     }
 
     @Override
     public synchronized void lifeCycleStarted(final LifeCycle lifeCycle) {
         logger.info("Jetty server started.");
-        setChanged();
-        notifyObservers(SIGNAL_STARTED);
+        notifyJettyStartListeners(SIGNAL_STARTED);
         status = STATUS_STARTED;
         notifyAll();
     }

exist-core/src/main/java/org/exist/jetty/JettyStartListener.java

@@ -0,0 +1,31 @@
+/*
+ * eXist-db Open Source Native XML Database
+ * Copyright (C) 2001 The eXist-db Authors
+ *
+ * info@exist-db.org
+ * http://www.exist-db.org
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+package org.exist.jetty;
+
+/**
+ * Receives lifecycle signals from {@link JettyStart} (replacing deprecated {@code java.util.Observer}).
+ */
+@FunctionalInterface
+public interface JettyStartListener {
+
+    void onJettyStartEvent(String signal);
+}