Add external OpenSearch support with E2E test

.github/workflows/external-opensearch-test.yml

@@ -0,0 +1,207 @@
+name: External OpenSearch Test
+
+on:
+    pull_request:
+        branches:
+            - main
+            - dev
+        paths:
+            - "Dockerfile"
+            - "src/**"
+            - "docker-compose*.yml"
+            - ".last_release"
+            - "pyproject.toml"
+            - "uv.lock"
+            - ".github/workflows/external-opensearch-test.yml"
+
+jobs:
+    test-external-opensearch:
+        runs-on: ubuntu-latest
+        env:
+            BASE_URL: "https://r2.koalasec.org/public"
+
+        steps:
+            - name: Checkout Repository
+              uses: actions/checkout@v6
+
+            - name: Set up Docker Buildx
+              uses: docker/setup-buildx-action@v3
+
+            - name: Read Photon version from .last_release
+              id: photon_version
+              run: |
+                  PHOTON_VERSION=$(cat .last_release | tr -d '[:space:]')
+                  if [[ -z "$PHOTON_VERSION" || ! "$PHOTON_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+                    echo "Error: .last_release is missing, empty, or contains an invalid version: '$PHOTON_VERSION'"
+                    exit 1
+                  fi
+                  echo "PHOTON_VERSION=$PHOTON_VERSION" >> "$GITHUB_ENV"
+                  echo "Photon Version: $PHOTON_VERSION"
+
+            - name: Build test image
+              uses: docker/build-push-action@v6
+              with:
+                  context: .
+                  file: ./Dockerfile
+                  build-args: |
+                      PHOTON_VERSION=${{ env.PHOTON_VERSION }}
+                  push: false
+                  load: true
+                  tags: photon-test:ext-os-${{ github.event.pull_request.number }}
+                  platforms: linux/amd64
+                  cache-from: type=gha
+                  cache-to: type=gha,mode=max
+
+            - name: Create Docker network
+              run: docker network create photon-ext-os-test
+
+            - name: Download and extract Andorra index
+              run: |
+                  INDEX_DB_VERSION="1.0"
+                  FILE_NAME="photon-db-andorra-${INDEX_DB_VERSION}-latest.tar.bz2"
+                  DOWNLOAD_URL="${BASE_URL}/europe/andorra/${FILE_NAME}"
+
+                  echo "Downloading Andorra index from ${DOWNLOAD_URL}..."
+                  mkdir -p /tmp/photon-data
+                  wget -q "${DOWNLOAD_URL}" -O "/tmp/${FILE_NAME}"
+                  tar -xjf "/tmp/${FILE_NAME}" -C /tmp/photon-data
+                  echo "Index extracted to /tmp/photon-data"
+                  ls -la /tmp/photon-data/
+
+            - name: Prepare OpenSearch data directory
+              run: |
+                  # Remap photon's node_1/ to OpenSearch standalone's nodes/0/
+                  mkdir -p /tmp/os-data/nodes/0
+                  cp -r /tmp/photon-data/photon_data/node_1/. /tmp/os-data/nodes/0/
+                  # OpenSearch container runs as uid 1000
+                  chmod -R 777 /tmp/os-data
+                  echo "OpenSearch data directory prepared:"
+                  ls -laR /tmp/os-data/nodes/0/ | head -20
+
+            - name: Start OpenSearch
+              run: |
+                  # OpenSearch 3.0.0 ignores gateway settings from -D JVM properties;
+                  # they must be in opensearch.yml.  cluster.name must match the
+                  # OPENSEARCH_CLUSTER value photon will use to connect.
+                  printf '%s\n' \
+                    'cluster.name: photon' \
+                    'network.host: 0.0.0.0' \
+                    'gateway.auto_import_dangling_indices: true' \
+                    > /tmp/opensearch.yml
+
+                  docker run -d \
+                    --name opensearch-test \
+                    --network photon-ext-os-test \
+                    -p 9200:9200 \
+                    -v /tmp/os-data:/usr/share/opensearch/data \
+                    -v /tmp/opensearch.yml:/usr/share/opensearch/config/opensearch.yml \
+                    -e "discovery.type=single-node" \
+                    -e "DISABLE_SECURITY_PLUGIN=true" \
+                    -e "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" \
+                    opensearchproject/opensearch:3.0.0
+
+            - name: Wait for OpenSearch cluster health
+              run: |
+                  echo "Waiting for OpenSearch cluster to be healthy (timeout: 120s)..."
+                  SECONDS=0
+                  TIMEOUT=120
+
+                  while [ $SECONDS -lt $TIMEOUT ]; do
+                    if curl -sf "http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=5s" > /dev/null 2>&1; then
+                      echo "OpenSearch cluster is healthy after ${SECONDS}s"
+                      curl -s "http://localhost:9200/_cluster/health" | python3 -m json.tool
+                      break
+                    fi
+                    echo "Waiting for OpenSearch... (elapsed: ${SECONDS}s)"
+                    sleep 5
+                    SECONDS=$((SECONDS + 5))
+                  done
+
+                  if [ $SECONDS -ge $TIMEOUT ]; then
+                    echo "OpenSearch failed to become healthy within ${TIMEOUT}s"
+                    docker logs opensearch-test
+                    exit 1
+                  fi
+
+            - name: Wait for photon index
+              run: |
+                  echo "Waiting for photon index to appear (timeout: 120s)..."
+                  SECONDS=0
+                  TIMEOUT=120
+
+                  while [ $SECONDS -lt $TIMEOUT ]; do
+                    if curl -sf "http://localhost:9200/photon" > /dev/null 2>&1; then
+                      echo "Photon index found after ${SECONDS}s"
+                      curl -s "http://localhost:9200/_cat/indices?v"
+                      break
+                    fi
+                    echo "Waiting for photon index... (elapsed: ${SECONDS}s)"
+                    sleep 5
+                    SECONDS=$((SECONDS + 5))
+                  done
+
+                  if [ $SECONDS -ge $TIMEOUT ]; then
+                    echo "Photon index did not appear within ${TIMEOUT}s"
+                    echo "OpenSearch indices:"
+                    curl -s "http://localhost:9200/_cat/indices?v" || true
+                    echo "Dangling indices:"
+                    curl -s "http://localhost:9200/_dangling" || true
+                    docker logs opensearch-test
+                    exit 1
+                  fi
+
+            - name: Start photon container
+              run: |
+                  docker run -d \
+                    --name photon-ext-os-test \
+                    --network photon-ext-os-test \
+                    -e OPENSEARCH_TRANSPORT_ADDRESSES=opensearch-test:9200 \
+                    -e OPENSEARCH_CLUSTER=photon \
+                    photon-test:ext-os-${{ github.event.pull_request.number }}
+
+            - name: Wait for photon to be healthy
+              run: |
+                  echo "Waiting for photon container to become healthy (timeout: 6 minutes)..."
+                  CONTAINER_NAME=photon-ext-os-test
+
+                  docker logs -f $CONTAINER_NAME &
+                  LOGS_PID=$!
+
+                  SECONDS=0
+                  TIMEOUT=360
+
+                  while [ $SECONDS -lt $TIMEOUT ]; do
+                    HEALTH_STATUS=$(docker inspect --format='{{.State.Health.Status}}' $CONTAINER_NAME 2>/dev/null || echo "unknown")
+
+                    if [ "$HEALTH_STATUS" = "healthy" ]; then
+                      echo "Photon container is healthy after $SECONDS seconds"
+                      kill $LOGS_PID 2>/dev/null || true
+                      exit 0
+                    fi
+
+                    echo "Health status: $HEALTH_STATUS (elapsed: ${SECONDS}s)"
+                    sleep 10
+                    SECONDS=$((SECONDS + 10))
+                  done
+
+                  kill $LOGS_PID 2>/dev/null || true
+                  echo "Photon container failed to become healthy within $TIMEOUT seconds"
+                  docker logs $CONTAINER_NAME
+                  exit 1
+
+            - name: Cleanup
+              if: always()
+              run: |
+                  docker stop photon-ext-os-test opensearch-test 2>/dev/null || true
+                  docker rm photon-ext-os-test opensearch-test 2>/dev/null || true
+                  docker network rm photon-ext-os-test 2>/dev/null || true
+                  docker rmi photon-test:ext-os-${{ github.event.pull_request.number }} 2>/dev/null || true
+
+            - name: Output summary
+              if: always()
+              run: |
+                  echo "## External OpenSearch Test Summary" >> $GITHUB_STEP_SUMMARY
+                  echo "- **PR Number:** ${{ github.event.pull_request.number }}" >> $GITHUB_STEP_SUMMARY
+                  echo "- **Photon Version:** ${{ env.PHOTON_VERSION }}" >> $GITHUB_STEP_SUMMARY
+                  echo "- **OpenSearch Version:** 3.0.0" >> $GITHUB_STEP_SUMMARY
+                  echo "- **Status:** ${{ job.status }}" >> $GITHUB_STEP_SUMMARY

docker-compose.external-opensearch.yml

@@ -0,0 +1,51 @@
+# Docker Compose for running photon with an external OpenSearch instance.
+#
+# Usage:
+#   docker compose -f docker-compose.external-opensearch.yml up -d
+#
+# The photon index must be loaded into OpenSearch before photon can serve
+# requests.  Common approaches:
+#   - Import from a Nominatim database using photon.jar:
+#       java -jar photon.jar import \
+#         -transport-addresses opensearch:9200 \
+#         -nominatim-db <dsn>
+#   - Import from a photon JSON dump:
+#       java -jar photon.jar import \
+#         -transport-addresses opensearch:9200 \
+#         -json-dump <path>
+#   - Restore an OpenSearch snapshot that contains the "photon" index.
+
+services:
+    opensearch:
+        image: opensearchproject/opensearch:3.0.0
+        container_name: "photon-opensearch"
+        environment:
+            - discovery.type=single-node
+            - DISABLE_SECURITY_PLUGIN=true
+            - OPENSEARCH_JAVA_OPTS=-Xms1g -Xmx1g
+        volumes:
+            - opensearch_data:/usr/share/opensearch/data
+        healthcheck:
+            test: ["CMD-SHELL", "curl -sf http://localhost:9200/_cluster/health || exit 1"]
+            interval: 30s
+            timeout: 10s
+            retries: 5
+            start_period: 60s
+        restart: unless-stopped
+
+    photon:
+        image: ghcr.io/rtuszik/photon-docker:latest
+        container_name: "photon-docker"
+        environment:
+            - OPENSEARCH_TRANSPORT_ADDRESSES=opensearch:9200
+            - OPENSEARCH_CLUSTER=photon
+            - LOG_LEVEL=INFO
+        depends_on:
+            opensearch:
+                condition: service_healthy
+        ports:
+            - "2322:2322"
+        restart: unless-stopped
+
+volumes:
+    opensearch_data:

src/entrypoint.py

@@ -36,6 +36,8 @@ def main():
         logger.info("APPRISE_URLS: REDACTED")
     else:
         logger.info("APPRISE_URLS: UNSET")
+    logger.info(f"OPENSEARCH_TRANSPORT_ADDRESSES: {config.OPENSEARCH_TRANSPORT_ADDRESSES}")
+    logger.info(f"OPENSEARCH_CLUSTER: {config.OPENSEARCH_CLUSTER}")
 
     logger.info("=== END CONFIG VARIABLES ===")
 
@@ -48,6 +50,10 @@ def main():
     if config.MIN_INDEX_DATE:
         logger.info(f"MIN_INDEX_DATE: {config.MIN_INDEX_DATE}")
 
+    if config.OPENSEARCH_TRANSPORT_ADDRESSES:
+        logger.info("External OpenSearch configured — skipping index download")
+        return
+
     if config.FORCE_UPDATE:
         logger.info("Starting forced update")
         try:

src/process_manager.py

@@ -107,7 +107,13 @@ def start_photon(self, max_startup_retries=3):
             if java_params:
                 cmd.extend(shlex.split(java_params))
 
-            cmd.extend(["-jar", "/photon/photon.jar", "serve", "-listen-ip", "0.0.0.0", "-data-dir", config.DATA_DIR]) #noqa S104
+            cmd.extend(["-jar", "/photon/photon.jar", "serve", "-listen-ip", "0.0.0.0"])  # noqa: S104
+
+            if config.OPENSEARCH_TRANSPORT_ADDRESSES:
+                cmd.extend(["-transport-addresses", config.OPENSEARCH_TRANSPORT_ADDRESSES])
+                cmd.extend(["-cluster", config.OPENSEARCH_CLUSTER])
+            else:
+                cmd.extend(["-data-dir", config.DATA_DIR])
 
             if enable_metrics:
                 cmd.extend(["-metrics-enable", "prometheus"])
@@ -177,6 +183,9 @@ def cleanup_orphaned_photon_processes(self):
             logger.debug(f"Error checking for orphaned processes: {e}")
 
     def _cleanup_lock_files(self):
+        if config.OPENSEARCH_TRANSPORT_ADDRESSES:
+            return
+
         lock_files = [
             os.path.join(config.OS_NODE_DIR, "node.lock"),
             os.path.join(config.OS_NODE_DIR, "data", "node.lock"),
@@ -292,7 +301,9 @@ def shutdown(self):
     def run(self):
         logger.info("Photon Manager starting...")
 
-        if not config.FORCE_UPDATE and os.path.isdir(config.OS_NODE_DIR):
+        if config.OPENSEARCH_TRANSPORT_ADDRESSES:
+            logger.info("External OpenSearch configured — skipping initial setup")
+        elif not config.FORCE_UPDATE and os.path.isdir(config.OS_NODE_DIR):
             logger.info("Existing index found, skipping initial setup")
         else:
             self.run_initial_setup()

src/utils/config.py

@@ -19,6 +19,10 @@
 APPRISE_URLS = os.getenv("APPRISE_URLS")
 MIN_INDEX_DATE = os.getenv("MIN_INDEX_DATE", "10.02.26")
 
+# EXTERNAL OPENSEARCH CONFIG
+OPENSEARCH_TRANSPORT_ADDRESSES = os.getenv("OPENSEARCH_TRANSPORT_ADDRESSES")
+OPENSEARCH_CLUSTER = os.getenv("OPENSEARCH_CLUSTER", "photon")
+
 # APP CONFIG
 INDEX_DB_VERSION = "1.0"
 INDEX_FILE_EXTENSION = "tar.bz2"
@@ -33,3 +37,6 @@
     UPDATE_STRATEGY = "DISABLED"
     if not MD5_URL:
         SKIP_MD5_CHECK = True
+
+if OPENSEARCH_TRANSPORT_ADDRESSES:
+    UPDATE_STRATEGY = "DISABLED"

src/utils/validate_config.py

@@ -25,6 +25,12 @@ def validate_config():
     if config.REGION and not is_valid_region(config.REGION):
         error_messages.append(f"Invalid REGION: '{config.REGION}'. Must be a valid continent, sub-region, or 'planet'.")
 
+    if config.OPENSEARCH_TRANSPORT_ADDRESSES and config.REGION:
+        logging.warning(
+            "REGION is set alongside OPENSEARCH_TRANSPORT_ADDRESSES — "
+            "REGION will be ignored (index data is managed by external OpenSearch)"
+        )
+
     if error_messages:
         full_error_message = "Configuration validation failed:\n" + "\n".join(error_messages)
         raise ValueError(full_error_message)