Merge pull request #1432 from swirlai/main_prerelease_3_8_0

Main prerelease 3 8 0

Author: erikspears

.github/workflows/docker-image.yml

@@ -1,4 +1,4 @@
-name: Docker Build and Push
+name: Docker Build
 
 on:
   # Manual trigger only

.github/workflows/qa-suite.yml

@@ -28,7 +28,7 @@ jobs:
       - name: Set Up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.12.4'
+          python-version: '3.12.5'
           cache: 'pip'
       - name: Install Swirl
         run: ./install.sh

.github/workflows/spell-checker.yml

@@ -8,9 +8,26 @@ on:
     paths:
       - "docs/**"
 
+  pull_request:
+    # Run for all PRs to develop - means PR cannot merge until unit tests pass
+    branches:
+      - develop
+      - main
+    # Skip non-code changes
+    paths-ignore:
+      - '.github/**'
+      - 'integrations/**'
+      - 'swirl-infra/**'
+      - 'db.sqlite3.dist'
   # Allows manual run of this workflow from the Actions tab (on any branch)
   workflow_dispatch:
 
+permissions:
+  contents: read
+  actions: read
+  checks: write
+  pull-requests: write
+
 jobs:
   build:
     runs-on: ubuntu-latest

.github/workflows/test-build-pipeline.yml

@@ -20,7 +20,7 @@ jobs:
         - name: Set Up Python
           uses: actions/setup-python@v5
           with:
-            python-version: '3.12.4'
+            python-version: '3.12.5'
             cache: 'pip'
         - name: Install Swirl
           run: ./install.sh
@@ -53,7 +53,7 @@ jobs:
         - name: Set Up Python
           uses: actions/setup-python@v5
           with:
-            python-version: '3.12.4'
+            python-version: '3.12.5'
             cache: 'pip'
         - name: Install Swirl
           run: ./install.sh

.github/workflows/testing-wip.yml

@@ -8,10 +8,14 @@ name: Testing WIP
 on:
   workflow_dispatch:
     inputs:
+      qa_image:  # Input the QA Repo image to use
+        description: 'QA Repo image to use'
+        required: true
+        default: 'automated-tests-master'  # Default image if none specified
       behave_tags:  # Input the Behave tag(s) to run
         description: 'Behave tag(s) to run'
         required: true
-        default: 'estest'  # Default tag if none specified
+        default: 'qa_suite,community'  # Default tag if none specified
 
 jobs:
   wip-tests:
@@ -37,7 +41,7 @@ jobs:
       - name: Set Up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.12.4'
+          python-version: '3.12.5'
           cache: 'pip'
       - name: Install Swirl
         run: ./install.sh
@@ -65,7 +69,7 @@ jobs:
           echo "========"
           cat .env.qa
           echo "========"
-          docker run --net=host --env-file .env.qa -t swirlai/swirl-search-qa:automated-tests-master sh -c "behave --tags=${{ github.event.inputs.behave_tags }}"
+          docker run --net=host --env-file .env.qa -t swirlai/swirl-search-qa:${{ github.event.inputs.qa_image }} sh -c "behave --tags=${{ github.event.inputs.behave_tags }}"
       - name: Upload Log Files
         if: always()
         uses: actions/upload-artifact@v4

.github/workflows/unit-tests.yml

@@ -1,12 +1,28 @@
 name: Unit Tests
 
 on:
-  # Manual trigger only
+  pull_request:
+    # Run for all PRs to develop - means PR cannot merge until unit tests pass
+    branches:
+      - develop
+    # Skip non-code changes
+    paths-ignore:
+      - '.github/**'
+      - 'integrations/**'
+      - 'swirl-infra/**'
+      - 'docs/**'
+      - 'README.md'
+      - 'db.sqlite3.dist'
   workflow_dispatch:
 
-jobs:
+permissions:
+  contents: read
+  actions: read
+  checks: write
+  pull-requests: write
 
-  build:
+jobs:
+  unit-tests:
     runs-on: ubuntu-latest
 
     steps:
@@ -15,14 +31,15 @@ jobs:
       - name: Set Up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.12.4'
+          python-version: '3.12.5'
           cache: 'pip'
       - name: Install Swirl
         run: ./install.sh
       - name: Install the Unit Tests
         run: ./install-test.sh
       - name: Run the Unit Tests
-        run: pytest
+        # generate a JUnit XML report for the test results
+        run: pytest --junitxml=reports/junit.xml --tb=short --ignore=integrations
       - name: Upload Log Files
         if: always()
         uses: actions/upload-artifact@v4
@@ -31,3 +48,9 @@ jobs:
           path: |
             logs/
             /var/log/syslog*
+
+      - name: Publish Unit Test Results
+        if: always()
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        with:
+          files: reports/junit.xml

DevUtils/docker/sw-start-sa.sh

@@ -1,59 +1,37 @@
-# Use an official Python runtime as a parent image
-FROM python:3.12.4-slim-bookworm
-
-# Upgrade pip to the specified version or higher
-RUN pip install --no-cache-dir --upgrade 'pip>=24.0'
-
-# try to upgrade to a more recent version of openssl
-RUN apt-get update
-RUN apt-get -y upgrade openssl
-
-# install jq
-RUN apt-get -y install jq
-
-# RUN sudo echo 'nameserver 8.8.8.8'>/etc/resolv.conf
-RUN apt-get update -y
-RUN apt-get install apt-file -y
-RUN apt-file update
-RUN apt-get install -y python3-dev build-essential
-RUN apt-get install -y procps
-RUN apt-get install -y libpq-dev
-
-RUN pip install --no-cache-dir --upgrade pip
-RUN pip install --no-cache-dir --upgrade grpcio
-
-ADD requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
-
-# install redis
-RUN apt-get install -y redis-server
-
-# install requirements
-RUN python -m spacy download en_core_web_lg
-RUN python -m nltk.downloader stopwords
-RUN python -m nltk.downloader punkt
-
-# Copy Swirl App to container
-RUN mkdir /app
+FROM python:3.12.5-slim-bookworm
+
+# Update, upgrade and install packages in a single RUN to reduce layers
+RUN apt-get update && apt-get install -y \
+    apt-file \
+    build-essential \
+    jq \
+    libpq-dev \
+    procps \
+    python3-dev \
+    redis-server \
+&& apt-file update \
+&& apt-get clean \
+&& rm -rf /var/lib/apt/lists/*
+
+# Copy application files (see .dockerignore for list of exclusions)
+COPY . /app
 COPY ./db.sqlite3.dist /app/db.sqlite3
 COPY ./.env.docker /app/.env
-COPY ./install-ui.sh /app/install-ui.sh
-ADD ./swirl /app/swirl
+COPY ./download-nltk-resources.sh /app/
 
-# Install Galaxy UI
-RUN mkdir -p /app/swirl/static/galaxy
-COPY --from=swirlai/spyglass:latest /usr/src/spyglass/ui/dist/spyglass/browser/. /app/swirl/static/galaxy
-COPY --from=swirlai/spyglass:latest /usr/src/spyglass/ui/config-swirl-demo.db.json /app/
+WORKDIR /app
 
-ADD ./swirl_server /app/swirl_server
-ADD ./SearchProviders /app/SearchProviders
-ADD ./DevUtils /app/DevUtils
-ADD ./Data /app/Data
-ADD ./uploads /app/uploads
-ADD ./swirl.py /app/swirl.py
-ADD ./swirl_load.py /app/swirl_load.py
-ADD ./manage.py /app/manage.py
+# Optimize pip and Python installations
+RUN pip install --no-cache-dir --upgrade pip \
+    && pip install --no-cache-dir -r requirements.txt \
+    && pip install --no-cache-dir --upgrade grpcio
 
-WORKDIR /app
+# Swirl install requirements
+RUN python -m spacy download en_core_web_lg && \
+     ./download-nltk-resources.sh
+
+# Install the Galaxy UI
+COPY --from=swirlai/spyglass:preview /usr/src/spyglass/ui/dist/spyglass/browser/. /app/swirl/static/galaxy
+COPY --from=swirlai/spyglass:preview /usr/src/spyglass/ui/config-swirl-demo.db.json /app/
 
 EXPOSE 8000

Dockerfile

@@ -1,10 +1,20 @@
 services:
   redis:
-    image: redis
+    image: redis:latest
+    ports:
+      - "6379:6379"
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
   app:
     image: swirlai/swirl-search:latest
     ports:
       - "8000:8000"
+    depends_on:
+      redis:
+        condition: service_healthy
     command: >
       sh -c 'rm -fr ./.swirl && python swirl.py setup && mkdir -p static/api/config &&
              /usr/bin/jq ".default" ./config-swirl-demo.db.json | sed -e "s/<msal-app-id>/$MSAL_APP_ID/" \