Improve build_container_image workflow to match other workflows

diraol · diraol · commit 1db31c35aceb · 2025-11-15T22:31:18.000-03:00
Aligned test-python-unit job with lint_and_unit and test_pull_request:
- Added plugins.security.disabled and OPENSEARCH_INITIAL_ADMIN_PASSWORD to OpenSearch
- Replaced MinIO service with manual docker run for better control
- Added apache-tika service
- Added Python setup and dependency installation
- Added MinIO bucket creation with architecture detection
- Added OpenSearch index initialization
- Added comprehensive environment variables for tests (OpenSearch, MinIO, Tika, etc.)
- Added timeout to test execution
- Added MinIO container cleanup

These changes ensure tests run with proper service configuration
and all required environment variables are passed to the container.
diff --git a/.github/workflows/build_container_image.yaml b/.github/workflows/build_container_image.yaml
@@ -181,6 +181,8 @@ jobs:
         env:
           discovery.type: single-node
           plugins.security.ssl.http.enabled: false
+          plugins.security.disabled: true
+          OPENSEARCH_INITIAL_ADMIN_PASSWORD: admin
         options: >-
           --health-cmd "curl -s http://localhost:9200 >/dev/null || exit 1"
           --health-interval 15s
@@ -189,19 +191,94 @@ jobs:
         ports:
           - 9200:9200
 
-      minio:
-        image: quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z
-        env:
-          MINIO_ACCESS_KEY: minio-access-key
-          MINIO_SECRET_KEY: minio-secret-key
-          MINIO_DEFAULT_BUCKETS: queridodiariobucket:public
+      apache-tika:
+        image: ghcr.io/${{ github.repository }}/apache-tika:latest
         ports:
-          - 9000:9000
+          - 9998:9998
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Start MinIO
+        run: |
+          docker run -d \
+            --name minio \
+            --network host \
+            --health-cmd "curl -f http://localhost:9000/minio/health/live || exit 1" \
+            --health-interval 10s \
+            --health-timeout 5s \
+            --health-retries 5 \
+            -e MINIO_ROOT_USER=minio-access-key \
+            -e MINIO_ROOT_PASSWORD=minio-secret-key \
+            quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z \
+            server /data --console-address :9001
+
+          # Wait for MinIO to be ready
+          echo "Waiting for MinIO to be ready..."
+          timeout 60 bash -c 'until curl -sf http://localhost:9000/minio/health/live > /dev/null 2>&1; do sleep 2; done'
+          echo "MinIO is ready"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+          cache-dependency-path: requirements.txt
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Create MinIO bucket
+        run: |
+          # Wait for MinIO to be ready
+          for i in {1..30}; do
+            if curl -sf http://localhost:9000/minio/health/live > /dev/null 2>&1; then
+              echo "MinIO is ready"
+              break
+            fi
+            echo "Waiting for MinIO... ($i/30)"
+            sleep 2
+          done
+
+          # Detect architecture and download appropriate MinIO client
+          ARCH=$(uname -m)
+          if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then
+            MC_URL="https://dl.min.io/client/mc/release/linux-arm64/mc"
+          else
+            MC_URL="https://dl.min.io/client/mc/release/linux-amd64/mc"
+          fi
+
+          echo "Downloading MinIO client for architecture: $ARCH"
+          curl -sLo /tmp/mc "$MC_URL"
+          chmod +x /tmp/mc
+
+          # Configure MinIO client
+          /tmp/mc alias set minio http://localhost:9000 minio-access-key minio-secret-key
+
+          # Create bucket if it doesn't exist
+          /tmp/mc mb minio/queridodiariobucket --ignore-existing || true
+
+          echo "MinIO bucket created successfully"
+
+      - name: Create OpenSearch index
+        timeout-minutes: 2
+        run: |
+          echo "=== Creating OpenSearch index ==="
+          echo "OpenSearch host: http://localhost:9200"
+          echo "Testing connectivity..."
+          curl -v http://localhost:9200/_cluster/health || echo "OpenSearch not responding"
+          echo "Running init script..."
+          chmod +x init-scripts/opensearch/create-opensearch-index.sh
+          OPENSEARCH_HOST=http://localhost:9200 \
+          OPENSEARCH_USER=admin \
+          OPENSEARCH_PASSWORD=admin \
+          INDEX_NAME=querido-diario \
+          ./init-scripts/opensearch/create-opensearch-index.sh
+          echo "=== OpenSearch index creation completed ==="
+
       - name: Free up disk space
         run: |
           # Remove unnecessary packages and clean up
@@ -258,28 +335,103 @@ jobs:
           done
 
       - name: Run Python unit tests
+        timeout-minutes: 5
+        env:
+          PYTHONPATH: /mnt/code
+          PYTHONUNBUFFERED: 1
+          POSTGRES_PASSWORD: queridodiario
+          POSTGRES_USER: queridodiario
+          POSTGRES_DB: queridodiariodb
+          POSTGRES_HOST: localhost
+          POSTGRES_PORT: 5432
+          STORAGE_REGION: us-east-1
+          STORAGE_ENDPOINT: http://localhost:9000
+          STORAGE_ACCESS_KEY: minio-access-key
+          STORAGE_ACCESS_SECRET: minio-secret-key
+          STORAGE_BUCKET: queridodiariobucket
+          OPENSEARCH_HOST: http://localhost:9200
+          OPENSEARCH_INDEX: querido-diario
+          OPENSEARCH_USER: admin
+          OPENSEARCH_PASSWORD: admin
+          APACHE_TIKA_SERVER: http://localhost:9998
+          DEBUG: 1
+          HF_HUB_OFFLINE: 1
+          TRANSFORMERS_OFFLINE: 1
+          SENTENCE_TRANSFORMERS_HOME: /tmp/sentence_transformers
         run: |
           docker run --rm \
             --network host \
-            -e PYTHONPATH=/mnt/code \
-            -e POSTGRES_PASSWORD=queridodiario \
-            -e POSTGRES_USER=queridodiario \
-            -e POSTGRES_DB=queridodiariodb \
-            -e POSTGRES_HOST=localhost \
-            -e POSTGRES_PORT=5432 \
+            -e PYTHONPATH \
+            -e PYTHONUNBUFFERED \
+            -e POSTGRES_PASSWORD \
+            -e POSTGRES_USER \
+            -e POSTGRES_DB \
+            -e POSTGRES_HOST \
+            -e POSTGRES_PORT \
+            -e STORAGE_REGION \
+            -e STORAGE_ENDPOINT \
+            -e STORAGE_ACCESS_KEY \
+            -e STORAGE_ACCESS_SECRET \
+            -e STORAGE_BUCKET \
+            -e OPENSEARCH_HOST \
+            -e OPENSEARCH_INDEX \
+            -e OPENSEARCH_USER \
+            -e OPENSEARCH_PASSWORD \
+            -e APACHE_TIKA_SERVER \
+            -e DEBUG \
+            -e HF_HUB_OFFLINE \
+            -e TRANSFORMERS_OFFLINE \
+            -e SENTENCE_TRANSFORMERS_HOME \
             test-data-processing:unittest-${{ matrix.arch }} \
             python -m unittest discover -s tests -p "*.py" -v
 
       - name: Run coverage report
+        env:
+          PYTHONPATH: /mnt/code
+          PYTHONUNBUFFERED: 1
+          POSTGRES_PASSWORD: queridodiario
+          POSTGRES_USER: queridodiario
+          POSTGRES_DB: queridodiariodb
+          POSTGRES_HOST: localhost
+          POSTGRES_PORT: 5432
+          STORAGE_REGION: us-east-1
+          STORAGE_ENDPOINT: http://localhost:9000
+          STORAGE_ACCESS_KEY: minio-access-key
+          STORAGE_ACCESS_SECRET: minio-secret-key
+          STORAGE_BUCKET: queridodiariobucket
+          OPENSEARCH_HOST: http://localhost:9200
+          OPENSEARCH_INDEX: querido-diario
+          OPENSEARCH_USER: admin
+          OPENSEARCH_PASSWORD: admin
+          APACHE_TIKA_SERVER: http://localhost:9998
+          DEBUG: 1
+          HF_HUB_OFFLINE: 1
+          TRANSFORMERS_OFFLINE: 1
+          SENTENCE_TRANSFORMERS_HOME: /tmp/sentence_transformers
         run: |
           docker run --rm \
             --network host \
-            -e PYTHONPATH=/mnt/code \
-            -e POSTGRES_PASSWORD=queridodiario \
-            -e POSTGRES_USER=queridodiario \
-            -e POSTGRES_DB=queridodiariodb \
-            -e POSTGRES_HOST=localhost \
-            -e POSTGRES_PORT=5432 \
+            -e PYTHONPATH \
+            -e PYTHONUNBUFFERED \
+            -e POSTGRES_PASSWORD \
+            -e POSTGRES_USER \
+            -e POSTGRES_DB \
+            -e POSTGRES_HOST \
+            -e POSTGRES_PORT \
+            -e STORAGE_REGION \
+            -e STORAGE_ENDPOINT \
+            -e STORAGE_ACCESS_KEY \
+            -e STORAGE_ACCESS_SECRET \
+            -e STORAGE_BUCKET \
+            -e OPENSEARCH_HOST \
+            -e OPENSEARCH_INDEX \
+            -e OPENSEARCH_USER \
+            -e OPENSEARCH_PASSWORD \
+            -e APACHE_TIKA_SERVER \
+            -e DEBUG \
+            -e HF_HUB_OFFLINE \
+            -e TRANSFORMERS_OFFLINE \
+            -e SENTENCE_TRANSFORMERS_HOME \
             test-data-processing:unittest-${{ matrix.arch }} \
             bash -c "coverage run -m unittest discover -s tests -p '*.py' && coverage report -m"
 
@@ -288,6 +440,8 @@ jobs:
         run: |
           docker stop tika-server-${{ matrix.arch }} || true
           docker rm tika-server-${{ matrix.arch }} || true
+          docker stop minio || true
+          docker rm minio || true
           docker system prune -f
           docker image prune -af
 
