Feature/pytest actions

.github/workflows/codeql-analysis.yml

@@ -41,7 +41,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         config-file: ./.github/codeql/codeql-config.yml
@@ -54,7 +54,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -68,4 +68,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

.github/workflows/cypress.yml

@@ -1,5 +1,8 @@
 name: Cypress
-on: [pull_request]
+on:
+#  pull_request:
+  workflow_call:
+  workflow_dispatch:
 env:
   NODE_VERSION: '16'
   PYTHON_VERSION: '3.9'
@@ -11,8 +14,8 @@ jobs:
   cypress:
     runs-on: ubuntu-latest
     services:
-      ckan-postgres:
-        image: postgres:12
+      ckan_postgres:
+        image: postgres:15
         ports:
           - 5432:5432
         options: >-
@@ -25,11 +28,11 @@ jobs:
           POSTGRES_PASSWORD: pass
           POSTGRES_DB: ckan_test
 
-      ckan-redis:
+      ckan_redis:
         image: redis
         ports:
           - 6379:6379
-      ckan-solr:
+      ckan_solr:
         image: ckan/ckan-solr:master
         ports:
           - 8983:8983
@@ -40,25 +43,32 @@ jobs:
       CKAN_REDIS_URL: redis://localhost:6379/1
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ env.PYTHON_VERSION }}
-      - uses: actions/setup-node@v3
+          cache: 'pip'
+      - uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODE_VERSION }}
+          cache: 'npm'
 
       - name: Install python deps
-        run: pip install -r requirements.txt -r dev-requirements.txt -e.
+        run: |
+         pip install -U pip
+         pip install -r requirements.txt -r dev-requirements.txt -e.
+         pip check
 
       - name: Init environment
         run: |
-          ckan -c test-core-cypress.ini db init
+          ckan -c test-core-github-actions.ini db init
+          #cypress testing requires activity plugin to be active
+          ckan config-tool test-core-github-actions.ini "ckan.plugins = activity"
 
       - name: Run Cypress
         uses: cypress-io/github-action@v6
         with:
-          start: ckan -c test-core-cypress.ini run
+          start: ckan -c test-core-github-actions.ini run
 
       - uses: actions/upload-artifact@v4
         if: failure()

.github/workflows/flake8.yml

@@ -1,5 +1,8 @@
 name: Lint
-on: [pull_request]
+on:
+#  pull_request:
+  workflow_call:
+  workflow_dispatch:
 
 permissions:
   contents: read

.github/workflows/pyright.yml

@@ -1,5 +1,8 @@
 name: Check types
-on: [pull_request]
+on:
+#  pull_request:
+  workflow_call:
+  workflow_dispatch:
 env:
   NODE_VERSION: '18'
   PYTHON_VERSION: '3.9'
@@ -15,9 +18,11 @@ jobs:
       - uses: actions/setup-python@v2
         with:
           python-version: ${{ env.PYTHON_VERSION }}
+          cache: 'pip'
       - uses: actions/setup-node@v3
         with:
           node-version: ${{ env.NODE_VERSION }}
+          cache: 'npm'
       - name: Install python deps
         run: pip install -r requirements.txt -r dev-requirements.txt -e.
       - name: Install node deps

.github/workflows/pytest.yml

@@ -0,0 +1,215 @@
+name: Pytest
+on:
+  #pull_request: #disabled as its called via test.yml uses delegation
+  workflow_call:
+  workflow_dispatch:
+    inputs:
+      split:
+        type: choice
+        options:
+          - '["1"]'
+          - '["1", "2", "3", "4"]'
+          - '["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"]'
+          - '["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16"]'
+          - '["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20"]'
+        description: 'Number of test splits (set to ["1"])'
+        required: false
+        default: '["1"]'
+      store-duration-generation:
+        type: boolean
+        default: True
+        description: "pytest create store-duration config, use split 1 for best results"
+env:
+  NODE_VERSION: '16'
+  PYTHON_VERSION: '3.9'
+
+permissions:
+  contents: read
+
+jobs:
+  define-matrix:
+    runs-on: ubuntu-latest
+
+    outputs:
+      SPLIT_LIST: ${{ steps.splits.outputs.SPLIT_LIST }}
+      SPLIT_COUNT: ${{ steps.splits.outputs.SPLIT_COUNT }}
+
+    steps:
+      # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/running-variations-of-jobs-in-a-workflow#example-using-an-output-to-define-two-matrice
+      - name: Define matrix split
+        id: splits
+        # default maxtrix for splits if we did not get an input
+        run: |
+          export SPLIT_LIST='${{ inputs.split || '["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"]' }}'
+          echo "this is my list: ${SPLIT_LIST}"
+          export SPLIT_COUNT="$(echo ${SPLIT_LIST} | jq 'length')"
+          echo "${SPLIT_COUNT} : ${SPLIT_LIST}"
+          echo "SPLIT_LIST=${SPLIT_LIST}" >> "$GITHUB_OUTPUT"
+          echo "SPLIT_COUNT=${SPLIT_COUNT}" >> "$GITHUB_OUTPUT"
+
+
+  pytest:
+    needs: define-matrix
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      #max-parallel: 16 #to restrict how many at once if required
+      matrix:
+        #split: [ 1, 2, 3, 4 ]
+        split: ${{ fromJSON(needs.define-matrix.outputs.SPLIT_LIST) }}
+
+    services:
+      ckan_postgres:
+        image: postgres:12 # should be postgres:15
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        env:
+          POSTGRES_USER: ckan
+          POSTGRES_PASSWORD: ckan
+          POSTGRES_DB: ckan
+          PGDATA: "/data"
+
+      ckan_redis:
+        image: redis:7
+        ports:
+          - 6379:6379
+      ckan_solr:
+        image: ckan/ckan-solr:master
+        ports:
+          - 8983:8983
+
+    env:
+      SPLIT_COUNT: ${{ needs.define-matrix.outputs.SPLIT_COUNT }}
+      CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@localhost/ckan_test
+      CKAN_SOLR_URL: http://localhost:8983/solr/ckan
+      CKAN_REDIS_URL: redis://localhost:6379/1
+      POSTGRES_USER: ckan
+      POSTGRES_PASSWORD: ckan
+      CKAN_DB: ckan_test
+      CKAN_DATASTORE_POSTGRES_DB: datastore_test
+      CKAN_DATASTORE_POSTGRES_READ_USER: datastore_read
+      CKAN_DATASTORE_POSTGRES_READ_PWD: pass
+      CKAN_DATASTORE_POSTGRES_WRITE_USER: datastore_write
+      CKAN_DATASTORE_POSTGRES_WRITE_PWD: pass
+      CKAN_POSTGRES_DB: ckan_test
+      CKAN_POSTGRES_USER: ckan_default
+      CKAN_POSTGRES_PWD: pass
+      PGPASSWORD: ckan
+      PGHOST: localhost
+      CKAN_STORAGE_PATH: /tmp/storage
+      PYTEST_COMMON_OPTIONS: -v --ckan-ini=test-core-github-actions.ini --cov=ckan --cov=ckanext --junitxml=./junit/result/junit.xml
+
+
+    steps:
+      - run: |
+          echo "${{ matrix.split }} of $SPLIT_COUNT"
+      - name: Setup Database
+        run: |
+          # Database Creation
+          psql     --host=${PGHOST} --username=${POSTGRES_USER} --command="CREATE USER ${CKAN_POSTGRES_USER} WITH PASSWORD '${CKAN_POSTGRES_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+          createdb --encoding=utf-8 --host=${PGHOST} --username=${POSTGRES_USER} --owner=${CKAN_POSTGRES_USER} ${CKAN_POSTGRES_DB}
+          psql     --host=${PGHOST} --username=${POSTGRES_USER} --command="CREATE USER ${CKAN_DATASTORE_POSTGRES_READ_USER} WITH PASSWORD '${CKAN_DATASTORE_POSTGRES_READ_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+          psql     --host=${PGHOST} --username=${POSTGRES_USER} --command="CREATE USER ${CKAN_DATASTORE_POSTGRES_WRITE_USER} WITH PASSWORD '${CKAN_DATASTORE_POSTGRES_WRITE_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+          createdb --encoding=utf-8 --host=${PGHOST} --username=${POSTGRES_USER} --owner=${CKAN_DATASTORE_POSTGRES_WRITE_USER} ${CKAN_DATASTORE_POSTGRES_DB}
+
+
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Fetch all history, including tags
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: 'pip'
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          cache: 'npm'
+
+      - name: Install python deps
+        run: |
+         pip install -U pip
+         pip install -r requirements.txt -r dev-requirements.txt -e .
+         pip check
+
+      - name: Init environment
+        run: |
+          export PGPASSWORD=${POSTGRES_PASSWORD}
+          # Database Initialization
+          ckan -c test-core-github-actions.ini datastore set-permissions | grep -v "Using configuration file" | psql --host=${PGHOST} --username=${POSTGRES_USER}
+          ckan -c test-core-github-actions.ini db init
+
+          # Note: to update this, you must run pytest --store-durations option, a manual run with store-duration-generation enabled should make said file
+          echo "unzip pytest test duration details"
+          gunzip .test_durations.gz
+
+      #as pytest runs test_building_the_docs, we might as well have an output
+      - name: Create Docs
+        if: matrix.split == '1'
+        run: |
+           sphinx-build doc build/sphinx
+
+      #as pytest runs test_building_the_docs, we might as well have an output
+      - name: Store docs
+        if: matrix.split == '1'
+        uses: actions/upload-artifact@v4
+        with:
+          name: docs-${{ matrix.split }}
+          path: ./build/sphinx
+
+      - name: Run pytest
+        if: inputs.store-duration-generation != 'true'
+        run: |
+          unset CKAN_SQLALCHEMY_URL CKAN_SOLR_URL CKAN_REDIS_URL POSTGRES_USER POSTGRES_PASSWORD
+          unset CKAN_DB CKAN_DATASTORE_POSTGRES_DB CKAN_DATASTORE_POSTGRES_READ_USER CKAN_DATASTORE_POSTGRES_READ_PWD
+          unset CKAN_DATASTORE_POSTGRES_WRITE_USER CKAN_DATASTORE_POSTGRES_WRITE_PWD CKAN_POSTGRES_DB
+          unset CKAN_POSTGRES_USER CKAN_POSTGRES_PWD PGPASSWORD PGHOST CKAN_STORAGE_PATH
+
+          mkdir -p ./junit/result
+          echo "${{ matrix.split }} of $SPLIT_COUNT"
+          pytest $PYTEST_COMMON_OPTIONS --splits $SPLIT_COUNT --group ${{ matrix.split }} --splitting-algorithm least_duration
+
+      - name: Run pytest (store durations)
+        if: inputs.store-duration-generation == 'true' && matrix.split == '1'
+        run: |
+          pytest $PYTEST_COMMON_OPTIONS --store-durations
+          gzip .test_durations
+
+      - name: Store test_durations results
+        if: inputs.store-duration-generation == 'true' && matrix.split == '1'
+        uses: actions/upload-artifact@v4
+        with:
+          name: test_durations-${{ matrix.split }}
+          path: ./.test_durations.gz
+
+      - name: Store test results
+        uses: actions/upload-artifact@v4
+        with:
+          name: junit-results-${{ matrix.split }}
+          path: ./junit
+
+      - name: Test Summary
+        uses: test-summary/action@v2
+        with:
+          paths: "./junit/result/*.xml"
+        if: always()
+
+      - name: Test Summary Local copy
+        uses: test-summary/action@v2
+        with:
+          paths: "./junit/result/*.xml"
+          output: "./junit/result/test-summary.md"
+        if: always()
+
+      - name: Store Test Summary results
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-summary-${{ matrix.split }}
+          path: "./junit/result/test-summary.md"
+        if: always()

.github/workflows/test.yml

@@ -0,0 +1,29 @@
+name: Test
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+  workflow_call:
+
+permissions:
+  contents: read
+
+jobs:
+
+  flake8:
+    name: Lint (Flake8)
+    uses: ./.github/workflows/flake8.yml # Call the reusable workflow
+
+  pyright:
+    name: Lint (PyRight)
+    uses: ./.github/workflows/pyright.yml # Call the reusable workflow
+
+  cypress:
+    name: Test (Cypress)
+    #needs: [flake8]
+    uses: ./.github/workflows/cypress.yml # Call the reusable workflow
+
+  pytest:
+    name: Test (Pytest)
+    #needs: [flake8]
+    uses: ./.github/workflows/pytest.yml # Call the reusable workflow

.gitignore

@@ -25,6 +25,11 @@ tmp/*
 solr_runtime/*
 fl_notes.txt
 *.ini
+!test-core-cypress.ini
+!test-core.ini
+!test-core-circle-ci.ini
+!test-core-docker.ini
+!test-core-github-actions.ini
 !ckan/migration/alembic.ini
 .noseids
 *~

changes/8731.misc

@@ -0,0 +1 @@
+Update CICD Infrastructure

ckan/lib/i18n.py

@@ -102,7 +102,7 @@ def get_locales_from_config() -> set[str]:
 
 def _get_locales() -> list[str]:
     # FIXME this wants cleaning up and merging with get_locales_from_config()
-    assert not config.get('lang'), \
+    assert not config.get('lang', None), \
         ('"lang" config option not supported - please use ckan.locale_default '
          'instead.')
     locales_offered = config.get('ckan.locales_offered')