refactor: Unify index directives #6339
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2025 Democratized Data Foundation | |
| # | |
| # Use of this software is governed by the Business Source License | |
| # included in the file licenses/BSL.txt. | |
| # | |
| # As of the Change Date specified in that file, in accordance with | |
| # the Business Source License, use of this software will be governed | |
| # by the Apache License, Version 2.0, included in the file | |
| # licenses/APL.txt. | |
| # Note: | |
| # - Unlike the [test-limited-resources.yml] workflow the coverage jobs in this | |
| # workflow run on faster custom runners. | |
| # - The coverage jobs in this workflow are required to pass within the PRs. | |
| # - All the coverage jobs will generate their respective coverage reports and | |
| # save them as artifacts. | |
| # - The [upload-coverage] job is responsible for gathering all the saved artifacts | |
| # and then uploading them to code-cov. | |
| # - The [upload-coverage] job is not required on the PRs as sometimes the uploading | |
| # can fail if code-cov api service is down, etc. | |
| name: Test Coverage Workflow | |
| on: | |
| pull_request: | |
| branches: | |
| - master | |
| - develop | |
| push: | |
| tags: | |
| - 'v[0-9]+.[0-9]+.[0-9]+' | |
| branches: | |
| - master | |
| - develop | |
| # Cancel the workflow if its in progress when there is a new commit. This will reduce unecessary resource usage. | |
| concurrency: | |
| group: ${{ github.workflow }}-\ | |
| ${{ github.ref == github.ref_protected && github.run_id || github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| # Default environment configuration settings. | |
| env: | |
| CGO_ENABLED: 1 | |
| DEFRA_CLIENT_GO: true | |
| DEFRA_CLIENT_HTTP: false | |
| DEFRA_CLIENT_CLI: false | |
| DEFRA_CLIENT_C: false | |
| DEFRA_BADGER_MEMORY: true | |
| DEFRA_BADGER_FILE: false | |
| DEFRA_BADGER_ENCRYPTION: false | |
| DEFRA_VECTOR_EMBEDDING: false | |
| DEFRA_MUTATION_TYPE: collection-save | |
| DEFRA_LENS_TYPE: wasm-time | |
| DEFRA_DOCUMENT_ACP_TYPE: local | |
| DEFRA_VIEW_TYPE: cacheless | |
| # We run all runners via the bash shell to provide us with a consistent set of env variables and commands | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| # The basic matrix job tests the combination of client, database and mutation types using | |
| # the default config settings for other options, all running on linux. | |
| test-coverage-basic: | |
| name: Test coverage job | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| client-type: [go, http, cli, c] | |
| database-type: [file, memory] | |
| mutation-type: [gql, collection-named, collection-save] | |
| runs-on: runs-on=${{ github.run_id }}-${{ github.run_attempt }}-${{ strategy.job-index }}/\ | |
| spot=pco/cpu=16+32/family=c6*+c7*/disk=large/extras=s3-cache | |
| # Overwrite the defaults based on the basic matrix | |
| env: | |
| DEFRA_CLIENT_GO: ${{ matrix.client-type == 'go' }} | |
| DEFRA_CLIENT_HTTP: ${{ matrix.client-type == 'http' }} | |
| DEFRA_CLIENT_CLI: ${{ matrix.client-type == 'cli' }} | |
| DEFRA_CLIENT_C: ${{ matrix.client-type == 'c' }} | |
| DEFRA_BADGER_MEMORY: ${{ matrix.database-type == 'memory' }} | |
| DEFRA_BADGER_FILE: ${{ matrix.database-type == 'file' }} | |
| DEFRA_MUTATION_TYPE: ${{ matrix.mutation-type }} | |
| DEFRA_VECTOR_EMBEDDING: true | |
| steps: | |
| - name: Enable RunsOn action | |
| uses: runs-on/action@v2 | |
| with: | |
| metrics: cpu,network,memory,disk,io | |
| - name: Checkout code into the directory | |
| uses: actions/checkout@v6 | |
| - name: Setup defradb | |
| uses: ./.github/composites/setup-defradb | |
| - name: Install Ollama | |
| run: make deps:ollama | |
| - name: Run Ollama | |
| run: make ollama | |
| - name: Pull LLM model | |
| run: make ollama:nomic | |
| - name: Test coverage & save coverage report in an artifact | |
| uses: ./.github/composites/test-coverage-with-artifact | |
| with: | |
| coverage-artifact-name: "coverage_basic\ | |
| _${{ matrix.client-type }}\ | |
| _${{ matrix.database-type }}\ | |
| _${{ matrix.mutation-type }}\ | |
| " | |
| coverage-path: coverage.txt | |
| # The document acp matrix job tests the combinations of source-hub acp and client types on linux. | |
| test-coverage-document-acp: | |
| name: Test coverage document acp job | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| client-type: [go, http, cli, c] | |
| document-acp-type: [source-hub] | |
| runs-on: runs-on=${{ github.run_id }}-${{ github.run_attempt }}-${{ strategy.job-index }}/\ | |
| spot=pco/cpu=16+32/family=c6*+c7*/disk=large/extras=s3-cache | |
| env: | |
| DEFRA_DOCUMENT_ACP_TYPE: ${{ matrix.document-acp-type }} | |
| DEFRA_CLIENT_GO: ${{ matrix.client-type == 'go' }} | |
| DEFRA_CLIENT_HTTP: ${{ matrix.client-type == 'http' }} | |
| DEFRA_CLIENT_CLI: ${{ matrix.client-type == 'cli' }} | |
| DEFRA_CLIENT_C: ${{ matrix.client-type == 'c' }} | |
| steps: | |
| - name: Enable RunsOn action | |
| uses: runs-on/action@v2 | |
| with: | |
| metrics: cpu,network,memory,disk,io | |
| - name: Checkout code into the directory | |
| uses: actions/checkout@v6 | |
| - name: Setup defradb | |
| uses: ./.github/composites/setup-defradb | |
| - name: Get SourceHub Docker Image Name | |
| id: sourcehub-image | |
| uses: ./.github/composites/get-sourcehub-image | |
| - name: Test coverage & save coverage report in an artifact | |
| uses: ./.github/composites/test-coverage-with-artifact | |
| env: | |
| DEFRA_SOURCEHUB_IMAGE: ${{ steps.sourcehub-image.outputs.image }} | |
| with: | |
| coverage-artifact-name: "coverage_document_acp\ | |
| _${{ matrix.document-acp-type }}\ | |
| _${{ matrix.client-type }}\ | |
| " | |
| coverage-path: coverage.txt | |
| # The lens matrix job tests the wazero and wasmer lens on linux. | |
| test-coverage-lens: | |
| name: Test coverage lens job | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| lens-type: [wazero] | |
| runs-on: runs-on=${{ github.run_id }}-${{ github.run_attempt }}-${{ strategy.job-index }}/\ | |
| spot=pco/cpu=16+32/family=c6*+c7*/disk=large/extras=s3-cache | |
| env: | |
| DEFRA_LENS_TYPE: ${{ matrix.lens-type }} | |
| steps: | |
| - name: Enable RunsOn action | |
| uses: runs-on/action@v2 | |
| with: | |
| metrics: cpu,network,memory,disk,io | |
| - name: Checkout code into the directory | |
| uses: actions/checkout@v6 | |
| - name: Setup defradb | |
| uses: ./.github/composites/setup-defradb | |
| - name: Test coverage & save coverage report in an artifact | |
| uses: ./.github/composites/test-coverage-with-artifact | |
| with: | |
| coverage-artifact-name: "coverage_lens_${{ matrix.lens-type }}" | |
| coverage-path: coverage.txt | |
| # This job runs the materialized view tests using default configuration, on linux. | |
| test-coverage-view: | |
| name: Test coverage view job | |
| runs-on: runs-on=${{ github.run_id }}-${{ github.run_attempt }}-${{ strategy.job-index }}/\ | |
| spot=pco/cpu=16+32/family=c6*+c7*/disk=large/extras=s3-cache | |
| env: | |
| DEFRA_VIEW_TYPE: materialized | |
| steps: | |
| - name: Enable RunsOn action | |
| uses: runs-on/action@v2 | |
| with: | |
| metrics: cpu,network,memory,disk,io | |
| - name: Checkout code into the directory | |
| uses: actions/checkout@v6 | |
| - name: Setup defradb | |
| uses: ./.github/composites/setup-defradb | |
| - name: Test coverage & save coverage report in an artifact | |
| uses: ./.github/composites/test-coverage-with-artifact | |
| with: | |
| coverage-artifact-name: "coverage_view_materialized" | |
| coverage-path: coverage.txt | |
| # This job runs the database with encryption tests using default configuration, on linux. | |
| test-coverage-encryption: | |
| name: Test coverage encryption job | |
| runs-on: runs-on=${{ github.run_id }}-${{ github.run_attempt }}-${{ strategy.job-index }}/\ | |
| spot=pco/cpu=16+32/family=c6*+c7*/disk=large/extras=s3-cache | |
| env: | |
| DEFRA_BADGER_ENCRYPTION: true | |
| steps: | |
| - name: Enable RunsOn action | |
| uses: runs-on/action@v2 | |
| with: | |
| metrics: cpu,network,memory,disk,io | |
| - name: Checkout code into the directory | |
| uses: actions/checkout@v6 | |
| - name: Setup defradb | |
| uses: ./.github/composites/setup-defradb | |
| - name: Test coverage & save coverage report in an artifact | |
| uses: ./.github/composites/test-coverage-with-artifact | |
| with: | |
| coverage-artifact-name: "coverage_encryption" | |
| coverage-path: coverage.txt | |
| # This job runs the database with telemetry tests using default configuration, on linux. | |
| test-coverage-telemetry: | |
| name: Test coverage telemetry job | |
| runs-on: runs-on=${{ github.run_id }}-${{ github.run_attempt }}-${{ strategy.job-index }}/\ | |
| spot=pco/cpu=16+32/family=c6*+c7*/disk=large/extras=s3-cache | |
| env: | |
| GOFLAGS: -tags=telemetry | |
| steps: | |
| - name: Enable RunsOn action | |
| uses: runs-on/action@v2 | |
| with: | |
| metrics: cpu,network,memory,disk,io | |
| - name: Checkout code into the directory | |
| uses: actions/checkout@v6 | |
| - name: Setup defradb | |
| uses: ./.github/composites/setup-defradb | |
| - name: Test coverage & save coverage report in an artifact | |
| uses: ./.github/composites/test-coverage-with-artifact | |
| with: | |
| coverage-artifact-name: "coverage_telemetry" | |
| coverage-path: coverage.txt | |
| # This job runs the database js tests using default configuration, on linux. | |
| test-coverage-js: | |
| name: Test coverage JS job | |
| runs-on: runs-on=${{ github.run_id }}-${{ github.run_attempt }}-${{ strategy.job-index }}/\ | |
| spot=pco/cpu=16+32/family=c6*+c7*/disk=large/extras=s3-cache | |
| env: | |
| CGO_ENABLED: 0 | |
| steps: | |
| - name: Enable RunsOn action | |
| uses: runs-on/action@v2 | |
| with: | |
| metrics: cpu,network,memory,disk,io | |
| - name: Checkout code into the directory | |
| uses: actions/checkout@v6 | |
| - name: Setup defradb | |
| uses: ./.github/composites/setup-defradb | |
| # NOTE: Uncomment the below if the headless browser tests start failing. | |
| # # Disable AppArmor for Ubuntu 23.10+. | |
| # # https://chromium.googlesource.com/chromium/src/+/main/docs/security/apparmor-userns-restrictions.md | |
| # - name: Disable AppArmor | |
| # run: echo 0 | sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns | |
| - name: Run integration tests | |
| run: make test:coverage-js | |
| - name: Save coverage report in an artifact | |
| uses: ./.github/composites/upload-coverage-artifact | |
| with: | |
| coverage-artifact-name: "coverage_js" | |
| coverage-path: coverage.txt | |
| # This job runs tests with secondary indexes automatically added to schemas, | |
| # to ensure query results are consistent regardless of whether indexes are present. | |
| test-coverage-secondary-index: | |
| name: Test coverage secondary index job | |
| runs-on: runs-on=${{ github.run_id }}-${{ github.run_attempt }}-${{ strategy.job-index }}/\ | |
| spot=pco/cpu=16+32/family=c6*+c7*/disk=large/extras=s3-cache | |
| env: | |
| DEFRA_MULTIPLIERS: secondary-index | |
| steps: | |
| - name: Enable RunsOn action | |
| uses: runs-on/action@v2 | |
| with: | |
| metrics: cpu,network,memory,disk,io | |
| - name: Checkout code into the directory | |
| uses: actions/checkout@v6 | |
| - name: Setup defradb | |
| uses: ./.github/composites/setup-defradb | |
| - name: Test coverage & save coverage report in an artifact | |
| uses: ./.github/composites/test-coverage-with-artifact | |
| with: | |
| coverage-artifact-name: "coverage_secondary_index" | |
| coverage-path: coverage.txt | |
| # This job runs tests with document signing automatically enabled, | |
| # to ensure every feature behaves correctly when all blocks are signed. | |
| test-coverage-signed-docs: | |
| name: Test coverage signed docs job | |
| runs-on: runs-on=${{ github.run_id }}-${{ github.run_attempt }}-${{ strategy.job-index }}/\ | |
| spot=pco/cpu=16+32/family=c6*+c7*/disk=large/extras=s3-cache | |
| env: | |
| DEFRA_MULTIPLIERS: signed-docs | |
| steps: | |
| - name: Enable RunsOn action | |
| uses: runs-on/action@v2 | |
| with: | |
| metrics: cpu,network,memory,disk,io | |
| - name: Checkout code into the directory | |
| uses: actions/checkout@v6 | |
| - name: Setup defradb | |
| uses: ./.github/composites/setup-defradb | |
| - name: Test coverage & save coverage report in an artifact | |
| uses: ./.github/composites/test-coverage-with-artifact | |
| with: | |
| coverage-artifact-name: "coverage_signed_docs" | |
| coverage-path: coverage.txt | |
| # This job tests the leveldb datastore. | |
| test-coverage-leveldb: | |
| name: Test coverage leveldb job | |
| runs-on: runs-on=${{ github.run_id }}-${{ github.run_attempt }}-${{ strategy.job-index }}/\ | |
| spot=pco/cpu=16+32/family=c6*+c7*/disk=large/extras=s3-cache | |
| env: | |
| DEFRA_LEVEL: true | |
| steps: | |
| - name: Enable RunsOn action | |
| uses: runs-on/action@v2 | |
| with: | |
| metrics: cpu,network,memory,disk,io | |
| - name: Checkout code into the directory | |
| uses: actions/checkout@v6 | |
| - name: Setup defradb | |
| uses: ./.github/composites/setup-defradb | |
| - name: Test coverage & save coverage report in an artifact | |
| uses: ./.github/composites/test-coverage-with-artifact | |
| with: | |
| coverage-artifact-name: "coverage_leveldb" | |
| coverage-path: coverage.txt | |
| ## This job gathers all the coverage reports and uploads them to code-cov | |
| upload-coverage: | |
| name: Upload test code coverage job | |
| needs: | |
| - test-coverage-basic # 24 test(s) | |
| - test-coverage-document-acp # 3 test(s) | |
| - test-coverage-lens # 2 test(s) | |
| - test-coverage-view # 1 test(s) | |
| - test-coverage-encryption # 1 test(s) | |
| - test-coverage-telemetry # 1 test(s) | |
| - test-coverage-js # 1 test(s) | |
| - test-coverage-secondary-index # 1 test(s) | |
| - test-coverage-signed-docs # 1 test(s) | |
| - test-coverage-leveldb # 1 test(s) | |
| # Important to know: | |
| # - We didn't use `if: always()` here, so this job doesn't run if we manually canceled. | |
| # - `if: success()` is always implied unless `always()` or `failure()` is specified. | |
| if: success() || failure() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code into the directory | |
| uses: actions/checkout@v6 | |
| - name: Download coverage reports | |
| uses: actions/download-artifact@v7 | |
| with: | |
| pattern: coverage_* | |
| # Note: https://github.com/actions/download-artifact/blob/main/docs/MIGRATION.md | |
| merge-multiple: false | |
| path: coverage_reports | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v5 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| name: defradb-codecov | |
| files: coverage_reports/**/*.txt | |
| flags: all-tests | |
| os: 'linux' | |
| fail_ci_if_error: true | |
| verbose: true |