Merge branch 'obsl-591-core-add-oracledb-support2' of github.com:soda… #645
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: CI pipeline | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - '*' | |
| tags: | |
| - "*" | |
| env: | |
| PROJECT_NAME: soda-core | |
| jobs: | |
| check: | |
| name: pre-commit | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Run pre-commit | |
| uses: pre-commit/action@v3.0.1 | |
| define-test-matrix: | |
| runs-on: ubuntu-24.04 | |
| needs: [check] | |
| outputs: | |
| modules: ${{ steps.modules.outputs.modules }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Define modules | |
| id: modules | |
| run: | | |
| if [ -z "${{ inputs.dataSource }}" ]; then | |
| echo modules=$(bash scripts/test_matrix.sh) >> "$GITHUB_OUTPUT" | |
| else | |
| echo modules=["${{ inputs.dataSource }}"] >> "$GITHUB_OUTPUT" | |
| fi | |
| test: | |
| runs-on: ubuntu-24.04 | |
| needs: [define-test-matrix] | |
| services: | |
| postgres: | |
| image: ${{ ( matrix.module == 'postgres' ) && 'postgres:15.10-alpine3.21' || '' }} | |
| env: | |
| POSTGRES_USER: soda_test | |
| POSTGRES_DB: soda_test | |
| POSTGRES_HOST_AUTH_METHOD: trust | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| --health-start-period 10s | |
| ports: | |
| - 5432:5432 | |
| sqlserver: | |
| image: ${{ ( matrix.module == 'sqlserver' ) && 'mcr.microsoft.com/mssql/server:2022-latest' || '' }} | |
| env: | |
| ACCEPT_EULA: Y | |
| SA_PASSWORD: Password1! | |
| ports: | |
| - 1433:1433 | |
| options: >- | |
| --health-cmd "/opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P Password1! -Q 'select 1' -C -b -o /dev/null" | |
| --health-interval 1s | |
| --health-timeout 2s | |
| --health-retries 10 | |
| --health-start-period 10s | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: | |
| - "3.10" | |
| - "3.11" | |
| module: ${{ fromJSON(needs.define-test-matrix.outputs.modules) }} | |
| env: | |
| SNOWFLAKE_ACCOUNT: ${{ vars.SNOWFLAKE_CI_ACCOUNT }} | |
| SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_CI_USERNAME }} | |
| SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_CI_DATABASE }} | |
| SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }} | |
| DATABRICKS_HOST: ${{ vars.DATABRICKS_CI_HOST }} | |
| DATABRICKS_HTTP_PATH: ${{ vars.DATABRICKS_CI_HTTP_PATH }} | |
| DATABRICKS_CATALOG: ${{ vars.DATABRICKS_CI_CATALOG }} | |
| DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_CI_TOKEN }} | |
| BIGQUERY_ACCOUNT_INFO_JSON: ${{ secrets.BIGQUERY_ACCOUNT_INFO_JSON }} | |
| SODA_CORE_TELEMETRY_LOCAL_DEBUG_MODE: "true" | |
| SQLSERVER_USERNAME: SA | |
| SQLSERVER_PASSWORD: Password1! | |
| SQLSERVER_DATABASE: master | |
| SYNAPSE_HOST: ${{ vars.MICROSOFT_SYNAPSE_CI_HOST }} | |
| SYNAPSE_DATABASE: sodacisynapse | |
| SYNAPSE_AUTHENTICATION_TYPE: activedirectoryserviceprincipal | |
| SYNAPSE_CLIENT_ID: ${{ secrets.MICROSOFT_SYNAPSE_CI_USER }} | |
| SYNAPSE_CLIENT_SECRET: ${{ secrets.MICROSOFT_SYNAPSE_CI_PASSWORD }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - | |
| curl https://packages.microsoft.com/config/ubuntu/21.04/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list > /dev/null | |
| sudo apt-get update | |
| ACCEPT_EULA=Y sudo apt-get install -y libsasl2-dev msodbcsql18 | |
| - name: Run tests | |
| run: | | |
| python -m venv .venv | |
| source .venv/bin/activate | |
| pip install -r dev-requirements.txt | |
| pip install -e soda-core | |
| pip install -e soda-tests | |
| if [ "${{ matrix.module }}" = "synapse" ]; then | |
| pip install -e soda-sqlserver | |
| fi | |
| pip install -e soda-${{ matrix.module }} | |
| export TEST_DATASOURCE=${{ matrix.module }} | |
| python -m pytest -ra soda-tests/tests/features | |
| if [ "${{ matrix.module }}" = "postgres" ]; then | |
| python -m pytest -ra soda-tests/tests/components | |
| fi | |
| python -m pytest -ra soda-${{ matrix.module }}/tests | |
| define-matrix: | |
| if: github.ref_name == 'v4' || contains(github.ref, 'refs/tags/') | |
| runs-on: ubuntu-24.04 | |
| needs: [test] | |
| outputs: | |
| modules: ${{ steps.modules.outputs.modules }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Define modules | |
| id: modules | |
| run: | | |
| echo modules=$(bash scripts/release_matrix.sh) >> "$GITHUB_OUTPUT" | |
| release: | |
| if: github.ref_name == 'v4' || contains(github.ref, 'refs/tags/') | |
| runs-on: ubuntu-24.04 | |
| needs: [define-matrix] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| module: ${{ fromJSON(needs.define-matrix.outputs.modules) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Debug GITHUB_REF | |
| run: echo "GITHUB_REF=$GITHUB_REF" | |
| - name: Release ${{ matrix.module }} | |
| run: | | |
| python -m venv .venv | |
| source .venv/bin/activate | |
| pip install tbump build twine | |
| if [ "${GITHUB_REF#refs/tags/}" != "$GITHUB_REF" ]; then | |
| VERSION="${GITHUB_REF#refs/tags/}" | |
| echo "Using tag version: $VERSION" | |
| else | |
| VERSION="4.0.0.dev${GITHUB_RUN_NUMBER}" | |
| echo "No tag found, bumping to dev version: $VERSION" | |
| tbump --only-patch --non-interactive "$VERSION" | |
| fi | |
| cd ${{ matrix.module }} | |
| python3 -m build | |
| - name: Publish package to pypi | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| packages-dir: ${{ matrix.module }}/dist | |
| user: ${{ secrets.DEV_PYPI_USERNAME }} | |
| password: ${{ secrets.DEV_PYPI_PASSWORD }} | |
| repository-url: ${{ secrets.DEV_PYPI_URL }} |