feat(catalog): support for multiple catalog pods #6621
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python workflows | |
| on: | |
| push: | |
| branches: | |
| - "main" | |
| workflow_dispatch: | |
| pull_request: | |
| paths-ignore: | |
| - "LICENSE*" | |
| - "**.gitignore" | |
| - "**.md" | |
| - "**.txt" | |
| - ".github/ISSUE_TEMPLATE/**" | |
| - ".github/dependabot.yml" | |
| - "docs/**" | |
| permissions: # set contents: read at top-level, per OpenSSF ScoreCard rule TokenPermissionsID | |
| contents: read | |
| jobs: | |
| lint: | |
| name: ${{ matrix.session }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python: ["3.10"] | |
| session: [lint, mypy] | |
| env: | |
| NOXSESSION: ${{ matrix.session }} | |
| FORCE_COLOR: "1" | |
| steps: | |
| - name: Check out the repository | |
| uses: actions/checkout@v6 | |
| - name: Setup Python, Poetry, and Nox | |
| uses: ./.github/actions/setup-python-poetry | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - name: Nox lint | |
| working-directory: clients/python | |
| run: | | |
| nox --python=${{ matrix.python }} | |
| check-autogen: | |
| name: Check autogenerated code is in sync | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python: ["3.12"] | |
| nodejs: ["20"] | |
| steps: | |
| - name: Check out the repository | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: ${{ matrix.nodejs }} | |
| - name: Upgrade pip | |
| run: | | |
| pip install --constraint=.github/workflows/constraints.txt pip | |
| pip --version | |
| - name: Install Poetry | |
| run: | | |
| pipx install --pip-args=--constraint=${{ github.workspace }}/.github/workflows/constraints.txt poetry | |
| poetry --version | |
| - name: Install openapi-generator-cli | |
| run: | | |
| make bin/openapi-generator-cli | |
| - name: Generate Python client | |
| working-directory: clients/python | |
| run: | | |
| make clean install tidy | |
| - name: Check if there are uncommitted file changes | |
| run: | | |
| clean=$(git status --porcelain) | |
| if [[ -z "$clean" ]]; then | |
| echo "Empty git status --porcelain: $clean" | |
| else | |
| echo "Uncommitted file changes detected: $clean" | |
| git diff | |
| exit 1 | |
| fi | |
| test: | |
| name: E2E Py ${{ matrix.python }} K8s ${{ matrix.kubernetes-version }} DB ${{ matrix.manifest-db }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| pull-requests: read # to use dorny/paths-filter (below) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python: ["3.12"] # see below for versions 3.10-3.11 | |
| kubernetes-version: | |
| ["v1.33.7", "v1.34.3"] | |
| manifest-db: ["db", "postgres"] # subdirectory of manifests/kustomize/overlays to select which database: 'db' (MySQL) or 'postgres' | |
| exclude: # on main merges (not PRs), use also different K8s versions for E2E testing | |
| - kubernetes-version: ${{ github.event_name != 'push' && 'v1.34.3' }} | |
| include: # test Py versions only with a reference K8s version, designated currently to kubernetes-version: v1.33.7 | |
| - python: "3.11" | |
| kubernetes-version: "v1.33.7" | |
| manifest-db: "db" | |
| - python: "3.10" | |
| kubernetes-version: "v1.33.7" | |
| manifest-db: "db" | |
| env: | |
| FORCE_COLOR: "1" | |
| IMG_REGISTRY: ghcr.io | |
| IMG_ORG: kubeflow | |
| IMG_REPO: model-registry | |
| DEPLOY_MANIFEST_DB: "${{ matrix.manifest-db }}" | |
| steps: | |
| - name: Check out the repository | |
| uses: actions/checkout@v6 | |
| - name: Setup Python, Poetry, and Nox | |
| uses: ./.github/actions/setup-python-poetry | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - name: Install Storage Clients | |
| # Oras should be available in the GH Action CI by default | |
| # See: https://oras.land/docs/installation/#runner-machine-of-azure-devops-and-github-actions | |
| run: | | |
| sudo apt-get -y install skopeo | |
| - name: Nox test | |
| working-directory: clients/python | |
| run: | | |
| nox --python=${{ matrix.python }} --session=tests | |
| - name: Generate Tag | |
| shell: bash | |
| id: tags | |
| run: | | |
| commit_sha=${{ github.event.after }} | |
| tag=main-${commit_sha:0:7} | |
| echo "tag=${tag}" >> $GITHUB_OUTPUT | |
| - name: Build Image | |
| shell: bash | |
| env: | |
| IMG_VERSION: ${{ steps.tags.outputs.tag }} | |
| run: make image/build | |
| - name: Start Kind Cluster | |
| uses: helm/[email protected] | |
| with: | |
| node_image: kindest/node:${{ matrix.kubernetes-version }} | |
| cluster_name: chart-testing-py-${{ matrix.python }} | |
| kubectl_version: ${{ matrix.kubernetes-version }} | |
| - name: Load Local Registry Test Image | |
| env: | |
| IMG: "${{ env.IMG_REGISTRY }}/${{ env.IMG_ORG }}/${{ env.IMG_REPO }}:${{ steps.tags.outputs.tag }}" | |
| run: | | |
| kind load docker-image -n chart-testing-py-${{ matrix.python }} ${IMG} | |
| - name: Deploy Model Registry using manifests | |
| env: | |
| IMG: "${{ env.IMG_REGISTRY }}/${{ env.IMG_ORG }}/${{ env.IMG_REPO }}:${{ steps.tags.outputs.tag }}" | |
| run: ./scripts/deploy_on_kind.sh | |
| - name: Deploy Minio using manifests | |
| run: | | |
| ./scripts/deploy_minio_on_kind.sh | |
| cat ./scripts/manifests/minio/.env >> $GITHUB_ENV | |
| - name: Deploy OCI Registry using manifests | |
| run: ./scripts/deploy_local_kind_registry.sh | |
| - name: Nox test end-to-end | |
| working-directory: clients/python | |
| run: | | |
| kubectl port-forward -n kubeflow service/model-registry-service 8080:8080 & | |
| kubectl port-forward -n minio svc/minio 9000:9000 & | |
| kubectl port-forward service/distribution-registry-test-service -n local-oci-registry-ns 5001:5001 & | |
| sleep 2 | |
| nox --python=${{ matrix.python }} --session=e2e -- --cov-report=xml | |
| - name: Check if OpenAPI files changed | |
| id: check-openapi-changed | |
| if: github.event_name == 'pull_request' | |
| uses: dorny/paths-filter@v3 | |
| with: | |
| filters: | | |
| openapi: | |
| - 'api/**' | |
| - name: Nox test fuzz (main only, or PRs with OpenAPI changes) | |
| if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && steps.check-openapi-changed.outputs.openapi == 'true') | |
| working-directory: clients/python | |
| run: | | |
| kubectl port-forward -n kubeflow service/model-registry-service 8080:8080 > /dev/null 2>&1 & | |
| kubectl port-forward -n minio svc/minio 9000:9000 > /dev/null 2>&1 & | |
| kubectl port-forward service/distribution-registry-test-service -n local-oci-registry-ns 5001:5001 > /dev/null 2>&1 & | |
| sleep 2 | |
| nox --python=${{ matrix.python }} --session=fuzz | |
| docs-build: | |
| name: ${{ matrix.session }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python: ["3.12"] | |
| session: [docs-build] | |
| env: | |
| NOXSESSION: ${{ matrix.session }} | |
| FORCE_COLOR: "1" | |
| steps: | |
| - name: Check out the repository | |
| uses: actions/checkout@v6 | |
| - name: Setup Python, Poetry, and Nox | |
| uses: ./.github/actions/setup-python-poetry | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - name: Run Nox | |
| working-directory: clients/python | |
| run: | | |
| nox --python=${{ matrix.python }} | |
| poetry build | |
| - name: Upload dist | |
| uses: actions/[email protected] | |
| with: | |
| name: py-dist | |
| path: clients/python/dist | |
| - name: Upload documentation | |
| uses: actions/[email protected] | |
| with: | |
| name: py-docs | |
| path: clients/python/docs/_build | |
| catalog-test: | |
| name: Catalog E2E | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| env: | |
| FORCE_COLOR: "1" | |
| CATALOG_NAMESPACE: "model-catalog" | |
| CATALOG_IMAGE: "model-registry:catalog-test" | |
| CLUSTER_NAME: "catalog-e2e" | |
| CATALOG_PORT: "8200" | |
| CATALOG_URL: "http://localhost:8200" | |
| PYTHON_VERSION: "3.12" | |
| KUBERNETES_VERSION: "v1.33.7" | |
| steps: | |
| - name: Check out the repository | |
| uses: actions/checkout@v6 | |
| - name: Setup Python, Poetry, and Nox | |
| uses: ./.github/actions/setup-python-poetry | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Start Kind Cluster | |
| uses: helm/[email protected] | |
| with: | |
| node_image: kindest/node:${{ env.KUBERNETES_VERSION }} | |
| cluster_name: ${{ env.CLUSTER_NAME }} | |
| kubectl_version: ${{ env.KUBERNETES_VERSION }} | |
| - name: Build Catalog Image | |
| working-directory: catalog/clients/python | |
| run: | | |
| make deploy-build | |
| - name: Load Catalog Image into Kind | |
| working-directory: catalog/clients/python | |
| run: | | |
| make deploy-load | |
| - name: Deploy Catalog to Kind | |
| working-directory: catalog/clients/python | |
| run: | | |
| make deploy-apply | |
| - name: Nox catalog E2E tests | |
| working-directory: catalog/clients/python | |
| run: | | |
| kubectl port-forward -n ${{ env.CATALOG_NAMESPACE }} svc/model-catalog ${{ env.CATALOG_PORT }}:8080 & | |
| sleep 2 | |
| nox --python=${{ env.PYTHON_VERSION }} --session=e2e -- --cov-report=xml |