diff --git a/.github/pull.yml b/.github/pull.yml new file mode 100644 index 0000000000..638294b627 --- /dev/null +++ b/.github/pull.yml @@ -0,0 +1,5 @@ +version: "1" +rules: + - base: main + upstream: kubeflow:main +label: "tide/merge-method-merge" diff --git a/.github/workflows/async-upload-test.yml b/.github/workflows/async-upload-test.yml index 214a0c661e..8a9c201e1c 100644 --- a/.github/workflows/async-upload-test.yml +++ b/.github/workflows/async-upload-test.yml @@ -21,14 +21,14 @@ permissions: # set contents: read at top-level, per OpenSSF ScoreCard rule Token env: # Async Job - JOB_IMG_REGISTRY: ghcr.io - JOB_IMG_ORG: kubeflow - JOB_IMG_NAME: model-registry/job/async-upload + JOB_IMG_REGISTRY: quay.io + JOB_IMG_ORG: opendatahub + JOB_IMG_NAME: model-registry-job-async-upload JOB_IMG_VERSION: cicd # MR Server - IMG_REGISTRY: ghcr.io - IMG_ORG: kubeflow - IMG_REPO: model-registry/server + IMG_REGISTRY: quay.io + IMG_ORG: opendatahub + IMG_REPO: model-registry IMG_VERSION: cicd PUSH_IMAGE: false diff --git a/.github/workflows/build-and-push-async-upload.yml b/.github/workflows/build-and-push-async-upload.yml index 891ebb33e2..ae0502eb64 100644 --- a/.github/workflows/build-and-push-async-upload.yml +++ b/.github/workflows/build-and-push-async-upload.yml @@ -4,8 +4,6 @@ on: push: branches: - 'main' - tags: - - 'v*' paths: - 'jobs/async-upload/**' - '!LICENSE*' @@ -18,12 +16,11 @@ on: permissions: read-all env: - IMG_REGISTRY: ghcr.io - IMG_ORG: kubeflow - IMG_NAME: model-registry/job/async-upload - REGISTRY_USER: ${{ github.actor }} - REGISTRY_PWD: ${{ secrets.GITHUB_TOKEN }} - PLATFORMS: linux/arm64,linux/amd64 + IMG_REGISTRY: quay.io + IMG_ORG: opendatahub + IMG_NAME: model-registry-job-async-upload + REGISTRY_USER: ${{ secrets.QUAY_USERNAME }} + REGISTRY_PWD: ${{ secrets.QUAY_PASSWORD }} jobs: build-and-push: diff --git a/.github/workflows/build-and-push-controller-image.yml b/.github/workflows/build-and-push-controller-image.yml index d335a56531..6ef9af4052 100644 --- a/.github/workflows/build-and-push-controller-image.yml +++ b/.github/workflows/build-and-push-controller-image.yml @@ -1,26 +1,28 @@ name: Controller container image build and tag on: - push: - branches: - - 'main' - tags: - - 'v*' - paths-ignore: - - 'LICENSE*' - - '**.gitignore' - - '**.md' - - '**.txt' - - '.github/ISSUE_TEMPLATE/**' - - '.github/dependabot.yml' - - 'docs/**' + workflow_dispatch: {} + # FIXME: Re-enable when the repo exists in opendatahub + #push: + # branches: + # - 'main' + # tags: + # - 'v*' + # paths-ignore: + # - 'LICENSE*' + # - '**.gitignore' + # - '**.md' + # - '**.txt' + # - '.github/ISSUE_TEMPLATE/**' + # - '.github/dependabot.yml' + # - 'docs/**' permissions: read-all env: - IMG_REGISTRY: ghcr.io - IMG_ORG: kubeflow - IMG_REPO: model-registry/controller + IMG_REGISTRY: quay.io + IMG_ORG: opendatahub + IMG_REPO: model-registry-controller PUSH_IMAGE: true - DOCKER_USER: ${{ github.actor }} - DOCKER_PWD: ${{ secrets.GITHUB_TOKEN }} + DOCKER_USER: ${{ secrets.QUAY_USERNAME }} + DOCKER_PWD: ${{ secrets.QUAY_PASSWORD }} PLATFORMS: linux/arm64,linux/amd64 jobs: build-controller-image: diff --git a/.github/workflows/build-and-push-csi-image.yml b/.github/workflows/build-and-push-csi-image.yml index c6b0603d20..47fd35ef70 100644 --- a/.github/workflows/build-and-push-csi-image.yml +++ b/.github/workflows/build-and-push-csi-image.yml @@ -15,12 +15,12 @@ on: - 'docs/**' permissions: read-all env: - IMG_REGISTRY: ghcr.io - IMG_ORG: kubeflow - IMG_REPO: model-registry/storage-initializer + IMG_REGISTRY: quay.io + IMG_ORG: opendatahub + IMG_REPO: model-registry-storage-initializer PUSH_IMAGE: true - DOCKER_USER: ${{ github.actor }} - DOCKER_PWD: ${{ secrets.GITHUB_TOKEN }} + DOCKER_USER: ${{ secrets.QUAY_USERNAME }} + DOCKER_PWD: ${{ secrets.QUAY_PASSWORD }} PLATFORMS: linux/arm64,linux/amd64 jobs: build-csi-image: diff --git a/.github/workflows/build-and-push-image.yml b/.github/workflows/build-and-push-image.yml index 02ca9f20a5..071e4c00a9 100644 --- a/.github/workflows/build-and-push-image.yml +++ b/.github/workflows/build-and-push-image.yml @@ -3,8 +3,6 @@ on: push: branches: - 'main' - tags: - - 'v*' paths-ignore: - 'LICENSE*' - '**.gitignore' @@ -14,12 +12,12 @@ on: - '.github/dependabot.yml' - 'docs/**' env: - IMG_REGISTRY: ghcr.io - IMG_ORG: kubeflow - IMG_REPO: model-registry/server + IMG_REGISTRY: quay.io + IMG_ORG: opendatahub + IMG_REPO: model-registry PUSH_IMAGE: true - DOCKER_USER: ${{ github.actor }} - DOCKER_PWD: ${{ secrets.GITHUB_TOKEN }} + DOCKER_USER: ${{ secrets.QUAY_USERNAME }} + DOCKER_PWD: ${{ secrets.QUAY_PASSWORD }} PLATFORMS: linux/arm64,linux/amd64 permissions: read-all diff --git a/.github/workflows/build-and-push-testops-image.yml b/.github/workflows/build-and-push-testops-image.yml new file mode 100644 index 0000000000..84578117ba --- /dev/null +++ b/.github/workflows/build-and-push-testops-image.yml @@ -0,0 +1,122 @@ +name: Build and Push testops container image + +on: + push: + branches: + - 'main' + - 'stable' + - 'stable-2.x' + paths: + - 'Dockerfile.testops' + - 'clients/python/**' + - 'api/**' + - 'manifests/**' + - 'scripts/**' + - '!LICENSE*' + - '!**.gitignore' + - '!**.md' + - '!**.txt' + - '.github/workflows/build-and-push-testops-image.yml' # self + +permissions: # set contents: read at top-level, per OpenSSF ScoreCard rule TokenPermissionsID + contents: read + +env: + IMG_REGISTRY: quay.io + IMG_ORG: opendatahub + IMG_NAME: model-registry-testops + REGISTRY_USER: ${{ secrets.QUAY_USERNAME }} + REGISTRY_PWD: ${{ secrets.QUAY_PASSWORD }} + +jobs: + build-and-push: + runs-on: ubuntu-latest + if: github.repository == 'opendatahub-io/model-registry' + permissions: + contents: read + packages: write + steps: + - name: Checkout repository + uses: actions/checkout@v5.0.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.IMG_REGISTRY }} + username: ${{ env.REGISTRY_USER }} + password: ${{ env.REGISTRY_PWD }} + + - name: Set branch-based environment + run: | + commit_sha=${{ github.sha }} + branch_name=${GITHUB_REF#refs/heads/} + tag=${branch_name}-${commit_sha:0:7} + # Calculate expiration date (7 days from now) + expiry_date=$(date -d '+7 days' -u +%Y-%m-%dT%H:%M:%SZ) + + echo "VERSION=${tag}" >> $GITHUB_ENV + echo "BRANCH_NAME=${branch_name}" >> $GITHUB_ENV + echo "EXPIRY_DATE=${expiry_date}" >> $GITHUB_ENV + + - name: Set tag environment # this is for v* tag image build + if: startsWith(github.ref, 'refs/tags/v') + run: | + echo "VERSION=${{ github.ref_name }}" >> $GITHUB_ENV + + - name: Build and push SHA-based image (expires in 7 days) + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile.testops + push: true + tags: | + ${{ env.IMG_REGISTRY }}/${{ env.IMG_ORG }}/${{ env.IMG_NAME }}:${{ env.VERSION }} + labels: | + org.opencontainers.image.source=https://github.com/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.created=${{ github.event.repository.updated_at }} + quay.expires-after=7d + cache-from: type=gha + cache-to: type=gha,mode=max + provenance: mode=max + sbom: true + + - name: Build and push persistent branch tags (main) + if: github.ref == 'refs/heads/main' + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile.testops + push: true + tags: | + ${{ env.IMG_REGISTRY }}/${{ env.IMG_ORG }}/${{ env.IMG_NAME }}:${{ env.BRANCH_NAME }} + ${{ env.IMG_REGISTRY }}/${{ env.IMG_ORG }}/${{ env.IMG_NAME }}:latest + labels: | + org.opencontainers.image.source=https://github.com/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.created=${{ github.event.repository.updated_at }} + cache-from: type=gha + cache-to: type=gha,mode=max + provenance: mode=max + sbom: true + + - name: Build and push persistent branch tags (non-main) + if: github.ref != 'refs/heads/main' + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile.testops + push: true + tags: | + ${{ env.IMG_REGISTRY }}/${{ env.IMG_ORG }}/${{ env.IMG_NAME }}:${{ env.BRANCH_NAME }} + labels: | + org.opencontainers.image.source=https://github.com/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.created=${{ github.event.repository.updated_at }} + cache-from: type=gha + cache-to: type=gha,mode=max + provenance: mode=max + sbom: true diff --git a/.github/workflows/build-image-pr.yml b/.github/workflows/build-image-pr.yml index 92bb2d1c34..80350cc0ae 100644 --- a/.github/workflows/build-image-pr.yml +++ b/.github/workflows/build-image-pr.yml @@ -16,9 +16,9 @@ permissions: # set contents: read at top-level, per OpenSSF ScoreCard rule Token contents: read env: - IMG_REGISTRY: ghcr.io - IMG_ORG: kubeflow - IMG_REPO: model-registry/server + IMG_REGISTRY: quay.io + IMG_ORG: opendatahub + IMG_REPO: model-registry PUSH_IMAGE: false BRANCH: ${{ github.base_ref }} jobs: @@ -47,13 +47,41 @@ jobs: IMG: "${{ env.IMG_REGISTRY }}/${{ env.IMG_ORG }}/${{ env.IMG_REPO }}:${{ steps.tags.outputs.tag }}" run: | kind load docker-image -n chart-testing ${IMG} - - name: Deploy Model Registry using manifests + - name: Deploy Operator With Test Image env: IMG: "${{ env.IMG_REGISTRY }}/${{ env.IMG_ORG }}/${{ env.IMG_REPO }}:${{ steps.tags.outputs.tag }}" - run: ./scripts/deploy_on_kind.sh - - name: Deployment logs run: | - kubectl logs -n kubeflow deployment/model-registry-deployment + echo "Deploying operator from model-registry-operator branch ${BRANCH}" + kubectl apply -k "https://github.com/opendatahub-io/model-registry-operator.git/config/default?ref=${BRANCH}" + kubectl set env -n model-registry-operator-system deployment/model-registry-operator-controller-manager REST_IMAGE="${IMG}" + kubectl wait --for=condition=Available=true -n model-registry-operator-system deployment/model-registry-operator-controller-manager --timeout=5m + - name: Display about MR Operator in KinD cluster status in any case + if: always() + run: | + kubectl get deployments -n model-registry-operator-system -o wide + kubectl get pods -n model-registry-operator-system -o wide --show-labels + kubectl events -A + kubectl describe -n model-registry-operator-system deployment/model-registry-operator-controller-manager + for p in $(kubectl get pods -n model-registry-operator-system -o name); do + echo "===== Logs for $p =====" + kubectl logs -n model-registry-operator-system $p + done + - name: Create Test Registry + run: | + kubectl apply -k "https://github.com/opendatahub-io/model-registry-operator.git/config/samples/mysql?ref=${BRANCH}" + kubectl get mr + - name: Wait for Test Registry Deployment + run: | + kubectl wait --for=condition=Available=true deployment/model-registry-db --timeout=5m + kubectl wait --for=condition=Available=true modelregistries/modelregistry-sample --timeout=5m + - name: Display KinD cluster status in any case + if: always() + run: | + kubectl get deployments -o wide + kubectl get pods -o wide + kubectl events + kubectl get modelregistries/modelregistry-sample -o wide + kubectl describe modelregistries/modelregistry-sample - name: Set up Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: @@ -73,6 +101,10 @@ jobs: pip install dist/*.whl - name: Connect with Python client run: | - kubectl port-forward -n kubeflow service/model-registry-service 8080:8080 & + kubectl port-forward service/modelregistry-sample 8080:8080 & sleep 5 python test/python/test_mr_conn.py http://localhost 8080 + - name: Check Pod Logs + if: always() + run: | + kubectl logs -l app=modelregistry-sample --all-containers --prefix diff --git a/.github/workflows/build-image-ui-pr.yml b/.github/workflows/build-image-ui-pr.yml index 843ef43e2a..c4d6b3d506 100644 --- a/.github/workflows/build-image-ui-pr.yml +++ b/.github/workflows/build-image-ui-pr.yml @@ -16,7 +16,7 @@ permissions: # set contents: read at top-level, per OpenSSF ScoreCard rule Token env: IMG_ORG: kubeflow - IMG_REPO: model-registry/ui + IMG_REPO: model-registry-ui PUSH_IMAGE: false BRANCH: ${{ github.base_ref }} jobs: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 578837df33..cf44c51509 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -36,3 +36,9 @@ jobs: run: make test-cover - name: Catalog unit tests run: make -C catalog test-cover + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4.5.0 + with: + files: coverage.txt + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: true diff --git a/.github/workflows/check-owners.yml b/.github/workflows/check-owners.yml deleted file mode 100644 index 19cc9f51f5..0000000000 --- a/.github/workflows/check-owners.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: "Check OWNERS" - -on: - pull_request: - paths: - - 'OWNERS' - workflow_dispatch: - -permissions: - contents: read - -jobs: - check-owners: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - - name: yq - portable yaml processor - uses: mikefarah/yq@751d8ad57b84f1794661bc70c0afb92a22ad7b3c # v4 - - - name: Check approvers against kubeflow-hub-team - continue-on-error: true - run: | - # Extract approvers from OWNERS (excluding noqa lines) - approvers=$(grep -v '# noqa(kubeflow-hub-team)' OWNERS | yq eval '.approvers[]') - - # Fetch kubeflow-hub-team members - hub_team=$(curl -s https://raw.githubusercontent.com/kubeflow/internal-acls/master/github-orgs/kubeflow/org.yaml | \ - yq '.orgs.kubeflow.teams.kubeflow-hub-team.members[]') - - echo -e "Approvers:\n$approvers\n" - echo -e "kubeflow-hub-team:\n$hub_team\n" - - # Find approvers not in hub team - missing=$(comm -23 <(echo "$approvers" | sort) <(echo "$hub_team" | sort)) - - echo "Approvers NOT in kubeflow-hub-team:" - echo "$missing" - - # Convert to space-separated for annotation - missing=$(echo "$missing" | tr '\n' ' ' | xargs) - - # Annotate if there are missing approvers - if [ -n "$missing" ]; then - echo "::warning file=OWNERS,line=1::Approvers not in kubeflow-hub-team: $missing" - else - echo "✅ All approvers are in kubeflow-hub-team!" - fi - - - name: Ensure workflow success # don't ever fail merging PR for this sanity check - run: exit 0 diff --git a/.github/workflows/csi-test.yml b/.github/workflows/csi-test.yml index 1d500d584d..34347f9981 100644 --- a/.github/workflows/csi-test.yml +++ b/.github/workflows/csi-test.yml @@ -23,10 +23,10 @@ permissions: # set contents: read at top-level, per OpenSSF ScoreCard rule Token contents: read env: - IMG_REGISTRY: ghcr.io - IMG_ORG: kubeflow - MODEL_REGISTRY_IMG: model-registry/server - MODEL_REGISTRY_CSI_IMG: model-registry/storage-initializer + IMG_REGISTRY: quay.io + IMG_ORG: opendatahub + MODEL_REGISTRY_IMG: model-registry + MODEL_REGISTRY_CSI_IMG: model-registry-storage-initializer PUSH_IMAGE: false BRANCH: ${{ github.base_ref }} jobs: diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index d182e59476..4de4ab96dc 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -118,9 +118,10 @@ jobs: manifest-db: "db" env: FORCE_COLOR: "1" - IMG_REGISTRY: ghcr.io - IMG_ORG: kubeflow + IMG_REGISTRY: quay.io + IMG_ORG: opendatahub IMG_REPO: model-registry + MR_NAMESPACE: kubeflow # used for kind testing, align to /manifest DEPLOY_MANIFEST_DB: "${{ matrix.manifest-db }}" steps: - name: Check out the repository @@ -174,11 +175,18 @@ jobs: - name: Nox test end-to-end working-directory: clients/python run: | - kubectl port-forward -n kubeflow service/model-registry-service 8080:8080 & + kubectl port-forward -n ${{ env.MR_NAMESPACE }} service/model-registry-service 8080:8080 & kubectl port-forward -n minio svc/minio 9000:9000 & kubectl port-forward service/distribution-registry-test-service -n local-oci-registry-ns 5001:5001 & sleep 2 nox --python=${{ matrix.python }} --session=e2e -- --cov-report=xml + - name: Upload coverage report # we upload coverage stats for py once, regardless if previous step failed + uses: codecov/codecov-action@v4.5.0 + if: always() && matrix.python == '3.12' + with: + files: coverage.xml + fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} - name: Check if OpenAPI files changed id: check-openapi-changed if: github.event_name == 'pull_request' @@ -191,7 +199,7 @@ jobs: if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && steps.check-openapi-changed.outputs.openapi == 'true') working-directory: clients/python run: | - kubectl port-forward -n kubeflow service/model-registry-service 8080:8080 > /dev/null 2>&1 & + kubectl port-forward -n ${{ env.MR_NAMESPACE }} service/model-registry-service 8080:8080 > /dev/null 2>&1 & kubectl port-forward -n minio svc/minio 9000:9000 > /dev/null 2>&1 & kubectl port-forward service/distribution-registry-test-service -n local-oci-registry-ns 5001:5001 > /dev/null 2>&1 & sleep 2 diff --git a/.github/workflows/sync-branch-stable.yml b/.github/workflows/sync-branch-stable.yml new file mode 100644 index 0000000000..49f9f5f52a --- /dev/null +++ b/.github/workflows/sync-branch-stable.yml @@ -0,0 +1,32 @@ +name: Sync Branch stable +on: + push: + branches: + - main + +jobs: + sync-branches: + runs-on: ubuntu-latest + name: Syncing branch stable + steps: + - name: Checkout + uses: actions/checkout@v5 + - name: Opening pull request + id: pull + uses: tretuna/sync-branches@1.4.0 + with: + TEAM_REVIEWERS: '["model-registry-maintainers"]' + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + FROM_BRANCH: "main" + TO_BRANCH: "stable" + - name: Add labels + if: ${{ steps.pull.outputs.PULL_REQUEST_NUMBER != '' }} + uses: actions/github-script@v7 + with: + script: | + github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: ${{steps.pull.outputs.PULL_REQUEST_NUMBER}}, + labels: ['Area/Jobs/Sync-stable', 'tide/merge-method-merge'] + }); diff --git a/.github/workflows/sync-branch-stable2x.yml b/.github/workflows/sync-branch-stable2x.yml new file mode 100644 index 0000000000..279cbdb423 --- /dev/null +++ b/.github/workflows/sync-branch-stable2x.yml @@ -0,0 +1,32 @@ +name: Sync Branch stable-2.x +on: + push: + branches: + - main + +jobs: + sync-branches: + runs-on: ubuntu-latest + name: Syncing branch stable-2.x + steps: + - name: Checkout + uses: actions/checkout@v5 + - name: Opening pull request + id: pull + uses: tretuna/sync-branches@1.4.0 + with: + TEAM_REVIEWERS: '["model-registry-maintainers"]' + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + FROM_BRANCH: "main" + TO_BRANCH: "stable-2.x" + - name: Add labels + if: ${{ steps.pull.outputs.PULL_REQUEST_NUMBER != '' }} + uses: actions/github-script@v7 + with: + script: | + github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: ${{steps.pull.outputs.PULL_REQUEST_NUMBER}}, + labels: ['Area/Jobs/Sync-stable2x', 'tide/merge-method-merge'] + }); diff --git a/.gitignore b/.gitignore index 22159ba411..c69b924a3f 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,8 @@ __pycache__ include/ readme.txt +# do not send certs +certs/ #OSX files **/.DS_Store diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 0000000000..11e0a3d957 --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,12 @@ +[allowlist] + description = "Global Allowlist" + + # Ignore based on any subset of the line + regexes = [ + + # Ignore specific database password + '''database-password\s*:\s*"The BlurstOfTimes"''', + + # Ignore specific database user + '''database-user\s*:\s*"mlmduser"''' + ] \ No newline at end of file diff --git a/.syft.yaml b/.syft.yaml new file mode 100644 index 0000000000..d43676b60b --- /dev/null +++ b/.syft.yaml @@ -0,0 +1,5 @@ +# Only the Go backend is part of the final container image. +# Other components like the Upstream MR UI are _not_ consumed midstream/downstream from this monorepo +# therefore, should not be part of any generated (see Syft config docs: https://github.com/anchore/syft/wiki/configuration#list-of-configurable-values). +exclude: + - ./clients/ui/** diff --git a/Dockerfile b/Dockerfile index 6618824151..77ae7e334e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,6 +36,7 @@ FROM registry.access.redhat.com/ubi9/ubi-minimal:latest WORKDIR / # copy the registry binary COPY --from=builder /workspace/model-registry . +COPY manifests/kustomize/options/catalog/default-catalog.yaml /default-catalog.yaml USER 65532:65532 ENTRYPOINT ["/model-registry"] diff --git a/Dockerfile.testops b/Dockerfile.testops new file mode 100644 index 0000000000..2f9ebeb979 --- /dev/null +++ b/Dockerfile.testops @@ -0,0 +1,57 @@ +FROM registry.access.redhat.com/ubi9/python-312 + +ARG TARGETARCH +# Switch to root to install packages and create user +USER 0 + +# Install skopeo +RUN dnf update -y && \ + dnf install -y skopeo && \ + dnf clean all + +# Create odh user +RUN useradd -m -s /bin/bash odh + +# Set working directory +WORKDIR /home/odh + +# Copy project files +COPY clients/python clients/python +COPY catalog/clients/python catalog/clients/python +COPY jobs/async-upload jobs/async-upload +COPY api/ api/ +COPY manifests/ manifests/ +COPY scripts/ scripts/ + +# Download kubectl binary +RUN ARCH="${TARGETARCH:-amd64}" && \ + KUBECTL_VERSION=$(curl -L -s https://dl.k8s.io/release/stable.txt) && \ + curl -fsSLO "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${ARCH}/kubectl" && \ + curl -fsSLO "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${ARCH}/kubectl.sha256" && \ + echo "$("] license = "Apache-2.0" readme = "README.md" homepage = "https://github.com/kubeflow/model-registry" +documentation = "https://model-registry.readthedocs.io/en/latest/" packages = [ { include = "model_registry", from = "src" }, { include = "mr_openapi", from = "src" }, diff --git a/docker-compose.yaml b/docker-compose.yaml index 366f8cbaf7..9412151ba9 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -34,7 +34,7 @@ services: - "localhost:host-gateway" model-catalog: - image: ghcr.io/kubeflow/model-registry/server:latest + image: quay.io/opendatahub/model-registry/server:latest pull_policy: always command: ["catalog", "--listen", "0.0.0.0:8081", "--catalogs-path", "/testdata/test-catalog-sources.yaml"] container_name: model-catalog @@ -53,7 +53,7 @@ services: - PGPASSWORD=demo model-registry: - image: ghcr.io/kubeflow/model-registry/server:latest + image: quay.io/opendatahub/model-registry:latest pull_policy: always entrypoint: ["/bin/sh"] command: diff --git a/jobs/async-upload/Makefile b/jobs/async-upload/Makefile index 81240dc6b9..b63080b0b5 100644 --- a/jobs/async-upload/Makefile +++ b/jobs/async-upload/Makefile @@ -1,15 +1,16 @@ JOB_IMG_VERSION ?= latest # when testing locally with KinD, override default to a tag of choice and you can opt-in to BUILD_IMAGE=false below. -JOB_IMG_REGISTRY ?= ghcr.io -JOB_IMG_ORG ?= kubeflow -JOB_IMG_NAME ?= model-registry/job/async-upload +JOB_IMG_REGISTRY ?= quay.io +JOB_IMG_ORG ?= opendatahub +JOB_IMG_NAME ?= model-registry-job-async-upload JOB_IMG ?= $(JOB_IMG_REGISTRY)/$(JOB_IMG_ORG)/$(JOB_IMG_NAME):$(JOB_IMG_VERSION) BUILD_IMAGE ?= true # whether to build the MR server image +include odh_rules.mk CLUSTER_NAME ?= mr-e2e # MR Server Params IMG_VERSION ?= latest # keep IMG consistent with root Makefile and ci/GHA: -IMG ?= ghcr.io/kubeflow/model-registry/server +IMG ?= quay.io/opendatahub/model-registry .PHONY: deploy-latest-mr deploy-latest-mr: @@ -100,3 +101,13 @@ install: install-poetry-export .PHONY: install-poetry-export install-poetry-export: poetry self add poetry-plugin-export + +.PHONY: undeploy-minio +undeploy-minio: + @echo "Undeploy Minio..." + cd ../../ && ./scripts/undeploy_minio.sh + +.PHONY: undeploy-local-kind-registry +undeploy-local-kind-registry: + @echo "Undeploy Local Kind Registry..." + cd ../../ && ./scripts/undeploy_local_registry.sh diff --git a/jobs/async-upload/README.md b/jobs/async-upload/README.md index f39c887428..8716d9ea49 100644 --- a/jobs/async-upload/README.md +++ b/jobs/async-upload/README.md @@ -14,7 +14,7 @@ docker build \ -t quay.io//async-upload-job:$(git rev-parse --short HEAD) . # 2 – Push to your registry -docker push quay.io//async-upload-job: +docker push quay.io//model-registry-job-async-upload: # TODO: Run locally... ``` diff --git a/jobs/async-upload/odh_rules.mk b/jobs/async-upload/odh_rules.mk new file mode 100644 index 0000000000..cf83986dbc --- /dev/null +++ b/jobs/async-upload/odh_rules.mk @@ -0,0 +1,31 @@ +.PHONY: deploy-mr-odh +deploy-mr-odh: + cd ../../ && ./scripts/deploy_on_odh.sh + +.PHONY: undeploy-mr-odh +undeploy-mr-odh: + cd ../../ && ./scripts/undeploy_on_odh.sh + +.PHONY: test-e2e-odh-async-jobs +test-e2e-odh-async-jobs: deploy-mr-odh deploy-local-registry deploy-test-minio + @echo "Running Async Jobs e2e tests..." + @( \ + trap 'rm -f ../../scripts/manifests/minio/.env' EXIT; \ + set -a; . ../../scripts/manifests/minio/.env; set +a; \ + mkdir -p ../../results; \ + . ../../scripts/odh_env.sh && \ + poetry install --all-extras --with integration && \ + poetry run pytest --e2e tests/integration/ -svvv -rA \ + --html=../../results/report.html \ + --junit-xml=../../results/xunit_report.xml \ + --self-contained-html \ + -o junit_suite_name=odh-async-upload \ + ) + +.PHONY: test-e2e-odh-async-jobs-cleanup +test-e2e-odh-async-jobs-cleanup: undeploy-mr-odh undeploy-minio undeploy-local-kind-registry + @echo "Cleaning up port-forward processes..." + @if [ -f .port-forwards.pid ]; then \ + kill $$(cat .port-forwards.pid) || true; \ + rm -f .port-forwards.pid; \ + fi diff --git a/jobs/async-upload/samples/sample_job_s3_to_oci.yaml b/jobs/async-upload/samples/sample_job_s3_to_oci.yaml index 07b968f1bb..c255781d81 100644 --- a/jobs/async-upload/samples/sample_job_s3_to_oci.yaml +++ b/jobs/async-upload/samples/sample_job_s3_to_oci.yaml @@ -66,7 +66,7 @@ spec: restartPolicy: Never containers: - name: async-upload - image: ghcr.io/kubeflow/model-registry/job/async-upload:latest + image: quay.io/opendatahub/model-registry-job-async-upload securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: false diff --git a/manifests/kustomize/OWNERS b/manifests/kustomize/OWNERS index 4e580875cc..ea99ec7ac6 100644 --- a/manifests/kustomize/OWNERS +++ b/manifests/kustomize/OWNERS @@ -1,8 +1,6 @@ approvers: - tarilabs - rareddy - - Tomcli reviewers: - tarilabs - rareddy - - Tomcli diff --git a/manifests/kustomize/base/model-registry-deployment.yaml b/manifests/kustomize/base/model-registry-deployment.yaml index e098c11650..ddb10bc260 100644 --- a/manifests/kustomize/base/model-registry-deployment.yaml +++ b/manifests/kustomize/base/model-registry-deployment.yaml @@ -29,7 +29,7 @@ spec: command: - /model-registry - proxy - image: ghcr.io/kubeflow/model-registry/server:latest + image: quay.io/opendatahub/model-registry:latest env: - name: DBCONFIG_USER valueFrom: diff --git a/manifests/kustomize/options/catalog/default-catalog.yaml b/manifests/kustomize/options/catalog/default-catalog.yaml new file mode 100644 index 0000000000..328e2ecccc --- /dev/null +++ b/manifests/kustomize/options/catalog/default-catalog.yaml @@ -0,0 +1,2 @@ +source: Default +models: [] diff --git a/manifests/kustomize/options/catalog/overlays/rhoai/README.md b/manifests/kustomize/options/catalog/overlays/rhoai/README.md new file mode 100644 index 0000000000..8b13a8c431 --- /dev/null +++ b/manifests/kustomize/options/catalog/overlays/rhoai/README.md @@ -0,0 +1,13 @@ +# RHOAI Dev Overlay + +These manifests deploy model catalog with the performance data from RHOAI. These are intended for development. + +The manifests expect a pull secret called `registry-redhat-io-credentials` to pull from `registry.redhat.io`. If you're authenticated to that registry with Docker locally, you can create the secret with: + +```sh +jq '{"auths": .auths | with_entries(select(.key == "registry.redhat.io"))}' ~/.docker/config.json | kubectl create secret generic registry-redhat-io-credentials --type=kubernetes.io/dockerconfigjson -n kubeflow --from-file=.dockerconfigjson=/dev/stdin +``` + +## Tilt + +To use these manifests with Tilt, follow the [instructions](../odh/README.md) for configuring the ODH overlay, but replace `../odh` with `../rhoai`. diff --git a/manifests/kustomize/options/catalog/overlays/rhoai/kustomization.yaml b/manifests/kustomize/options/catalog/overlays/rhoai/kustomization.yaml new file mode 100644 index 0000000000..dbf6d9ae93 --- /dev/null +++ b/manifests/kustomize/options/catalog/overlays/rhoai/kustomization.yaml @@ -0,0 +1,37 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../odh + +patches: +- target: + group: apps + version: v1 + kind: Deployment + name: model-catalog-server + patch: |- + - op: add + path: /spec/template/spec/containers/0/args/1 + value: "--performance-metrics=/shared-data/performance" + - op: add + path: /spec/template/spec/imagePullSecrets + value: + - name: registry-redhat-io-credentials + - op: add + path: /spec/template/spec/initContainers/1 + value: + name: benchmark-data-init + image: registry.redhat.io/rhoai/odh-model-performance-data-rhel9:v3.0.0 + command: + - /bin/sh + - -c + - mkdir /shared-data/performance && cp -r /app/benchmarks/* /shared-data/performance/ + volumeMounts: + - name: shared-data + mountPath: /shared-data + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL diff --git a/manifests/kustomize/options/csi/clusterstoragecontainer.yaml b/manifests/kustomize/options/csi/clusterstoragecontainer.yaml index aee0be3d90..b14cc2db94 100644 --- a/manifests/kustomize/options/csi/clusterstoragecontainer.yaml +++ b/manifests/kustomize/options/csi/clusterstoragecontainer.yaml @@ -5,7 +5,7 @@ metadata: spec: container: name: storage-initializer - image: ghcr.io/kubeflow/model-registry/storage-initializer:latest + image: quay.io/opendatahub/model-registry-storage-initializer:latest env: - name: MODEL_REGISTRY_BASE_URL value: "model-registry-service.kubeflow.svc.cluster.local:8080" diff --git a/manifests/kustomize/overlays/db-odh/kustomization.yaml b/manifests/kustomize/overlays/db-odh/kustomization.yaml new file mode 100644 index 0000000000..9b9e00aafb --- /dev/null +++ b/manifests/kustomize/overlays/db-odh/kustomization.yaml @@ -0,0 +1,66 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- model-registry-db-pvc.yaml +- model-registry-db-deployment.yaml +- model-registry-db-service.yaml + + +configMapGenerator: +- envs: + - params.env + name: model-registry-db-parameters +secretGenerator: +- envs: + - secrets.env + name: model-registry-db-secrets +generatorOptions: + disableNameSuffixHash: true + + +images: +- name: mysql + newName: mysql + newTag: 8.3.0 + +replacements: +- source: + fieldPath: metadata.name + kind: Service + name: model-registry-db + version: v1 + targets: + - fieldPaths: + - spec.template.spec.containers.1.args.1 + options: + delimiter: = + index: 1 + select: + group: apps + kind: Deployment + name: model-registry-deployment + version: v1 +- source: + fieldPath: data.MYSQL_PORT + kind: ConfigMap + name: model-registry-db-parameters + version: v1 + targets: + - fieldPaths: + - spec.template.spec.containers.1.args.3 + options: + delimiter: = + index: 1 + select: + group: apps + kind: Deployment + name: model-registry-deployment + version: v1 + - fieldPaths: + - spec.template.metadata.annotations.[traffic.sidecar.istio.io/excludeOutboundPorts] + select: + group: apps + kind: Deployment + name: model-registry-deployment + version: v1 diff --git a/manifests/kustomize/overlays/db-odh/model-registry-db-deployment.yaml b/manifests/kustomize/overlays/db-odh/model-registry-db-deployment.yaml new file mode 100644 index 0000000000..61eb78539f --- /dev/null +++ b/manifests/kustomize/overlays/db-odh/model-registry-db-deployment.yaml @@ -0,0 +1,60 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: model-registry-db + labels: + component: db +spec: + selector: + matchLabels: + component: db + replicas: 1 + strategy: + type: Recreate + template: + metadata: + name: db + labels: + component: db + sidecar.istio.io/inject: "false" + spec: + securityContext: + seccompProfile: + type: RuntimeDefault + runAsNonRoot: true + containers: + - name: db-container + image: mysql:8.3.0 + args: + - --datadir + - /var/lib/mysql/datadir + - --default-authentication-plugin=mysql_native_password + envFrom: + - configMapRef: + name: model-registry-db-parameters + - secretRef: + name: model-registry-db-secrets + ports: + - name: dbapi + containerPort: 3306 + readinessProbe: + exec: + command: + - "/bin/bash" + - "-c" + - "mysql -D $$MYSQL_DATABASE -u$$MYSQL_USER_NAME -p$$MYSQL_ROOT_PASSWORD -e 'SELECT 1'" + initialDelaySeconds: 5 + periodSeconds: 2 + timeoutSeconds: 1 + volumeMounts: + - name: metadata-mysql + mountPath: /var/lib/mysql + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumes: + - name: metadata-mysql + persistentVolumeClaim: + claimName: metadata-mysql diff --git a/manifests/kustomize/overlays/db-odh/model-registry-db-pvc.yaml b/manifests/kustomize/overlays/db-odh/model-registry-db-pvc.yaml new file mode 100644 index 0000000000..b1c083d9f4 --- /dev/null +++ b/manifests/kustomize/overlays/db-odh/model-registry-db-pvc.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: metadata-mysql +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/manifests/kustomize/overlays/db-odh/model-registry-db-service.yaml b/manifests/kustomize/overlays/db-odh/model-registry-db-service.yaml new file mode 100644 index 0000000000..f27c8c767b --- /dev/null +++ b/manifests/kustomize/overlays/db-odh/model-registry-db-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: model-registry-db + labels: + component: db +spec: + type: ClusterIP + ports: + - port: 3306 + protocol: TCP + name: dbapi + selector: + component: db diff --git a/manifests/kustomize/overlays/db-odh/params.env b/manifests/kustomize/overlays/db-odh/params.env new file mode 100644 index 0000000000..5ab2adb3bb --- /dev/null +++ b/manifests/kustomize/overlays/db-odh/params.env @@ -0,0 +1,3 @@ +MYSQL_DATABASE=metadb +MYSQL_PORT=3306 +MYSQL_ALLOW_EMPTY_PASSWORD=true \ No newline at end of file diff --git a/manifests/kustomize/overlays/db-odh/secrets.env b/manifests/kustomize/overlays/db-odh/secrets.env new file mode 100644 index 0000000000..44ac2ee398 --- /dev/null +++ b/manifests/kustomize/overlays/db-odh/secrets.env @@ -0,0 +1,2 @@ +MYSQL_USER_NAME=root +MYSQL_ROOT_PASSWORD=test \ No newline at end of file diff --git a/openshift-ci/README.md b/openshift-ci/README.md new file mode 100644 index 0000000000..149e036db5 --- /dev/null +++ b/openshift-ci/README.md @@ -0,0 +1,34 @@ +# Model Registry Deployment and Deployment Test + +This deployment of model-registry is deployed via Opendatahub and used the ODH nightly images deployed to an openshfit cluster. + +The script will do the following: +* Create a catalogue source pointing to the latest successful nightly ODH image. +* Deploy the opendatahub operator using the catalogue source. +* Deploy a Data Science Cluster. +* Test the model-registry-operator-contoller-manager pods are running. +* Clone the model-registry-operator repository. +* Deploy model-registry using config/samples/mysql configuration files. +* Test the model-registry-db mysql database pod is running +* Test the modelregistry-sample pods are running + +## Pre-requisites: + +You will need to have an openshift cluster deployed and be oc logged into you cluster as admin. + +## Runing the script: + +From the root of the repository +``` +./openshift-ci/scripts/oc-model-registry-deploy.sh +``` + +## Runing the openshift-ci + +You can start the openshift-ci job to test changes in your Pull Request. To do so put the following command into a comment in your Pull Request +``` +/test e2e-odh-mro-optional +``` + +Previous jobs can be seen [here](https://prow.ci.openshift.org/job-history/gs/test-platform-results/pr-logs/directory/rehearse-49999-pull-ci-opendatahub-io-model-registry-main-e2e-odh-mro-optional) + diff --git a/openshift-ci/resources/model-registry-DSCInitialization.yaml b/openshift-ci/resources/model-registry-DSCInitialization.yaml new file mode 100644 index 0000000000..cd9deaaca1 --- /dev/null +++ b/openshift-ci/resources/model-registry-DSCInitialization.yaml @@ -0,0 +1,33 @@ +apiVersion: dscinitialization.opendatahub.io/v1 +kind: DSCInitialization +metadata: + creationTimestamp: '2024-03-27T16:16:55Z' + finalizers: + - dscinitialization.opendatahub.io/finalizer + generation: 1 + labels: + app.kubernetes.io/created-by: opendatahub-operator + app.kubernetes.io/instance: default + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: dscinitialization + app.kubernetes.io/part-of: opendatahub-operator + name: default-dsci +spec: + applicationsNamespace: opendatahub + devFlags: + logmode: production + monitoring: + managementState: Managed + namespace: opendatahub + serviceMesh: + auth: + audiences: + - https://kubernetes.default.svc + controlPlane: + metricsCollection: Istio + name: data-science-smcp + namespace: istio-system + managementState: Managed + trustedCABundle: + customCABundle: "" + managementState: Managed \ No newline at end of file diff --git a/openshift-ci/resources/model-registry-operator/modelregistry_v1alpha1_modelregistry.yaml b/openshift-ci/resources/model-registry-operator/modelregistry_v1alpha1_modelregistry.yaml new file mode 100644 index 0000000000..b77ecfdd17 --- /dev/null +++ b/openshift-ci/resources/model-registry-operator/modelregistry_v1alpha1_modelregistry.yaml @@ -0,0 +1,24 @@ +apiVersion: modelregistry.opendatahub.io/v1alpha1 +kind: ModelRegistry +metadata: + labels: + app.kubernetes.io/name: modelregistry + app.kubernetes.io/instance: modelregistry-sample + app.kubernetes.io/part-of: model-registry-operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: model-registry-operator + name: modelregistry-sample +spec: + # TODO(user): Add fields here + grpc: + port: 9090 + rest: + port: 8080 + serviceRoute: enabled + mysql: + host: model-registry-db + database: model_registry + username: mlmduser + passwordSecret: + name: model-registry-db + key: database-password diff --git a/openshift-ci/resources/model-registry-operator/mysql-db.yaml b/openshift-ci/resources/model-registry-operator/mysql-db.yaml new file mode 100644 index 0000000000..a1a741f6e3 --- /dev/null +++ b/openshift-ci/resources/model-registry-operator/mysql-db.yaml @@ -0,0 +1,126 @@ +apiVersion: v1 +items: +- apiVersion: v1 + kind: Service + metadata: + annotations: + template.openshift.io/expose-uri: mysql://{.spec.clusterIP}:{.spec.ports[?(.name==\mysql\)].port} + name: model-registry-db + spec: + ports: + - name: mysql + nodePort: 0 + port: 3306 + protocol: TCP + targetPort: 3306 + selector: + name: model-registry-db + sessionAffinity: None + type: ClusterIP +- apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: model-registry-db + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi +- apiVersion: apps/v1 + kind: Deployment + metadata: + annotations: + template.alpha.openshift.io/wait-for-ready: "true" + name: model-registry-db + spec: + replicas: 1 + revisionHistoryLimit: 0 + selector: + matchLabels: + name: model-registry-db + strategy: + type: Recreate + template: + metadata: + annotations: + sidecar.istio.io/inject: "false" + labels: + name: model-registry-db + spec: + containers: + - env: + - name: MYSQL_USER + valueFrom: + secretKeyRef: + key: database-user + name: model-registry-db + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + key: database-password + name: model-registry-db + - name: MYSQL_ROOT_PASSWORD + valueFrom: + secretKeyRef: + key: database-password + name: model-registry-db + - name: MYSQL_DATABASE + valueFrom: + secretKeyRef: + key: database-name + name: model-registry-db + args: + - --datadir + - /var/lib/mysql/datadir + - --default-authentication-plugin=mysql_native_password + image: mysql:8.3.0 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - /bin/bash + - -c + - mysqladmin -u${MYSQL_USER} -p${MYSQL_ROOT_PASSWORD} ping + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 5 + name: mysql + ports: + - containerPort: 3306 + protocol: TCP + readinessProbe: + exec: + command: + - /bin/bash + - -c + - mysql -D ${MYSQL_DATABASE} -u${MYSQL_USER} -p${MYSQL_ROOT_PASSWORD} -e 'SELECT 1' + initialDelaySeconds: 10 + timeoutSeconds: 5 + securityContext: + capabilities: {} + privileged: false + terminationMessagePath: /dev/termination-log + volumeMounts: + - mountPath: /var/lib/mysql + name: model-registry-db-data + dnsPolicy: ClusterFirst + restartPolicy: Always + volumes: + - name: model-registry-db-data + persistentVolumeClaim: + claimName: model-registry-db +- apiVersion: v1 + kind: Secret + metadata: + annotations: + template.openshift.io/expose-database_name: '{.data[''database-name'']}' + template.openshift.io/expose-password: '{.data[''database-password'']}' + template.openshift.io/expose-username: '{.data[''database-user'']}' + name: model-registry-db + stringData: + database-name: "model_registry" + database-password: "TheBlurstOfTimes" # notsecret + database-user: "mlmduser" # notsecret +kind: List +metadata: {} diff --git a/openshift-ci/resources/opendatahub-data-science-cluster.yaml b/openshift-ci/resources/opendatahub-data-science-cluster.yaml new file mode 100644 index 0000000000..b5d56c09e1 --- /dev/null +++ b/openshift-ci/resources/opendatahub-data-science-cluster.yaml @@ -0,0 +1,38 @@ +kind: DataScienceCluster +apiVersion: datasciencecluster.opendatahub.io/v1 +metadata: + labels: + app.kubernetes.io/created-by: opendatahub-operator + app.kubernetes.io/instance: default + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: datasciencecluster + app.kubernetes.io/part-of: opendatahub-operator + name: default-dsc +spec: + components: + codeflare: + managementState: Removed + dashboard: + managementState: Managed + datasciencepipelines: + managementState: Managed + kserve: + managementState: Managed + serving: + ingressGateway: + certificate: + type: SelfSigned + managementState: Managed + name: knative-serving + kueue: + managementState: Removed + modelmeshserving: + managementState: Managed + modelregistry: + managementState: Managed + ray: + managementState: Removed + trustyai: + managementState: Removed + workbenches: + managementState: Managed diff --git a/openshift-ci/resources/opendatahub-subscription.yaml b/openshift-ci/resources/opendatahub-subscription.yaml new file mode 100644 index 0000000000..86dbe37de6 --- /dev/null +++ b/openshift-ci/resources/opendatahub-subscription.yaml @@ -0,0 +1,11 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: model-registry-test-source-subscription + namespace: openshift-operators +spec: + channel: fast + name: opendatahub-operator + source: community-operators + sourceNamespace: openshift-marketplace + \ No newline at end of file diff --git a/openshift-ci/resources/samples/authorino-subscription.yaml b/openshift-ci/resources/samples/authorino-subscription.yaml new file mode 100644 index 0000000000..b67e2b59ae --- /dev/null +++ b/openshift-ci/resources/samples/authorino-subscription.yaml @@ -0,0 +1,10 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: authorino-operator + namespace: openshift-operators +spec: + channel: stable + name: authorino-operator + source: community-operators + sourceNamespace: openshift-marketplace \ No newline at end of file diff --git a/openshift-ci/resources/samples/istio/components/authconfig-labels.yaml b/openshift-ci/resources/samples/istio/components/authconfig-labels.yaml new file mode 100644 index 0000000000..8fea57de4f --- /dev/null +++ b/openshift-ci/resources/samples/istio/components/authconfig-labels.yaml @@ -0,0 +1,4 @@ +- op: replace + path: /spec/istio/authConfigLabels + value: + security.opendatahub.io/authorization-group: default diff --git a/openshift-ci/resources/samples/istio/components/istio.env b/openshift-ci/resources/samples/istio/components/istio.env new file mode 100644 index 0000000000..f4d118429a --- /dev/null +++ b/openshift-ci/resources/samples/istio/components/istio.env @@ -0,0 +1,5 @@ +AUTH_PROVIDER=opendatahub-auth-provider +ISTIO_INGRESS=ingressgateway +DOMAIN= +REST_CREDENTIAL_NAME=modelregistry-sample-rest-credential +GRPC_CREDENTIAL_NAME=modelregistry-sample-grpc-credential diff --git a/openshift-ci/resources/samples/istio/components/istio_modelregistry.yaml b/openshift-ci/resources/samples/istio/components/istio_modelregistry.yaml new file mode 100644 index 0000000000..9ce5493fed --- /dev/null +++ b/openshift-ci/resources/samples/istio/components/istio_modelregistry.yaml @@ -0,0 +1,15 @@ +apiVersion: modelregistry.opendatahub.io/v1alpha1 +kind: ModelRegistry +metadata: + name: modelregistry-sample +spec: + istio: + authProvider: AUTH_PROVIDER + authConfigLabels: {} + gateway: + domain: DOMAIN + istioIngress: ISTIO_INGRESS + rest: + gatewayRoute: enabled + grpc: + gatewayRoute: enabled diff --git a/openshift-ci/resources/samples/istio/components/kustomization.yaml b/openshift-ci/resources/samples/istio/components/kustomization.yaml new file mode 100644 index 0000000000..0480731d9a --- /dev/null +++ b/openshift-ci/resources/samples/istio/components/kustomization.yaml @@ -0,0 +1,26 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +# Istio config patch +patches: + - path: istio_modelregistry.yaml + - target: + group: modelregistry.opendatahub.io + version: v1alpha1 + kind: ModelRegistry + name: modelregistry-sample + path: authconfig-labels.yaml + +# Config map and replacements to use istio.env for cluster specific config +configMapGenerator: + - envs: + - istio.env + files: + - authconfig-labels.yaml + name: modelregistry-sample-environment +generatorOptions: + disableNameSuffixHash: true +replacements: + - path: replacements.yaml + +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/openshift-ci/resources/samples/istio/components/replacements.yaml b/openshift-ci/resources/samples/istio/components/replacements.yaml new file mode 100644 index 0000000000..6afad827f2 --- /dev/null +++ b/openshift-ci/resources/samples/istio/components/replacements.yaml @@ -0,0 +1,32 @@ +# replacements from configmap using istio.env for cluster specific config +- source: + kind: ConfigMap + name: modelregistry-sample-environment + fieldPath: data.AUTH_PROVIDER + targets: + - select: + apiVersion: modelregistry.opendatahub.io/v1alpha1 + kind: ModelRegistry + name: modelregistry-sample + fieldPaths: + - spec.istio.authProvider +- source: + kind: ConfigMap + name: modelregistry-sample-environment + fieldPath: data.DOMAIN + targets: + - select: + kind: ModelRegistry + name: modelregistry-sample + fieldPaths: + - spec.istio.gateway.domain +- source: + kind: ConfigMap + name: modelregistry-sample-environment + fieldPath: data.ISTIO_INGRESS + targets: + - select: + kind: ModelRegistry + name: modelregistry-sample + fieldPaths: + - spec.istio.gateway.istioIngress diff --git a/openshift-ci/resources/samples/istio/components/tls/istio_tls_modelregistry.yaml b/openshift-ci/resources/samples/istio/components/tls/istio_tls_modelregistry.yaml new file mode 100644 index 0000000000..e1f80fb3af --- /dev/null +++ b/openshift-ci/resources/samples/istio/components/tls/istio_tls_modelregistry.yaml @@ -0,0 +1,13 @@ +apiVersion: modelregistry.opendatahub.io/v1alpha1 +kind: ModelRegistry +metadata: + name: modelregistry-sample +spec: + istio: + gateway: + rest: + tls: + credentialName: REST_CREDENTIAL_NAME + grpc: + tls: + credentialName: GRPC_CREDENTIAL_NAME diff --git a/openshift-ci/resources/samples/istio/components/tls/kustomization.yaml b/openshift-ci/resources/samples/istio/components/tls/kustomization.yaml new file mode 100644 index 0000000000..ff53383d5d --- /dev/null +++ b/openshift-ci/resources/samples/istio/components/tls/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +## Append samples of your project ## + +# Istio tls config patch +patches: + - path: istio_tls_modelregistry.yaml + +replacements: + - path: tls-replacements.yaml + +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/openshift-ci/resources/samples/istio/components/tls/tls-replacements.yaml b/openshift-ci/resources/samples/istio/components/tls/tls-replacements.yaml new file mode 100644 index 0000000000..12f490e691 --- /dev/null +++ b/openshift-ci/resources/samples/istio/components/tls/tls-replacements.yaml @@ -0,0 +1,21 @@ +# replacements from configmap using istio.env for cluster specific config +- source: + kind: ConfigMap + name: modelregistry-sample-environment + fieldPath: data.REST_CREDENTIAL_NAME + targets: + - select: + kind: ModelRegistry + name: modelregistry-sample + fieldPaths: + - spec.istio.gateway.rest.tls.credentialName +- source: + kind: ConfigMap + name: modelregistry-sample-environment + fieldPath: data.GRPC_CREDENTIAL_NAME + targets: + - select: + kind: ModelRegistry + name: modelregistry-sample + fieldPaths: + - spec.istio.gateway.grpc.tls.credentialName diff --git a/openshift-ci/resources/samples/istio/mysql-tls/kustomization.yaml b/openshift-ci/resources/samples/istio/mysql-tls/kustomization.yaml new file mode 100644 index 0000000000..1536bd9f0e --- /dev/null +++ b/openshift-ci/resources/samples/istio/mysql-tls/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +## Append samples of your project ## +resources: + - ../mysql + +components: + - ../components/tls + +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/openshift-ci/resources/samples/istio/mysql/kustomization.yaml b/openshift-ci/resources/samples/istio/mysql/kustomization.yaml new file mode 100644 index 0000000000..eaa1000d30 --- /dev/null +++ b/openshift-ci/resources/samples/istio/mysql/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +## Append samples of your project ## +resources: +- ../../mysql + +components: +- ../components + +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/openshift-ci/resources/samples/istio/postgres-tls/kustomization.yaml b/openshift-ci/resources/samples/istio/postgres-tls/kustomization.yaml new file mode 100644 index 0000000000..4f97e9c2fc --- /dev/null +++ b/openshift-ci/resources/samples/istio/postgres-tls/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +## Append samples of your project ## +resources: + - ../postgres + +components: + - ../components/tls + +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/openshift-ci/resources/samples/istio/postgres/kustomization.yaml b/openshift-ci/resources/samples/istio/postgres/kustomization.yaml new file mode 100644 index 0000000000..8f773cc925 --- /dev/null +++ b/openshift-ci/resources/samples/istio/postgres/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +## Append samples of your project ## +resources: + - ../../postgres + +components: + - ../components + +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/openshift-ci/resources/samples/mysql/kustomization.yaml b/openshift-ci/resources/samples/mysql/kustomization.yaml new file mode 100644 index 0000000000..a988c2e8f8 --- /dev/null +++ b/openshift-ci/resources/samples/mysql/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +## Append samples of your project ## +resources: +- mysql-db.yaml +- modelregistry_v1alpha1_modelregistry.yaml +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/openshift-ci/resources/samples/mysql/modelregistry_v1alpha1_modelregistry.yaml b/openshift-ci/resources/samples/mysql/modelregistry_v1alpha1_modelregistry.yaml new file mode 100644 index 0000000000..81d26e5c77 --- /dev/null +++ b/openshift-ci/resources/samples/mysql/modelregistry_v1alpha1_modelregistry.yaml @@ -0,0 +1,24 @@ +apiVersion: modelregistry.opendatahub.io/v1alpha1 +kind: ModelRegistry +metadata: + labels: + app.kubernetes.io/name: modelregistry + app.kubernetes.io/instance: modelregistry-sample + app.kubernetes.io/part-of: model-registry-operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: model-registry-operator + name: modelregistry-sample +spec: + # TODO(user): Add fields here + grpc: + port: 9090 + rest: + port: 8080 + serviceRoute: disabled + mysql: + host: model-registry-db + database: model_registry + username: mlmduser + passwordSecret: + name: model-registry-db + key: database-password diff --git a/openshift-ci/resources/samples/mysql/mysql-db.yaml b/openshift-ci/resources/samples/mysql/mysql-db.yaml new file mode 100644 index 0000000000..14296dac50 --- /dev/null +++ b/openshift-ci/resources/samples/mysql/mysql-db.yaml @@ -0,0 +1,126 @@ +apiVersion: v1 +items: +- apiVersion: v1 + kind: Service + metadata: + annotations: + template.openshift.io/expose-uri: mysql://{.spec.clusterIP}:{.spec.ports[?(.name==\mysql\)].port} + name: model-registry-db + spec: + ports: + - name: mysql + nodePort: 0 + port: 3306 + protocol: TCP + appProtocol: tcp + targetPort: 3306 + selector: + name: model-registry-db + sessionAffinity: None + type: ClusterIP +- apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: model-registry-db + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi +- apiVersion: apps/v1 + kind: Deployment + metadata: + annotations: + template.alpha.openshift.io/wait-for-ready: "true" + name: model-registry-db + spec: + replicas: 1 + revisionHistoryLimit: 0 + selector: + matchLabels: + name: model-registry-db + strategy: + type: Recreate + template: + metadata: + labels: + name: model-registry-db + sidecar.istio.io/inject: "false" + spec: + containers: + - env: + - name: MYSQL_USER + valueFrom: + secretKeyRef: + key: database-user + name: model-registry-db + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + key: database-password + name: model-registry-db + - name: MYSQL_ROOT_PASSWORD + valueFrom: + secretKeyRef: + key: database-password + name: model-registry-db + - name: MYSQL_DATABASE + valueFrom: + secretKeyRef: + key: database-name + name: model-registry-db + args: + - --datadir + - /var/lib/mysql/datadir + - --default-authentication-plugin=mysql_native_password + image: mysql:8.3.0 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - /bin/bash + - -c + - mysqladmin -u${MYSQL_USER} -p${MYSQL_ROOT_PASSWORD} ping + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 5 + name: mysql + ports: + - containerPort: 3306 + protocol: TCP + readinessProbe: + exec: + command: + - /bin/bash + - -c + - mysql -D ${MYSQL_DATABASE} -u${MYSQL_USER} -p${MYSQL_ROOT_PASSWORD} -e 'SELECT 1' + initialDelaySeconds: 10 + timeoutSeconds: 5 + securityContext: + capabilities: {} + privileged: false + terminationMessagePath: /dev/termination-log + volumeMounts: + - mountPath: /var/lib/mysql + name: model-registry-db-data + dnsPolicy: ClusterFirst + restartPolicy: Always + volumes: + - name: model-registry-db-data + persistentVolumeClaim: + claimName: model-registry-db +- apiVersion: v1 + kind: Secret + metadata: + annotations: + template.openshift.io/expose-database_name: '{.data[''database-name'']}' + template.openshift.io/expose-password: '{.data[''database-password'']}' + template.openshift.io/expose-username: '{.data[''database-user'']}' + name: model-registry-db + stringData: + database-name: "model_registry" + database-password: "TheBlurstOfTimes" # notsecret + database-user: "mlmduser" # notsecret +kind: List +metadata: {} diff --git a/openshift-ci/resources/samples/postgres/kustomization.yaml b/openshift-ci/resources/samples/postgres/kustomization.yaml new file mode 100644 index 0000000000..557a9df75a --- /dev/null +++ b/openshift-ci/resources/samples/postgres/kustomization.yaml @@ -0,0 +1,5 @@ +## Append samples of your project ## +resources: +- postgres-db.yaml +- modelregistry_v1alpha1_modelregistry.yaml +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/openshift-ci/resources/samples/postgres/modelregistry_v1alpha1_modelregistry.yaml b/openshift-ci/resources/samples/postgres/modelregistry_v1alpha1_modelregistry.yaml new file mode 100644 index 0000000000..4a89f54510 --- /dev/null +++ b/openshift-ci/resources/samples/postgres/modelregistry_v1alpha1_modelregistry.yaml @@ -0,0 +1,24 @@ +apiVersion: modelregistry.opendatahub.io/v1alpha1 +kind: ModelRegistry +metadata: + labels: + app.kubernetes.io/name: modelregistry + app.kubernetes.io/instance: modelregistry-sample + app.kubernetes.io/part-of: model-registry-operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: model-registry-operator + name: modelregistry-sample +spec: + # TODO(user): Add fields here + grpc: + port: 9090 + rest: + port: 8080 + serviceRoute: disabled + postgres: + host: model-registry-db + database: model-registry + username: mlmduser + passwordSecret: + name: model-registry-db + key: database-password diff --git a/openshift-ci/resources/samples/postgres/postgres-db.yaml b/openshift-ci/resources/samples/postgres/postgres-db.yaml new file mode 100644 index 0000000000..7877bdd2b7 --- /dev/null +++ b/openshift-ci/resources/samples/postgres/postgres-db.yaml @@ -0,0 +1,118 @@ +apiVersion: v1 +items: +- apiVersion: v1 + kind: Service + metadata: + annotations: + template.openshift.io/expose-uri: postgres://{.spec.clusterIP}:{.spec.ports[?(.name==\postgresql\)].port} + name: model-registry-db + spec: + ports: + - name: postgresql + nodePort: 0 + port: 5432 + protocol: TCP + appProtocol: tcp + targetPort: 5432 + selector: + name: model-registry-db + sessionAffinity: None + type: ClusterIP +- apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: model-registry-db + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi +- apiVersion: apps/v1 + kind: Deployment + metadata: + annotations: + template.alpha.openshift.io/wait-for-ready: "true" + name: model-registry-db + spec: + replicas: 1 + revisionHistoryLimit: 0 + selector: + matchLabels: + name: model-registry-db + strategy: + type: Recreate + template: + metadata: + labels: + name: model-registry-db + sidecar.istio.io/inject: "false" + spec: + containers: + - env: + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + key: database-user + name: model-registry-db + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + key: database-password + name: model-registry-db + - name: POSTGRES_DB + valueFrom: + secretKeyRef: + key: database-name + name: model-registry-db + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + image: postgres:16 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - bash + - "-c" + - "/usr/bin/pg_isready -U $POSTGRES_USER -d $POSTGRES_DB" + initialDelaySeconds: 30 + timeoutSeconds: 2 + name: postgresql + ports: + - containerPort: 5432 + protocol: TCP + readinessProbe: + exec: + command: + - bash + - "-c" + - "psql -w -U $POSTGRES_USER -d $POSTGRES_DB -c 'SELECT 1'" + initialDelaySeconds: 10 + timeoutSeconds: 5 + securityContext: + capabilities: {} + privileged: false + terminationMessagePath: /dev/termination-log + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: model-registry-db-data + dnsPolicy: ClusterFirst + restartPolicy: Always + volumes: + - name: model-registry-db-data + persistentVolumeClaim: + claimName: model-registry-db +- apiVersion: v1 + kind: Secret + metadata: + annotations: + template.openshift.io/expose-database_name: '{.data[''database-name'']}' + template.openshift.io/expose-password: '{.data[''database-password'']}' + template.openshift.io/expose-username: '{.data[''database-user'']}' + name: model-registry-db + stringData: + database-name: "model-registry" + database-password: "TheBlurstOfTimes" # notsecret + database-user: "mlmduser" # notsecret +kind: List +metadata: {} diff --git a/openshift-ci/resources/samples/samples/istio/components/authconfig-labels.yaml b/openshift-ci/resources/samples/samples/istio/components/authconfig-labels.yaml new file mode 100644 index 0000000000..8fea57de4f --- /dev/null +++ b/openshift-ci/resources/samples/samples/istio/components/authconfig-labels.yaml @@ -0,0 +1,4 @@ +- op: replace + path: /spec/istio/authConfigLabels + value: + security.opendatahub.io/authorization-group: default diff --git a/openshift-ci/resources/samples/samples/istio/components/istio.env b/openshift-ci/resources/samples/samples/istio/components/istio.env new file mode 100644 index 0000000000..6008f81b82 --- /dev/null +++ b/openshift-ci/resources/samples/samples/istio/components/istio.env @@ -0,0 +1,5 @@ +AUTH_PROVIDER=opendatahub-auth-provider +DOMAIN=my-domain +ISTIO_INGRESS=ingressgateway +REST_CREDENTIAL_NAME=modelregistry-sample-rest-credential +GRPC_CREDENTIAL_NAME=modelregistry-sample-grpc-credential diff --git a/openshift-ci/resources/samples/samples/istio/components/istio_modelregistry.yaml b/openshift-ci/resources/samples/samples/istio/components/istio_modelregistry.yaml new file mode 100644 index 0000000000..9ce5493fed --- /dev/null +++ b/openshift-ci/resources/samples/samples/istio/components/istio_modelregistry.yaml @@ -0,0 +1,15 @@ +apiVersion: modelregistry.opendatahub.io/v1alpha1 +kind: ModelRegistry +metadata: + name: modelregistry-sample +spec: + istio: + authProvider: AUTH_PROVIDER + authConfigLabels: {} + gateway: + domain: DOMAIN + istioIngress: ISTIO_INGRESS + rest: + gatewayRoute: enabled + grpc: + gatewayRoute: enabled diff --git a/openshift-ci/resources/samples/samples/istio/components/kustomization.yaml b/openshift-ci/resources/samples/samples/istio/components/kustomization.yaml new file mode 100644 index 0000000000..0480731d9a --- /dev/null +++ b/openshift-ci/resources/samples/samples/istio/components/kustomization.yaml @@ -0,0 +1,26 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +# Istio config patch +patches: + - path: istio_modelregistry.yaml + - target: + group: modelregistry.opendatahub.io + version: v1alpha1 + kind: ModelRegistry + name: modelregistry-sample + path: authconfig-labels.yaml + +# Config map and replacements to use istio.env for cluster specific config +configMapGenerator: + - envs: + - istio.env + files: + - authconfig-labels.yaml + name: modelregistry-sample-environment +generatorOptions: + disableNameSuffixHash: true +replacements: + - path: replacements.yaml + +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/openshift-ci/resources/samples/samples/istio/components/replacements.yaml b/openshift-ci/resources/samples/samples/istio/components/replacements.yaml new file mode 100644 index 0000000000..6afad827f2 --- /dev/null +++ b/openshift-ci/resources/samples/samples/istio/components/replacements.yaml @@ -0,0 +1,32 @@ +# replacements from configmap using istio.env for cluster specific config +- source: + kind: ConfigMap + name: modelregistry-sample-environment + fieldPath: data.AUTH_PROVIDER + targets: + - select: + apiVersion: modelregistry.opendatahub.io/v1alpha1 + kind: ModelRegistry + name: modelregistry-sample + fieldPaths: + - spec.istio.authProvider +- source: + kind: ConfigMap + name: modelregistry-sample-environment + fieldPath: data.DOMAIN + targets: + - select: + kind: ModelRegistry + name: modelregistry-sample + fieldPaths: + - spec.istio.gateway.domain +- source: + kind: ConfigMap + name: modelregistry-sample-environment + fieldPath: data.ISTIO_INGRESS + targets: + - select: + kind: ModelRegistry + name: modelregistry-sample + fieldPaths: + - spec.istio.gateway.istioIngress diff --git a/openshift-ci/resources/samples/samples/istio/components/tls/istio_tls_modelregistry.yaml b/openshift-ci/resources/samples/samples/istio/components/tls/istio_tls_modelregistry.yaml new file mode 100644 index 0000000000..e1f80fb3af --- /dev/null +++ b/openshift-ci/resources/samples/samples/istio/components/tls/istio_tls_modelregistry.yaml @@ -0,0 +1,13 @@ +apiVersion: modelregistry.opendatahub.io/v1alpha1 +kind: ModelRegistry +metadata: + name: modelregistry-sample +spec: + istio: + gateway: + rest: + tls: + credentialName: REST_CREDENTIAL_NAME + grpc: + tls: + credentialName: GRPC_CREDENTIAL_NAME diff --git a/openshift-ci/resources/samples/samples/istio/components/tls/kustomization.yaml b/openshift-ci/resources/samples/samples/istio/components/tls/kustomization.yaml new file mode 100644 index 0000000000..ff53383d5d --- /dev/null +++ b/openshift-ci/resources/samples/samples/istio/components/tls/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +## Append samples of your project ## + +# Istio tls config patch +patches: + - path: istio_tls_modelregistry.yaml + +replacements: + - path: tls-replacements.yaml + +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/openshift-ci/resources/samples/samples/istio/components/tls/tls-replacements.yaml b/openshift-ci/resources/samples/samples/istio/components/tls/tls-replacements.yaml new file mode 100644 index 0000000000..12f490e691 --- /dev/null +++ b/openshift-ci/resources/samples/samples/istio/components/tls/tls-replacements.yaml @@ -0,0 +1,21 @@ +# replacements from configmap using istio.env for cluster specific config +- source: + kind: ConfigMap + name: modelregistry-sample-environment + fieldPath: data.REST_CREDENTIAL_NAME + targets: + - select: + kind: ModelRegistry + name: modelregistry-sample + fieldPaths: + - spec.istio.gateway.rest.tls.credentialName +- source: + kind: ConfigMap + name: modelregistry-sample-environment + fieldPath: data.GRPC_CREDENTIAL_NAME + targets: + - select: + kind: ModelRegistry + name: modelregistry-sample + fieldPaths: + - spec.istio.gateway.grpc.tls.credentialName diff --git a/openshift-ci/resources/samples/samples/istio/mysql-tls/kustomization.yaml b/openshift-ci/resources/samples/samples/istio/mysql-tls/kustomization.yaml new file mode 100644 index 0000000000..1536bd9f0e --- /dev/null +++ b/openshift-ci/resources/samples/samples/istio/mysql-tls/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +## Append samples of your project ## +resources: + - ../mysql + +components: + - ../components/tls + +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/openshift-ci/resources/samples/samples/istio/mysql/kustomization.yaml b/openshift-ci/resources/samples/samples/istio/mysql/kustomization.yaml new file mode 100644 index 0000000000..eaa1000d30 --- /dev/null +++ b/openshift-ci/resources/samples/samples/istio/mysql/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +## Append samples of your project ## +resources: +- ../../mysql + +components: +- ../components + +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/openshift-ci/resources/samples/samples/istio/postgres-tls/kustomization.yaml b/openshift-ci/resources/samples/samples/istio/postgres-tls/kustomization.yaml new file mode 100644 index 0000000000..4f97e9c2fc --- /dev/null +++ b/openshift-ci/resources/samples/samples/istio/postgres-tls/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +## Append samples of your project ## +resources: + - ../postgres + +components: + - ../components/tls + +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/openshift-ci/resources/samples/samples/istio/postgres/kustomization.yaml b/openshift-ci/resources/samples/samples/istio/postgres/kustomization.yaml new file mode 100644 index 0000000000..8f773cc925 --- /dev/null +++ b/openshift-ci/resources/samples/samples/istio/postgres/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +## Append samples of your project ## +resources: + - ../../postgres + +components: + - ../components + +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/openshift-ci/resources/samples/samples/mysql/kustomization.yaml b/openshift-ci/resources/samples/samples/mysql/kustomization.yaml new file mode 100644 index 0000000000..a988c2e8f8 --- /dev/null +++ b/openshift-ci/resources/samples/samples/mysql/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +## Append samples of your project ## +resources: +- mysql-db.yaml +- modelregistry_v1alpha1_modelregistry.yaml +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/openshift-ci/resources/samples/samples/mysql/modelregistry_v1alpha1_modelregistry.yaml b/openshift-ci/resources/samples/samples/mysql/modelregistry_v1alpha1_modelregistry.yaml new file mode 100644 index 0000000000..81d26e5c77 --- /dev/null +++ b/openshift-ci/resources/samples/samples/mysql/modelregistry_v1alpha1_modelregistry.yaml @@ -0,0 +1,24 @@ +apiVersion: modelregistry.opendatahub.io/v1alpha1 +kind: ModelRegistry +metadata: + labels: + app.kubernetes.io/name: modelregistry + app.kubernetes.io/instance: modelregistry-sample + app.kubernetes.io/part-of: model-registry-operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: model-registry-operator + name: modelregistry-sample +spec: + # TODO(user): Add fields here + grpc: + port: 9090 + rest: + port: 8080 + serviceRoute: disabled + mysql: + host: model-registry-db + database: model_registry + username: mlmduser + passwordSecret: + name: model-registry-db + key: database-password diff --git a/openshift-ci/resources/samples/samples/mysql/mysql-db.yaml b/openshift-ci/resources/samples/samples/mysql/mysql-db.yaml new file mode 100644 index 0000000000..14296dac50 --- /dev/null +++ b/openshift-ci/resources/samples/samples/mysql/mysql-db.yaml @@ -0,0 +1,126 @@ +apiVersion: v1 +items: +- apiVersion: v1 + kind: Service + metadata: + annotations: + template.openshift.io/expose-uri: mysql://{.spec.clusterIP}:{.spec.ports[?(.name==\mysql\)].port} + name: model-registry-db + spec: + ports: + - name: mysql + nodePort: 0 + port: 3306 + protocol: TCP + appProtocol: tcp + targetPort: 3306 + selector: + name: model-registry-db + sessionAffinity: None + type: ClusterIP +- apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: model-registry-db + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi +- apiVersion: apps/v1 + kind: Deployment + metadata: + annotations: + template.alpha.openshift.io/wait-for-ready: "true" + name: model-registry-db + spec: + replicas: 1 + revisionHistoryLimit: 0 + selector: + matchLabels: + name: model-registry-db + strategy: + type: Recreate + template: + metadata: + labels: + name: model-registry-db + sidecar.istio.io/inject: "false" + spec: + containers: + - env: + - name: MYSQL_USER + valueFrom: + secretKeyRef: + key: database-user + name: model-registry-db + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + key: database-password + name: model-registry-db + - name: MYSQL_ROOT_PASSWORD + valueFrom: + secretKeyRef: + key: database-password + name: model-registry-db + - name: MYSQL_DATABASE + valueFrom: + secretKeyRef: + key: database-name + name: model-registry-db + args: + - --datadir + - /var/lib/mysql/datadir + - --default-authentication-plugin=mysql_native_password + image: mysql:8.3.0 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - /bin/bash + - -c + - mysqladmin -u${MYSQL_USER} -p${MYSQL_ROOT_PASSWORD} ping + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 5 + name: mysql + ports: + - containerPort: 3306 + protocol: TCP + readinessProbe: + exec: + command: + - /bin/bash + - -c + - mysql -D ${MYSQL_DATABASE} -u${MYSQL_USER} -p${MYSQL_ROOT_PASSWORD} -e 'SELECT 1' + initialDelaySeconds: 10 + timeoutSeconds: 5 + securityContext: + capabilities: {} + privileged: false + terminationMessagePath: /dev/termination-log + volumeMounts: + - mountPath: /var/lib/mysql + name: model-registry-db-data + dnsPolicy: ClusterFirst + restartPolicy: Always + volumes: + - name: model-registry-db-data + persistentVolumeClaim: + claimName: model-registry-db +- apiVersion: v1 + kind: Secret + metadata: + annotations: + template.openshift.io/expose-database_name: '{.data[''database-name'']}' + template.openshift.io/expose-password: '{.data[''database-password'']}' + template.openshift.io/expose-username: '{.data[''database-user'']}' + name: model-registry-db + stringData: + database-name: "model_registry" + database-password: "TheBlurstOfTimes" # notsecret + database-user: "mlmduser" # notsecret +kind: List +metadata: {} diff --git a/openshift-ci/resources/samples/samples/postgres/kustomization.yaml b/openshift-ci/resources/samples/samples/postgres/kustomization.yaml new file mode 100644 index 0000000000..557a9df75a --- /dev/null +++ b/openshift-ci/resources/samples/samples/postgres/kustomization.yaml @@ -0,0 +1,5 @@ +## Append samples of your project ## +resources: +- postgres-db.yaml +- modelregistry_v1alpha1_modelregistry.yaml +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/openshift-ci/resources/samples/samples/postgres/modelregistry_v1alpha1_modelregistry.yaml b/openshift-ci/resources/samples/samples/postgres/modelregistry_v1alpha1_modelregistry.yaml new file mode 100644 index 0000000000..4a89f54510 --- /dev/null +++ b/openshift-ci/resources/samples/samples/postgres/modelregistry_v1alpha1_modelregistry.yaml @@ -0,0 +1,24 @@ +apiVersion: modelregistry.opendatahub.io/v1alpha1 +kind: ModelRegistry +metadata: + labels: + app.kubernetes.io/name: modelregistry + app.kubernetes.io/instance: modelregistry-sample + app.kubernetes.io/part-of: model-registry-operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: model-registry-operator + name: modelregistry-sample +spec: + # TODO(user): Add fields here + grpc: + port: 9090 + rest: + port: 8080 + serviceRoute: disabled + postgres: + host: model-registry-db + database: model-registry + username: mlmduser + passwordSecret: + name: model-registry-db + key: database-password diff --git a/openshift-ci/resources/samples/samples/postgres/postgres-db.yaml b/openshift-ci/resources/samples/samples/postgres/postgres-db.yaml new file mode 100644 index 0000000000..7877bdd2b7 --- /dev/null +++ b/openshift-ci/resources/samples/samples/postgres/postgres-db.yaml @@ -0,0 +1,118 @@ +apiVersion: v1 +items: +- apiVersion: v1 + kind: Service + metadata: + annotations: + template.openshift.io/expose-uri: postgres://{.spec.clusterIP}:{.spec.ports[?(.name==\postgresql\)].port} + name: model-registry-db + spec: + ports: + - name: postgresql + nodePort: 0 + port: 5432 + protocol: TCP + appProtocol: tcp + targetPort: 5432 + selector: + name: model-registry-db + sessionAffinity: None + type: ClusterIP +- apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: model-registry-db + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi +- apiVersion: apps/v1 + kind: Deployment + metadata: + annotations: + template.alpha.openshift.io/wait-for-ready: "true" + name: model-registry-db + spec: + replicas: 1 + revisionHistoryLimit: 0 + selector: + matchLabels: + name: model-registry-db + strategy: + type: Recreate + template: + metadata: + labels: + name: model-registry-db + sidecar.istio.io/inject: "false" + spec: + containers: + - env: + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + key: database-user + name: model-registry-db + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + key: database-password + name: model-registry-db + - name: POSTGRES_DB + valueFrom: + secretKeyRef: + key: database-name + name: model-registry-db + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + image: postgres:16 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - bash + - "-c" + - "/usr/bin/pg_isready -U $POSTGRES_USER -d $POSTGRES_DB" + initialDelaySeconds: 30 + timeoutSeconds: 2 + name: postgresql + ports: + - containerPort: 5432 + protocol: TCP + readinessProbe: + exec: + command: + - bash + - "-c" + - "psql -w -U $POSTGRES_USER -d $POSTGRES_DB -c 'SELECT 1'" + initialDelaySeconds: 10 + timeoutSeconds: 5 + securityContext: + capabilities: {} + privileged: false + terminationMessagePath: /dev/termination-log + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: model-registry-db-data + dnsPolicy: ClusterFirst + restartPolicy: Always + volumes: + - name: model-registry-db-data + persistentVolumeClaim: + claimName: model-registry-db +- apiVersion: v1 + kind: Secret + metadata: + annotations: + template.openshift.io/expose-database_name: '{.data[''database-name'']}' + template.openshift.io/expose-password: '{.data[''database-password'']}' + template.openshift.io/expose-username: '{.data[''database-user'']}' + name: model-registry-db + stringData: + database-name: "model-registry" + database-password: "TheBlurstOfTimes" # notsecret + database-user: "mlmduser" # notsecret +kind: List +metadata: {} diff --git a/openshift-ci/resources/samples/samples/secure-db/components/kustomization.yaml b/openshift-ci/resources/samples/samples/secure-db/components/kustomization.yaml new file mode 100644 index 0000000000..b3a3f5dc73 --- /dev/null +++ b/openshift-ci/resources/samples/samples/secure-db/components/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +# Istio config patch +patches: + - target: + group: apps + version: v1 + kind: Deployment + name: model-registry-db + path: model-registry-db-secrets.yaml + +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/openshift-ci/resources/samples/samples/secure-db/components/model-registry-db-secrets.yaml b/openshift-ci/resources/samples/samples/secure-db/components/model-registry-db-secrets.yaml new file mode 100644 index 0000000000..7d54ac2287 --- /dev/null +++ b/openshift-ci/resources/samples/samples/secure-db/components/model-registry-db-secrets.yaml @@ -0,0 +1,15 @@ +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + # name must match the volume name below + name: server-cert + mountPath: /etc/server-cert + readOnly: true +- op: add + path: /spec/template/spec/volumes/- + value: + # The secret data is exposed to Containers in the Pod through a Volume. + name: server-cert + secret: + secretName: model-registry-db-credential + defaultMode: 0600 diff --git a/openshift-ci/resources/samples/samples/secure-db/components/mysql/kustomization.yaml b/openshift-ci/resources/samples/samples/secure-db/components/mysql/kustomization.yaml new file mode 100644 index 0000000000..130e84e87e --- /dev/null +++ b/openshift-ci/resources/samples/samples/secure-db/components/mysql/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +## Append samples of your project ## + +# MySQL tls args patch +patches: + - path: mysql-ssl-args.yaml + target: + group: apps + version: v1 + kind: Deployment + name: model-registry-db + - path: secure_mysql_modelregistry.yaml + +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/openshift-ci/resources/samples/samples/secure-db/components/mysql/mysql-ssl-args.yaml b/openshift-ci/resources/samples/samples/secure-db/components/mysql/mysql-ssl-args.yaml new file mode 100644 index 0000000000..62793f86e9 --- /dev/null +++ b/openshift-ci/resources/samples/samples/secure-db/components/mysql/mysql-ssl-args.yaml @@ -0,0 +1,16 @@ +- op: add + path: /spec/template/spec/containers/0/args/- + value: + --ssl-ca=/etc/server-cert/ca.crt +- op: add + path: /spec/template/spec/containers/0/args/- + value: + --ssl-cert=/etc/server-cert/tls.crt +- op: add + path: /spec/template/spec/containers/0/args/- + value: + --ssl-key=/etc/server-cert/tls.key +- op: add + path: /spec/template/spec/containers/0/args/- + value: + --require-secure-transport diff --git a/openshift-ci/resources/samples/samples/secure-db/components/mysql/secure_mysql_modelregistry.yaml b/openshift-ci/resources/samples/samples/secure-db/components/mysql/secure_mysql_modelregistry.yaml new file mode 100644 index 0000000000..9c273b74d3 --- /dev/null +++ b/openshift-ci/resources/samples/samples/secure-db/components/mysql/secure_mysql_modelregistry.yaml @@ -0,0 +1,9 @@ +apiVersion: modelregistry.opendatahub.io/v1alpha1 +kind: ModelRegistry +metadata: + name: modelregistry-sample +spec: + mysql: + sslRootCertificateSecret: + name: model-registry-db-credential + key: ca.crt diff --git a/openshift-ci/resources/samples/samples/secure-db/mysql-tls/kustomization.yaml b/openshift-ci/resources/samples/samples/secure-db/mysql-tls/kustomization.yaml new file mode 100644 index 0000000000..ce2c3b64da --- /dev/null +++ b/openshift-ci/resources/samples/samples/secure-db/mysql-tls/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +## Append samples of your project ## +resources: + - ../../istio/mysql-tls + +components: + - ../components + - ../components/mysql + +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/openshift-ci/resources/samples/samples/secure-db/mysql/kustomization.yaml b/openshift-ci/resources/samples/samples/secure-db/mysql/kustomization.yaml new file mode 100644 index 0000000000..9188f771d4 --- /dev/null +++ b/openshift-ci/resources/samples/samples/secure-db/mysql/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +## Append samples of your project ## +resources: + - ../../mysql + +components: + - ../components/ + - ../components/mysql + +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/openshift-ci/resources/samples/secure-db/components/kustomization.yaml b/openshift-ci/resources/samples/secure-db/components/kustomization.yaml new file mode 100644 index 0000000000..b3a3f5dc73 --- /dev/null +++ b/openshift-ci/resources/samples/secure-db/components/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +# Istio config patch +patches: + - target: + group: apps + version: v1 + kind: Deployment + name: model-registry-db + path: model-registry-db-secrets.yaml + +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/openshift-ci/resources/samples/secure-db/components/model-registry-db-secrets.yaml b/openshift-ci/resources/samples/secure-db/components/model-registry-db-secrets.yaml new file mode 100644 index 0000000000..7d54ac2287 --- /dev/null +++ b/openshift-ci/resources/samples/secure-db/components/model-registry-db-secrets.yaml @@ -0,0 +1,15 @@ +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + # name must match the volume name below + name: server-cert + mountPath: /etc/server-cert + readOnly: true +- op: add + path: /spec/template/spec/volumes/- + value: + # The secret data is exposed to Containers in the Pod through a Volume. + name: server-cert + secret: + secretName: model-registry-db-credential + defaultMode: 0600 diff --git a/openshift-ci/resources/samples/secure-db/components/mysql/kustomization.yaml b/openshift-ci/resources/samples/secure-db/components/mysql/kustomization.yaml new file mode 100644 index 0000000000..130e84e87e --- /dev/null +++ b/openshift-ci/resources/samples/secure-db/components/mysql/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +## Append samples of your project ## + +# MySQL tls args patch +patches: + - path: mysql-ssl-args.yaml + target: + group: apps + version: v1 + kind: Deployment + name: model-registry-db + - path: secure_mysql_modelregistry.yaml + +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/openshift-ci/resources/samples/secure-db/components/mysql/mysql-ssl-args.yaml b/openshift-ci/resources/samples/secure-db/components/mysql/mysql-ssl-args.yaml new file mode 100644 index 0000000000..62793f86e9 --- /dev/null +++ b/openshift-ci/resources/samples/secure-db/components/mysql/mysql-ssl-args.yaml @@ -0,0 +1,16 @@ +- op: add + path: /spec/template/spec/containers/0/args/- + value: + --ssl-ca=/etc/server-cert/ca.crt +- op: add + path: /spec/template/spec/containers/0/args/- + value: + --ssl-cert=/etc/server-cert/tls.crt +- op: add + path: /spec/template/spec/containers/0/args/- + value: + --ssl-key=/etc/server-cert/tls.key +- op: add + path: /spec/template/spec/containers/0/args/- + value: + --require-secure-transport diff --git a/openshift-ci/resources/samples/secure-db/components/mysql/secure_mysql_modelregistry.yaml b/openshift-ci/resources/samples/secure-db/components/mysql/secure_mysql_modelregistry.yaml new file mode 100644 index 0000000000..9c273b74d3 --- /dev/null +++ b/openshift-ci/resources/samples/secure-db/components/mysql/secure_mysql_modelregistry.yaml @@ -0,0 +1,9 @@ +apiVersion: modelregistry.opendatahub.io/v1alpha1 +kind: ModelRegistry +metadata: + name: modelregistry-sample +spec: + mysql: + sslRootCertificateSecret: + name: model-registry-db-credential + key: ca.crt diff --git a/openshift-ci/resources/samples/secure-db/mysql-tls/kustomization.yaml b/openshift-ci/resources/samples/secure-db/mysql-tls/kustomization.yaml new file mode 100644 index 0000000000..ce2c3b64da --- /dev/null +++ b/openshift-ci/resources/samples/secure-db/mysql-tls/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +## Append samples of your project ## +resources: + - ../../istio/mysql-tls + +components: + - ../components + - ../components/mysql + +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/openshift-ci/resources/samples/secure-db/mysql/kustomization.yaml b/openshift-ci/resources/samples/secure-db/mysql/kustomization.yaml new file mode 100644 index 0000000000..9188f771d4 --- /dev/null +++ b/openshift-ci/resources/samples/secure-db/mysql/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +## Append samples of your project ## +resources: + - ../../mysql + +components: + - ../components/ + - ../components/mysql + +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/openshift-ci/resources/samples/service-mesh-subscription.yaml b/openshift-ci/resources/samples/service-mesh-subscription.yaml new file mode 100644 index 0000000000..083b972bcf --- /dev/null +++ b/openshift-ci/resources/samples/service-mesh-subscription.yaml @@ -0,0 +1,10 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: servicemeshoperator + namespace: openshift-operators +spec: + channel: stable + name: servicemeshoperator + source: redhat-operators + sourceNamespace: openshift-marketplace \ No newline at end of file diff --git a/openshift-ci/scripts/colour_text_variables.sh b/openshift-ci/scripts/colour_text_variables.sh new file mode 100644 index 0000000000..c1be2c092f --- /dev/null +++ b/openshift-ci/scripts/colour_text_variables.sh @@ -0,0 +1,4 @@ +export RED='\033[0;31m' +export GREEN='\033[0;32m' +export NC='\033[0m' +export YELLOW='\033[0;33m' \ No newline at end of file diff --git a/openshift-ci/scripts/generate_certs.sh b/openshift-ci/scripts/generate_certs.sh new file mode 100755 index 0000000000..86053161a1 --- /dev/null +++ b/openshift-ci/scripts/generate_certs.sh @@ -0,0 +1,16 @@ +DOMAIN=$1 +mkdir -p certs +# create CA cert +openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -subj "/O=modelregistry Inc./CN=$DOMAIN" -keyout certs/domain.key -out certs/domain.crt +# create rest cert and private key +echo "subjectAltName = DNS:modelregistry-sample-rest.$DOMAIN" > certs/modelregistry-sample-rest.domain.ext +openssl req -out certs/modelregistry-sample-rest.domain.csr -newkey rsa:2048 -nodes -keyout certs/modelregistry-sample-rest.domain.key -subj "/CN=modelregistry-sample-rest/O=modelregistry organization" -addext "subjectAltName = DNS:modelregistry-sample-rest.$DOMAIN" +openssl x509 -req -sha256 -days 365 -CA certs/domain.crt -CAkey certs/domain.key -set_serial 0 -in certs/modelregistry-sample-rest.domain.csr -out certs/modelregistry-sample-rest.domain.crt -extfile certs/modelregistry-sample-rest.domain.ext +# create grpc cert and private key +echo "subjectAltName = DNS:modelregistry-sample-grpc.$DOMAIN" > certs/modelregistry-sample-grpc.domain.ext +openssl req -out certs/modelregistry-sample-grpc.domain.csr -newkey rsa:2048 -nodes -keyout certs/modelregistry-sample-grpc.domain.key -subj "/CN=modelregistry-sample-grpc/O=modelregistry organization" -addext "subjectAltName = DNS:modelregistry-sample-grpc.$DOMAIN" +openssl x509 -req -sha256 -days 365 -CA certs/domain.crt -CAkey certs/domain.key -set_serial 0 -in certs/modelregistry-sample-grpc.domain.csr -out certs/modelregistry-sample-grpc.domain.crt -extfile certs/modelregistry-sample-grpc.domain.ext + +# create DB service cert and private key +openssl req -out certs/model-registry-db.csr -newkey rsa:2048 -nodes -keyout certs/model-registry-db.key -subj "/CN=model-registry-db/O=modelregistry organization" +openssl x509 -req -sha256 -days 365 -CA certs/domain.crt -CAkey certs/domain.key -set_serial 0 -in certs/model-registry-db.csr -out certs/model-registry-db.crt diff --git a/openshift-ci/scripts/oci-model-registry-deploy.sh b/openshift-ci/scripts/oci-model-registry-deploy.sh new file mode 100755 index 0000000000..6b78c835f8 --- /dev/null +++ b/openshift-ci/scripts/oci-model-registry-deploy.sh @@ -0,0 +1,225 @@ +#!/bin/bash + +# Define variables for ODH deployment deployment +OPENDATAHUB_SUBSCRIPTION="openshift-ci/resources/opendatahub-subscription.yaml" +DSC_INITIALIZATION_MANIFEST="openshift-ci/resources/model-registry-DSCInitialization.yaml" +DATA_SCIENCE_CLUSTER_MANIFEST="openshift-ci/resources/opendatahub-data-science-cluster.yaml" +MODEL_REGISTRY_DB_MANIFEST="openshift-ci/resources/model-registry-operator/mysql-db.yaml" +MODEL_REGISTRY_SAMPLE_MANIFEST="openshift-ci/resources/model-registry-operator/modelregistry_v1alpha1_modelregistry.yaml" +OPENDATAHUB_CRDS="datascienceclusters.datasciencecluster.opendatahub.io,dscinitializations.dscinitialization.opendatahub.io,featuretrackers.features.opendatahub.io" +DATA_SCIENCE_CLUSTER_CRDS="acceleratorprofiles.dashboard.opendatahub.io,datasciencepipelinesapplications.datasciencepipelinesapplications.opendatahub.io,odhapplications.dashboard.opendatahub.io,odhdashboardconfigs.opendatahub.io,odhdocuments.dashboard.opendatahub.io" +MODEL_REGISTRY_CRDS="modelregistries.modelregistry.opendatahub.io" +source "openshift-ci/scripts/colour_text_variables.sh" + +# Function to monitor CRDS creation and deployment. +# The function takes two arguments, reference to manifest and a wait time in seconds. +monitoring_crds_installation() { + IFS=',' read -ra crds_array <<< "$1" + local timeout=$2 + + echo "Monitoring the installation of CRDs: ${crds_array[*]}" + echo "Timeout set to ${timeout}s" + + local start_time=$(date +%s) + + # Loop until all specified CRDs are installed or timeout is reached + while true; do + local elapsed_time=$(($(date +%s) - start_time)) + + # Check if timeout has been reached + if [ "$elapsed_time" -ge "$timeout" ]; then + echo -e "${RED}X Error:${NC} Timeout reached. Installation of CRDs failed." + return 1 + fi + + # Get the list of installed CRDs + local installed_crds=($(oc get crd -o=name | cut -d '/' -f2)) + + # Check if all CRDs are installed + local all_installed=true + for crd in "${crds_array[@]}"; do + if ! [[ " ${installed_crds[@]} " =~ " ${crd} " ]]; then + all_installed=false + break + fi + done + + # If all CRDs are installed, break out of the loop + if [ "$all_installed" = true ]; then + echo -e "${GREEN}✔ Success:${NC} All specified CRDs are installed." + return 0 + fi + + # Print the status of each CRD + for crd in "${crds_array[@]}"; do + if [[ " ${installed_crds[@]} " =~ " ${crd} " ]]; then + echo "CRD '$crd' is installed." + else + echo "CRD '$crd' is not installed." + fi + done + + # Wait for a few seconds before checking again + sleep 5 + done +} + +# Function to deploy and wait for deployment +# The function takes two arguments, reference to manifest and a wait time in seconds. +deploy_and_wait() { + local namespace=$1 + local manifest=$2 + local resource_name=$(basename -s .yaml $manifest) + local wait_time=$3 + + sleep $wait_time + + echo "Deploying $resource_name from $manifest..." + + if oc apply -f $manifest $namespace --wait=true --timeout=300s; then + echo -e "${GREEN}✔ Success:${NC} Deployment of $resource_name succeeded." + else + echo -e "${RED}X Error:${NC} Deployment of $resource_name failed or timed out." >&2 + return 1 + fi +} + +check_deployment_availability() { + local namespace="$1" + local deployment="$2" + local timeout=300 # Timeout in seconds + local start_time=$(date +%s) + + # Loop until timeout + while (( $(date +%s) - start_time < timeout )); do + # Get the availability status of the deployment + local deployment_status=$(oc get deployment "$deployment" -n "$namespace" --no-headers -o custom-columns=:.status.availableReplicas) + + # Check if the deployment is available + if [[ $deployment_status != "" ]]; then + echo -e "${GREEN}✔ Success:${NC} Deployment $deployment is available" + return 0 # Success + fi + + sleep 5 # Wait for 5 seconds before checking again + done + + echo -e "${RED}X Error:${NC} Timeout reached. Deployment $deployment did not become available within $timeout seconds" + return 1 # Failure +} + +# Function to check the status of deploying pods +# The function takes three arguments, namespace, descriptor to identify the component and number of containers expected. +check_pod_status() { + local namespace="$1" + local pod_selector="$2" + local expected_ready_containers="$3" + local timeout=300 # Timeout in seconds + local start_time=$(date +%s) + + # Loop until timeout + while (( $(date +%s) - start_time < timeout )); do + # Get the list of pods in the specified namespace matching the provided partial names + local pod_list=$(oc get pods -n $namespace $pod_selector --no-headers -o custom-columns=NAME:.metadata.name) + + # Iterate over each pod in the list + while IFS= read -r pod_name; do + # Get the pod info + local pod_info=$(oc get pod "$pod_name" -n "$namespace" --no-headers) + + # Extract pod status and ready status from the info + local pod_name=$(echo "$pod_info" | awk '{print $1}') + local pod_status=$(echo "$pod_info" | awk '{print $3}') + local pod_ready=$(echo "$pod_info" | awk '{print $2}') + local ready_containers=$(echo "$pod_ready" | cut -d'/' -f1) + + # Check if the pod is Running and all containers are ready + if [[ $pod_status == "Running" ]] && [[ $ready_containers -eq $expected_ready_containers ]]; then + echo -e "${GREEN}✔ Success:${NC} Pod $pod_name is running and $ready_containers out of $expected_ready_containers containers are ready" + return 0 # Success + else + echo -e "${YELLOW}! INFO:${NC} Pod $pod_name is not running or does not have $expected_ready_containers containers ready" + fi + done <<< "$pod_list" + + sleep 5 # Wait for 5 seconds before checking again + done + + echo -e "${RED}X Failure:${NC} Timeout reached. No pod matching '$pod_name_partial' became ready within $timeout seconds" + return 1 # Failure +} + +# Function to check the status of a route +# The function takes two arguments, namespace and route name. +check_route_status() { + local namespace="$1" + local route_name="$2" + local key="items" + local interval=5 + local timeout=300 + local start_time=$(date +%s) + + while (( $(date +%s) - start_time < timeout )); do + # Get the route URL + local route=$(oc get route -n "$namespace" "$route_name" -o jsonpath='{.spec.host}') + local route_url="http://$route" + + if [[ -z "$route_url" ]]; then + echo -e "${RED}X Error:${NC} Route '$route_name' does not exist in namespace '$namespace'" + return 1 + else + echo -e "${GREEN}✔ Success:${NC} Route '$route_name' exists in namespace '$namespace'" + fi + + # Test if the route is live + local response=$(curl -s -o /dev/null -w "%{http_code}" "$route_url/api/model_registry/v1alpha3/registered_models") + + # Check if the response status code is 200 OK or 404 Not Found + if [[ "$response" == "200" ]]; then + echo -e "${GREEN}✔ Success:${NC} Route server is reachable. Status code: 200 OK" + return 0 + elif [[ "$response" == "404" ]]; then + echo -e "${YELLOW}! WARNING:${NC} Route server is reachable, but service is not. Status code: 404 Not Found" + return 0 + else + echo -e "${RED}X Error:${NC} Route server is unreachable. Status code: $response" + fi + + sleep "$interval" + done + + echo -e "${RED}X Error:${NC} Timeout reached. Route '$route_name' did not become live within $timeout seconds." + return 1 +} + +# Function to source the rest api tests and run them. +run_api_tests() { + ./test/scripts/rest.sh "-n opendatahub" +} + +# Run the deployment tests. +run_deployment_tests() { + check_deployment_availability "opendatahub" model-registry-db + check_deployment_availability "opendatahub" modelregistry-sample + check_pod_status "opendatahub" "-l name=model-registry-db" 1 + check_pod_status "opendatahub" "-l app=modelregistry-sample" 2 + check_route_status "opendatahub" "modelregistry-sample-http" +} + +# Main function for orchestrating deployments +main() { + deploy_and_wait "" $OPENDATAHUB_SUBSCRIPTION 0 + monitoring_crds_installation $OPENDATAHUB_CRDS 120 + deploy_and_wait "" $DSC_INITIALIZATION_MANIFEST 20 + deploy_and_wait "" $DATA_SCIENCE_CLUSTER_MANIFEST 10 + monitoring_crds_installation $DATA_SCIENCE_CLUSTER_CRDS 120 + check_pod_status "opendatahub" "-l component.opendatahub.io/name=model-registry-operator" 2 + deploy_and_wait "-n opendatahub" $MODEL_REGISTRY_SAMPLE_MANIFEST 20 + monitoring_crds_installation $MODEL_REGISTRY_CRDS 120 + deploy_and_wait "-n opendatahub" $MODEL_REGISTRY_DB_MANIFEST 20 + run_deployment_tests + run_api_tests "-n opendatahub" +} + +# Execute main function +main \ No newline at end of file diff --git a/openshift-ci/scripts/oci-model-registry-istio-tls-deploy.sh b/openshift-ci/scripts/oci-model-registry-istio-tls-deploy.sh new file mode 100755 index 0000000000..1051f216e7 --- /dev/null +++ b/openshift-ci/scripts/oci-model-registry-istio-tls-deploy.sh @@ -0,0 +1,311 @@ +#!/bin/bash + +# Define variables for ODH deployment deployment +OPENDATAHUB_SUBSCRIPTION="openshift-ci/resources/opendatahub-subscription.yaml" +ATHORINO_SUBSCRIPTION="openshift-ci/resources/samples/authorino-subscription.yaml" +SERVICE_MESH_SUBSCRIPTION="openshift-ci/resources/samples/service-mesh-subscription.yaml" +DSC_INITIALIZATION_MANIFEST="openshift-ci/resources/model-registry-DSCInitialization.yaml" +DATA_SCIENCE_CLUSTER_MANIFEST="openshift-ci/resources/opendatahub-data-science-cluster.yaml" +MODEL_REGISTRY_SAMPLE_MANIFEST="openshift-ci/resources/samples/istio/mysql-tls" +OPENDATAHUB_CRDS="datascienceclusters.datasciencecluster.opendatahub.io,dscinitializations.dscinitialization.opendatahub.io,featuretrackers.features.opendatahub.io" +AUTHORINO_CRDS="authconfigs.authorino.kuadrant.io,authconfigs.authorino.kuadrant.io,authorinos.operator.authorino.kuadrant.io" +SERVICE_MESH_CRDS="servicemeshcontrolplanes.maistra.io,servicemeshmembers.maistra.io,servicemeshmemberrolls.maistra.io" +DATA_SCIENCE_CLUSTER_CRDS="acceleratorprofiles.dashboard.opendatahub.io,datasciencepipelinesapplications.datasciencepipelinesapplications.opendatahub.io,odhapplications.dashboard.opendatahub.io,odhdashboardconfigs.opendatahub.io,odhdocuments.dashboard.opendatahub.io" +MODEL_REGISTRY_CRDS="modelregistries.modelregistry.opendatahub.io" +export DOMAIN=$(oc get ingresses.config/cluster -o jsonpath='{.spec.domain}') +export TOKEN=$(oc whoami -t) +source "openshift-ci/scripts/colour_text_variables.sh" + +# Set up environment +set_up_env() { + create_istio_env + oc new-project odh-model-registries +} + +# Function to update isto.env file with Domain +create_istio_env() { + local file_path="openshift-ci/resources/samples/istio/components/istio.env" + + cat < "$file_path" +AUTH_PROVIDER=opendatahub-auth-provider +ISTIO_INGRESS=ingressgateway +DOMAIN=$DOMAIN +REST_CREDENTIAL_NAME=modelregistry-sample-rest-credential +GRPC_CREDENTIAL_NAME=modelregistry-sample-grpc-credential +EOF +} + +# Function to deploy and set certs +deploy_certs() { + create_secret "istio-system" "generic" "modelregistry-sample-rest-credential" "modelregistry-sample-rest.domain" + create_secret "istio-system" "generic" "modelregistry-sample-grpc-credential" "modelregistry-sample-grpc.domain" + create_secret "odh-model-registries" "generic" "model-registry-db-credential" "model-registry-db" +} + +# Function to deloy certs +create_certs() { + echo $DOMAIN + source openshift-ci/scripts/generate_certs.sh "$DOMAIN" +} + +# Function to add namespace to the control plane +add_namespace_to_istio_cp() { + oc apply -f -< /dev/null; then + echo -e "${GREEN}✔ Success:${NC} Secret '$secret_name' created successfully in namespace '$namespace'." + else + echo -e "${RED}X Error:${NC} Failed to create secret '$secret_name' in namespace '$namespace'." + fi +} + +# Function to monitor CRDS creation and deployment. +# The function takes two arguments, reference to manifest and a wait time in seconds. +monitoring_crds_installation() { + IFS=',' read -ra crds_array <<< "$1" + local timeout=$2 + + echo "Monitoring the installation of CRDs: ${crds_array[*]}" + echo "Timeout set to ${timeout}s" + + local start_time=$(date +%s) + + # Loop until all specified CRDs are installed or timeout is reached + while true; do + local elapsed_time=$(($(date +%s) - start_time)) + + # Check if timeout has been reached + if [ "$elapsed_time" -ge "$timeout" ]; then + echo -e "${RED}X Error:${NC} Timeout reached. Installation of CRDs failed." + return 1 + fi + + # Get the list of installed CRDs + local installed_crds=($(oc get crd -o=name | cut -d '/' -f2)) + + # Check if all CRDs are installed + local all_installed=true + for crd in "${crds_array[@]}"; do + if ! [[ " ${installed_crds[@]} " =~ " ${crd} " ]]; then + all_installed=false + break + fi + done + + # If all CRDs are installed, break out of the loop + if [ "$all_installed" = true ]; then + echo -e "${GREEN}✔ Success:${NC} All specified CRDs are installed." + return 0 + fi + + # Print the status of each CRD + for crd in "${crds_array[@]}"; do + if [[ " ${installed_crds[@]} " =~ " ${crd} " ]]; then + echo "CRD '$crd' is installed." + else + echo "CRD '$crd' is not installed." + fi + done + + # Wait for a few seconds before checking again + sleep 5 + done +} + +# Function to deploy and wait for deployment +# The function takes four arguments, namespace, reference to manifest, wait time in seconds and a flag. +deploy_and_wait() { + local namespace=$1 + local manifest=$2 + local resource_name=$(basename -s .yaml $manifest) + local wait_time=$3 + local apply_flag="${4:--f}" + + echo "Deploying $resource_name from $manifest..." + + if oc apply $apply_flag $manifest $namespace --wait=true --timeout=1500s; then + echo -e "${GREEN}✔ Success:${NC} Deployment of $resource_name succeeded." + else + echo -e "${RED}X Error:${NC} Deployment of $resource_name failed or timed out." >&2 + return 1 + fi + + sleep $wait_time +} + +check_deployment_availability() { + local namespace="$1" + local deployment="$2" + local timeout=300 # Timeout in seconds + local start_time=$(date +%s) + + # Loop until timeout + while (( $(date +%s) - start_time < timeout )); do + # Get the availability status of the deployment + local deployment_status=$(oc get deployment "$deployment" -n "$namespace" --no-headers -o custom-columns=:.status.availableReplicas) + + # Check if the deployment is available + if [[ $deployment_status != "" ]]; then + echo -e "${GREEN}✔ Success:${NC} Deployment $deployment is available" + return 0 # Success + fi + + sleep 5 # Wait for 5 seconds before checking again + done + + echo -e "${RED}X Error:${NC} Timeout reached. Deployment $deployment did not become available within $timeout seconds" + return 1 # Failure +} + +# Function to check the status of deploying pods +# The function takes three arguments, namespace, descriptor to identify the component and number of containers expected. +check_pod_status() { + local namespace="$1" + local pod_selector="$2" + local expected_ready_containers="$3" + local timeout=900 # Timeout in seconds + local start_time=$(date +%s) + + # Loop until timeout + while (( $(date +%s) - start_time < timeout )); do + # Get the list of pods in the specified namespace matching the provided partial names + local pod_list=$(oc get pods -n $namespace $pod_selector --no-headers -o custom-columns=NAME:.metadata.name) + + # Iterate over each pod in the list + while IFS= read -r pod_name; do + # Get the pod info + local pod_info=$(oc get pod "$pod_name" -n "$namespace" --no-headers) + + # Extract pod status and ready status from the info + local pod_name=$(echo "$pod_info" | awk '{print $1}') + local pod_status=$(echo "$pod_info" | awk '{print $3}') + local pod_ready=$(echo "$pod_info" | awk '{print $2}') + local ready_containers=$(echo "$pod_ready" | cut -d'/' -f1) + + # Check if the pod is Running and all containers are ready + if [[ $pod_status == "Running" ]] && [[ $ready_containers -eq $expected_ready_containers ]]; then + echo -e "${GREEN}✔ Success:${NC} Pod $pod_name is running and $ready_containers out of $expected_ready_containers containers are ready" + return 0 # Success + else + echo -e "${YELLOW}! INFO:${NC} Pod $pod_name is not running or does not have $expected_ready_containers containers ready" + fi + done <<< "$pod_list" + + sleep 5 # Wait for 5 seconds before checking again + done + + echo -e "${RED}X Failure:${NC} Timeout reached. No pod matching '$pod_name_partial' became ready within $timeout seconds" + return 1 # Failure +} + +# Function to check the status of a route +# The function takes two arguments, namespace and route name. +check_route_status() { + local namespace="$1" + local route_name="$2" + local key="items" + local interval=5 + local timeout=300 + local start_time=$(date +%s) + + while (( $(date +%s) - start_time < timeout )); do + # Get the route URL + local route=$(oc get route -n "$namespace" "$route_name" -o jsonpath='{.spec.host}') + local route_url="http://$route" + + if [[ -z "$route_url" ]]; then + echo -e "${RED}X Error:${NC} Route '$route_name' does not exist in namespace '$namespace'" + return 1 + else + echo -e "${GREEN}✔ Success:${NC} Route '$route_name' exists in namespace '$namespace'" + fi + + # Test if the route is live + local response=$(curl -v -H "Authorization: Bearer $TOKEN" --cacert certs/domain.crt -o /dev/null -s -w "%{http_code}" "https://modelregistry-sample-rest.$DOMAIN/api/model_registry/v1alpha3/registered_models") + + # Check if the response status code is 200 OK or 404 Not Found + if [[ "$response" == "200" ]]; then + echo -e "${GREEN}✔ Success:${NC} Route server is reachable. Status code: 200 OK" + return 0 + elif [[ "$response" == "404" ]]; then + echo -e "${YELLOW}! WARNING:${NC} Route server is reachable, but service is not. Status code: 404 Not Found" + return 0 + else + echo -e "${RED}X Error:${NC} Route server is unreachable. Status code: $response" + fi + + sleep "$interval" + done + + echo -e "${RED}X Error:${NC} Timeout reached. Route '$route_name' did not become live within $timeout seconds." + return 1 +} + +# Function to source the rest api tests and run them. +run_api_tests() { + ./test/scripts/istio-tls-rest.sh "-n istio-system" "$TOKEN" ../certs/domain.crt +} + +# Run the deployment tests. +run_deployment_tests() { + check_deployment_availability "odh-model-registries" model-registry-db + check_deployment_availability "odh-model-registries" modelregistry-sample + check_pod_status "odh-model-registries" "-l name=model-registry-db" 1 + check_pod_status "odh-model-registries" "-l app=modelregistry-sample" 3 + check_route_status "istio-system" "odh-model-registries-modelregistry-sample-rest" +} + +# Main function for orchestrating deployments +main() { + set_up_env + deploy_and_wait "" $ATHORINO_SUBSCRIPTION 60 + monitoring_crds_installation $AUTHORINO_CRDS 120 + deploy_and_wait "" $SERVICE_MESH_SUBSCRIPTION 120 + monitoring_crds_installation $SERVICE_MESH_CRDS 120 + deploy_and_wait "" $OPENDATAHUB_SUBSCRIPTION 60 + monitoring_crds_installation $OPENDATAHUB_CRDS 120 + deploy_and_wait "" $DSC_INITIALIZATION_MANIFEST 20 + deploy_and_wait "" $DATA_SCIENCE_CLUSTER_MANIFEST 10 + monitoring_crds_installation $DATA_SCIENCE_CLUSTER_CRDS 120 + check_pod_status "opendatahub" "-l component.opendatahub.io/name=model-registry-operator" 2 + create_certs + deploy_certs + add_namespace_to_istio_cp + deploy_and_wait "-n odh-model-registries" $MODEL_REGISTRY_SAMPLE_MANIFEST 20 "-k" + monitoring_crds_installation $MODEL_REGISTRY_CRDS 120 + run_deployment_tests + run_api_tests "-n istio-system" +} + +# Execute main function +main \ No newline at end of file diff --git a/proposals/README.md b/proposals/README.md deleted file mode 100644 index 4f89165979..0000000000 --- a/proposals/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Proposals - -Kubeflow uses the KEP process to document large scale changes to the project. - -Details on the process (including the KEP template, recommendations, etc.) can be found at -[kubeflow/community/proposals](https://github.com/kubeflow/community/blob/master/proposals/README.md) diff --git a/scripts/build_deploy.sh b/scripts/build_deploy.sh index 653e86a755..f3428244d8 100755 --- a/scripts/build_deploy.sh +++ b/scripts/build_deploy.sh @@ -3,9 +3,9 @@ set -e # see Makefile for the IMG_ variables semantic -IMG_REGISTRY="${IMG_REGISTRY:-ghcr.io}" -IMG_ORG="${IMG_ORG:-kubeflow}" -IMG_REPO="${IMG_REPO:-model-registry/server}" +IMG_REGISTRY="${IMG_REGISTRY:-quay.io}" +IMG_ORG="${IMG_ORG:-opendatahub}" +IMG_REPO="${IMG_REPO:-model-registry}" DOCKER_USER="${DOCKER_USER}" DOCKER_PWD="${DOCKER_PWD}" diff --git a/scripts/deploy_mcp_catalog_on_odh.sh b/scripts/deploy_mcp_catalog_on_odh.sh new file mode 100755 index 0000000000..886659c92e --- /dev/null +++ b/scripts/deploy_mcp_catalog_on_odh.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash + +set -e + +DSC_NAME="default-dsc" + +echo "Check if DataScienceCluster exists" +if kubectl get datasciencecluster "$DSC_NAME" &> /dev/null; then + echo "DataScienceCluster '$DSC_NAME' exists." +else + echo "DataScienceCluster '$DSC_NAME' does NOT exist." + exit 1 +fi + +echo "Check if Model Registry is enabled in DSC" +MR_STATE=$(kubectl get datasciencecluster "$DSC_NAME" -o jsonpath='{.spec.components.modelregistry.managementState}' 2>/dev/null) +if [ "$MR_STATE" != "Managed" ]; then + echo "Model Registry is not enabled (managementState='$MR_STATE'). Expected 'Managed'." + exit 1 +fi +echo "Model Registry is enabled (managementState='Managed')." + +MR_NAMESPACE=$(kubectl get datasciencecluster "$DSC_NAME" -o jsonpath='{.spec.components.modelregistry.registriesNamespace}' 2>/dev/null) +if [ -z "$MR_NAMESPACE" ]; then + echo "Could not determine registriesNamespace from DSC." + exit 1 +fi +echo "Model Registry namespace: '$MR_NAMESPACE'" + +echo "Looking for catalog sources ConfigMap in namespace '$MR_NAMESPACE'" +if kubectl get configmap mcp-catalog-sources -n "$MR_NAMESPACE" &> /dev/null; then + CATALOG_CONFIGMAP="mcp-catalog-sources" + echo "ConfigMap 'mcp-catalog-sources' found." +elif kubectl get configmap model-catalog-sources -n "$MR_NAMESPACE" &> /dev/null; then + CATALOG_CONFIGMAP="model-catalog-sources" + echo "ConfigMap 'model-catalog-sources' found (fallback)." +else + echo "Neither 'mcp-catalog-sources' nor 'model-catalog-sources' ConfigMap found in namespace '$MR_NAMESPACE'." + exit 1 +fi + +SCRIPT_DIR="$(dirname "$(realpath "$BASH_SOURCE")")" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +MCP_SERVERS_FILE="${REPO_ROOT}/manifests/kustomize/options/catalog/overlays/e2e/test-mcp-servers.yaml" + +if [ ! -f "$MCP_SERVERS_FILE" ]; then + echo "Required file not found: $MCP_SERVERS_FILE" + exit 1 +fi + +echo "Fetching current sources.yaml from ConfigMap" +CURRENT_SOURCES=$(kubectl get configmap "$CATALOG_CONFIGMAP" -n "$MR_NAMESPACE" -o jsonpath='{.data.sources\.yaml}') + +if echo "$CURRENT_SOURCES" | grep -q "test_mcp_servers"; then + echo "test_mcp_servers already present in sources.yaml, skipping patch." +else + echo "Patching ConfigMap to add mcp_catalogs, labels, and namedQueries" + + # Read the MCP servers test data file content + MCP_SERVERS_CONTENT=$(cat "$MCP_SERVERS_FILE") + + # Replace empty mcp_catalogs: [] and append test config + UPDATED_SOURCES=$(echo "$CURRENT_SOURCES" | sed 's/mcp_catalogs: \[\]//') + UPDATED_SOURCES="${UPDATED_SOURCES} +mcp_catalogs: + - name: Test MCP Servers + id: test_mcp_servers + type: yaml + enabled: true + properties: + yamlCatalogPath: test-mcp-servers.yaml + labels: + - Test MCP Servers + +labels: + - name: mcp-label + assetType: mcp_servers + +namedQueries: + production_ready: + assetType: mcp_servers + filters: + verifiedSource: + operator: '=' + value: true + security_focused: + assetType: mcp_servers + filters: + sast: + operator: '=' + value: true + readOnlyTools: + operator: '=' + value: true +" + + # Patch the existing ConfigMap: update sources.yaml and add the MCP servers YAML as a data key + kubectl patch configmap "$CATALOG_CONFIGMAP" -n "$MR_NAMESPACE" --type merge -p "$(cat </dev/null) + if [ -z "$CATALOG_DEPLOYMENT" ]; then + echo "Could not find model-catalog deployment in namespace '$MR_NAMESPACE'." + exit 1 + fi + kubectl delete pod -l app.kubernetes.io/name=model-catalog -n "$MR_NAMESPACE" --wait=true + kubectl wait --for=condition=Available deployment/"$CATALOG_DEPLOYMENT" -n "$MR_NAMESPACE" --timeout=5m + echo "model-catalog pod is ready." +fi diff --git a/scripts/deploy_on_kind.sh b/scripts/deploy_on_kind.sh index da87b16d4c..f8726f9a19 100755 --- a/scripts/deploy_on_kind.sh +++ b/scripts/deploy_on_kind.sh @@ -4,7 +4,7 @@ set -e DIR="$(dirname "$0")" MR_NAMESPACE="${MR_NAMESPACE:-kubeflow}" -IMG="${IMG:-ghcr.io/kubeflow/model-registry/server:latest}" # TODO RHOAIENG-30453 align consistency ./scripts/deploy_on_kind.sh uses IMG with :tag, Vs, Makefile(s) and ci/GHA we use IMG without trailing :tag +IMG="${IMG:-quay.io/opendatahub/model-registry:latest}" DEPLOY_MANIFEST_DB="${DEPLOY_MANIFEST_DB:-db}" # subdirectory of manifests/kustomize/overlays to select which database: 'db' (MySQL) or 'postgres' source ./${DIR}/utils.sh diff --git a/scripts/deploy_on_odh.sh b/scripts/deploy_on_odh.sh new file mode 100755 index 0000000000..55dd3d1715 --- /dev/null +++ b/scripts/deploy_on_odh.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -e + +SCRIPT_DIR="$(dirname "$(realpath "$BASH_SOURCE")")" + +echo "Check if Data Science Cluster exists" +DSC_NAME="default-dsc" + +if kubectl get datasciencecluster "$DSC_NAME" &> /dev/null; then + echo "DataScienceCluster '$DSC_NAME' exists." +else + echo "DataScienceCluster '$DSC_NAME' does NOT exist." + exit 1 +fi +echo "Update Data Science Cluster" +kubectl patch datasciencecluster default-dsc -p '{"spec":{"components":{"modelregistry":{"managementState":"Managed"}}}}' --type=merge -o yaml +MR_NAMESPACE=$(kubectl get datasciencecluster "$DSC_NAME" -o jsonpath='{.spec.components.modelregistry.registriesNamespace}' 2>/dev/null) + +echo 'Check if Namespace '$MR_NAMESPACE' exists:' +kubectl wait --for=create namespace/"$MR_NAMESPACE" --timeout="1m" +kubectl wait --for=jsonpath='{.status.phase}=Active' namespace/"$MR_NAMESPACE" --timeout="5m" +echo "Namespace '$MR_NAMESPACE' is Active." + +kubectl apply -k manifests/kustomize/overlays/db-odh -n "$MR_NAMESPACE" + +if ! kubectl wait --for=condition=available -n "$MR_NAMESPACE" deployment/model-registry-db --timeout=5m ; then + kubectl events -A + kubectl describe deployment/model-registry-db -n "$MR_NAMESPACE" + kubectl logs deployment/model-registry-db -n "$MR_NAMESPACE" + exit 1 +fi + +MYSQL_PORT_STR=$(kubectl get configmap model-registry-db-parameters -n "$MR_NAMESPACE" -o jsonpath='{.data.MYSQL_PORT}') + +sed -e "s/DB_PORT_PLACEHOLDER/$MYSQL_PORT_STR/" -e "s/\$MR_NAMESPACE/$MR_NAMESPACE/" "${SCRIPT_DIR}/manifests/model_registry_resource/model_registry_resource.yaml" | kubectl apply -n "$MR_NAMESPACE" -f - +kubectl wait --for=create service/model-registry -n "$MR_NAMESPACE" --timeout=5m +kubectl wait --for=condition=Available -n "$MR_NAMESPACE" deployment/model-registry --timeout=5m diff --git a/scripts/get_async_upload_image.sh b/scripts/get_async_upload_image.sh new file mode 100755 index 0000000000..e814d8b3da --- /dev/null +++ b/scripts/get_async_upload_image.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Reads the async-upload job image from the model-registry-operator-parameters +# ConfigMap in the applications namespace (from DSCInitialization). +set -e + +APPS_NS=$(kubectl get dscinitializations default-dsci \ + -o jsonpath='{.spec.applicationsNamespace}') + +kubectl get configmap model-registry-operator-parameters \ + -n "$APPS_NS" \ + -o jsonpath='{.data.IMAGES_JOBS_ASYNC_UPLOAD}' diff --git a/scripts/manifests/model_registry_resource/model_registry_resource.yaml b/scripts/manifests/model_registry_resource/model_registry_resource.yaml new file mode 100644 index 0000000000..2a4f83d34f --- /dev/null +++ b/scripts/manifests/model_registry_resource/model_registry_resource.yaml @@ -0,0 +1,22 @@ +apiVersion: modelregistry.opendatahub.io/v1beta1 +kind: ModelRegistry +metadata: + name: model-registry + labels: + app.kubernetes.io/name: model-registry + app.kubernetes.io/instance: model-registry + app.kubernetes.io/part-of: model-registry-operator + app.kubernetes.io/created-by: model-registry-operator +spec: + grpc: {} + rest: {} + oauthProxy: {} + mysql: + host: model-registry-db.$MR_NAMESPACE.svc.cluster.local + database: metadb + passwordSecret: + key: MYSQL_ROOT_PASSWORD + name: model-registry-db-secrets + port: DB_PORT_PLACEHOLDER + skipDBCreation: False + username: root diff --git a/scripts/odh_env.sh b/scripts/odh_env.sh new file mode 100755 index 0000000000..8caa29d1ec --- /dev/null +++ b/scripts/odh_env.sh @@ -0,0 +1,30 @@ +# Common ODH environment variables for test targets. +# Usage: . scripts/odh_env.sh +# +# Exported variables: +# AUTH_TOKEN, VERIFY_SSL, MR_NAMESPACE, MR_HOST_URL, MR_ENDPOINT, +# MODEL_SYNC_REGISTRY_SERVER_ADDRESS, MODEL_SYNC_REGISTRY_PORT, +# MODEL_SYNC_REGISTRY_IS_SECURE, MODEL_SYNC_REGISTRY_USER_TOKEN, +# CONTAINER_IMAGE_URI + +SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" + +AUTH_TOKEN=$(kubectl config view --raw -o jsonpath="{.users[?(@.name==\"$(kubectl config view -o jsonpath="{.contexts[?(@.name==\"$(kubectl config current-context)\")].context.user}")\")].user.token}") +export AUTH_TOKEN + +export VERIFY_SSL=False + +MR_NAMESPACE=$(kubectl get datasciencecluster default-dsc -o jsonpath='{.spec.components.modelregistry.registriesNamespace}') +export MR_NAMESPACE + +MR_ENDPOINT=$(kubectl get service -n "${MR_NAMESPACE}" model-registry -o jsonpath='{.metadata.annotations.routing\.opendatahub\.io\/external-address-rest}') +export MR_HOST_URL="https://${MR_ENDPOINT}" +MR_ENDPOINT="${MR_ENDPOINT%%:*}" +export MR_ENDPOINT + +export MODEL_SYNC_REGISTRY_SERVER_ADDRESS="https://${MR_ENDPOINT}" +export MODEL_SYNC_REGISTRY_PORT="443" +export MODEL_SYNC_REGISTRY_IS_SECURE="false" +export MODEL_SYNC_REGISTRY_USER_TOKEN="${AUTH_TOKEN}" +CONTAINER_IMAGE_URI=$("${SCRIPT_DIR}/get_async_upload_image.sh") +export CONTAINER_IMAGE_URI diff --git a/scripts/undeploy_catalog_on_odh.sh b/scripts/undeploy_catalog_on_odh.sh new file mode 100755 index 0000000000..259dae46cb --- /dev/null +++ b/scripts/undeploy_catalog_on_odh.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -e + +DSC_NAME="default-dsc" + +echo "Check if DataScienceCluster exists" +if kubectl get datasciencecluster "$DSC_NAME" &> /dev/null; then + echo "DataScienceCluster '$DSC_NAME' exists." +else + echo "DataScienceCluster '$DSC_NAME' does NOT exist." + exit 1 +fi + +echo "Check if Model Registry is enabled in DSC" +MR_STATE=$(kubectl get datasciencecluster "$DSC_NAME" -o jsonpath='{.spec.components.modelregistry.managementState}' 2>/dev/null) +if [ "$MR_STATE" != "Managed" ]; then + echo "Model Registry is not enabled (managementState='$MR_STATE'). Expected 'Managed'." + exit 1 +fi +echo "Model Registry is enabled (managementState='Managed')." + +MR_NAMESPACE=$(kubectl get datasciencecluster "$DSC_NAME" -o jsonpath='{.spec.components.modelregistry.registriesNamespace}' 2>/dev/null) +if [ -z "$MR_NAMESPACE" ]; then + echo "Could not determine registriesNamespace from DSC." + exit 1 +fi +echo "Model Registry namespace: '$MR_NAMESPACE'" + +echo "Deleting ConfigMaps to trigger operator recreation..." +for cm in mcp-catalog-sources model-catalog-sources; do + if kubectl get configmap "$cm" -n "$MR_NAMESPACE" &> /dev/null; then + kubectl delete configmap "$cm" -n "$MR_NAMESPACE" + echo "ConfigMap '$cm' deleted." + else + echo "ConfigMap '$cm' not found, skipping." + fi +done + +echo "Restarting model-catalog pods to pick up recreated ConfigMaps" +CATALOG_DEPLOYMENT=$(kubectl get deployment -n "$MR_NAMESPACE" -l app.kubernetes.io/name=model-catalog -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) +if [ -z "$CATALOG_DEPLOYMENT" ]; then + echo "Could not find model-catalog deployment in namespace '$MR_NAMESPACE'." + exit 1 +fi +kubectl delete pod -l app.kubernetes.io/name=model-catalog -n "$MR_NAMESPACE" --wait=true +echo "Waiting for model-catalog deployment to be ready..." +kubectl wait --for=condition=Available deployment/"$CATALOG_DEPLOYMENT" -n "$MR_NAMESPACE" --timeout=5m +echo "model-catalog pod is ready." + +echo "Undeploy complete." diff --git a/scripts/undeploy_on_odh.sh b/scripts/undeploy_on_odh.sh new file mode 100755 index 0000000000..79aba6f786 --- /dev/null +++ b/scripts/undeploy_on_odh.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -e +SCRIPT_DIR="$(dirname "$(realpath "$BASH_SOURCE")")" +echo "Check if Data Science Cluster exists" +DSC_NAME="default-dsc" +FILE_NAME="${SCRIPT_DIR}/manifests/model_registry_resource/.port-forwards.pid" +if [ -f "$FILE_NAME" ] ; then + echo "File $FILE_NAME exists. Deleting it..." + while IFS= read -r pid; do + if [[ "$pid" =~ ^[0-9]+$ ]]; then + echo "Killing process with PID: $pid" + kill $pid || true + fi + done < "$FILE_NAME" + rm -f "$FILE_NAME" +else + echo "File $FILE_NAME port-forward.pid does NOT exist." +fi +if kubectl get datasciencecluster "$DSC_NAME" &> /dev/null; then + echo "DataScienceCluster '$DSC_NAME' exists." +else + echo "DataScienceCluster '$DSC_NAME' does NOT exist." + exit 1 +fi + +MR_NAMESPACE=$(kubectl get datasciencecluster "$DSC_NAME" -o jsonpath='{.spec.components.modelregistry.registriesNamespace}' 2>/dev/null) +echo "Delete modelregistry resource in namespace '$MR_NAMESPACE'." + +kubectl delete modelregistry.modelregistry.opendatahub.io model-registry -n "$MR_NAMESPACE" || true +echo "Update Data Science Cluster" +kubectl delete -k manifests/kustomize/overlays/db-odh -n "$MR_NAMESPACE" diff --git a/test/csi/e2e_test.sh b/test/csi/e2e_test.sh index e91c93e045..4302617316 100755 --- a/test/csi/e2e_test.sh +++ b/test/csi/e2e_test.sh @@ -15,16 +15,16 @@ KUBECTL=${KUBECTL:-"kubectl"} # You can provide a local version of the model registry storage initializer # In that case, assure that is visible to the local k8s env, e.g., using # `kind load docker-image $MRCSI_IMG` -MRCSI_IMG=${MRCSI_IMG:-"ghcr.io/kubeflow/model-registry/storage-initializer:main"} +MRCSI_IMG=${MRCSI_IMG:-"kubeflow/model-registry-storage-initializer:main"} KSERVE_VERSION=${KSERVE_VERSION:-"0.16"} MODELREGISTRY_VERSION=${MODELREGISTRY_VERSION:-"v0.3.2"} MODELREGISTRY_CSI=${MODELREGISTRY_CSI:-"v0.3.2"} # You can provide a local model registry container image -MR_IMG=${MR_IMG:-"ghcr.io/kubeflow/model-registry/server:$MODELREGISTRY_VERSION"} +MR_IMG=${MR_IMG:-"kubeflow/model-registry:$MODELREGISTRY_VERSION"} # You can provide a local model registry storage initializer container image -MR_CSI_IMG=${MR_CSI_IMG:-"ghcr.io/kubeflow/model-registry/storage-initializer:$MODELREGISTRY_CSI"} +MR_CSI_IMG=${MR_CSI_IMG:-"kubeflow/model-registry-storage-initializer:$MODELREGISTRY_CSI"} # Export variables so they're available to setup_test_env.sh export KSERVE_VERSION diff --git a/test/csi/setup_test_env.sh b/test/csi/setup_test_env.sh index a54b448a03..89b624aacb 100755 --- a/test/csi/setup_test_env.sh +++ b/test/csi/setup_test_env.sh @@ -14,16 +14,16 @@ CLUSTER=${CLUSTER:-"kind"} # You can provide a local version of the model registry storage initializer # In that case, assure that is visible to the local k8s env, e.g., using # `kind load docker-image $MRCSI_IMG` -MRCSI_IMG=${MRCSI_IMG:-"ghcr.io/kubeflow/model-registry/storage-initializer:main"} +MRCSI_IMG=${MRCSI_IMG:-"kubeflow/model-registry-storage-initializer:main"} KSERVE_VERSION=${KSERVE_VERSION:-"0.16"} MODELREGISTRY_VERSION=${MODELREGISTRY_VERSION:-"v0.3.2"} MODELREGISTRY_CSI=${MODELREGISTRY_CSI:-"v0.3.2"} # You can provide a local model registry container image -MR_IMG=${MR_IMG:-"ghcr.io/kubeflow/model-registry/server:$MODELREGISTRY_VERSION"} +MR_IMG=${MR_IMG:-"kubeflow/model-registry:$MODELREGISTRY_VERSION"} # You can provide a local model registry storage initializer container image -MR_CSI_IMG=${MR_CSI_IMG:-"ghcr.io/kubeflow/model-registry/storage-initializer:$MODELREGISTRY_CSI"} +MR_CSI_IMG=${MR_CSI_IMG:-"kubeflow/model-registry-storage-initializer:$MODELREGISTRY_CSI"} # Check if KUBECTL is a valid command if [ ! command -v "$KUBECTL" > /dev/null 2>&1 ]; then diff --git a/test/scripts/install_modelregistry.sh b/test/scripts/install_modelregistry.sh index 447963462b..4034f6b179 100755 --- a/test/scripts/install_modelregistry.sh +++ b/test/scripts/install_modelregistry.sh @@ -18,7 +18,7 @@ Help() { MR_ROOT="$(dirname "$0")/../.." namespace=kubeflow -image=ghcr.io/kubeflow/model-registry/server:latest +image=quay.io/opendatahub/model-registry:latest while getopts ":hn:i:" option; do case $option in h) # display Help @@ -40,7 +40,7 @@ if ! kubectl get namespace "$namespace" &> /dev/null; then fi # Apply model-registry kustomize manifests echo Using model registry image: $image -cd $MR_ROOT/manifests/kustomize/base && kustomize edit set image ghcr.io/kubeflow/model-registry/server:latest=${image} && \ +cd $MR_ROOT/manifests/kustomize/base && kustomize edit set image quay.io/opendatahub/model-registry:latest=${image} && \ kustomize edit set namespace $namespace && cd - cd $MR_ROOT/manifests/kustomize/overlays/db && kustomize edit set namespace $namespace && cd - kubectl -n $namespace apply -k "$MR_ROOT/manifests/kustomize/overlays/db" diff --git a/test/scripts/istio-tls-rest.sh b/test/scripts/istio-tls-rest.sh new file mode 100755 index 0000000000..19f97c6ef0 --- /dev/null +++ b/test/scripts/istio-tls-rest.sh @@ -0,0 +1,126 @@ +#!/bin/bash +FLAG_WITH_NAMESPACE=$1 +TOKEN=$2 +CERT="certs/domain.crt" +MR_HOSTNAME="https://$(oc get route odh-model-registries-modelregistry-sample-rest $FLAG_WITH_NAMESPACE --template='{{.spec.host}}')" + +# Function to send data and extract ID from response +make_post_extract_id() { + local url="$1" + local data="$2" + local id=$(curl -s -X POST -H "Authorization: Bearer $TOKEN" --cacert certs/domain.crt "$url" \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d "$data" | jq -r '.id') + + if [ -z "$id" ]; then + echo -e "Error: Failed to extract ID from response" + exit 1 + else + echo "$id" + fi +} + +# Function to post model registry data +post_model_registry_data() { + timestamp=$(date +"%Y%m%d%H%M%S") + rm_name="demo-$timestamp" + + rm_id=$(make_post_extract_id "$MR_HOSTNAME/api/model_registry/v1alpha3/registered_models" '{ + "description": "lorem ipsum registered model", + "name": "'"$rm_name"'" + }') + + if [ $? -ne 0 ]; then + echo -e "Error: Registered Model ID not returned" + exit 1 + else + echo -e "Success: Registered Model ID: $rm_id" + fi + + mv_id=$(make_post_extract_id "$MR_HOSTNAME/api/model_registry/v1alpha3/model_versions" '{ + "description": "lorem ipsum model version", + "name": "v1", + "author": "John Doe", + "registeredModelId": "'"$rm_id"'" + }') + + if [ $? -ne 0 ]; then + echo -e "Error: Model Version ID not returned" + exit 1 + else + echo -e "Success: Model Version ID: $mv_id" + fi + + RAW_ML_MODEL_URI='https://huggingface.co/tarilabs/mnist/resolve/v1.nb20231206162408/mnist.onnx' + ma_id=$(make_post_extract_id "$MR_HOSTNAME/api/model_registry/v1alpha3/model_versions/$mv_id/artifacts" '{ + "description": "lorem ipsum model artifact", + "uri": "'"$RAW_ML_MODEL_URI"'", + "name": "mnist", + "modelFormatName": "onnx", + "modelFormatVersion": "1", + "storageKey": "aws-connection-unused", + "storagePath": "unused just demo", + "artifactType": "model-artifact" + }') + + if [ $? -ne 0 ]; then + echo -e "Error: Model Artifact ID not returned" + exit 1 + else + echo -e "Success: Model Artifact ID: $ma_id" + fi +} + +# Function to test deployment of second odh-project +odh_project_b() { + ISVC_TARGET_NS=odh-project-b + MODEL_SERVER_NAME=modelserverb + + oc apply -n $ISVC_TARGET_NS -f - < +# Example : Putting 5 entries into the database : ./test/scripts/populateDatabase.sh 5 + +LOOPS=$1 +MR_HOSTNAME="http://$(oc get route modelregistry-sample-http -n opendatahub --template='{{.spec.host}}')" + +# Function to send data and extract ID from response +make_post_extract_id() { + local url="$1" + local data="$2" + local id=$(curl -s -X POST "$url" \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d "$data" | jq -r '.id') + + if [ -z "$id" ]; then + echo -e "Error: Failed to extract ID from response" + exit 1 + else + echo "$id" + fi +} + +# Function to post model registry data +post_model_registry_data() { + test_data_number=$1 + timestamp=$(date +"%Y%m%d%H%M%S") + rm_name="test-data-2-$test_data_number" + + rm_id=$(make_post_extract_id "$MR_HOSTNAME/api/model_registry/v1alpha3/registered_models" '{ + "description": "lorem ipsum registered model", + "name": "'"$rm_name"'" + }') + + if [ $? -ne 0 ]; then + echo -e "Error: Registered Model ID not returned" + exit 1 + else + echo -e "Success: Registered Model ID: $rm_id" + fi + + mv_id=$(make_post_extract_id "$MR_HOSTNAME/api/model_registry/v1alpha3/model_versions" '{ + "description": "lorem ipsum model version", + "name": "v1", + "author": "John Doe", + "registeredModelId": "'"$rm_id"'" + }') + + if [ $? -ne 0 ]; then + echo -e "Error: Model Version ID not returned" + exit 1 + else + echo -e "Success: Model Version ID: $mv_id" + fi + + RAW_ML_MODEL_URI='https://huggingface.co/tarilabs/mnist/resolve/v1.nb20231206162408/mnist.onnx' + ma_id=$(make_post_extract_id "$MR_HOSTNAME/api/model_registry/v1alpha3/model_versions/$mv_id/artifacts" '{ + "description": "lorem ipsum model artifact", + "uri": "'"$RAW_ML_MODEL_URI"'", + "name": "mnist", + "modelFormatName": "onnx", + "modelFormatVersion": "1", + "storageKey": "aws-connection-unused", + "storagePath": "unused just demo", + "artifactType": "model-artifact" + }') + + if [ $? -ne 0 ]; then + echo -e "Error: Model Artifact ID not returned" + exit 1 + else + echo -e "Success: Model Artifact ID: $ma_id" + fi +} + +# Main function for populating database +main() { + for i in {$(seq 1 $LOOPS)}; do + post_model_registry_data $i + done +} + +# Execute main function +main diff --git a/test/scripts/rest.sh b/test/scripts/rest.sh index 81e36f4443..13551debe5 100755 --- a/test/scripts/rest.sh +++ b/test/scripts/rest.sh @@ -1,5 +1,8 @@ #!/bin/bash +FLAG_WITH_NAMESPACE=$1 +MR_HOSTNAME="http://$(oc get route modelregistry-sample-http $FLAG_WITH_NAMESPACE --template='{{.spec.host}}')" +# Function to send data and extract ID from response make_post_extract_id() { local url="$1" local data="$2" @@ -9,89 +12,113 @@ make_post_extract_id() { -d "$data" | jq -r '.id') if [ -z "$id" ]; then - echo "Error: Failed to extract ID from response" + echo -e "Error: Failed to extract ID from response" exit 1 + else + echo "$id" fi - - echo "$id" } -# TODO: finalize using openshift-ci values. -OCP_CLUSTER_NAME="PROVIDE OCP CLUSTER NAME FOR OPENSHIFT-CI" -MR_NAMESPACE="shared-modelregistry-ns" -MR_HOSTNAME="http://modelregistry-sample-http-$MR_NAMESPACE.apps.$OCP_CLUSTER_NAME" - -timestamp=$(date +"%Y%m%d%H%M%S") -rm_name="demo-$timestamp" +# Function to post model registry data +post_model_registry_data() { + timestamp=$(date +"%Y%m%d%H%M%S") + rm_name="demo-$timestamp" -rm_id=$(make_post_extract_id "$MR_HOSTNAME/api/model_registry/v1alpha3/registered_models" '{ - "description": "lorem ipsum registered model", - "name": "'"$rm_name"'" -}') + rm_id=$(make_post_extract_id "$MR_HOSTNAME/api/model_registry/v1alpha3/registered_models" '{ + "description": "lorem ipsum registered model", + "name": "'"$rm_name"'" + }') -if [ $? -ne 0 ]; then - exit 1 -fi -echo "Registered Model ID: $rm_id" + if [ $? -ne 0 ]; then + echo -e "Error: Registered Model ID not returned" + exit 1 + else + echo -e "Success: Registered Model ID: $rm_id" + fi -mv_id=$(make_post_extract_id "$MR_HOSTNAME/api/model_registry/v1alpha3/model_versions" '{ - "description": "lorem ipsum model version", - "name": "v1", - "author": "John Doe", - "registeredModelId": "'"$rm_id"'" -}') + mv_id=$(make_post_extract_id "$MR_HOSTNAME/api/model_registry/v1alpha3/model_versions" '{ + "description": "lorem ipsum model version", + "name": "v1", + "author": "John Doe", + "registeredModelId": "'"$rm_id"'" + }') -if [ $? -ne 0 ]; then - exit 1 -fi -echo "Model Version ID: $mv_id" + if [ $? -ne 0 ]; then + echo -e "Error: Model Version ID not returned" + exit 1 + else + echo -e "Success: Model Version ID: $mv_id" + fi -RAW_ML_MODEL_URI='https://huggingface.co/tarilabs/mnist/resolve/v1.nb20231206162408/mnist.onnx' -ma_id=$(make_post_extract_id "$MR_HOSTNAME/api/model_registry/v1alpha3/model_versions/$mv_id/artifacts" '{ - "description": "lorem ipsum model artifact", - "uri": "'"$RAW_ML_MODEL_URI"'", - "name": "mnist", - "modelFormatName": "onnx", - "modelFormatVersion": "1", - "storageKey": "aws-connection-unused", - "storagePath": "unused just demo", - "artifactType": "model-artifact" -}') + RAW_ML_MODEL_URI='https://huggingface.co/tarilabs/mnist/resolve/v1.nb20231206162408/mnist.onnx' + ma_id=$(make_post_extract_id "$MR_HOSTNAME/api/model_registry/v1alpha3/model_versions/$mv_id/artifacts" '{ + "description": "lorem ipsum model artifact", + "uri": "'"$RAW_ML_MODEL_URI"'", + "name": "mnist", + "modelFormatName": "onnx", + "modelFormatVersion": "1", + "storageKey": "aws-connection-unused", + "storagePath": "unused just demo", + "artifactType": "model-artifact" + }') -if [ $? -ne 0 ]; then - exit 1 -fi -echo "Model Artifact ID: $ma_id" + if [ $? -ne 0 ]; then + echo -e "Error: Model Artifact ID not returned" + exit 1 + else + echo -e "Success: Model Artifact ID: $ma_id" + fi +} -ISVC_TARGET_NS=odh-project-b -MODEL_SERVER_NAME=modelserverb +# Function to test deployment of second odh-project +odh_project_b() { + ISVC_TARGET_NS=odh-project-b + MODEL_SERVER_NAME=modelserverb -oc apply -n $ISVC_TARGET_NS -f - <