Sync kubeflow/pipelines manifests 1.8.2 (#2223)

surajkota · kimwnasptd · commit 9fc7538c118c · 2022-06-15T21:35:53.000+03:00
* Update kubeflow/pipelines manifests from 1.8.2

* update example, tests
diff --git a/README.md b/README.md
@@ -53,7 +53,7 @@ This repo periodically syncs all official Kubeflow components from their respect
 | Katib | apps/katib/upstream | [v0.13.0](https://github.com/kubeflow/katib/tree/v0.13.0/manifests/v1beta1) |
 | KFServing | apps/kfserving/upstream | [v0.6.1](https://github.com/kubeflow/kfserving/releases/tag/v0.6.1) |
 | KServe | contrib/kserve/upstream | [v0.7.0](https://github.com/kserve/kserve/tree/v0.7.0) |
-| Kubeflow Pipelines | apps/pipeline/upstream | [1.8.1](https://github.com/kubeflow/pipelines/tree/1.8.1/manifests/kustomize) |
+| Kubeflow Pipelines | apps/pipeline/upstream | [1.8.2](https://github.com/kubeflow/pipelines/tree/1.8.2/manifests/kustomize) |
 | Kubeflow Tekton Pipelines | apps/kfp-tekton/upstream | [v1.1.1](https://github.com/kubeflow/kfp-tekton/tree/v1.1.1/manifests/kustomize) |
 
 The following is also a matrix with versions from common components that are
@@ -217,7 +217,7 @@ kustomize build common/istio-1-11/kubeflow-istio-resources/base | kubectl apply
 Install the [Multi-User Kubeflow Pipelines](https://www.kubeflow.org/docs/components/pipelines/multi-user/) official Kubeflow component:
 
 ```sh
-kustomize build apps/pipeline/upstream/env/platform-agnostic-multi-user | kubectl apply -f -
+kustomize build apps/pipeline/upstream/env/cert-manager/platform-agnostic-multi-user | kubectl apply -f -
 ```
 
 If your container runtime is not docker, use pns executor instead:
diff --git a/apps/pipeline/upstream/base/cache-deployer/kustomization.yaml b/apps/pipeline/upstream/base/cache-deployer/kustomization.yaml
@@ -8,4 +8,4 @@ commonLabels:
   app: cache-deployer
 images:
   - name: gcr.io/ml-pipeline/cache-deployer
-    newTag: 1.8.1
+    newTag: 1.8.2
diff --git a/apps/pipeline/upstream/base/cache/kustomization.yaml b/apps/pipeline/upstream/base/cache/kustomization.yaml
@@ -10,4 +10,4 @@ commonLabels:
   app: cache-server
 images:
   - name: gcr.io/ml-pipeline/cache-server
-    newTag: 1.8.1
+    newTag: 1.8.2
diff --git a/apps/pipeline/upstream/base/installs/generic/pipeline-install-config.yaml b/apps/pipeline/upstream/base/installs/generic/pipeline-install-config.yaml
@@ -11,7 +11,7 @@ data:
     until the changes take effect. A quick way to restart all deployments in a
     namespace: `kubectl rollout restart deployment -n <your-namespace>`.
   appName: pipeline
-  appVersion: 1.8.1
+  appVersion: 1.8.2
   dbHost: mysql
   dbPort: "3306"
   mlmdDb: metadb
diff --git a/apps/pipeline/upstream/base/metadata/base/kustomization.yaml b/apps/pipeline/upstream/base/metadata/base/kustomization.yaml
@@ -9,4 +9,4 @@ resources:
   - metadata-grpc-sa.yaml
 images:
   - name: gcr.io/ml-pipeline/metadata-envoy
-    newTag: 1.8.1
+    newTag: 1.8.2
diff --git a/apps/pipeline/upstream/base/pipeline/kustomization.yaml b/apps/pipeline/upstream/base/pipeline/kustomization.yaml
@@ -37,14 +37,14 @@ resources:
   - kfp-launcher-configmap.yaml
 images:
   - name: gcr.io/ml-pipeline/api-server
-    newTag: 1.8.1
+    newTag: 1.8.2
   - name: gcr.io/ml-pipeline/persistenceagent
-    newTag: 1.8.1
+    newTag: 1.8.2
   - name: gcr.io/ml-pipeline/scheduledworkflow
-    newTag: 1.8.1
+    newTag: 1.8.2
   - name: gcr.io/ml-pipeline/frontend
-    newTag: 1.8.1
+    newTag: 1.8.2
   - name: gcr.io/ml-pipeline/viewer-crd-controller
-    newTag: 1.8.1
+    newTag: 1.8.2
   - name: gcr.io/ml-pipeline/visualization-server
-    newTag: 1.8.1
+    newTag: 1.8.2
diff --git a/apps/pipeline/upstream/base/pipeline/metadata-writer/kustomization.yaml b/apps/pipeline/upstream/base/pipeline/metadata-writer/kustomization.yaml
@@ -7,4 +7,4 @@ resources:
   - metadata-writer-sa.yaml
 images:
   - name: gcr.io/ml-pipeline/metadata-writer
-    newTag: 1.8.1
+    newTag: 1.8.2
diff --git a/apps/pipeline/upstream/env/cert-manager/base/cache-cert-issuer.yaml b/apps/pipeline/upstream/env/cert-manager/base/cache-cert-issuer.yaml
@@ -0,0 +1,6 @@
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: kfp-cache-selfsigned-issuer
+spec:
+  selfSigned: {}
diff --git a/apps/pipeline/upstream/env/cert-manager/base/cache-cert.yaml b/apps/pipeline/upstream/env/cert-manager/base/cache-cert.yaml
@@ -0,0 +1,15 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: kfp-cache-cert
+spec:
+  commonName: kfp-cache-cert
+  isCA: true
+  dnsNames:
+  - cache-server
+  - cache-server.$(kfp-namespace)
+  - cache-server.$(kfp-namespace).svc
+  issuerRef:
+    kind: Issuer
+    name: kfp-cache-selfsigned-issuer
+  secretName: webhook-server-tls
diff --git a/apps/pipeline/upstream/env/cert-manager/base/cache-webhook-config.yaml b/apps/pipeline/upstream/env/cert-manager/base/cache-webhook-config.yaml
@@ -0,0 +1,25 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: cache-webhook-kubeflow
+  annotations:
+    cert-manager.io/inject-ca-from: $(kfp-namespace)/kfp-cache-cert
+webhooks:
+  - name: cache-server.$(kfp-namespace).svc
+    clientConfig:
+      service:
+        name: cache-server
+        namespace: $(kfp-namespace)
+        path: "/mutate"
+    failurePolicy: Ignore
+    rules:
+    - operations: [ "CREATE" ]
+      apiGroups: [""]
+      apiVersions: ["v1"]
+      resources: ["pods"]
+    sideEffects: None
+    timeoutSeconds: 5
+    objectSelector:
+      matchLabels:
+        pipelines.kubeflow.org/cache_enabled: "true"
+    admissionReviewVersions: ["v1beta1"]
diff --git a/apps/pipeline/upstream/env/cert-manager/base/kustomization.yaml b/apps/pipeline/upstream/env/cert-manager/base/kustomization.yaml
@@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kubeflow 
+
+resources:
+  - cache-cert-issuer.yaml
+  - cache-cert.yaml
+  - cache-webhook-config.yaml
+commonLabels:
+  app: cache-server-cert-manager
+
+configurations:
+- params.yaml
diff --git a/apps/pipeline/upstream/env/cert-manager/base/params.yaml b/apps/pipeline/upstream/env/cert-manager/base/params.yaml
@@ -0,0 +1,13 @@
+varReference:
+  - path: spec/commonName
+    kind: Certificate
+  - path: spec/dnsNames
+    kind: Certificate
+  - path: spec/issuerRef/name
+    kind: Certificate
+  - path: metadata/annotations
+    kind: MutatingWebhookConfiguration
+  - path: webhooks/clientConfig/service/namespace
+    kind: MutatingWebhookConfiguration
+  - path: webhooks/name
+    kind: MutatingWebhookConfiguration
diff --git a/apps/pipeline/upstream/env/cert-manager/cluster-scoped-resources/kustomization.yaml b/apps/pipeline/upstream/env/cert-manager/cluster-scoped-resources/kustomization.yaml
@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+bases:
+- ../../../third-party/application/cluster-scoped
+- ../../../third-party/argo/installs/namespace/cluster-scoped
+- ../../../base/pipeline/cluster-scoped
diff --git a/apps/pipeline/upstream/env/cert-manager/dev/delete-cache-deployer.yaml b/apps/pipeline/upstream/env/cert-manager/dev/delete-cache-deployer.yaml
@@ -0,0 +1,18 @@
+# Delete cache deployer related resources as we use the cert-manager instead 
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cache-deployer-deployment
+$patch: delete
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: kubeflow-pipelines-cache-deployer-role
+$patch: delete
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kubeflow-pipelines-cache-deployer-rolebinding
+$patch: delete
diff --git a/apps/pipeline/upstream/env/cert-manager/dev/kustomization.yaml b/apps/pipeline/upstream/env/cert-manager/dev/kustomization.yaml
@@ -0,0 +1,47 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+bases:
+  - ../../dev
+  - ../base
+namespace: kubeflow
+
+# Delete the cache deployer as we use the cert-manager instead 
+patchesStrategicMerge:
+  - ./delete-cache-deployer.yaml
+
+resources:
+- namespace.yaml
+
+vars:
+# NOTE: var name must be unique globally to allow composition of multiple kustomize
+# packages. Therefore, we added prefix `kfp-dev-` to distinguish it from
+# others.
+- name: kfp-dev-namespace
+  objref:
+    # ml-pipeline sa's metadata.namespace will be first transformed by namespace field in kustomization.yaml
+    # so that we only need to change kustomization.yaml's namespace field for namespace customization.
+    kind: ServiceAccount
+    name: ml-pipeline
+    apiVersion: v1
+  fieldref:
+    fieldpath: metadata.namespace
+configurations:
+- params.yaml
+
+# Pass proper arguments to cache-server to use cert-manager certificate
+patches:
+- patch: |-
+    - op: add
+      path: /spec/template/spec/containers/0/args/-
+      value: "--tls_cert_filename=tls.crt"
+  target:
+    kind: Deployment
+    name: cache-server
+- patch: |-
+    - op: add
+      path: /spec/template/spec/containers/0/args/-
+      value: "--tls_key_filename=tls.key"
+  target:
+    kind: Deployment
+    name: cache-server
diff --git a/apps/pipeline/upstream/env/cert-manager/dev/namespace.yaml b/apps/pipeline/upstream/env/cert-manager/dev/namespace.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: '$(kfp-dev-namespace)'
diff --git a/apps/pipeline/upstream/env/cert-manager/dev/params.yaml b/apps/pipeline/upstream/env/cert-manager/dev/params.yaml
@@ -0,0 +1,4 @@
+# Allow Kustomize var to replace following fields.
+varReference:
+- path: metadata/name
+  kind: Namespace
diff --git a/apps/pipeline/upstream/env/cert-manager/platform-agnostic-multi-user/delete-cache-deployer.yaml b/apps/pipeline/upstream/env/cert-manager/platform-agnostic-multi-user/delete-cache-deployer.yaml
@@ -0,0 +1,36 @@
+# Delete cache deployer as we use the cert-manager instead 
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kubeflow-pipelines-cache-deployer-clusterrole
+$patch: delete
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kubeflow-pipelines-cache-deployer-clusterrolebinding
+$patch: delete
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kubeflow-pipelines-cache-deployer-sa
+$patch: delete
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cache-deployer-deployment
+$patch: delete
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: kubeflow-pipelines-cache-deployer-role
+$patch: delete
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kubeflow-pipelines-cache-deployer-rolebinding
+$patch: delete
diff --git a/apps/pipeline/upstream/env/cert-manager/platform-agnostic-multi-user/kustomization.yaml b/apps/pipeline/upstream/env/cert-manager/platform-agnostic-multi-user/kustomization.yaml
@@ -0,0 +1,28 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+bases:
+  - ../../platform-agnostic-multi-user
+  - ../base
+namespace: kubeflow
+
+# Delete the cache deployer as we use the cert-manager instead 
+patchesStrategicMerge:
+  - ./delete-cache-deployer.yaml
+
+# Pass proper arguments to cache-server to use cert-manager certificate
+patches:
+- patch: |-
+    - op: add
+      path: /spec/template/spec/containers/0/args/-
+      value: "--tls_cert_filename=tls.crt"
+  target:
+    kind: Deployment
+    name: cache-server
+- patch: |-
+    - op: add
+      path: /spec/template/spec/containers/0/args/-
+      value: "--tls_key_filename=tls.key"
+  target:
+    kind: Deployment
+    name: cache-server
diff --git a/apps/pipeline/upstream/env/gcp/inverse-proxy/kustomization.yaml b/apps/pipeline/upstream/env/gcp/inverse-proxy/kustomization.yaml
@@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
   - name: gcr.io/ml-pipeline/inverse-proxy-agent
-    newTag: 1.8.1
+    newTag: 1.8.2
 resources:
   - proxy-configmap.yaml
   - proxy-deployment.yaml
diff --git a/example/kustomization.yaml b/example/kustomization.yaml
@@ -26,7 +26,7 @@ resources:
 
 
 # Kubeflow Pipelines
-- ../apps/pipeline/upstream/env/platform-agnostic-multi-user
+- ../apps/pipeline/upstream/env/cert-manager/platform-agnostic-multi-user
 # KFServing
 - ../apps/kfserving/upstream/overlays/kubeflow
 # Katib
@@ -35,9 +35,9 @@ resources:
 - ../apps/centraldashboard/upstream/overlays/kserve
 # Admission Webhook
 - ../apps/admission-webhook/upstream/overlays/cert-manager
-# Notebook Controller
-- ../apps/jupyter/jupyter-web-app/upstream/overlays/istio
 # Jupyter Web App
+- ../apps/jupyter/jupyter-web-app/upstream/overlays/istio
+# Notebook Controller
 - ../apps/jupyter/notebook-controller/upstream/overlays/kubeflow
 # Profiles + KFAM
 - ../apps/profiles/upstream/overlays/kubeflow
diff --git a/hack/setup-kubeflow-light.sh b/hack/setup-kubeflow-light.sh
@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# This shell script is used to setup Katib deployment.
+# This shell script is used to setup Kubeflow deployment.
 set -euo pipefail
 
 TIMEOUT=600s  # 10mins
@@ -58,7 +58,7 @@ kubectl wait --timeout=${TIMEOUT} -n knative-serving --all --for=condition=Ready
 
 echo "Deploying KFP."
 function install_kfp {
-    kustomize build apps/pipeline/upstream/env/platform-agnostic-multi-user | kubectl apply -f - --validate=false
+    kustomize build apps/pipeline/upstream/env/cert-manager/platform-agnostic-multi-user | kubectl apply -f - --validate=false
 }
 
 while ! install_kfp;
diff --git a/tests/README.md b/tests/README.md
@@ -4,7 +4,7 @@
 
 There are 2 goals for the e2e tests:
 1. Verify that core components can be applied and become Ready
-2. Virify that core CRDs can be created and succeed
+2. Verify that core CRDs can be created and succeed
 
 ### Components
 
