Implement (simplified) Deployment controller and its model

.github/workflows/ci.yml

@@ -405,13 +405,7 @@ jobs:
       - name: Install kind
         run: go install sigs.k8s.io/kind@v$kind_version
       - name: Deploy vreplicaset admission controller
-        run: |
-          VERUS_DIR="${HOME}/verus" ./build.sh v2_vreplicaset_admission_controller.rs --no-verify --time
-          docker build -f docker/controller/Dockerfile.local -t local/admission-controller:v0.1.0 --build-arg APP=v2_vreplicaset_admission .
-          kind create cluster --config deploy/kind.yaml
-          kind load docker-image local/admission-controller:v0.1.0
-          cd e2e
-          ./admission_setup.sh vreplicaset
+        run: VERUS_DIR="${HOME}/verus" ./local-test.sh v2-vreplicaset-admission --build
       - name: Run vreplicaset e2e tests for admission
         run: . "$HOME/.cargo/env" && cd e2e && cargo run -- v2-vreplicaset-admission
   vstatefulset-admission-e2e-test:
@@ -436,13 +430,7 @@ jobs:
       - name: Install kind
         run: go install sigs.k8s.io/kind@v$kind_version
       - name: Deploy vstatefulset admission controller
-        run: |
-          VERUS_DIR="${HOME}/verus" ./build.sh v2_vstatefulset_admission_controller.rs --no-verify --time
-          docker build -f docker/controller/Dockerfile.local -t local/admission-controller:v0.1.0 --build-arg APP=v2_vstatefulset_admission .
-          kind create cluster --config deploy/kind.yaml
-          kind load docker-image local/admission-controller:v0.1.0
-          cd e2e
-          ./admission_setup.sh vstatefulset
+        run: VERUS_DIR="${HOME}/verus" ./local-test.sh v2-vstatefulset-admission --build
       - name: Run vstatefulset e2e tests for admission
         run: . "$HOME/.cargo/env" && cd e2e && cargo run -- v2-vstatefulset-admission
   vreplicaset-e2e-test:
@@ -470,3 +458,29 @@ jobs:
         run: VERUS_DIR="${HOME}/verus" ./local-test.sh v2-vreplicaset --build-remote
       - name: Run vreplicaset e2e tests
         run: . "$HOME/.cargo/env" && cd e2e && cargo run -- v2-vreplicaset
+  vdeployment-e2e-test:
+    needs:
+      - build-and-cache-verus
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Restore Verus cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ${{ env.home_dir }}/verus/source
+            ${{ env.home_dir }}/verus/dependencies
+            ${{ env.home_dir }}/.cargo
+            ${{ env.home_dir }}/.rustup
+          key: verus-${{ runner.os }}-${{ env.verus_commit }}-${{ hashFiles('rust-toolchain.toml') }}
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "^1.20"
+      - name: Install kind
+        run: go install sigs.k8s.io/[email protected]
+      - name: Deploy v2-vdeployment controller
+        run: VERUS_DIR="${HOME}/verus" ./local-test.sh v2-vdeployment --build
+      - name: Run vdeployment e2e tests
+        run: . "$HOME/.cargo/env" && cd e2e && cargo run -- v2-vdeployment
+

build.md

@@ -60,14 +60,25 @@ Make sure `VERUS_DIR` points to verus repo location and built binary exists, `<c
 3. Setup cluster, apply controller image using [kind](https://kind.sigs.k8s.io/).
 4. Apply test specified in `e2e/src` and workload in `deploy` by `deploy.sh`
 
-This process can be automated with
+This process can be automated with:
+
+**1-3**
 
 ```
 ./local-test.sh <controller_name> [--build|--build-remote]
 Usage:
 	--build:		Call ./build.sh to build the controller before test, should have VERUS_DIR speccified
 	--build-remote:		Call ./build.sh to build the controller image using Verus builder. This is useful when host has different runtime environment from image (Ubuntu 22.04), for example, different glibc version
-	unspecified:		Just use existing host built controller to setup the controller image. Assume binary is ready in `src/<controller_name>`
+	unspecified:		Just use existing built controller image to set up kind cluster. Assume the image is named as `local/$app-controller:v0.1.0`
+```
+
+If deployment/test failed, you can manually run `./deploy.sh <controller_name> [local|remote]` to reset the e2e test environment.
+
+**4**
+```
+cd e2e
+# ./admission_setup.sh <controller_name> for admission test
+cargo run -- <controller_name>
 ```
 
 > You can find more examples in `.github/workflows/ci.yml`

deploy.sh

@@ -4,7 +4,7 @@
 ##
 ## Requires a running Kubernetes cluster and kubectl to be installed.
 
-set -xeu
+set -xu
 
 YELLOW='\033[1;33m'
 GREEN='\033[1;32m'
@@ -13,10 +13,50 @@ NC='\033[0m'
 
 app=$(echo "$1" | tr '_' '-') # should be the controller's name (with words separated by dashes)
 app_filename=$(echo "$app" | tr '-' '_')
+cluster_name="${app}-e2e"
 registry=$2 # should be either remote or local
 
-## use imperative management for CRDs since metadata for PodTemplateSpec is too long.
-if cd deploy/$app_filename && kubectl create -f crd.yaml && kubectl apply -f rbac.yaml && kubectl apply -f deploy_$registry.yaml; then
+kind get clusters | grep $cluster_name > /dev/null 2>&1
+if [ $? -eq 0 ]; then
+    echo -e "${YELLOW}A kind cluster named \"$cluster_name\" already exists. Deleting...${NC}"
+    kind delete cluster --name $cluster_name
+fi
+
+set -xeu
+# Set up the kind cluster and load the image into the cluster
+kind create cluster --config deploy/kind.yaml --name $cluster_name
+kind load docker-image local/$app-controller:v0.1.0 --name $cluster_name
+
+# for VDeployment, need to deploy VReplicaSet as a dependency
+if [ "$app" == "v2-vdeployment" ]; then
+    kind load docker-image local/v2-vreplicaset-controller:v0.1.0 --name $cluster_name
+fi
+
+# admission controller has a different deployment process
+if [ $(echo $app | awk -F'-' '{print $NF}') == "admission" ]; then
+    app=${app%-admission}
+    app_filename=${app_filename%_admission}
+    set -o pipefail
+    kubectl create -f deploy/${app_filename}/crd.yaml
+    echo "Creating Webhook Server Certs"
+    mkdir -p certs
+    openssl genrsa -out certs/tls.key 2048
+    openssl req -new -key certs/tls.key -out certs/tls.csr -subj "/CN=admission-server.default.svc"
+    openssl x509 -req -extfile <(printf "subjectAltName=DNS:admission-server.default.svc") -in certs/tls.csr -signkey certs/tls.key -out certs/tls.crt
+
+    echo "Creating Webhook Server TLS Secret"
+    kubectl create secret tls admission-server-tls \
+        --cert "certs/tls.crt" \
+        --key "certs/tls.key"
+    echo "Creating Webhook Server Deployment"
+    sed -e 's@${APP}@'"${app}-admission-controller"'@g' <"e2e/manifests/admission_server.yaml" | kubectl create -f -
+    CA_PEM64="$(openssl base64 -A < certs/tls.crt)"
+    echo "Creating K8s Webhooks"
+    sed -e 's@${CA_PEM_B64}@'"$CA_PEM64"'@g' -e 's@${RESOURCE}@'"${app#v2-}"s'@g' <"e2e/manifests/admission_webhooks.yaml" | kubectl create -f -
+    exit 0
+fi
+
+if cd deploy/$app_filename && { for crd in $(ls crd*.yaml); do kubectl create -f "$crd"; done } && kubectl apply -f rbac.yaml && kubectl apply -f deploy_$registry.yaml; then
     echo ""
     echo -e "${GREEN}The $app controller is deployed in your Kubernetes cluster in namespace \"$app\".${NC}"
     echo -e "${GREEN}Run \"kubectl get pod -n $app\" to check the controller pod.${NC}"

deploy/vdeployment/crd.yaml → deploy/v2_vdeployment/crd.yaml

@@ -9,7 +9,7 @@ spec:
     kind: VDeployment
     plural: vdeployments
     shortNames:
-    - vrs
+    - vd
     singular: vdeployment
   scope: Namespaced
   versions:
@@ -4743,3 +4743,4 @@ spec:
     served: true
     storage: true
     subresources: {}
+

deploy/v2_vdeployment/crd_vreplicaset.yaml

@@ -0,0 +1 @@
+../v2_vreplicaset/crd.yaml

deploy/v2_vdeployment/deploy_local.yaml

@@ -0,0 +1,45 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: v2-vdeployment-controller
+  namespace: v2-vdeployment
+  labels:
+    app.kubernetes.io/name: v2-vdeployment-controller
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: v2-vdeployment-controller
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: v2-vdeployment-controller
+    spec:
+      containers:
+        - image: local/v2-vdeployment-controller:v0.1.0
+          imagePullPolicy: IfNotPresent
+          name: controller
+      serviceAccountName: v2-vdeployment-controller
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: v2-vreplicaset-controller
+  namespace: v2-vdeployment
+  labels:
+    app.kubernetes.io/name: v2-vreplicaset-controller
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: v2-vreplicaset-controller
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: v2-vreplicaset-controller
+    spec:
+      containers:
+        - image: local/v2-vreplicaset-controller:v0.1.0
+          imagePullPolicy: IfNotPresent
+          name: controller
+      serviceAccountName: v2-vdeployment-controller

deploy/v2_vdeployment/deploy_remote.yaml

@@ -0,0 +1,43 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: v2-vdeployment-controller
+  namespace: v2-vdeployment
+  labels:
+    app.kubernetes.io/name: v2-vdeployment-controller
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: v2-vdeployment-controller
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: v2-vdeployment-controller
+    spec:
+      containers:
+        - image: ghcr.io/anvil-verifier/anvil/v2-vdeployment-controller:latest
+          name: controller
+      serviceAccountName: v2-vdeployment-controller
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: v2-vreplicaset-controller
+  namespace: v2-vdeployment
+  labels:
+    app.kubernetes.io/name: v2-vreplicaset-controller
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: v2-vreplicaset-controller
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: v2-vreplicaset-controller
+    spec:
+      containers:
+        - image: ghcr.io/anvil-verifier/anvil/v2-vreplicaset-controller:latest
+          name: controller
+      serviceAccountName: v2-vdeployment-controller

deploy/v2_vdeployment/rbac.yaml

@@ -0,0 +1,55 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    app.kubernetes.io/name: v2-vdeployment
+  name: v2-vdeployment
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: v2-vdeployment-controller
+  namespace: v2-vdeployment
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/name: v2-vdeployment-controller
+  name: v2-vdeployment-controller-role
+rules:
+  - apiGroups:
+      - anvil.dev
+    resources:
+      - "*"
+    verbs:
+      - "*"
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - replicasets
+      - services
+      - endpoints
+      - persistentvolumeclaims
+      - events
+      - configmaps
+      - secrets
+      - serviceaccounts
+    verbs:
+      - "*"
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/name: v2-vdeployment-controller
+  name: v2-vdeployment-controller-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: v2-vdeployment-controller-role
+subjects:
+  - kind: ServiceAccount
+    name: v2-vdeployment-controller
+    namespace: v2-vdeployment

deploy/v2_vdeployment/v2_vdeployment.yaml

@@ -0,0 +1,19 @@
+apiVersion: anvil.dev/v1
+kind: VDeployment
+metadata:
+  name: pause-deployment
+  labels:
+    app: pause-demo
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: pause-demo
+  template:
+    metadata:
+      labels:
+        app: pause-demo
+    spec:
+      containers:
+      - name: pause
+        image: k8s.gcr.io/pause:3.9

discussion/kubernetes-model/deployment_controller.md

@@ -0,0 +1,30 @@
+# Deployment Controller (DC)
+
+## Reconciliation Login
+
+> In `pkg/controller/deployment/deployment_controller.go`
+
+Reconciliation is performed by `syncDeployment`, which can be modeled as state machine:
+
+0. Orphaning, adoption. This one is not considered in the model
+
+1. [List all replicasets](https://github.com/kubernetes/kubernetes/blob/cdc807a9e849b651fb48c962cc18e25d39ec5edf/pkg/controller/deployment/deployment_controller.go#L629) (RS) owned by this DC from API Server
+
+2. [Get all pods](https://github.com/kubernetes/kubernetes/blob/cdc807a9e849b651fb48c962cc18e25d39ec5edf/pkg/controller/deployment/deployment_controller.go#L638) managed by controlled RS and create a RS-pod map from API Server
+
+3. Make decision on how to manage controlled RS:
+   x. Rollback, not considered in the model
+   Compare managed RS template and DC template,
+   **A**. if the number of replica is different, this is a [scaling event](https://github.com/kubernetes/kubernetes/blob/cdc807a9e849b651fb48c962cc18e25d39ec5edf/pkg/controller/deployment/deployment_controller.go#L665), just scale up/down the single newest controlled RS
+   **B**. The application managed* by DC should be updated (`spec.template.container`). Based on the configured update policy**:
+
+   ​	I. [rollout](https://github.com/kubernetes/kubernetes/blob/cdc807a9e849b651fb48c962cc18e25d39ec5edf/pkg/controller/deployment/rolling.go#L31): (Create if non-existing and) scale up new RS, then scale down old RS
+   ​	II. [recreate](https://github.com/kubernetes/kubernetes/blob/cdc807a9e849b651fb48c962cc18e25d39ec5edf/pkg/controller/deployment/recreate.go#L29): Scale down old RS, then (create if non-existing and) scale up the new RS.
+
+---
+
+*: The deployment controller will only control at most one new replica set, and multiple old replica sets
+
+> In k8s implementation it's [possible](https://github.com/kubernetes/kubernetes/blob/cdc807a9e849b651fb48c962cc18e25d39ec5edf/pkg/controller/deployment/util/deployment_util.go#L633-L634) to have multiple new replica sets in rare cases
+
+**: The scale up/down process should satisfy `maxSurge` and `maxUnavailable` properties of DC, which means DC may not scale the controlled new & old RS to desired replicas in one step. Instead DC will scale up & down gradually in turn. Currently we do not model this feature.