Add ceph-provider and rook-managed ceph cluster

Makefile

@@ -39,6 +39,12 @@ setup-network: metalbond metalbond-client dpservice metalnet ## Customize the ne
 	$(KUBECTL) rollout status daemonset/dpservice -n dpservice-system --timeout=360s && \
 	$(KIND) get nodes | xargs -I {} sh -c '$(CRE) cp hack/setup-network.sh {}:/setup-network.sh && $(CRE) exec {} bash -c "bash /setup-network.sh"'
 
+setup-storage:
+	$(KIND) get nodes | xargs -I {} sh -c '$(CRE) cp hack/setup-storage.sh {}:/setup-storage.sh && $(CRE) exec {} bash -c "bash /setup-storage.sh"'
+
+cleanup-storage:
+	$(KIND) get nodes | xargs -I {} sh -c '$(CRE) cp hack/cleanup-storage.sh {}:/cleanup-storage.sh && $(CRE) exec {} bash -c "bash /cleanup-storage.sh"'
+
 delete: ## Delete the kind cluster
 	$(KIND) delete cluster
 
@@ -73,6 +79,15 @@ dpservice: kubectl ## Install dpservice
 metalnet: kubectl ## Install metalnet
 	$(KUBECTL) apply -k cluster/local/metalnet
 
+rook: kubectl setup-storage ## Install rook
+	$(KUBECTL) apply -k cluster/local/rook
+	$(KUBECTL) apply -k cluster/local/rook-cluster
+
+ceph-volume-provider: kubectl ## Install the ceph-volume-provider
+	$(KUBECTL) apply -k cluster/local/ceph-volume-provider
+
+volumepoollet-broker: kubectl ## Install the ceph-volume-provider
+	$(KUBECTL) apply -k cluster/local/volumepoollet-broker
 
 libvirt-provider: kubectl ## Install the libvirt-provider
 	$(KUBECTL) apply -k cluster/local/libvirt-provider
@@ -104,6 +119,16 @@ remove-dpservice: kubectl ## Remove dpservice
 remove-metalnet: kubectl ## Remove metalnet
 	$(KUBECTL) delete -k cluster/local/metalnet
 
+remove-rook: kubectl cleanup-storage ## Remove rook
+	$(KUBECTL) delete -k cluster/local/rook-cluster
+	$(KUBECTL) delete -k cluster/local/rook
+
+remove-ceph-volume-provider: kubectl ## Remove the ceph-volume-provider
+	$(KUBECTL) delete -k cluster/local/ceph-volume-provider
+
+remove-volumepoollet-broker: kubectl ## Remove the volumepoollet-broker
+	$(KUBECTL) delete -k cluster/local/volumepoollet-broker
+
 remove-libvirt-provider: kubectl ## Remove libvirt-provider
 	$(KUBECTL) delete -k cluster/local/libvirt-provider
 

base/ceph-volume-provider/kustomization.yaml

@@ -0,0 +1,25 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: ceph-volume-provider-system
+
+resources:
+  - poollet-rbac
+  - [email protected]/ironcore-dev/ceph-provider/config/ceph-volume-provider/default?ref=b0d8926754a53510dd0a7dd4d7a9c69de4e128f2
+
+images:
+  - name: volumepoollet
+    newName: ghcr.io/ironcore-dev/ironcore-volumepoollet
+    newTag: sha-8a0b30c
+  - name: ceph-volume-provider
+    newName: ghcr.io/ironcore-dev/ceph-volume-provider
+    newTag: sha-ada684d
+
+#patches:
+#  - path: patch-manager-deployment.yaml
+#  - patch: |-
+#      apiVersion: v1
+#      kind: Namespace
+#      metadata:
+#        name: ceph-volume-provider-system
+#      $patch: delete

base/ceph-volume-provider/patch-manager-deployment.yaml

@@ -0,0 +1,38 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      # TODO: fix later when RBAC config is refactored in ceph-provider
+      serviceAccountName: ceph-volume-provider-controller-manager
+      containers:
+        - name: manager
+          resources:
+            limits:
+              cpu: 200m
+              memory: 256Mi
+        - name: ceph-volume-provider
+          resources:
+            limits:
+              cpu: 200m
+              memory: 256Mi
+          volumeMounts:
+            - mountPath: /var/cfg/classes
+              name: supported-volume-classes
+            - mountPath: /var/cfg/ceph/
+              name: ceph-keyring
+            - mountPath: /var/cfg/kek
+              name: ceph-kek
+      volumes:
+        - name: supported-volume-classes
+          configMap:
+              name: supported-volume-classes
+        - name: ceph-keyring
+          secret:
+            secretName: rook-ceph-admin-keyring
+        - name: ceph-kek
+          secret:
+            secretName: ceph-kek

base/ceph-volume-provider/poollet-rbac/kustomization.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namePrefix: ceph-volume-provider-
+
+resources:
+  # TODO: remove once RBAC in ceph provider is cleaned up
+  - leader_election_role.yaml
+  - leader_election_role_binding.yaml
+  - github.com/ironcore-dev/ironcore/config/volumepoollet-broker/poollet-rbac?ref=v0.2.0

base/ceph-volume-provider/poollet-rbac/leader_election_role.yaml

@@ -0,0 +1,37 @@
+# permissions to do leader election.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: leader-election-role
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - patch
+      - delete
+  - apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - patch
+      - delete
+  - apiGroups:
+      - ""
+    resources:
+      - events
+    verbs:
+      - create
+      - patch

base/ceph-volume-provider/poollet-rbac/leader_election_role_binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: leader-election-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: leader-election-role
+subjects:
+  - kind: ServiceAccount
+    name: controller-manager
+    namespace: ceph-volume-provider-system

base/rook-cluster/cluster.yaml

@@ -0,0 +1,62 @@
+apiVersion: ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: rook-ceph
+  namespace: rook-ceph # namespace:cluster
+spec:
+  dataDirHostPath: /var/lib/rook
+  cephVersion:
+    image: quay.io/ceph/ceph:v19
+    allowUnsupported: true
+  mon:
+    count: 1
+    allowMultiplePerNode: true
+  # test environments can skip ok-to-stop checks during upgrades
+  skipUpgradeChecks: true
+  mgr:
+    count: 1
+    allowMultiplePerNode: true
+    modules:
+      - name: rook
+        enabled: true
+  dashboard:
+    enabled: true
+  crashCollector:
+    disable: true
+  storage:
+    useAllNodes: true
+    useAllDevices: false
+    allowDeviceClassUpdate: true
+    allowOsdCrushWeightUpdate: false
+    devices:
+      - name: /dev/loop0
+  monitoring:
+    enabled: false
+  healthCheck:
+    daemonHealth:
+      mon:
+        interval: 45s
+        timeout: 600s
+  priorityClassNames:
+    all: system-node-critical
+    mgr: system-cluster-critical
+  disruptionManagement:
+    managePodBudgets: true
+  cephConfig:
+    global:
+      osd_pool_default_size: "1"
+      mon_warn_on_pool_no_redundancy: "false"
+      bdev_flock_retry: "20"
+      bluefs_buffered_io: "false"
+      mon_data_avail_warn: "10"
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: builtin-mgr
+  namespace: rook-ceph # namespace:cluster
+spec:
+  name: .mgr
+  replicated:
+    size: 1
+    requireSafeReplicaSize: false

base/rook-cluster/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - cluster.yaml
+  - https://raw.githubusercontent.com/rook/rook/v1.18.4/deploy/examples/toolbox.yaml

base/rook/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - https://raw.githubusercontent.com/rook/rook/v1.18.4/deploy/examples/crds.yaml
+  - https://raw.githubusercontent.com/rook/rook/v1.18.4/deploy/examples/common.yaml
+  - https://raw.githubusercontent.com/rook/rook/v1.18.4/deploy/examples/csi-operator.yaml
+  - https://raw.githubusercontent.com/rook/rook/v1.18.4/deploy/examples/operator.yaml
+
+patches:
+  - path: patch-operator.yaml

base/rook/patch-operator.yaml

@@ -0,0 +1,7 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: rook-ceph-operator-config
+  namespace: rook-ceph
+data:
+  ROOK_CEPH_ALLOW_LOOP_DEVICES: "true"

base/volumepoollet-broker/kustomization.yaml

@@ -0,0 +1,14 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - [email protected]/ironcore-dev/ironcore/config/volumepoollet-broker/default?ref=07323fc92ea6da00acc0b9ef1c0b5502ff591b6b
+
+images:
+  - name: volumepoollet
+    newName: ghcr.io/ironcore-dev/ironcore-volumepoollet
+    digest: sha256:1b4a354edaca8426cab1c50ecbe545fe47322de43135a3ae07638c577cb72fdb
+  - name: volumebroker
+    newName: ghcr.io/ironcore-dev/ironcore-volumebroker
+    digest: sha256:c67095cfe874ea9e569c5a7ecafd05d46d9602de2a6f8d5ebd35a598e179a18f
+    #digest: sha256:fa56521ccbca4e15335648221a323dda96496e683ac347f85742f7c398dae533

cluster/local/ceph-volume-provider/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../base/ceph-volume-provider

cluster/local/rook-cluster/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../base/rook-cluster

cluster/local/rook/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../base/rook

cluster/local/volumepoollet-broker/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../base/volumepoollet-broker

hack/cleanup-storage.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -e
+
+echo "Delete storage disks..."
+losetup -d /dev/loop0
+rm /var/tmp/osd-disk

hack/setup-storage.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+echo "Prepare storage disks..."
+# OSD disk must have a minimum size of 5GB
+dd if=/dev/zero of=/var/tmp/osd-disk bs=1M count=5120
+losetup /dev/loop0 /var/tmp/osd-disk