diff --git a/Makefile b/Makefile index e2b08ed..62576a0 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,12 @@ setup-network: metalbond metalbond-client dpservice metalnet ## Customize the ne $(KUBECTL) rollout status daemonset/dpservice -n dpservice-system --timeout=360s && \ $(KIND) get nodes | xargs -I {} sh -c '$(CRE) cp hack/setup-network.sh {}:/setup-network.sh && $(CRE) exec {} bash -c "bash /setup-network.sh"' +setup-storage: + $(KIND) get nodes | xargs -I {} sh -c '$(CRE) cp hack/setup-storage.sh {}:/setup-storage.sh && $(CRE) exec {} bash -c "bash /setup-storage.sh"' + +cleanup-storage: + $(KIND) get nodes | xargs -I {} sh -c '$(CRE) cp hack/cleanup-storage.sh {}:/cleanup-storage.sh && $(CRE) exec {} bash -c "bash /cleanup-storage.sh"' + delete: ## Delete the kind cluster $(KIND) delete cluster @@ -73,6 +79,15 @@ dpservice: kubectl ## Install dpservice metalnet: kubectl ## Install metalnet $(KUBECTL) apply -k cluster/local/metalnet +rook: kubectl setup-storage ## Install rook + $(KUBECTL) apply -k cluster/local/rook + $(KUBECTL) apply -k cluster/local/rook-cluster + +ceph-volume-provider: kubectl ## Install the ceph-volume-provider + $(KUBECTL) apply -k cluster/local/ceph-volume-provider + +volumepoollet-broker: kubectl ## Install the ceph-volume-provider + $(KUBECTL) apply -k cluster/local/volumepoollet-broker libvirt-provider: kubectl ## Install the libvirt-provider $(KUBECTL) apply -k cluster/local/libvirt-provider @@ -104,6 +119,16 @@ remove-dpservice: kubectl ## Remove dpservice remove-metalnet: kubectl ## Remove metalnet $(KUBECTL) delete -k cluster/local/metalnet +remove-rook: kubectl cleanup-storage ## Remove rook + $(KUBECTL) delete -k cluster/local/rook-cluster + $(KUBECTL) delete -k cluster/local/rook + +remove-ceph-volume-provider: kubectl ## Remove the ceph-volume-provider + $(KUBECTL) delete -k cluster/local/ceph-volume-provider + +remove-volumepoollet-broker: kubectl ## Remove the volumepoollet-broker + $(KUBECTL) delete -k cluster/local/volumepoollet-broker + remove-libvirt-provider: kubectl ## Remove libvirt-provider $(KUBECTL) delete -k cluster/local/libvirt-provider diff --git a/base/ceph-volume-provider/kustomization.yaml b/base/ceph-volume-provider/kustomization.yaml new file mode 100644 index 0000000..4d11839 --- /dev/null +++ b/base/ceph-volume-provider/kustomization.yaml @@ -0,0 +1,25 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: ceph-volume-provider-system + +resources: + - poollet-rbac + - git@github.com/ironcore-dev/ceph-provider/config/ceph-volume-provider/default?ref=b0d8926754a53510dd0a7dd4d7a9c69de4e128f2 + +images: + - name: volumepoollet + newName: ghcr.io/ironcore-dev/ironcore-volumepoollet + newTag: sha-8a0b30c + - name: ceph-volume-provider + newName: ghcr.io/ironcore-dev/ceph-volume-provider + newTag: sha-ada684d + +#patches: +# - path: patch-manager-deployment.yaml +# - patch: |- +# apiVersion: v1 +# kind: Namespace +# metadata: +# name: ceph-volume-provider-system +# $patch: delete \ No newline at end of file diff --git a/base/ceph-volume-provider/patch-manager-deployment.yaml b/base/ceph-volume-provider/patch-manager-deployment.yaml new file mode 100644 index 0000000..449e8b3 --- /dev/null +++ b/base/ceph-volume-provider/patch-manager-deployment.yaml @@ -0,0 +1,38 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + # TODO: fix later when RBAC config is refactored in ceph-provider + serviceAccountName: ceph-volume-provider-controller-manager + containers: + - name: manager + resources: + limits: + cpu: 200m + memory: 256Mi + - name: ceph-volume-provider + resources: + limits: + cpu: 200m + memory: 256Mi + volumeMounts: + - mountPath: /var/cfg/classes + name: supported-volume-classes + - mountPath: /var/cfg/ceph/ + name: ceph-keyring + - mountPath: /var/cfg/kek + name: ceph-kek + volumes: + - name: supported-volume-classes + configMap: + name: supported-volume-classes + - name: ceph-keyring + secret: + secretName: rook-ceph-admin-keyring + - name: ceph-kek + secret: + secretName: ceph-kek diff --git a/base/ceph-volume-provider/poollet-rbac/kustomization.yaml b/base/ceph-volume-provider/poollet-rbac/kustomization.yaml new file mode 100644 index 0000000..42f02be --- /dev/null +++ b/base/ceph-volume-provider/poollet-rbac/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namePrefix: ceph-volume-provider- + +resources: + # TODO: remove once RBAC in ceph provider is cleaned up + - leader_election_role.yaml + - leader_election_role_binding.yaml + - github.com/ironcore-dev/ironcore/config/volumepoollet-broker/poollet-rbac?ref=v0.2.0 diff --git a/base/ceph-volume-provider/poollet-rbac/leader_election_role.yaml b/base/ceph-volume-provider/poollet-rbac/leader_election_role.yaml new file mode 100644 index 0000000..9221419 --- /dev/null +++ b/base/ceph-volume-provider/poollet-rbac/leader_election_role.yaml @@ -0,0 +1,37 @@ +# permissions to do leader election. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: leader-election-role +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch diff --git a/base/ceph-volume-provider/poollet-rbac/leader_election_role_binding.yaml b/base/ceph-volume-provider/poollet-rbac/leader_election_role_binding.yaml new file mode 100644 index 0000000..2878b49 --- /dev/null +++ b/base/ceph-volume-provider/poollet-rbac/leader_election_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: leader-election-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election-role +subjects: + - kind: ServiceAccount + name: controller-manager + namespace: ceph-volume-provider-system diff --git a/base/rook-cluster/cluster.yaml b/base/rook-cluster/cluster.yaml new file mode 100644 index 0000000..887da9a --- /dev/null +++ b/base/rook-cluster/cluster.yaml @@ -0,0 +1,62 @@ +apiVersion: ceph.rook.io/v1 +kind: CephCluster +metadata: + name: rook-ceph + namespace: rook-ceph # namespace:cluster +spec: + dataDirHostPath: /var/lib/rook + cephVersion: + image: quay.io/ceph/ceph:v19 + allowUnsupported: true + mon: + count: 1 + allowMultiplePerNode: true + # test environments can skip ok-to-stop checks during upgrades + skipUpgradeChecks: true + mgr: + count: 1 + allowMultiplePerNode: true + modules: + - name: rook + enabled: true + dashboard: + enabled: true + crashCollector: + disable: true + storage: + useAllNodes: true + useAllDevices: false + allowDeviceClassUpdate: true + allowOsdCrushWeightUpdate: false + devices: + - name: /dev/loop0 + monitoring: + enabled: false + healthCheck: + daemonHealth: + mon: + interval: 45s + timeout: 600s + priorityClassNames: + all: system-node-critical + mgr: system-cluster-critical + disruptionManagement: + managePodBudgets: true + cephConfig: + global: + osd_pool_default_size: "1" + mon_warn_on_pool_no_redundancy: "false" + bdev_flock_retry: "20" + bluefs_buffered_io: "false" + mon_data_avail_warn: "10" +--- +apiVersion: ceph.rook.io/v1 +kind: CephBlockPool +metadata: + name: builtin-mgr + namespace: rook-ceph # namespace:cluster +spec: + name: .mgr + replicated: + size: 1 + requireSafeReplicaSize: false \ No newline at end of file diff --git a/base/rook-cluster/kustomization.yaml b/base/rook-cluster/kustomization.yaml new file mode 100644 index 0000000..3d3ebae --- /dev/null +++ b/base/rook-cluster/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - cluster.yaml + - https://raw.githubusercontent.com/rook/rook/v1.18.4/deploy/examples/toolbox.yaml diff --git a/base/rook/kustomization.yaml b/base/rook/kustomization.yaml new file mode 100644 index 0000000..72596a5 --- /dev/null +++ b/base/rook/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - https://raw.githubusercontent.com/rook/rook/v1.18.4/deploy/examples/crds.yaml + - https://raw.githubusercontent.com/rook/rook/v1.18.4/deploy/examples/common.yaml + - https://raw.githubusercontent.com/rook/rook/v1.18.4/deploy/examples/csi-operator.yaml + - https://raw.githubusercontent.com/rook/rook/v1.18.4/deploy/examples/operator.yaml + +patches: + - path: patch-operator.yaml diff --git a/base/rook/patch-operator.yaml b/base/rook/patch-operator.yaml new file mode 100644 index 0000000..5a1ea9a --- /dev/null +++ b/base/rook/patch-operator.yaml @@ -0,0 +1,7 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: rook-ceph-operator-config + namespace: rook-ceph +data: + ROOK_CEPH_ALLOW_LOOP_DEVICES: "true" \ No newline at end of file diff --git a/base/volumepoollet-broker/kustomization.yaml b/base/volumepoollet-broker/kustomization.yaml new file mode 100644 index 0000000..172e4a1 --- /dev/null +++ b/base/volumepoollet-broker/kustomization.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - git@github.com/ironcore-dev/ironcore/config/volumepoollet-broker/default?ref=07323fc92ea6da00acc0b9ef1c0b5502ff591b6b + +images: + - name: volumepoollet + newName: ghcr.io/ironcore-dev/ironcore-volumepoollet + digest: sha256:1b4a354edaca8426cab1c50ecbe545fe47322de43135a3ae07638c577cb72fdb + - name: volumebroker + newName: ghcr.io/ironcore-dev/ironcore-volumebroker + digest: sha256:c67095cfe874ea9e569c5a7ecafd05d46d9602de2a6f8d5ebd35a598e179a18f + #digest: sha256:fa56521ccbca4e15335648221a323dda96496e683ac347f85742f7c398dae533 diff --git a/cluster/local/ceph-volume-provider/kustomization.yaml b/cluster/local/ceph-volume-provider/kustomization.yaml new file mode 100644 index 0000000..857594d --- /dev/null +++ b/cluster/local/ceph-volume-provider/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../../base/ceph-volume-provider diff --git a/cluster/local/rook-cluster/kustomization.yaml b/cluster/local/rook-cluster/kustomization.yaml new file mode 100644 index 0000000..22393d0 --- /dev/null +++ b/cluster/local/rook-cluster/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../../base/rook-cluster diff --git a/cluster/local/rook/kustomization.yaml b/cluster/local/rook/kustomization.yaml new file mode 100644 index 0000000..5bd2a7e --- /dev/null +++ b/cluster/local/rook/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../../base/rook diff --git a/cluster/local/volumepoollet-broker/kustomization.yaml b/cluster/local/volumepoollet-broker/kustomization.yaml new file mode 100644 index 0000000..b64d7eb --- /dev/null +++ b/cluster/local/volumepoollet-broker/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../../base/volumepoollet-broker diff --git a/hack/cleanup-storage.sh b/hack/cleanup-storage.sh new file mode 100644 index 0000000..d68b0d7 --- /dev/null +++ b/hack/cleanup-storage.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +echo "Delete storage disks..." +losetup -d /dev/loop0 +rm /var/tmp/osd-disk diff --git a/hack/setup-storage.sh b/hack/setup-storage.sh new file mode 100644 index 0000000..4530c3a --- /dev/null +++ b/hack/setup-storage.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -e + +echo "Prepare storage disks..." +# OSD disk must have a minimum size of 5GB +dd if=/dev/zero of=/var/tmp/osd-disk bs=1M count=5120 +losetup /dev/loop0 /var/tmp/osd-disk