SPLAT-2295: Setup additional disks via machine configs

go.mod

@@ -85,7 +85,7 @@ require (
 	github.com/nutanix-cloud-native/cluster-api-provider-nutanix v1.5.4-0.20250116153252-296a5347104c
 	github.com/nutanix-cloud-native/prism-go-client v0.5.0
 	github.com/onsi/gomega v1.36.2
-	github.com/openshift/api v0.0.0-20250527072845-f5e205b58365
+	github.com/openshift/api v0.0.0-20250704153732-ad766c4e6d8e
 	github.com/openshift/assisted-image-service v0.0.0-20240607085136-02df2e56dde6
 	github.com/openshift/assisted-service/api v0.0.0
 	github.com/openshift/assisted-service/client v0.0.0

go.sum

@@ -722,8 +722,8 @@ github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQ
 github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
 github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk=
 github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/openshift/api v0.0.0-20250527072845-f5e205b58365 h1:WfJTorFO5mJP6DLhK84K83TWuSqmeC3jCN436stKRZk=
-github.com/openshift/api v0.0.0-20250527072845-f5e205b58365/go.mod h1:yk60tHAmHhtVpJQo3TwVYq2zpuP70iJIFDCmeKMIzPw=
+github.com/openshift/api v0.0.0-20250704153732-ad766c4e6d8e h1:9hzClGu+YAziWP6X93S/To86Q6P8aIfAeasl7zIPYA8=
+github.com/openshift/api v0.0.0-20250704153732-ad766c4e6d8e/go.mod h1:yk60tHAmHhtVpJQo3TwVYq2zpuP70iJIFDCmeKMIzPw=
 github.com/openshift/assisted-image-service v0.0.0-20240607085136-02df2e56dde6 h1:U6ve+dnHlHhAELoxX+rdFOHVhoaYl0l9qtxwYtsO6C0=
 github.com/openshift/assisted-image-service v0.0.0-20240607085136-02df2e56dde6/go.mod h1:o2H5VwQhUD8P6XsK6dRmKpCCJqVvv12KJQZBXmcCXCU=
 github.com/openshift/assisted-service v1.0.10-0.20230830164851-6573b5d7021d h1:CKw2Y4EdaFsMoqAdr2Tq0nlYTaaXmCRdP0gOu7pN64U=

pkg/asset/machines/machineconfig/disks.go

@@ -0,0 +1,164 @@
+package machineconfig
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+	"text/template"
+
+	igntypes "github.com/coreos/ignition/v2/config/v3_2/types"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/utils/ptr"
+
+	mcfgv1 "github.com/openshift/api/machineconfiguration/v1"
+	"github.com/openshift/installer/pkg/asset/ignition"
+	"github.com/openshift/installer/pkg/types"
+)
+
+// DiskMountUnit is used to supply the template the proper fields to produce the unit string.
+type DiskMountUnit struct {
+	MountPath string
+	Label     string
+}
+
+const diskMountUnit = `
+[Unit]
+Requires=systemd-fsck@dev-disk-by\x2dpartlabel-{{.Label}}.service
+After=systemd-fsck@dev-disk-by\x2dpartlabel-{{.Label}}.service
+
+[Mount]
+Where={{.MountPath}}
+What=/dev/disk/by-partlabel/{{.Label}}
+Type=xfs
+Options=defaults,prjquota
+
+[Install]
+RequiredBy=local-fs.target
+`
+
+const swapMountUnit = `
+[Swap]
+What=/dev/disk/by-partlabel/{{.Label}}
+`
+
+// ForDiskSetup generates a machine config for the three disk setup types, etcd, swap or user-defined.
+func ForDiskSetup(role, device, label, path string, diskType types.DiskType) (*mcfgv1.MachineConfig, error) {
+	ignConfig := igntypes.Config{
+		Ignition: igntypes.Ignition{
+			Version: igntypes.MaxVersion.String(),
+		},
+	}
+
+	// todo: jcallen: the label needs to be sanitized for special characters
+
+	mountUnit := DiskMountUnit{
+		MountPath: path,
+		Label:     label,
+	}
+
+	var templateStringToParse string
+	switch diskType {
+	case types.Etcd, types.UserDefined:
+		templateStringToParse = diskMountUnit
+	case types.Swap:
+		templateStringToParse = swapMountUnit
+	}
+
+	diskMountUnitTemplate, err := template.New("mountUnit").Parse(templateStringToParse)
+	if err != nil {
+		return nil, err
+	}
+
+	var dmu bytes.Buffer
+	err = diskMountUnitTemplate.Execute(&dmu, mountUnit)
+	if err != nil {
+		return nil, err
+	}
+
+	units := dmu.String()
+
+	var rawExt runtime.RawExtension
+	switch diskType {
+	case types.Etcd, types.UserDefined:
+		rawExt, err = getDiskIgnition(ignConfig, device, label, path, units)
+		if err != nil {
+			return nil, err
+		}
+	case types.Swap:
+		rawExt, err = getSwapIgnition(ignConfig, device, label, units)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return &mcfgv1.MachineConfig{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: mcfgv1.SchemeGroupVersion.String(),
+			Kind:       "MachineConfig",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: fmt.Sprintf("01-disk-setup-%s-%s", label, role),
+			Labels: map[string]string{
+				"machineconfiguration.openshift.io/role": role,
+			},
+		},
+		Spec: mcfgv1.MachineConfigSpec{
+			Config: rawExt,
+		},
+	}, nil
+}
+func getDiskIgnition(ignConfig igntypes.Config, device, label, path, units string) (runtime.RawExtension, error) {
+	unitName := strings.Trim(path, "/")
+	unitName = strings.ReplaceAll(unitName, "/", "-")
+
+	ignConfig.Storage.Disks = append(ignConfig.Storage.Disks, igntypes.Disk{
+		Device: device,
+		Partitions: []igntypes.Partition{{
+			Label:    ptr.To(label),
+			StartMiB: ptr.To(0),
+			SizeMiB:  ptr.To(0),
+		}},
+		WipeTable: ptr.To(true),
+	})
+
+	ignConfig.Storage.Filesystems = append(ignConfig.Storage.Filesystems, igntypes.Filesystem{
+		Device:         fmt.Sprintf("/dev/disk/by-partlabel/%s", label),
+		Format:         ptr.To("xfs"),
+		Label:          ptr.To(label),
+		MountOptions:   []igntypes.MountOption{"defaults", "prjquota"},
+		Path:           ptr.To(path),
+		WipeFilesystem: ptr.To(true),
+	})
+	ignConfig.Systemd.Units = append(ignConfig.Systemd.Units, igntypes.Unit{
+		Name:     fmt.Sprintf("%s.mount", unitName),
+		Enabled:  ptr.To(true),
+		Contents: &units,
+	})
+	return ignition.ConvertToRawExtension(ignConfig)
+}
+
+func getSwapIgnition(ignConfig igntypes.Config, device, label, units string) (runtime.RawExtension, error) {
+	unitName := "dev-disk-byx2dpartlabel-swap.swap"
+	ignConfig.Storage.Disks = append(ignConfig.Storage.Disks, igntypes.Disk{
+		Device: device,
+		Partitions: []igntypes.Partition{{
+			Label:    ptr.To(label),
+			StartMiB: ptr.To(0),
+			SizeMiB:  ptr.To(0),
+		}},
+		WipeTable: ptr.To(true),
+	})
+
+	ignConfig.Storage.Filesystems = append(ignConfig.Storage.Filesystems, igntypes.Filesystem{
+		Device: fmt.Sprintf("/dev/disk/by-partlabel/%s", label),
+		Format: ptr.To("swap"),
+		Label:  ptr.To(label),
+	})
+	ignConfig.Systemd.Units = append(ignConfig.Systemd.Units, igntypes.Unit{
+		Name:     fmt.Sprintf("%s.mount", unitName),
+		Enabled:  ptr.To(true),
+		Contents: &units,
+	})
+	return ignition.ConvertToRawExtension(ignConfig)
+}

pkg/asset/machines/master.go

@@ -20,6 +20,7 @@ import (
 	"sigs.k8s.io/yaml"
 
 	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/api/features"
 	machinev1 "github.com/openshift/api/machine/v1"
 	machinev1alpha1 "github.com/openshift/api/machine/v1alpha1"
 	machinev1beta1 "github.com/openshift/api/machine/v1beta1"
@@ -615,6 +616,24 @@ func (m *Master) Generate(ctx context.Context, dependencies asset.Parents) error
 		machineConfigs = append(machineConfigs, ignIPv6)
 	}
 
+	if installConfig.Config.EnabledFeatureGates().Enabled(features.FeatureGateMultiDiskSetup) {
+		for i, diskSetup := range installConfig.Config.ControlPlane.DiskSetup {
+			var dataDisk any
+			switch ic.Platform.Name() {
+			case azuretypes.Name:
+				azureControlPlaneMachinePool := ic.ControlPlane.Platform.Azure
+				dataDisk = azureControlPlaneMachinePool.DataDisks[i]
+			case vspheretypes.Name:
+				// todo, putting it here to get the linter to stop bugging me
+			}
+			diskSetupIgn, err := NodeDiskSetup(installConfig, "master", diskSetup, dataDisk)
+			if err != nil {
+				return errors.Wrap(err, "failed to create ignition to setup disks for control plane")
+			}
+			machineConfigs = append(machineConfigs, diskSetupIgn)
+		}
+	}
+
 	m.MachineConfigFiles, err = machineconfig.Manifests(machineConfigs, "master", directory)
 	if err != nil {
 		return errors.Wrap(err, "failed to create MachineConfig manifests for master machines")

pkg/asset/machines/util.go

@@ -0,0 +1,58 @@
+package machines
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
+
+	"github.com/openshift/api/features"
+	v1 "github.com/openshift/api/machineconfiguration/v1"
+	"github.com/openshift/installer/pkg/asset/installconfig"
+	"github.com/openshift/installer/pkg/asset/machines/machineconfig"
+	"github.com/openshift/installer/pkg/types"
+	azuretypes "github.com/openshift/installer/pkg/types/azure"
+	vspheretypes "github.com/openshift/installer/pkg/types/vsphere"
+)
+
+// NodeDiskSetup determines the path per disk type, and per platform and role, runs ForDiskSetup.
+func NodeDiskSetup(installConfig *installconfig.InstallConfig, role string, diskSetup types.Disk, dataDisk any) (*v1.MachineConfig, error) {
+	var path string
+
+	ic := installConfig.Config
+
+	label := string(diskSetup.Type)
+
+	switch diskSetup.Type {
+	case types.Etcd:
+		path = "/var/lib/etcd"
+	case types.Swap:
+		path = ""
+	case types.UserDefined:
+		path = diskSetup.UserDefined.MountPath
+		label = diskSetup.UserDefined.PlatformDiskID
+	}
+
+	switch ic.Platform.Name() {
+	case azuretypes.Name:
+		logrus.Warn("in nodedisksetup")
+		if installConfig.Config.EnabledFeatureGates().Enabled(features.FeatureGateAzureMultiDisk) {
+			if azureDataDisk, ok := dataDisk.(v1beta1.DataDisk); ok {
+				device := fmt.Sprintf("/dev/disk/azure/scsi1/lun%d", *azureDataDisk.Lun)
+
+				logrus.Warnf("lun %d, role %s, type %s", *azureDataDisk.Lun, role, string(diskSetup.Type))
+
+				diskSetupIgn, err := machineconfig.ForDiskSetup(role, device, label, path, diskSetup.Type)
+				if err != nil {
+					return nil, errors.Wrap(err, "failed to create ignition to setup disks for master machines")
+				}
+				return diskSetupIgn, nil
+			}
+			return nil, errors.Errorf("unsupported azure data disk type")
+		}
+	case vspheretypes.Name:
+		// adding it here to avoid the linter and this will be defined later
+	}
+	return nil, nil
+}

pkg/asset/machines/worker.go

@@ -18,6 +18,7 @@ import (
 	"sigs.k8s.io/yaml"
 
 	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/api/features"
 	machinev1 "github.com/openshift/api/machine/v1"
 	machinev1alpha1 "github.com/openshift/api/machine/v1alpha1"
 	machinev1beta1 "github.com/openshift/api/machine/v1beta1"
@@ -372,6 +373,29 @@ func (w *Worker) Generate(ctx context.Context, dependencies asset.Parents) error
 				}
 				machineConfigs = append(machineConfigs, ignRoutes)
 			}
+
+		}
+		logrus.Warn("before worker disksetup")
+		if installConfig.Config.EnabledFeatureGates().Enabled(features.FeatureGateMultiDiskSetup) {
+
+			logrus.Warn("in disksetup")
+			for i, diskSetup := range pool.DiskSetup {
+
+				var dataDisk any
+				switch ic.Platform.Name() {
+				// Each platform has their unique dataDisk type
+				case azuretypes.Name:
+					dataDisk = pool.Platform.Azure.DataDisks[i]
+				case vspheretypes.Name:
+					// todo, putting it here to get the linter to stop bugging me
+				}
+
+				diskSetupIgn, err := NodeDiskSetup(installConfig, "worker", diskSetup, dataDisk)
+				if err != nil {
+					return errors.Wrap(err, "failed to create ignition to setup disks for compute")
+				}
+				machineConfigs = append(machineConfigs, diskSetupIgn)
+			}
 		}
 		// The maximum number of networks supported on ServiceNetwork is two, one IPv4 and one IPv6 network.
 		// The cluster-network-operator handles the validation of this field.

pkg/types/azure/validation/machinepool.go

@@ -110,7 +110,7 @@ func ValidateMachinePool(p *azure.MachinePool, poolName string, platform *azure.
 	}
 
 	if pool != nil {
-		if len(p.DataDisks) != 0 {
+		if len(p.DataDisks) != 0 && len(pool.DiskSetup) != 0 {
 			allErrs = append(allErrs, validateDataDiskSetup(p, pool, fldPath.Child("dataDisks"))...)
 		}
 	}
@@ -124,7 +124,19 @@ func ValidateMachinePool(p *azure.MachinePool, poolName string, platform *azure.
 func validateDataDiskSetup(azurePool *azure.MachinePool, pool *types.MachinePool, fldPath *field.Path) field.ErrorList {
 	var allErrs field.ErrorList
 
-	for _, d := range azurePool.DataDisks {
+	if len(azurePool.DataDisks) > len(pool.DiskSetup) {
+		allErrs = append(allErrs, field.TooLong(fldPath, azurePool.DataDisks, len(pool.DiskSetup)))
+	} else if len(azurePool.DataDisks) < len(pool.DiskSetup) {
+		allErrs = append(allErrs, field.TooLong(fldPath, pool.DiskSetup, len(azurePool.DataDisks)))
+	}
+
+	// return early if disksetup and datadisks don't match lengths
+	if allErrs != nil {
+		return allErrs
+	}
+
+	for i, d := range azurePool.DataDisks {
+		setup := pool.DiskSetup[i]
 		if d.Lun == nil {
 			allErrs = append(allErrs, field.Required(fldPath.Child("Lun"), fmt.Sprintf("%s must have lun id", d.NameSuffix)))
 		} else if *(d.Lun) < 0 || *(d.Lun) > 63 {
@@ -134,6 +146,21 @@ func validateDataDiskSetup(azurePool *azure.MachinePool, pool *types.MachinePool
 		if d.DiskSizeGB == 0 {
 			allErrs = append(allErrs, field.Invalid(fldPath.Child("DiskSizeGB"), d.DiskSizeGB, "diskSizeGB must be greater than zero"))
 		}
+
+		switch setup.Type {
+		case types.Etcd:
+			if setup.Etcd.PlatformDiskID != d.NameSuffix {
+				allErrs = append(allErrs, field.Invalid(fldPath.Child("NameSuffix"), d.NameSuffix, fmt.Sprintf("does not match etcd PlatformDiskID %s", setup.Etcd.PlatformDiskID)))
+			}
+		case types.Swap:
+			if setup.Swap.PlatformDiskID != d.NameSuffix {
+				allErrs = append(allErrs, field.Invalid(fldPath.Child("NameSuffix"), d.NameSuffix, fmt.Sprintf("does not match swap PlatformDiskID %s", setup.Swap.PlatformDiskID)))
+			}
+		case types.UserDefined:
+			if setup.UserDefined.PlatformDiskID != d.NameSuffix {
+				allErrs = append(allErrs, field.Invalid(fldPath.Child("NameSuffix"), d.NameSuffix, fmt.Sprintf("does not match user defined PlatformDiskID %s", setup.UserDefined.PlatformDiskID)))
+			}
+		}
 	}
 
 	return allErrs