machines.go

// Package azure generates Machine objects for azure.
package azure

import (
	"fmt"
	"strings"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	capz "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"

	v1 "github.com/openshift/api/config/v1"
	machinev1 "github.com/openshift/api/machine/v1"
	machineapi "github.com/openshift/api/machine/v1beta1"
	icazure "github.com/openshift/installer/pkg/asset/installconfig/azure"
	"github.com/openshift/installer/pkg/types"
	"github.com/openshift/installer/pkg/types/azure"
	utils "github.com/openshift/installer/pkg/utils"
)

const (
	cloudsSecret          = "azure-cloud-credentials"
	cloudsSecretNamespace = "openshift-machine-api"
	controlPlaneRoleName  = "master"
)

// Machines returns a list of machines for a machinepool.
func Machines(clusterID string, config *types.InstallConfig, pool *types.MachinePool, osImage, role, userDataSecret string, capabilities map[string]string, session *icazure.Session) ([]machineapi.Machine, *machinev1.ControlPlaneMachineSet, error) {
	if configPlatform := config.Platform.Name(); configPlatform != azure.Name {
		return nil, nil, fmt.Errorf("non-Azure configuration: %q", configPlatform)
	}
	if poolPlatform := pool.Platform.Name(); poolPlatform != azure.Name {
		return nil, nil, fmt.Errorf("non-Azure machine-pool: %q", poolPlatform)
	}
	platform := config.Platform.Azure
	mpool := pool.Platform.Azure

	if len(mpool.Zones) == 0 {
		// if no azs are given we set to []string{""} for convenience over later operations.
		// It means no-zoned for the machine API
		mpool.Zones = []string{""}
	}
	azs := mpool.Zones

	total := int64(1)
	if pool.Replicas != nil {
		total = *pool.Replicas
	}
	var machines []machineapi.Machine
	machineSetProvider := &machineapi.AzureMachineProviderSpec{}

	networkResourceGroup, virtualNetworkName, subnets, err := getNetworkInfo(platform, clusterID, role, nil)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to get subnets for role %s : %w", role, err)
	}

	for idx := int64(0); idx < total; idx++ {
		var azIndex int
		if len(azs) > 0 {
			azIndex = int(idx) % len(azs)
		}
		subnetIndex := int(idx) % len(subnets)
		provider, err := provider(platform, mpool, osImage, userDataSecret, clusterID, role, &azIndex, capabilities, session, networkResourceGroup, virtualNetworkName, subnets[subnetIndex])
		if err != nil {
			return nil, nil, errors.Wrap(err, "failed to create provider")
		}
		machine := machineapi.Machine{
			TypeMeta: metav1.TypeMeta{
				APIVersion: "machine.openshift.io/v1beta1",
				Kind:       "Machine",
			},
			ObjectMeta: metav1.ObjectMeta{
				Namespace: "openshift-machine-api",
				Name:      fmt.Sprintf("%s-%s-%d", clusterID, pool.Name, idx),
				Labels: map[string]string{
					"machine.openshift.io/cluster-api-cluster":      clusterID,
					"machine.openshift.io/cluster-api-machine-role": role,
					"machine.openshift.io/cluster-api-machine-type": role,
				},
			},
			Spec: machineapi.MachineSpec{
				ProviderSpec: machineapi.ProviderSpec{
					Value: &runtime.RawExtension{Object: provider},
				},
				// we don't need to set Versions, because we control those via operators.
			},
		}
		*machineSetProvider = *provider
		utils.SetMachineOSStreamLabels(&machine, config)
		machines = append(machines, machine)
	}

	replicas := int32(total)
	failureDomains := []machinev1.AzureFailureDomain{}

	if len(mpool.Zones) > 1 {
		for _, zone := range mpool.Zones {
			domain := machinev1.AzureFailureDomain{
				Zone: zone,
			}

			failureDomains = append(failureDomains, domain)
		}
		machineSetProvider.Zone = ""
	} else if len(mpool.Zones) == 1 {
		machineSetProvider.Zone = mpool.Zones[0]
	}

	controlPlaneMachineSet := &machinev1.ControlPlaneMachineSet{
		TypeMeta: metav1.TypeMeta{
			APIVersion: "machine.openshift.io/v1",
			Kind:       "ControlPlaneMachineSet",
		},
		ObjectMeta: metav1.ObjectMeta{
			Namespace: "openshift-machine-api",
			Name:      "cluster",
			Labels: map[string]string{
				"machine.openshift.io/cluster-api-cluster": clusterID,
			},
		},
		Spec: machinev1.ControlPlaneMachineSetSpec{
			Replicas: &replicas,
			State:    machinev1.ControlPlaneMachineSetStateActive,
			Selector: metav1.LabelSelector{
				MatchLabels: map[string]string{
					"machine.openshift.io/cluster-api-machine-role": role,
					"machine.openshift.io/cluster-api-machine-type": role,
					"machine.openshift.io/cluster-api-cluster":      clusterID,
				},
			},
			Template: machinev1.ControlPlaneMachineSetTemplate{
				MachineType: machinev1.OpenShiftMachineV1Beta1MachineType,
				OpenShiftMachineV1Beta1Machine: &machinev1.OpenShiftMachineV1Beta1MachineTemplate{
					ObjectMeta: machinev1.ControlPlaneMachineSetTemplateObjectMeta{
						Labels: map[string]string{
							"machine.openshift.io/cluster-api-cluster":      clusterID,
							"machine.openshift.io/cluster-api-machine-role": role,
							"machine.openshift.io/cluster-api-machine-type": role,
						},
					},
					Spec: machineapi.MachineSpec{
						ProviderSpec: machineapi.ProviderSpec{
							Value: &runtime.RawExtension{Object: machineSetProvider},
						},
					},
				},
			},
		},
	}
	utils.SetCPMSOSStreamLabels(controlPlaneMachineSet, config)

	if len(failureDomains) > 0 {
		controlPlaneMachineSet.Spec.Template.OpenShiftMachineV1Beta1Machine.FailureDomains = &machinev1.FailureDomains{
			Platform: v1.AzurePlatformType,
			Azure:    &failureDomains,
		}
	}

	return machines, controlPlaneMachineSet, nil
}

func provider(platform *azure.Platform, mpool *azure.MachinePool, osImage string, userDataSecret string, clusterID string, role string, azIdx *int, capabilities map[string]string, session *icazure.Session, networkResourceGroup, virtualNetwork, subnet string) (*machineapi.AzureMachineProviderSpec, error) {
	var az string
	if len(mpool.Zones) > 0 && azIdx != nil {
		az = mpool.Zones[*azIdx]
	}
	hyperVGen, err := icazure.GetHyperVGenerationVersion(capabilities, "")
	if err != nil {
		return nil, err
	}

	if mpool.VMNetworkingType == "" {
		acceleratedNetworking := icazure.GetVMNetworkingCapability(capabilities)
		if acceleratedNetworking {
			mpool.VMNetworkingType = string(azure.VMnetworkingTypeAccelerated)
		} else {
			logrus.Infof("Instance type %s does not support Accelerated Networking. Using Basic Networking instead.", mpool.InstanceType)
		}
	}
	rg := platform.ClusterResourceGroupName(clusterID)

	confidentialVM := mpool.Settings != nil && mpool.Settings.SecurityType != ""
	image := mapiImage(mpool.OSImage, platform.CloudName, confidentialVM, hyperVGen, rg, session.Credentials.SubscriptionID, clusterID, osImage)

	if mpool.OSDisk.DiskType == "" {
		mpool.OSDisk.DiskType = "Premium_LRS"
	}

	publicLB := clusterID
	if platform.OutboundType == azure.UserDefinedRoutingOutboundType {
		publicLB = ""
	}

	managedIdentity := ""
	if len(mpool.Identity.UserAssignedIdentities) > 0 {
		managedIdentity = mpool.Identity.UserAssignedIdentities[0].ProviderID()
	} else if mpool.Identity.Type == capz.VMIdentityUserAssigned {
		// In this case, the installer will create the user-assigned identity.
		managedIdentity = fmt.Sprintf("%s-identity", clusterID)
	}

	var diskEncryptionSet *machineapi.DiskEncryptionSetParameters
	if mpool.OSDisk.DiskEncryptionSet != nil {
		diskEncryptionSet = &machineapi.DiskEncryptionSetParameters{
			ID: mpool.OSDisk.DiskEncryptionSet.ToID(),
		}
	}

	var diskSecurityProfile machineapi.VMDiskSecurityProfile
	if mpool.OSDisk.SecurityProfile != nil && mpool.OSDisk.SecurityProfile.SecurityEncryptionType != "" {
		diskSecurityProfile = machineapi.VMDiskSecurityProfile{
			SecurityEncryptionType: machineapi.SecurityEncryptionTypes(mpool.OSDisk.SecurityProfile.SecurityEncryptionType),
		}

		if mpool.OSDisk.SecurityProfile.DiskEncryptionSet != nil {
			diskSecurityProfile.DiskEncryptionSet = machineapi.DiskEncryptionSetParameters{
				ID: mpool.OSDisk.SecurityProfile.DiskEncryptionSet.ToID(),
			}
		}
	}

	securityProfile := generateSecurityProfile(mpool)

	ultraSSDCapability := machineapi.AzureUltraSSDCapabilityState(mpool.UltraSSDCapability)

	dataDisks := make([]machineapi.DataDisk, 0, len(mpool.DataDisks))

	for _, disk := range mpool.DataDisks {
		dataDisk := machineapi.DataDisk{
			NameSuffix:     disk.NameSuffix,
			DiskSizeGB:     disk.DiskSizeGB,
			CachingType:    machineapi.CachingTypeOption(disk.CachingType),
			DeletionPolicy: machineapi.DiskDeletionPolicyTypeDelete,
		}

		if disk.Lun != nil {
			dataDisk.Lun = *disk.Lun
		}

		if disk.ManagedDisk != nil {
			dataDisk.ManagedDisk = machineapi.DataDiskManagedDiskParameters{
				StorageAccountType: machineapi.StorageAccountType(disk.ManagedDisk.StorageAccountType),
			}

			if disk.ManagedDisk.DiskEncryptionSet != nil {
				dataDisk.ManagedDisk.DiskEncryptionSet = (*machineapi.DiskEncryptionSetParameters)(disk.ManagedDisk.SecurityProfile.DiskEncryptionSet)
			}
		}

		dataDisks = append(dataDisks, dataDisk)
	}

	spec := &machineapi.AzureMachineProviderSpec{
		TypeMeta: metav1.TypeMeta{
			APIVersion: "machine.openshift.io/v1beta1",
			Kind:       "AzureMachineProviderSpec",
		},
		UserDataSecret:    &corev1.SecretReference{Name: userDataSecret},
		CredentialsSecret: &corev1.SecretReference{Name: cloudsSecret, Namespace: cloudsSecretNamespace},
		Location:          platform.Region,
		VMSize:            mpool.InstanceType,
		Image:             image,
		OSDisk: machineapi.OSDisk{
			OSType:     "Linux",
			DiskSizeGB: mpool.OSDisk.DiskSizeGB,
			ManagedDisk: machineapi.OSDiskManagedDiskParameters{
				StorageAccountType: mpool.OSDisk.DiskType,
				DiskEncryptionSet:  diskEncryptionSet,
				SecurityProfile:    diskSecurityProfile,
			},
		},
		SecurityProfile:       securityProfile,
		UltraSSDCapability:    ultraSSDCapability,
		Zone:                  az,
		Subnet:                subnet,
		ManagedIdentity:       managedIdentity,
		Vnet:                  virtualNetwork,
		ResourceGroup:         rg,
		NetworkResourceGroup:  networkResourceGroup,
		PublicLoadBalancer:    publicLB,
		AcceleratedNetworking: getVMNetworkingType(mpool.VMNetworkingType),
		Tags:                  platform.UserTags,
		DataDisks:             dataDisks,
	}
	var bootDiagnostics *machineapi.AzureDiagnostics
	if platform.DefaultMachinePlatform != nil {
		bootDiagnostics = getBootDiagnosticObject(platform.DefaultMachinePlatform.BootDiagnostics, session.Environment.StorageEndpointSuffix, role)
	}
	tempDiagnostics := getBootDiagnosticObject(mpool.BootDiagnostics, session.Environment.StorageEndpointSuffix, role)
	if tempDiagnostics != nil {
		bootDiagnostics = tempDiagnostics
	}
	if bootDiagnostics != nil {
		spec.Diagnostics.Boot = bootDiagnostics.Boot
	}

	if platform.CloudName == azure.StackCloud {
		spec.AvailabilitySet = fmt.Sprintf("%s_control-plane-as", clusterID)
	}

	return spec, nil
}

func getBootDiagnosticObject(diag *azure.BootDiagnostics, cloudName string, role string) *machineapi.AzureDiagnostics {
	if diag == nil {
		if role == controlPlaneRoleName {
			return &machineapi.AzureDiagnostics{Boot: &machineapi.AzureBootDiagnostics{StorageAccountType: machineapi.AzureManagedAzureDiagnosticsStorage}}
		}
		return nil
	}
	if diag.Type == capz.DisabledDiagnosticsStorage {
		return nil
	}
	bootDiagnostics := &machineapi.AzureDiagnostics{Boot: &machineapi.AzureBootDiagnostics{}}
	if diag.Type == capz.ManagedDiagnosticsStorage {
		bootDiagnostics.Boot.StorageAccountType = machineapi.AzureManagedAzureDiagnosticsStorage
	} else {
		bootDiagnostics.Boot.StorageAccountType = machineapi.CustomerManagedAzureDiagnosticsStorage
		bootDiagnostics.Boot.CustomerManaged = &machineapi.AzureCustomerManagedBootDiagnostics{
			StorageAccountURI: bootDiagStorageURIBuilder(diag, cloudName),
		}
	}
	return bootDiagnostics
}

// ConfigMasters sets the PublicIP flag and assigns a set of load balancers to the given machines
func ConfigMasters(machines []machineapi.Machine, controlPlane *machinev1.ControlPlaneMachineSet, clusterID string) error {
	internalLB := fmt.Sprintf("%s-internal", clusterID)

	for _, machine := range machines {
		providerSpec := machine.Spec.ProviderSpec.Value.Object.(*machineapi.AzureMachineProviderSpec)
		providerSpec.InternalLoadBalancer = internalLB
	}
	providerSpec, ok := controlPlane.Spec.Template.OpenShiftMachineV1Beta1Machine.Spec.ProviderSpec.Value.Object.(*machineapi.AzureMachineProviderSpec)
	if !ok {
		return errors.New("Unable to set internal load balancers to control plane machine set")
	}
	providerSpec.InternalLoadBalancer = internalLB
	return nil
}

func getNetworkInfo(platform *azure.Platform, clusterID, role string, subnetZones []string) (string, string, []string, error) {
	networkResourceGroupName := platform.NetworkResourceGroupName
	if platform.VirtualNetwork == "" {
		networkResourceGroupName = platform.ClusterResourceGroupName(clusterID)
	}
	virtualNetworkName := platform.VirtualNetworkName(clusterID)
	var subnetRole capz.SubnetRole
	var defaultSubnet string
	switch role {
	case "worker":
		subnetRole = capz.SubnetNode
		defaultSubnet = platform.ComputeSubnetName(clusterID)
	case controlPlaneRoleName:
		subnetRole = capz.SubnetControlPlane
		defaultSubnet = platform.ControlPlaneSubnetName(clusterID)
	default:
		return "", "", nil, fmt.Errorf("unrecognized machine role %s", role)
	}

	subnets := []string{}
	for _, subnetSpec := range platform.Subnets {
		if subnetSpec.Role == subnetRole {
			subnets = append(subnets, subnetSpec.Name)
		}
	}

	if len(subnets) == 0 {
		subnets = append(subnets, defaultSubnet)
		if platform.OutboundType == azure.NATGatewayMultiZoneOutboundType && subnetRole == capz.SubnetNode {
			// Starting from 2 here since there is one already added. For default installs, there has to
			// be one guaranteed and then for multi zone, we need to add extra per availability zone.
			// This code will only run if multi zone so the first one is already set and we start from 2.
			if subnetZones != nil {
				for i := 2; i <= len(subnetZones); i++ {
					subnets = append(subnets, fmt.Sprintf("%s-%d", defaultSubnet, i))
				}
			}
		}
	}
	return networkResourceGroupName, virtualNetworkName, subnets, nil
}

// getVMNetworkingType should set the correct capability for instance type
func getVMNetworkingType(value string) bool {
	return value == string(azure.VMnetworkingTypeAccelerated)
}

func generateSecurityProfile(mpool *azure.MachinePool) *machineapi.SecurityProfile {
	securityProfile := &machineapi.SecurityProfile{}

	if mpool.EncryptionAtHost {
		securityProfile.EncryptionAtHost = &mpool.EncryptionAtHost
	}

	if mpool.Settings != nil && mpool.Settings.SecurityType != "" {
		securityProfile.Settings = machineapi.SecuritySettings{
			SecurityType: machineapi.SecurityTypes(mpool.Settings.SecurityType),
		}

		var uefiSettings machineapi.UEFISettings
		if securityProfile.Settings.SecurityType == machineapi.SecurityTypesTrustedLaunch {
			if mpool.Settings.TrustedLaunch != nil && mpool.Settings.TrustedLaunch.UEFISettings != nil {
				if sb := mpool.Settings.TrustedLaunch.UEFISettings.SecureBoot; sb != nil {
					uefiSettings.SecureBoot = machineapi.SecureBootPolicy(*sb)
				}
				if vtpm := mpool.Settings.TrustedLaunch.UEFISettings.VirtualizedTrustedPlatformModule; vtpm != nil {
					uefiSettings.VirtualizedTrustedPlatformModule = machineapi.VirtualizedTrustedPlatformModulePolicy(*vtpm)
				}

				securityProfile.Settings.TrustedLaunch = &machineapi.TrustedLaunch{
					UEFISettings: uefiSettings,
				}
			}
		} else if securityProfile.Settings.SecurityType == machineapi.SecurityTypesConfidentialVM {
			if mpool.Settings.ConfidentialVM != nil && mpool.Settings.ConfidentialVM.UEFISettings != nil {
				if sb := mpool.Settings.ConfidentialVM.UEFISettings.SecureBoot; sb != nil {
					uefiSettings.SecureBoot = machineapi.SecureBootPolicy(*sb)
				}
				if vtpm := mpool.Settings.ConfidentialVM.UEFISettings.VirtualizedTrustedPlatformModule; vtpm != nil {
					uefiSettings.VirtualizedTrustedPlatformModule = machineapi.VirtualizedTrustedPlatformModulePolicy(*vtpm)
				}

				securityProfile.Settings.ConfidentialVM = &machineapi.ConfidentialVM{
					UEFISettings: uefiSettings,
				}
			}
		}
	}

	return securityProfile
}

func mapiImage(osImage azure.OSImage, azEnv azure.CloudEnvironment, confidentialVM bool, gen, rg, sub, infraID, rhcosImg string) machineapi.Image {
	mImg := machineapi.Image{}
	cImg := capzImage(osImage, azEnv, confidentialVM, gen, rg, sub, infraID, rhcosImg)

	if cImg.ID != nil {
		mImg.ResourceID = trimSubscriptionPrefix(*cImg.ID)
	} else if cImg.Marketplace != nil {
		mImg.Publisher = cImg.Marketplace.Publisher
		mImg.Offer = cImg.Marketplace.Offer
		mImg.SKU = cImg.Marketplace.SKU
		mImg.Version = cImg.Marketplace.Version
		mImg.Type = machineapi.AzureImageTypeMarketplaceNoPlan
		if cImg.Marketplace.ThirdPartyImage {
			mImg.Type = machineapi.AzureImageTypeMarketplaceWithPlan
		}
	}
	return mImg
}

// trimSubscriptionPrefix takes an image id string
// formatted for CAPZ and returns a string formatted
// for MAPI, by removing the /subspcription/ prefix.
func trimSubscriptionPrefix(image string) string {
	rgIndex := strings.Index(image, "/resourceGroups/")
	// Allow invalid inputs to fail downstream
	if rgIndex < 0 {
		return image
	}
	return image[rgIndex:]
}