create_fastdeploy.go

// © Broadcom. All Rights Reserved.
// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.
// SPDX-License-Identifier: Apache-2.0

package vmlifecycle

import (
	"context"
	"errors"
	"fmt"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/go-logr/logr"
	"github.com/vmware/govmomi/fault"
	"github.com/vmware/govmomi/object"
	"github.com/vmware/govmomi/vim25"
	vimtypes "github.com/vmware/govmomi/vim25/types"

	pkgcfg "github.com/vmware-tanzu/vm-operator/pkg/config"
	pkgconst "github.com/vmware-tanzu/vm-operator/pkg/constants"
	pkgctx "github.com/vmware-tanzu/vm-operator/pkg/context"
	pkglog "github.com/vmware-tanzu/vm-operator/pkg/log"
	pkgutil "github.com/vmware-tanzu/vm-operator/pkg/util"
	"github.com/vmware-tanzu/vm-operator/pkg/util/ptr"
)

//nolint:gocyclo
func fastDeploy(
	vmCtx pkgctx.VirtualMachineContext,
	vimClient *vim25.Client,
	createArgs *CreateArgs) (vmRef *vimtypes.ManagedObjectReference, retErr error) {

	logger := vmCtx.Logger.WithName("fastDeploy")

	if len(createArgs.Datastores) == 0 {
		return nil, errors.New("no compatible datastores")
	}

	vmDir := path.Dir(createArgs.ConfigSpec.Files.VmPathName)
	logger.Info("Got vm dir", "vmDir", vmDir)

	var dp object.DatastorePath
	if !dp.FromString(vmDir) {
		return nil, fmt.Errorf("invalid datastore path: %s", vmDir)
	}
	vmDirName := dp.Path
	logger.Info("Got vm dir name", "vmDirName", vmDirName)

	srcFilePaths := createArgs.FilePaths
	logger.Info("Got source file paths", "srcFilePaths", srcFilePaths)

	srcDiskPaths := createArgs.DiskPaths
	logger.Info("Got source disk paths", "srcDiskPaths", srcDiskPaths)

	dstDiskFormat := pkgutil.GetPreferredDiskFormat(
		createArgs.Datastores[0].DiskFormats...)
	logger.Info("Got destination disk format", "dstDiskFormat", dstDiskFormat)

	datacenter := object.NewDatacenter(
		vimClient,
		vimtypes.ManagedObjectReference{
			Type:  string(vimtypes.ManagedObjectTypeDatacenter),
			Value: createArgs.DatacenterMoID,
		})
	logger.Info("Got datacenter", "datacenter", datacenter.Reference())

	// Create the directory where the VM will be created.
	var (
		vmDirPath     string
		vmDirUUIDPath string
		nm            *object.DatastoreNamespaceManager
		fm            = object.NewFileManager(vimClient)
	)

	if createArgs.Datastores[0].TopLevelDirectoryCreateSupported {
		//
		// The datastore supports TLD creation, so just use file manager.
		//
		if err := fm.MakeDirectory(vmCtx, vmDir, datacenter, true); err != nil {
			return nil, fmt.Errorf("failed to create vm dir %q: %w", vmDir, err)
		}
		vmDirPath = vmDir
	} else {
		//
		// The datastore does NOT support TLD creation, so use the datastore
		// namespace manager to create the new directory.
		//
		nm = object.NewDatastoreNamespaceManager(vimClient)
		vdp, err := nm.CreateDirectory(
			vmCtx,
			object.NewDatastore(vimClient, createArgs.Datastores[0].MoRef),
			vmDirName,
			"")
		if err != nil {
			return nil, fmt.Errorf("failed to create vm dir: %w", err)
		}

		// The vmDirUUIDPath value will look something like the following:
		//
		//     /vmfs/volumes/vsan:a5982d36bd6d11f0-9fd50050569d3d0d/dfc05687-5880-431c-9924-a3a34837443e
		vmDirUUIDPath = vdp
		logger.Info("Got vm dir UUID path", "vmDirUUIDPath", vmDirUUIDPath)

		// Get the name of the newly created directory. From the above example
		// this would be dfc05687-5880-431c-9924-a3a34837443e.
		vmDirUUIDName := path.Base(vmDirUUIDPath)

		// Update the original VM directory path to use the newly created dir
		// name. If the original path was:
		//
		//     [vsanDatastore-1] bdefaf58-234d-49c0-a152-55a592f37e34
		//
		// Then using the above example, the new path would be:
		//
		//     [vsanDatastore-1] dfc05687-5880-431c-9924-a3a34837443e
		//
		// The original path also exists, but as a symlink to the real one.
		vmDirPath = strings.ReplaceAll(vmDir, vmDirName, vmDirUUIDName)

		// Update the path to the VMX file with the newly created directory
		// name.  If the original path was:
		//
		//     [vsanDatastore-1] bdefaf58-234d-49c0-a152-55a592f37e34/my-vm-1.vmx
		//
		// Then using the above example, the new path would be:
		//
		//     [vsanDatastore-1] dfc05687-5880-431c-9924-a3a34837443e/my-vm-1.vmx
		//
		// The original path also exists, but as a symlink to the real one.
		createArgs.ConfigSpec.Files.VmPathName = strings.ReplaceAll(
			createArgs.ConfigSpec.Files.VmPathName, vmDirName, vmDirUUIDName)
	}

	logger.Info("Updated vm dir paths",
		"vmDirPath", vmDirPath,
		"vmPathName", createArgs.ConfigSpec.Files.VmPathName)

	// Register cleanup defer immediately after directory creation.
	defer func() {
		if retErr == nil {
			// Do not delete the VM directory if this function succeeded.
			return
		}

		// Use a context that is not cancelled when the parent is, so cleanup
		// runs even if the request is cancelled. Preserves the VC opID so
		// cleanup is correlated with the failed create in VC logs.
		ctx := context.WithoutCancel(vmCtx.Context)

		// Delete the VM directory and its contents.
		// Always use FileManager.DeleteDatastoreFile() first as it can delete
		// non-empty directories recursively. Note that DeleteDirectory() on a
		// non-empty directory will return an error, which is why we delete the
		// directory contents first using DeleteDatastoreFile().
		t, err := fm.DeleteDatastoreFile(ctx, vmDirPath, datacenter)
		if err != nil {
			retErr = fmt.Errorf(
				"failed to call delete api for vm dir %q: %w,%w",
				vmDirPath, err, retErr)
			return
		}

		// Wait for the delete call to return.
		if err := t.Wait(ctx); err != nil &&
			!fault.Is(err, &vimtypes.FileNotFound{}) {

			retErr = fmt.Errorf(
				"failed to delete vm dir %q: %w,%w",
				vmDirPath, err, retErr)
		}

		// For non-TLD datastores, also clean up the namespace mapping.
		if !createArgs.Datastores[0].TopLevelDirectoryCreateSupported {
			if err := nm.DeleteDirectory(
				ctx,
				datacenter,
				vmDirUUIDPath); err != nil &&
				!fault.Is(err, &vimtypes.FileNotFound{}) {

				retErr = fmt.Errorf(
					"failed to delete vm dir namespace mapping %q: %w,%w",
					vmDirUUIDPath, err, retErr)
			}
		}
	}()

	dstDiskPaths := make([]string, len(srcDiskPaths))
	for i := 0; i < len(dstDiskPaths); i++ {
		dstDiskPaths[i] = fmt.Sprintf("%s/disk-%d.vmdk", vmDirPath, i)
	}
	logger.Info("Got destination disk paths", "dstDiskPaths", dstDiskPaths)

	dstFilePaths := make([]string, len(srcFilePaths))
	for i := 0; i < len(dstFilePaths); i++ {
		// Rename file using vm name + same extension
		ext := path.Ext(srcFilePaths[i])
		name := createArgs.ConfigSpec.Name + ext

		dstFilePaths[i] = path.Join(vmDirPath, name)
	}
	logger.Info("Got destination file paths", "dstFilePaths", dstFilePaths)

	var (
		disks     []*vimtypes.VirtualDisk
		diskSpecs []*vimtypes.VirtualDeviceConfigSpec
	)
	for i := range createArgs.ConfigSpec.DeviceChange {
		dc := createArgs.ConfigSpec.DeviceChange[i].GetVirtualDeviceConfigSpec()
		if d, ok := dc.Device.(*vimtypes.VirtualDisk); ok {
			d.VirtualDiskFormat = string(dstDiskFormat)
			disks = append(disks, d)
			diskSpecs = append(diskSpecs, dc)
		}
	}

	if a, b := len(srcDiskPaths), len(disks); a != b {
		logger.Info("Got disks", "disks", disks)
		return nil, fmt.Errorf(
			"invalid disk count: len(srcDiskPaths)=%d, len(disks)=%d", a, b)
	}

	// Update the disks with their expected file names.
	for i := range disks {
		d := disks[i]
		if bfb, ok := d.Backing.(vimtypes.BaseVirtualDeviceFileBackingInfo); ok {
			fb := bfb.GetVirtualDeviceFileBackingInfo()
			fb.Datastore = &createArgs.Datastores[0].MoRef
			fb.FileName = dstDiskPaths[i]
		}
	}
	logger.Info("Got disks", "disks", disks)

	// Copy any non-disk files to the target directory.
	for i := range dstFilePaths {
		task, err := fm.CopyDatastoreFile(
			vmCtx,
			srcFilePaths[i],
			datacenter,
			dstFilePaths[i],
			datacenter, false)
		if err != nil {
			return nil, err
		}
		if err = task.Wait(vmCtx); err != nil {
			return nil, err
		}
	}

	// Update the config spec to know about a potential NVRAM file.
	for i := range createArgs.ConfigSpec.ExtraConfig {
		ov := createArgs.ConfigSpec.ExtraConfig[i].GetOptionValue()

		if ov != nil && ov.Key == "nvram" {

			if v, ok := ov.Value.(string); ok {

				oldNvramPath := v

				for j := range dstFilePaths {
					if strings.EqualFold(".nvram", path.Ext(dstFilePaths[j])) {
						newNvramPath := path.Base(dstFilePaths[j])
						logger.Info("Updated NVRAM in ExtraConfig",
							"oldValue", oldNvramPath,
							"newValue", newNvramPath)
						createArgs.ConfigSpec.ExtraConfig[i] = &vimtypes.OptionValue{
							Key:   ov.Key,
							Value: newNvramPath,
						}
					}
				}
			}
		}
	}

	folder := object.NewFolder(vimClient, vimtypes.ManagedObjectReference{
		Type:  "Folder",
		Value: createArgs.FolderMoID,
	})
	logger.Info("Got folder", "folder", folder.Reference())

	pool := object.NewResourcePool(vimClient, vimtypes.ManagedObjectReference{
		Type:  "ResourcePool",
		Value: createArgs.ResourcePoolMoID,
	})
	logger.Info("Got pool", "pool", pool.Reference())

	var host *object.HostSystem
	if createArgs.HostMoID != "" {
		host = object.NewHostSystem(vimClient, vimtypes.ManagedObjectReference{
			Type:  "HostSystem",
			Value: createArgs.HostMoID,
		})
		logger.Info("Got host", "host", host.Reference())
	}

	// Determine the type of fast deploy operation.
	var (
		fastDeployMode       string
		fastDeployModeReason string
	)

	switch {
	case createArgs.IsEncryptedStorageProfile:
		fastDeployMode = pkgconst.FastDeployModeDirect
		fastDeployModeReason = "encrypted storage profile"
	case !pkgutil.IsOnlinePromoteDisksSupported(createArgs.ConfigSpec):
		fastDeployMode = pkgconst.FastDeployModeDirect
		fastDeployModeReason = "online promote disks not supported"
	default:
		fastDeployMode = vmCtx.VM.Annotations[pkgconst.FastDeployAnnotationKey]
		if fastDeployMode == "" {
			fastDeployMode = pkgcfg.FromContext(vmCtx).FastDeployMode
			fastDeployModeReason = "global default"
		} else {
			fastDeployModeReason = "from annotation"
		}
	}

	logger.Info(
		"Deploying VM with Fast Deploy",
		"mode", fastDeployMode,
		"reason", fastDeployModeReason,
		"k8sVMCreateTimestamp", vmCtx.VM.CreationTimestamp.Format(time.RFC3339))

	if strings.EqualFold(fastDeployMode, pkgconst.FastDeployModeLinked) {
		return fastDeployLinked(
			vmCtx,
			folder,
			pool,
			host,
			createArgs.ConfigSpec,
			createArgs.Datastores[0].MoRef,
			disks,
			diskSpecs,
			srcDiskPaths)
	}

	return fastDeployDirect(
		vmCtx,
		datacenter,
		folder,
		pool,
		host,
		createArgs.ConfigSpec,
		diskSpecs,
		dstDiskFormat,
		dstDiskPaths,
		srcDiskPaths)
}

func fastDeployLinked(
	ctx context.Context,
	folder *object.Folder,
	pool *object.ResourcePool,
	host *object.HostSystem,
	configSpec vimtypes.VirtualMachineConfigSpec,
	datastoreRef vimtypes.ManagedObjectReference,
	disks []*vimtypes.VirtualDisk,
	diskSpecs []*vimtypes.VirtualDeviceConfigSpec,
	srcDiskPaths []string) (*vimtypes.ManagedObjectReference, error) {

	logger := pkglog.FromContextOrDefault(ctx).WithName("fastDeployLinked")

	// Linked clones do not fully support encryption, so remove the possible
	// crypto information from the VM's disks.
	for i := range diskSpecs {
		ds := diskSpecs[i]
		if ds.Backing != nil {
			ds.Backing.Crypto = nil
		}
	}

	// Ensure all the parent disks are added to the ExtraConfig as the special
	// vmprov.keepDisks value to prevent the parent disk from being deleted with
	// the VM when it is deleted and/or promoted.
	srcDiskNames := make([]string, len(srcDiskPaths))
	for i := range srcDiskPaths {
		srcDiskNames[i] = path.Base(srcDiskPaths[i])
	}
	exConfigKeyVal := &vimtypes.OptionValue{
		Key:   pkgconst.VMProvKeepDisksExtraConfigKey,
		Value: strings.Join(srcDiskNames, ","),
	}
	configSpec.ExtraConfig = append(configSpec.ExtraConfig, exConfigKeyVal)
	logger.Info("Preserving linked parents on delete/promote via extraConfig",
		"extraConfigKey", exConfigKeyVal.Key,
		"extraConfigValue", exConfigKeyVal.Value)

	for i := range disks {
		fileBackingInfo := vimtypes.VirtualDeviceFileBackingInfo{
			Datastore: &datastoreRef,
			FileName:  srcDiskPaths[i],
		}
		switch tBack := disks[i].Backing.(type) {
		case *vimtypes.VirtualDiskFlatVer2BackingInfo:
			// Point the disk to its parent.
			tBack.Parent = &vimtypes.VirtualDiskFlatVer2BackingInfo{
				VirtualDeviceFileBackingInfo: fileBackingInfo,
				DiskMode:                     string(vimtypes.VirtualDiskModePersistent),
				ThinProvisioned:              ptr.To(true),
			}
		case *vimtypes.VirtualDiskSeSparseBackingInfo:
			// Point the disk to its parent.
			tBack.Parent = &vimtypes.VirtualDiskSeSparseBackingInfo{
				VirtualDeviceFileBackingInfo: fileBackingInfo,
				DiskMode:                     string(vimtypes.VirtualDiskModePersistent),
			}
		case *vimtypes.VirtualDiskSparseVer2BackingInfo:
			// Point the disk to its parent.
			tBack.Parent = &vimtypes.VirtualDiskSparseVer2BackingInfo{
				VirtualDeviceFileBackingInfo: fileBackingInfo,
				DiskMode:                     string(vimtypes.VirtualDiskModePersistent),
			}
		}
	}

	return fastDeployCreateVM(ctx, logger, folder, pool, host, configSpec)
}

func fastDeployDirect(
	ctx context.Context,
	datacenter *object.Datacenter,
	folder *object.Folder,
	pool *object.ResourcePool,
	host *object.HostSystem,
	configSpec vimtypes.VirtualMachineConfigSpec,
	diskSpecs []*vimtypes.VirtualDeviceConfigSpec,
	diskFormat vimtypes.DatastoreSectorFormat,
	dstDiskPaths,
	srcDiskPaths []string) (*vimtypes.ManagedObjectReference, error) {

	logger := pkglog.FromContextOrDefault(ctx).WithName("fastDeployDirect")

	// Copy each disk into the VM directory.
	if err := fastDeployDirectCopyDisks(
		ctx,
		logger,
		datacenter,
		configSpec,
		srcDiskPaths,
		dstDiskPaths,
		diskFormat); err != nil {

		return nil, err
	}

	for i := range diskSpecs {
		ds := diskSpecs[i]

		// Set the file operation to an empty string since the disk already
		// exists.
		ds.FileOperation = ""
	}

	return fastDeployCreateVM(ctx, logger, folder, pool, host, configSpec)
}

func fastDeployCreateVM(
	ctx context.Context,
	logger logr.Logger,
	folder *object.Folder,
	pool *object.ResourcePool,
	host *object.HostSystem,
	configSpec vimtypes.VirtualMachineConfigSpec) (*vimtypes.ManagedObjectReference, error) {

	logger.Info("Creating VM", "configSpec", vimtypes.ToString(configSpec))

	createTask, err := folder.CreateVM(
		ctx,
		configSpec,
		pool,
		host)
	if err != nil {
		return nil, fmt.Errorf("failed to call create task: %w", err)
	}

	createTaskInfo, err := createTask.WaitForResult(ctx)
	if err != nil {
		return nil, fmt.Errorf("failed to wait for create task: %w", err)
	}

	vmRefVal, ok := createTaskInfo.Result.(vimtypes.ManagedObjectReference)
	if !ok {
		return nil, fmt.Errorf(
			"failed to assert create task result is ref: %[1]T %+[1]v",
			createTaskInfo.Result)
	}

	return &vmRefVal, nil
}

func fastDeployDirectCopyDisks(
	ctx context.Context,
	logger logr.Logger,
	datacenter *object.Datacenter,
	configSpec vimtypes.VirtualMachineConfigSpec,
	srcDiskPaths,
	dstDiskPaths []string,
	diskFormat vimtypes.DatastoreSectorFormat) error {

	var (
		wg            sync.WaitGroup
		copyDiskTasks = make([]*object.Task, len(srcDiskPaths))
		copyDiskErrs  = make(chan error, len(srcDiskPaths))
		copyDiskSpec  = vimtypes.FileBackedVirtualDiskSpec{
			VirtualDiskSpec: vimtypes.VirtualDiskSpec{
				AdapterType: string(vimtypes.VirtualDiskAdapterTypeLsiLogic),
				DiskType:    string(vimtypes.VirtualDiskTypeThin),
			},
			SectorFormat: string(diskFormat),
			Profile:      configSpec.VmProfile,
			Crypto:       configSpec.Crypto,
		}
		diskManager = object.NewVirtualDiskManager(datacenter.Client())
	)

	for i := range srcDiskPaths {
		s := srcDiskPaths[i]
		d := dstDiskPaths[i]

		logger.Info(
			"Copying disk",
			"dstDiskPath", d,
			"srcDiskPath", s,
			"copyDiskSpec", copyDiskSpec)

		t, err := diskManager.CopyVirtualDisk(
			ctx,
			s,
			datacenter,
			d,
			datacenter,
			&copyDiskSpec,
			false)
		if err != nil {
			logger.Error(err, "failed to copy disk, cancelling other tasks")

			// Cancel any other outstanding disk copies.
			for _, t := range copyDiskTasks {
				if t != nil {
					// Cancel the task using a background context to ensure it
					// goes through.
					_ = t.Cancel(context.Background())
				}
			}

			logger.Info("waiting on other copy tasks to complete")
			// Wait on any other outstanding disk copies to complete before
			// returning to ensure the parent folder can be cleaned up.
			wg.Wait()
			logger.Info("waited on other copy tasks to complete")

			return fmt.Errorf("failed to call copy disk %q to %q: %w", s, d, err)
		}

		wg.Add(1)
		copyDiskTasks[i] = t

		go func() {
			defer wg.Done()
			if err := t.Wait(context.Background()); err != nil {
				copyDiskErrs <- fmt.Errorf(
					"failed to copy disk %q to %q: %w", s, d, err)
			}
		}()
	}

	// Wait on all the disk copies to complete before proceeding.
	go func() {
		wg.Wait()
		close(copyDiskErrs)
	}()
	var copyDiskErr error
	for err := range copyDiskErrs {
		if err != nil {
			if copyDiskErr == nil {
				copyDiskErr = err
			} else {
				copyDiskErr = fmt.Errorf("%w,%w", copyDiskErr, err)
			}
		}
	}

	return copyDiskErr
}