diff --git a/deploy/00crds.yaml b/deploy/00crds.yaml index bd6747770..40bf2c985 100644 --- a/deploy/00crds.yaml +++ b/deploy/00crds.yaml @@ -1337,6 +1337,11 @@ spec: - windowsGuest - linuxGuest type: string + proxyVMName: + description: |- + ProxyVMName is the name of the VM in vCenter that will receive the source disk + via SCSI HotAdd. Required when StorageCopyMethod is HotAddCopy. + type: string source: description: Source is the source details for the virtual machine properties: @@ -1351,10 +1356,11 @@ spec: default: normal description: |- StorageCopyMethod indicates the method to use for storage migration - Valid values: "normal" (default), "StorageAcceleratedCopy" + Valid values: "normal" (default), "StorageAcceleratedCopy", "HotAddCopy" enum: - normal - StorageAcceleratedCopy + - HotAddCopy type: string storageMapping: description: |- diff --git a/deploy/installer.yaml b/deploy/installer.yaml index 5dc62ebbc..dbf137af9 100644 --- a/deploy/installer.yaml +++ b/deploy/installer.yaml @@ -1337,6 +1337,11 @@ spec: - windowsGuest - linuxGuest type: string + proxyVMName: + description: |- + ProxyVMName is the name of the VM in vCenter that will receive the source disk + via SCSI HotAdd. Required when StorageCopyMethod is HotAddCopy. + type: string source: description: Source is the source details for the virtual machine properties: @@ -1351,10 +1356,11 @@ spec: default: normal description: |- StorageCopyMethod indicates the method to use for storage migration - Valid values: "normal" (default), "StorageAcceleratedCopy" + Valid values: "normal" (default), "StorageAcceleratedCopy", "HotAddCopy" enum: - normal - StorageAcceleratedCopy + - HotAddCopy type: string storageMapping: description: |- diff --git a/k8s/migration/api/v1alpha1/migrationtemplate_types.go b/k8s/migration/api/v1alpha1/migrationtemplate_types.go index cb5298c2d..71cc872d7 100644 --- a/k8s/migration/api/v1alpha1/migrationtemplate_types.go +++ b/k8s/migration/api/v1alpha1/migrationtemplate_types.go @@ -49,11 +49,15 @@ type MigrationTemplateSpec struct { // +optional ArrayCredsMapping string `json:"arrayCredsMapping,omitempty"` // StorageCopyMethod indicates the method to use for storage migration - // Valid values: "normal" (default), "StorageAcceleratedCopy" - // +kubebuilder:validation:Enum=normal;StorageAcceleratedCopy + // Valid values: "normal" (default), "StorageAcceleratedCopy", "HotAddCopy" + // +kubebuilder:validation:Enum=normal;StorageAcceleratedCopy;HotAddCopy // +kubebuilder:default:=normal // +optional StorageCopyMethod string `json:"storageCopyMethod,omitempty"` + // ProxyVMName is the name of the VM in vCenter that will receive the source disk + // via SCSI HotAdd. Required when StorageCopyMethod is HotAddCopy. + // +optional + ProxyVMName string `json:"proxyVMName,omitempty"` // Source is the source details for the virtual machine Source MigrationTemplateSource `json:"source"` // Destination is the destination details for the virtual machine diff --git a/k8s/migration/config/crd/bases/vjailbreak.k8s.pf9.io_migrationtemplates.yaml b/k8s/migration/config/crd/bases/vjailbreak.k8s.pf9.io_migrationtemplates.yaml index c5d24ca8c..98a5db7b4 100644 --- a/k8s/migration/config/crd/bases/vjailbreak.k8s.pf9.io_migrationtemplates.yaml +++ b/k8s/migration/config/crd/bases/vjailbreak.k8s.pf9.io_migrationtemplates.yaml @@ -71,6 +71,11 @@ spec: - windowsGuest - linuxGuest type: string + proxyVMName: + description: |- + ProxyVMName is the name of the VM in vCenter that will receive the source disk + via SCSI HotAdd. Required when StorageCopyMethod is HotAddCopy. + type: string source: description: Source is the source details for the virtual machine properties: @@ -85,10 +90,11 @@ spec: default: normal description: |- StorageCopyMethod indicates the method to use for storage migration - Valid values: "normal" (default), "StorageAcceleratedCopy" + Valid values: "normal" (default), "StorageAcceleratedCopy", "HotAddCopy" enum: - normal - StorageAcceleratedCopy + - HotAddCopy type: string storageMapping: description: |- diff --git a/k8s/migration/internal/controller/migrationplan_controller.go b/k8s/migration/internal/controller/migrationplan_controller.go index 7a1087451..f1e995548 100644 --- a/k8s/migration/internal/controller/migrationplan_controller.go +++ b/k8s/migration/internal/controller/migrationplan_controller.go @@ -72,6 +72,9 @@ const VDDKDirectory = "/home/ubuntu/vmware-vix-disklib-distrib" // StorageCopyMethod is the storage copy method value for Storage Accelerated copy const StorageCopyMethod = "StorageAcceleratedCopy" +// HotAddCopyMethod is the storage copy method value for VMware HotAdd SCSI transport +const HotAddCopyMethod = "HotAddCopy" + // MigrationPlanReconciler reconciles a MigrationPlan object type MigrationPlanReconciler struct { client.Client @@ -465,7 +468,8 @@ func GetVMwareMachineForVM(ctx context.Context, r *MigrationPlanReconciler, vm s //nolint:gocyclo func (r *MigrationPlanReconciler) ReconcileMigrationPlanJob(ctx context.Context, migrationplan *vjailbreakv1alpha1.MigrationPlan, - scope *scope.MigrationPlanScope) (ctrl.Result, error) { + scope *scope.MigrationPlanScope, +) (ctrl.Result, error) { totalVMs := 0 for _, group := range migrationplan.Spec.VirtualMachines { totalVMs += len(group) @@ -531,7 +535,6 @@ func (r *MigrationPlanReconciler) ReconcileMigrationPlanJob(ctx context.Context, latest.Status.MigrationMessage = "" return r.Status().Update(ctx, latest) }) - if err != nil { return ctrl.Result{}, errors.Wrap(err, "failed to reset status for retry") } @@ -1555,10 +1558,14 @@ func (r *MigrationPlanReconciler) setOSFamilyAndStorageFields( configMapData["OS_FAMILY"] = migrationtemplate.Spec.OSFamily } - if migrationtemplate.Spec.StorageCopyMethod == StorageCopyMethod { + switch migrationtemplate.Spec.StorageCopyMethod { + case StorageCopyMethod: configMapData["STORAGE_COPY_METHOD"] = StorageCopyMethod configMapData["VENDOR_TYPE"] = arraycreds.Spec.VendorType configMapData["ARRAY_CREDS_MAPPING"] = migrationtemplate.Spec.ArrayCredsMapping + case HotAddCopyMethod: + configMapData["STORAGE_COPY_METHOD"] = HotAddCopyMethod + configMapData["PROXY_VM_NAME"] = migrationtemplate.Spec.ProxyVMName } return nil diff --git a/pkg/common/constants/constants.go b/pkg/common/constants/constants.go index 2f917cdcd..938282718 100644 --- a/pkg/common/constants/constants.go +++ b/pkg/common/constants/constants.go @@ -376,6 +376,15 @@ const ( // StorageCopyMethod is the default value for storage copy method StorageCopyMethod = "StorageAcceleratedCopy" + // HotAddCopy is the storage copy method that uses VMware HotAdd SCSI transport. + // A proxy VM on the same ESXi host or shared datastore receives the source disk + // directly via the SCSI fabric, bypassing the ESXi NFC management-network path. + // Beta: cold migration only, SCSI disks only. + HotAddCopy = "HotAddCopy" + + // ProxyVMNameKey is the ConfigMap key for the proxy VM name used in HotAdd copy + ProxyVMNameKey = "PROXY_VM_NAME" + // MaxPowerOffRetryLimit is the max number of retries for power off status check MaxPowerOffRetryLimit = 3 diff --git a/ui/src/features/migration/MigrationForm.tsx b/ui/src/features/migration/MigrationForm.tsx index 3e7a4f02b..238af7d3a 100644 --- a/ui/src/features/migration/MigrationForm.tsx +++ b/ui/src/features/migration/MigrationForm.tsx @@ -115,7 +115,7 @@ export interface FormValues extends Record { networkMappings?: { source: string; target: string }[] storageMappings?: { source: string; target: string }[] arrayCredsMappings?: { source: string; target: string }[] - storageCopyMethod?: 'normal' | 'StorageAcceleratedCopy' + storageCopyMethod?: 'normal' | 'StorageAcceleratedCopy' | 'HotAddCopy' // Cluster selection fields vmwareCluster?: string // Format: "credName:datacenter:clusterName" pcdCluster?: string // PCD cluster ID diff --git a/ui/src/features/migration/NetworkAndStorageMappingStep.tsx b/ui/src/features/migration/NetworkAndStorageMappingStep.tsx index 1b0c2bb10..0a969584d 100644 --- a/ui/src/features/migration/NetworkAndStorageMappingStep.tsx +++ b/ui/src/features/migration/NetworkAndStorageMappingStep.tsx @@ -8,7 +8,8 @@ import { FormControlLabel, Radio, Alert, - Chip + Chip, + TextField } from '@mui/material' import { useEffect, useMemo, useCallback, useRef } from 'react' import { ResourceMappingTableNew as ResourceMappingTable } from './components' @@ -35,7 +36,8 @@ export interface ResourceMap { // Storage copy method options export const STORAGE_COPY_METHOD_OPTIONS = [ { value: 'normal', label: 'Standard Copy' }, - { value: 'StorageAcceleratedCopy', label: 'Storage Accelerated Copy' } + { value: 'StorageAcceleratedCopy', label: 'Storage Accelerated Copy' }, + { value: 'HotAddCopy', label: 'HotAdd Copy' } ] as const export type StorageCopyMethod = (typeof STORAGE_COPY_METHOD_OPTIONS)[number]['value'] @@ -50,6 +52,7 @@ interface NetworkAndStorageMappingStepProps { storageMappings?: ResourceMap[] arrayCredsMappings?: ResourceMap[] storageCopyMethod?: StorageCopyMethod + proxyVMName?: string } onChange: (key: string) => (value: any) => void networkMappingError?: string @@ -330,7 +333,7 @@ export default function NetworkAndStorageMappingStep({ value={option.value} control={} label={ - option.value === 'StorageAcceleratedCopy' ? ( + option.value === 'StorageAcceleratedCopy' || option.value === 'HotAddCopy' ? ( {option.label} - {storageCopyMethod === 'normal' ? ( - <> - - Select source and target storage to automatically create mappings. All storage - devices must be mapped in order to proceed. - - onChange('storageMappings')(value)} - oneToManyMapping - fieldPrefix="storageMapping" - /> - - ) : ( + {storageCopyMethod === 'StorageAcceleratedCopy' ? ( <> Map datastores to storage array credentials for storage array data copy. @@ -413,6 +399,53 @@ export default function NetworkAndStorageMappingStep({ )} + ) : storageCopyMethod === 'HotAddCopy' ? ( + <> + + HotAdd attaches the source disk directly to a proxy VM on the same ESXi host, + bypassing the management network. SCSI disks only. Cold migration only (Beta). + + onChange('proxyVMName')(e.target.value)} + fullWidth + size="small" + helperText="Name of the VM in vCenter that will receive the source disk via SCSI HotAdd" + sx={{ mb: 2 }} + /> + + Select source and target storage for Cinder volume creation: + + onChange('storageMappings')(value)} + oneToManyMapping + fieldPrefix="storageMapping" + /> + + ) : ( + <> + + Select source and target storage to automatically create mappings. All storage + devices must be mapped in order to proceed. + + onChange('storageMappings')(value)} + oneToManyMapping + fieldPrefix="storageMapping" + /> + )} {storageMappingError && {storageMappingError}} diff --git a/ui/src/features/migration/RollingMigrationForm.tsx b/ui/src/features/migration/RollingMigrationForm.tsx index a77af5175..51e62a12c 100644 --- a/ui/src/features/migration/RollingMigrationForm.tsx +++ b/ui/src/features/migration/RollingMigrationForm.tsx @@ -100,12 +100,8 @@ interface FormValues extends Record { cutoverEndTime?: string postMigrationScript?: string osFamily?: string - useGPU?: boolean - useFlavorless?: boolean - disconnectSourceNetwork?: boolean - fallbackToDHCP?: boolean - networkPersistence?: boolean - storageCopyMethod?: 'normal' | 'StorageAcceleratedCopy' + storageCopyMethod?: 'normal' | 'StorageAcceleratedCopy' | 'HotAddCopy' + proxyVMName?: string } type RollingMigrationRHFValues = { @@ -1107,7 +1103,7 @@ export default function RollingMigrationFormDrawer({ const handleMappingsChange = (key: string) => (value: unknown) => { markTouched('mapResources') - if (!Array.isArray(value) && key !== 'storageCopyMethod') { + if (!Array.isArray(value) && key !== 'storageCopyMethod' && key !== 'proxyVMName') { return } @@ -1138,6 +1134,11 @@ export default function RollingMigrationFormDrawer({ getParamsUpdater('storageCopyMethod')(value) } break + case 'proxyVMName': + if (typeof value === 'string') { + getParamsUpdater('proxyVMName')(value) + } + break default: break } @@ -1212,6 +1213,7 @@ export default function RollingMigrationFormDrawer({ const storageCopyMethod = (params.storageCopyMethod || 'normal') as | 'normal' | 'StorageAcceleratedCopy' + | 'HotAddCopy' if (selectedVMs.length > 0) { if ( @@ -1392,6 +1394,10 @@ export default function RollingMigrationFormDrawer({ ...(storageCopyMethod !== 'StorageAcceleratedCopy' && storageMappingResponse?.metadata?.name && { storageMapping: storageMappingResponse.metadata.name + }), + ...(storageCopyMethod === 'HotAddCopy' && + params.proxyVMName && { + proxyVMName: params.proxyVMName }) } }) @@ -3560,7 +3566,8 @@ export default function RollingMigrationFormDrawer({ networkMappings: networkMappings, storageMappings: storageMappings, arrayCredsMappings: arrayCredsMappings, - storageCopyMethod: params.storageCopyMethod as any + storageCopyMethod: params.storageCopyMethod as any, + proxyVMName: params.proxyVMName }} onChange={handleMappingsChange} networkMappingError={networkMappingError} diff --git a/v2v-helper/esxi-ssh/client.go b/v2v-helper/esxi-ssh/client.go index ca3a42255..cccf056be 100644 --- a/v2v-helper/esxi-ssh/client.go +++ b/v2v-helper/esxi-ssh/client.go @@ -6,6 +6,7 @@ import ( "context" "encoding/xml" "fmt" + "io" "net" "strconv" "strings" @@ -202,6 +203,64 @@ func (c *Client) TestConnection() error { return nil } +// TestConnectionGeneric tests SSH connectivity using a command that works on any Linux host. +// Use this instead of TestConnection() when connecting to a proxy VM (not an ESXi host). +func (c *Client) TestConnectionGeneric() error { + if c.sshClient == nil { + return fmt.Errorf("not connected") + } + + output, err := c.ExecuteCommand("hostname") + if err != nil { + return fmt.Errorf("connection test failed: %w", err) + } + + if output == "" { + return fmt.Errorf("connection test returned no output") + } + + utils.PrintLog(fmt.Sprintf("SSH connection verified, remote hostname: %s", strings.TrimSpace(output))) + return nil +} + +// RunCommandToWriter runs a command on the remote host and streams its stdout +// directly into dest without buffering the entire output in memory. +// Used to stream large disk data (e.g. dd if=/dev/sdb bs=4M) to a local writer. +func (c *Client) RunCommandToWriter(ctx context.Context, command string, dest io.Writer) error { + if c.sshClient == nil { + return fmt.Errorf("not connected to SSH host") + } + + session, err := c.sshClient.NewSession() + if err != nil { + return fmt.Errorf("failed to create SSH session: %w", err) + } + defer session.Close() + + session.Stdout = dest + + // Capture stderr separately so errors are visible in logs + var stderrBuf strings.Builder + session.Stderr = &stderrBuf + + done := make(chan error, 1) + go func() { + done <- session.Run(command) + }() + + select { + case <-ctx.Done(): + _ = session.Signal(ssh.SIGKILL) + return fmt.Errorf("command cancelled: %w", ctx.Err()) + case err := <-done: + if err != nil { + return fmt.Errorf("command %q failed: %w (stderr: %s)", command, err, strings.TrimSpace(stderrBuf.String())) + } + } + + return nil +} + // StartVmkfstoolsClone starts a vmkfstools clone operation from source VMDK to target LUN // This will automatically use StorageAcceleratedCopy if available on the storage array // The clone runs in the background and returns immediately diff --git a/v2v-helper/main.go b/v2v-helper/main.go index 324a25525..0958158e1 100644 --- a/v2v-helper/main.go +++ b/v2v-helper/main.go @@ -172,6 +172,7 @@ func main() { Reporter: eventReporter, FallbackToDHCP: migrationparams.FallbackToDHCP, StorageCopyMethod: migrationparams.StorageCopyMethod, + ProxyVMName: migrationparams.ProxyVMName, ArrayHost: arrayHost, ArrayUser: arrayUser, ArrayPassword: arrayPassword, diff --git a/v2v-helper/migrate/hotadd_copy.go b/v2v-helper/migrate/hotadd_copy.go new file mode 100644 index 000000000..7753c8e5c --- /dev/null +++ b/v2v-helper/migrate/hotadd_copy.go @@ -0,0 +1,533 @@ +// Copyright © 2025 The vjailbreak authors + +package migrate + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + "github.com/pkg/errors" + + "github.com/platform9/vjailbreak/pkg/common/constants" + esxissh "github.com/platform9/vjailbreak/v2v-helper/esxi-ssh" + "github.com/platform9/vjailbreak/v2v-helper/pkg/utils" + "github.com/platform9/vjailbreak/v2v-helper/vcenter" + "github.com/platform9/vjailbreak/v2v-helper/vm" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" +) + +// hotAddGetVCenterClient extracts the VCenterClient from VMops using the same +// pattern as vaai_copy.go so we do not need to change the VMOperations interface. +func (migobj *Migrate) hotAddGetVCenterClient() *vcenter.VCenterClient { + type vcenterClientGetter interface { + GetVCenterClient() *vcenter.VCenterClient + } + if g, ok := migobj.VMops.(vcenterClientGetter); ok { + return g.GetVCenterClient() + } + return nil +} + +// hotAddCreateLinkedClone creates a powered-off linked clone of sourceVM at the +// given snapshot. The clone is placed in the same folder and datastore as the +// source VM — it shares the parent disk, so creation is instant and uses no +// extra storage. Returns the govmomi object for the new clone VM. +func hotAddCreateLinkedClone( + ctx context.Context, + vcClient *vcenter.VCenterClient, + sourceVM *object.VirtualMachine, + snapshotRef types.ManagedObjectReference, + cloneName string, +) (*object.VirtualMachine, error) { + // Resolve the source VM's parent folder so the clone lands in the same place. + var vmMo mo.VirtualMachine + if err := sourceVM.Properties(ctx, sourceVM.Reference(), []string{"parent"}, &vmMo); err != nil { + return nil, fmt.Errorf("failed to get source VM parent folder: %w", err) + } + if vmMo.Parent == nil { + return nil, fmt.Errorf("source VM has no parent folder") + } + folder := object.NewFolder(vcClient.VCClient, *vmMo.Parent) + + cloneSpec := types.VirtualMachineCloneSpec{ + // MoveAllDiskBackingsAndAllowSharing keeps the clone on the same datastore + // and marks the disks as shared with the parent — this is the linked clone. + Location: types.VirtualMachineRelocateSpec{ + DiskMoveType: string(types.VirtualMachineRelocateDiskMoveOptionsMoveAllDiskBackingsAndAllowSharing), + }, + Snapshot: &snapshotRef, + PowerOn: false, + Template: false, + } + + task, err := sourceVM.Clone(ctx, folder, cloneName, cloneSpec) + if err != nil { + return nil, fmt.Errorf("CloneVM_Task failed: %w", err) + } + + info, err := task.WaitForResult(ctx) + if err != nil { + return nil, fmt.Errorf("linked clone task failed: %w", err) + } + + ref, ok := info.Result.(types.ManagedObjectReference) + if !ok { + return nil, fmt.Errorf("unexpected result type from CloneVM_Task: %T", info.Result) + } + + clone := object.NewVirtualMachine(vcClient.VCClient, ref) + return clone, nil +} + +// hotAddDestroyVM deletes a VM and all of its disk files. Used to clean up the +// linked clone after the migration copy is complete (or on failure). +func hotAddDestroyVM(ctx context.Context, vm *object.VirtualMachine) error { + task, err := vm.Destroy(ctx) + if err != nil { + return fmt.Errorf("Destroy_Task failed: %w", err) + } + if err := task.Wait(ctx); err != nil { + return fmt.Errorf("destroy task failed: %w", err) + } + return nil +} + +// hotAddGetVMDisks returns all VirtualDisk devices attached to a VM. +func hotAddGetVMDisks(ctx context.Context, vm *object.VirtualMachine) ([]*types.VirtualDisk, error) { + var vmMo mo.VirtualMachine + if err := vm.Properties(ctx, vm.Reference(), []string{"config.hardware.device"}, &vmMo); err != nil { + return nil, fmt.Errorf("failed to get VM device list: %w", err) + } + + var disks []*types.VirtualDisk + for _, device := range vmMo.Config.Hardware.Device { + if d, ok := device.(*types.VirtualDisk); ok { + disks = append(disks, d) + } + } + return disks, nil +} + +// hotAddGetVMIP returns the first non-link-local IPv4 address reported by +// VMware Tools for the given VM. +func hotAddGetVMIP(ctx context.Context, targetVM *object.VirtualMachine) (string, error) { + var vmMo mo.VirtualMachine + if err := targetVM.Properties(ctx, targetVM.Reference(), []string{"guest.net"}, &vmMo); err != nil { + return "", fmt.Errorf("failed to get VM guest info: %w", err) + } + + for _, nic := range vmMo.Guest.Net { + for _, ip := range nic.IpAddress { + if strings.Contains(ip, ":") || strings.HasPrefix(ip, "169.254.") { + continue + } + return ip, nil + } + } + + return "", fmt.Errorf("no IPv4 address found for VM (is VMware Tools running?)") +} + +// ValidateHotAddPrerequisites checks that all requirements for HotAdd copy are +// met before the migration starts, failing fast with a clear error message. +func (migobj *Migrate) ValidateHotAddPrerequisites(ctx context.Context) error { + migobj.logMessage("[HotAdd] Validating prerequisites") + + if migobj.ProxyVMName == "" { + return fmt.Errorf("PROXY_VM_NAME is required for HotAdd copy method") + } + + if len(migobj.ESXiSSHPrivateKey) == 0 { + if err := migobj.LoadESXiSSHKey(ctx); err != nil { + return errors.Wrap(err, "failed to load SSH private key") + } + } + + vcClient := migobj.hotAddGetVCenterClient() + if vcClient == nil { + return fmt.Errorf("cannot access vCenter client from VMops") + } + + proxyVM, err := vcClient.GetVMByName(ctx, migobj.ProxyVMName) + if err != nil { + return errors.Wrapf(err, "proxy VM %q not found in vCenter", migobj.ProxyVMName) + } + + state, err := proxyVM.PowerState(ctx) + if err != nil { + return errors.Wrap(err, "failed to get proxy VM power state") + } + if state != types.VirtualMachinePowerStatePoweredOn { + return fmt.Errorf("proxy VM %q must be powered on (current state: %s)", migobj.ProxyVMName, state) + } + + proxyIP, err := hotAddGetVMIP(ctx, proxyVM) + if err != nil { + return errors.Wrapf(err, "cannot determine IP of proxy VM %q (is VMware Tools running?)", migobj.ProxyVMName) + } + + sshClient := esxissh.NewClient() + defer sshClient.Disconnect() + if err := sshClient.Connect(ctx, proxyIP, "root", migobj.ESXiSSHPrivateKey); err != nil { + return errors.Wrapf(err, "cannot SSH to proxy VM %q at %s", migobj.ProxyVMName, proxyIP) + } + if err := sshClient.TestConnectionGeneric(); err != nil { + return errors.Wrapf(err, "SSH test failed for proxy VM %q", migobj.ProxyVMName) + } + + migobj.logMessage(fmt.Sprintf("[HotAdd] Proxy VM %q validated (IP: %s)", migobj.ProxyVMName, proxyIP)) + return nil +} + +// hotAddAddDiskToVM hot-adds a disk (identified by its VMDK backing path) to a +// running VM. It finds an existing SCSI controller and a free unit number, then +// issues a ReconfigVM_Task. Returns the device key assigned by vCenter so the +// disk can be removed later with hotAddRemoveDiskFromVM. +func hotAddAddDiskToVM( + ctx context.Context, + targetVM *object.VirtualMachine, + backingPath string, +) (int32, error) { + var vmMo mo.VirtualMachine + if err := targetVM.Properties(ctx, targetVM.Reference(), []string{"config.hardware.device"}, &vmMo); err != nil { + return 0, fmt.Errorf("failed to get proxy VM devices: %w", err) + } + + // Find the first SCSI controller on the proxy VM. + var controllerKey int32 + usedUnits := map[int32]bool{7: true} // unit 7 is reserved for the controller itself + + for _, dev := range vmMo.Config.Hardware.Device { + switch sc := dev.(type) { + case *types.VirtualLsiLogicController, + *types.VirtualLsiLogicSASController, + *types.ParaVirtualSCSIController, + *types.VirtualBusLogicController: + vc := sc.(types.BaseVirtualDevice).GetVirtualDevice() + if controllerKey == 0 { + controllerKey = vc.Key + } + } + // Track used unit numbers on the chosen controller. + vd := dev.GetVirtualDevice() + if controllerKey != 0 && vd.ControllerKey == controllerKey { + if vd.UnitNumber != nil { + usedUnits[*vd.UnitNumber] = true + } + } + } + + if controllerKey == 0 { + return 0, fmt.Errorf("no SCSI controller found on proxy VM %s", targetVM.Reference().Value) + } + + // Find the lowest free unit number (0-15, skip 7). + var unitNumber int32 + for usedUnits[unitNumber] { + unitNumber++ + if unitNumber > 15 { + return 0, fmt.Errorf("no free SCSI slots on proxy VM (controller key=%d)", controllerKey) + } + } + + disk := &types.VirtualDisk{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualDiskFlatVer2BackingInfo{ + VirtualDeviceFileBackingInfo: types.VirtualDeviceFileBackingInfo{ + FileName: backingPath, + }, + DiskMode: string(types.VirtualDiskModePersistent), + }, + ControllerKey: controllerKey, + UnitNumber: &unitNumber, + }, + } + + spec := types.VirtualMachineConfigSpec{ + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: disk, + }, + }, + } + + task, err := targetVM.Reconfigure(ctx, spec) + if err != nil { + return 0, fmt.Errorf("ReconfigVM hot-add failed: %w", err) + } + if err := task.Wait(ctx); err != nil { + return 0, fmt.Errorf("hot-add task failed: %w", err) + } + + // Re-read devices to find the key vCenter assigned to the new disk. + var after mo.VirtualMachine + if err := targetVM.Properties(ctx, targetVM.Reference(), []string{"config.hardware.device"}, &after); err != nil { + return 0, fmt.Errorf("failed to read devices after hot-add: %w", err) + } + for _, dev := range after.Config.Hardware.Device { + if d, ok := dev.(*types.VirtualDisk); ok { + if b, ok := d.Backing.(*types.VirtualDiskFlatVer2BackingInfo); ok { + if b.FileName == backingPath { + return d.Key, nil + } + } + } + } + + return 0, fmt.Errorf("disk was hot-added but could not find its device key (backing: %s)", backingPath) +} + +// hotAddRemoveDiskFromVM hot-removes a disk from a VM by its device key. +// The disk file is NOT deleted — only the device attachment is removed. +func hotAddRemoveDiskFromVM( + ctx context.Context, + targetVM *object.VirtualMachine, + deviceKey int32, +) error { + var vmMo mo.VirtualMachine + if err := targetVM.Properties(ctx, targetVM.Reference(), []string{"config.hardware.device"}, &vmMo); err != nil { + return fmt.Errorf("failed to get proxy VM devices: %w", err) + } + + var diskToRemove types.BaseVirtualDevice + for _, dev := range vmMo.Config.Hardware.Device { + if dev.GetVirtualDevice().Key == deviceKey { + diskToRemove = dev + break + } + } + if diskToRemove == nil { + return fmt.Errorf("device key %d not found on VM %s", deviceKey, targetVM.Reference().Value) + } + + spec := types.VirtualMachineConfigSpec{ + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationRemove, + Device: diskToRemove, + }, + }, + } + + task, err := targetVM.Reconfigure(ctx, spec) + if err != nil { + return fmt.Errorf("ReconfigVM hot-remove failed: %w", err) + } + if err := task.Wait(ctx); err != nil { + return fmt.Errorf("hot-remove task failed: %w", err) + } + return nil +} + +// hotAddDetectBlockDevice polls lsblk on the proxy VM until a block device +// matching the expected size appears. Returns the device path (e.g. /dev/sdb). +// Polls every 2 seconds for up to 30 seconds. +func hotAddDetectBlockDevice(sshClient *esxissh.Client, diskSizeBytes int64) (string, error) { + diskSizeKiB := diskSizeBytes / 1024 + deadline := time.Now().Add(30 * time.Second) + for time.Now().Before(deadline) { + out, err := sshClient.ExecuteCommand( + fmt.Sprintf("lsblk -bdn -o NAME,SIZE | awk '$2==%d {print \"/dev/\" $1}' | head -1", diskSizeBytes), + ) + if err == nil { + dev := strings.TrimSpace(out) + if dev != "" { + return dev, nil + } + } + // Fallback: match by size in KiB (lsblk -bn reports bytes but some versions differ). + out, err = sshClient.ExecuteCommand( + fmt.Sprintf("lsblk -dn -o NAME,SIZE | awk '$2==\"%dK\" {print \"/dev/\" $1}' | head -1", diskSizeKiB), + ) + if err == nil { + dev := strings.TrimSpace(out) + if dev != "" { + return dev, nil + } + } + time.Sleep(2 * time.Second) + } + return "", fmt.Errorf("block device with size %d bytes not found on proxy VM after 30s", diskSizeBytes) +} + +// HotAddCopyDisks orchestrates the full HotAdd disk copy for all VM disks. +// It is called from StartMigration after CreateVolumes has already created +// and set vminfo.VMDisks[idx].OpenstackVol for each disk. +func (migobj *Migrate) HotAddCopyDisks(ctx context.Context, vminfo *vm.VMInfo) error { + migobj.logMessage("[HotAdd] *** BETA: HotAdd SCSI transport ***") + migobj.logMessage("[HotAdd] Limitations: SCSI disks only, cold migration only, proxy VM must share datastore with source VM") + + vcClient := migobj.hotAddGetVCenterClient() + if vcClient == nil { + return fmt.Errorf("[HotAdd] cannot access vCenter client from VMops") + } + + // Step 1: Power off source VM (cold migration only for Beta). + migobj.logMessage("[HotAdd] Powering off source VM") + if err := migobj.VMops.VMPowerOff(); err != nil { + return errors.Wrap(err, "failed to power off source VM") + } + migobj.logMessage("[HotAdd] Source VM powered off") + + // Step 2: Clean up any leftover snapshots, then take a fresh one. + migobj.logMessage("[HotAdd] Cleaning up existing snapshots") + if err := migobj.VMops.CleanUpSnapshots(false); err != nil { + return errors.Wrap(err, "failed to clean up snapshots") + } + + migobj.logMessage(fmt.Sprintf("[HotAdd] Taking snapshot %q", constants.MigrationSnapshotName)) + if err := migobj.VMops.TakeSnapshot(constants.MigrationSnapshotName); err != nil { + return errors.Wrap(err, "failed to take snapshot") + } + if err := migobj.VMops.UpdateDisksInfo(vminfo); err != nil { + return errors.Wrap(err, "failed to update disk info after snapshot") + } + migobj.logMessage(fmt.Sprintf("[HotAdd] Snapshot taken, %d disk(s) found", len(vminfo.VMDisks))) + for i, d := range vminfo.VMDisks { + migobj.logMessage(fmt.Sprintf("[HotAdd] disk[%d] %s: backing=%s", i, d.Name, d.SnapBackingDisk)) + } + + // Step 3: Get snapshot reference for linked clone creation. + snapRef, err := migobj.VMops.GetSnapshot(constants.MigrationSnapshotName) + if err != nil { + return errors.Wrap(err, "failed to get snapshot reference") + } + + // Step 4: Locate proxy VM and get its IP. + proxyVM, err := vcClient.GetVMByName(ctx, migobj.ProxyVMName) + if err != nil { + return errors.Wrapf(err, "failed to find proxy VM %q", migobj.ProxyVMName) + } + proxyIP, err := hotAddGetVMIP(ctx, proxyVM) + if err != nil { + return errors.Wrapf(err, "failed to get IP of proxy VM %q", migobj.ProxyVMName) + } + migobj.logMessage(fmt.Sprintf("[HotAdd] Proxy VM %q found at %s", migobj.ProxyVMName, proxyIP)) + + // Step 5: Create linked clone — instant, shares parent disk, never powered on. + cloneName := fmt.Sprintf("vjailbreak-hotadd-%s-%d", vminfo.Name, time.Now().Unix()) + migobj.logMessage(fmt.Sprintf("[HotAdd] Creating linked clone %q", cloneName)) + linkedClone, err := hotAddCreateLinkedClone(ctx, vcClient, migobj.VMops.GetVMObj(), *snapRef, cloneName) + if err != nil { + return errors.Wrap(err, "failed to create linked clone") + } + migobj.logMessage(fmt.Sprintf("[HotAdd] Linked clone %q created", cloneName)) + + defer func() { + migobj.logMessage(fmt.Sprintf("[HotAdd] Destroying linked clone %q", cloneName)) + if err := hotAddDestroyVM(ctx, linkedClone); err != nil { + migobj.logMessage(fmt.Sprintf("[HotAdd] WARNING: failed to destroy linked clone: %v", err)) + } + migobj.logMessage("[HotAdd] Cleaning up snapshot") + if err := migobj.VMops.CleanUpSnapshots(true); err != nil { + migobj.logMessage(fmt.Sprintf("[HotAdd] WARNING: failed to clean up snapshot: %v", err)) + } + }() + + // Step 6: Get disk backing paths from the linked clone. + cloneDisks, err := hotAddGetVMDisks(ctx, linkedClone) + if err != nil { + return errors.Wrap(err, "failed to get linked clone disk list") + } + + if len(cloneDisks) != len(vminfo.VMDisks) { + return fmt.Errorf("[HotAdd] disk count mismatch: source has %d, clone has %d", len(vminfo.VMDisks), len(cloneDisks)) + } + for i, d := range cloneDisks { + if b, ok := d.Backing.(*types.VirtualDiskFlatVer2BackingInfo); ok { + migobj.logMessage(fmt.Sprintf("[HotAdd] clone disk[%d]: key=%d backing=%s", i, d.Key, b.FileName)) + } + } + + // Step 7: Per-disk: attach Cinder volume, hot-add, detect device, copy, hot-remove. + hotAddedKeys := []int32{} + defer func() { + for _, key := range hotAddedKeys { + migobj.logMessage(fmt.Sprintf("[HotAdd] Hot-removing disk key=%d from proxy VM", key)) + if err := hotAddRemoveDiskFromVM(ctx, proxyVM, key); err != nil { + migobj.logMessage(fmt.Sprintf("[HotAdd] WARNING: hot-remove key=%d failed: %v", key, err)) + } + } + }() + for idx := range vminfo.VMDisks { + disk := &vminfo.VMDisks[idx] + cloneDisk := cloneDisks[idx] + + backing, ok := cloneDisk.Backing.(*types.VirtualDiskFlatVer2BackingInfo) + if !ok { + return fmt.Errorf("[HotAdd] disk %s has unsupported backing type — only flat SCSI disks are supported", disk.Name) + } + migobj.logMessage(fmt.Sprintf("[HotAdd] === Disk %d/%d: %s ===", idx+1, len(vminfo.VMDisks), disk.Name)) + + // Attach the Cinder volume to this pod to get its device path. + cinderDevPath, err := migobj.AttachVolume(ctx, *disk) + if err != nil { + return errors.Wrapf(err, "failed to attach Cinder volume for disk %s", disk.Name) + } + + disk.Path = cinderDevPath + migobj.logMessage(fmt.Sprintf("[HotAdd] Cinder volume attached at %s", cinderDevPath)) + + // Hot-add the linked clone's disk to the proxy VM. + migobj.logMessage(fmt.Sprintf("[HotAdd] Hot-adding disk to proxy VM (backing: %s)", backing.FileName)) + deviceKey, err := hotAddAddDiskToVM(ctx, proxyVM, backing.FileName) + if err != nil { + return errors.Wrapf(err, "failed to hot-add disk %s to proxy VM", disk.Name) + } + hotAddedKeys = append(hotAddedKeys, deviceKey) + migobj.logMessage(fmt.Sprintf("[HotAdd] Disk hot-added (device key=%d)", deviceKey)) + + // Give the proxy VM OS a moment to register the new SCSI device. + time.Sleep(3 * time.Second) + + // SSH to proxy VM and detect the block device by size. + sshClient := esxissh.NewClientWithTimeout(2 * time.Hour) + if err := sshClient.Connect(ctx, proxyIP, "root", migobj.ESXiSSHPrivateKey); err != nil { + return errors.Wrapf(err, "failed to SSH to proxy VM at %s", proxyIP) + } + + migobj.logMessage(fmt.Sprintf("[HotAdd] Detecting block device for %s (size %d bytes)", disk.Name, disk.Size)) + remoteDevice, err := hotAddDetectBlockDevice(sshClient, disk.Size) + if err != nil { + sshClient.Disconnect() + return errors.Wrapf(err, "failed to detect block device on proxy VM for disk %s", disk.Name) + } + migobj.logMessage(fmt.Sprintf("[HotAdd] Block device detected: %s", remoteDevice)) + + // Open local Cinder device for writing. + destFile, err := os.OpenFile(cinderDevPath, os.O_WRONLY, 0) + if err != nil { + sshClient.Disconnect() + return errors.Wrapf(err, "failed to open Cinder device %s for writing", cinderDevPath) + } + + // Stream: proxy VM dd → SSH pipe → local Cinder device. + copyCmd := fmt.Sprintf("dd if=%s bs=4M 2>/dev/null", remoteDevice) + migobj.logMessage(fmt.Sprintf("[HotAdd] Starting copy: %s (proxy) -> %s (Cinder)", remoteDevice, cinderDevPath)) + startTime := time.Now() + + copyErr := sshClient.RunCommandToWriter(ctx, copyCmd, destFile) + destFile.Close() + sshClient.Disconnect() + + if copyErr != nil { + return errors.Wrapf(copyErr, "disk copy failed for %s", disk.Name) + } + + duration := time.Since(startTime).Round(time.Second) + throughputMBps := float64(disk.Size) / duration.Seconds() / 1024 / 1024 + migobj.logMessage(fmt.Sprintf("[HotAdd] Disk %s copied in %s (%.1f MB/s)", disk.Name, duration, throughputMBps)) + + utils.PrintLog(fmt.Sprintf("[HotAdd] disk %d/%d done: %s in %s (%.1f MB/s)", + idx+1, len(vminfo.VMDisks), disk.Name, duration, throughputMBps)) + } + + migobj.logMessage("[HotAdd] All disks copied successfully") + return nil +} diff --git a/v2v-helper/migrate/migrate.go b/v2v-helper/migrate/migrate.go index eaf78e2ae..d56ba6033 100644 --- a/v2v-helper/migrate/migrate.go +++ b/v2v-helper/migrate/migrate.go @@ -72,6 +72,8 @@ type Migrate struct { Reporter *reporter.Reporter FallbackToDHCP bool StorageCopyMethod string + // ProxyVMName is the name of the proxy VM used for HotAdd transport + ProxyVMName string // Array credentials for StorageAcceleratedCopy storage migration ArrayHost string ArrayUser string @@ -1791,7 +1793,8 @@ func (migobj *Migrate) MigrateVM(ctx context.Context) error { return errors.Wrap(err, "failed to get vcenter settings") } - if migobj.StorageCopyMethod == constants.StorageCopyMethod { + switch migobj.StorageCopyMethod { + case constants.StorageCopyMethod: // StorageAcceleratedCopy // Initialize storage provider if using StorageAcceleratedCopy migration if err := migobj.InitializeStorageProvider(ctx); err != nil { return errors.Wrap(err, "failed to initialize storage provider") @@ -1804,13 +1807,26 @@ func (migobj *Migrate) MigrateVM(ctx context.Context) error { if err := migobj.ValidateStorageAcceleratedCopyPrerequisites(ctx); err != nil { return errors.Wrap(err, "StorageAcceleratedCopy prerequisites validation failed") } - - // Perform the copy here. if _, err := migobj.StorageAcceleratedCopyCopyDisks(ctx, vminfo); err != nil { return errors.Wrap(err, "failed to perform StorageAcceleratedCopy copy") } - } else { + case constants.HotAddCopy: + if err := migobj.ValidateHotAddPrerequisites(ctx); err != nil { + return errors.Wrap(err, "HotAdd prerequisites validation failed") + } + vminfo, err = migobj.CreateVolumes(ctx, vminfo) + if err != nil { + return errors.Wrap(err, "failed to create Cinder volumes for HotAdd") + } + if err := migobj.HotAddCopyDisks(ctx, &vminfo); err != nil { + if cleanuperror := migobj.cleanup(ctx, vminfo, fmt.Sprintf("HotAdd copy failed: %s", err), portids, nil); cleanuperror != nil { + return errors.Wrapf(err, "HotAdd copy failed, also cleanup failed: %s", cleanuperror) + } + return errors.Wrap(err, "HotAdd copy failed") + } + + default: // NBD // Create and Add Volumes to Host vminfo, err = migobj.CreateVolumes(ctx, vminfo) diff --git a/v2v-helper/pkg/utils/vcenterutils.go b/v2v-helper/pkg/utils/vcenterutils.go index d9cb20da7..ad8cadb86 100644 --- a/v2v-helper/pkg/utils/vcenterutils.go +++ b/v2v-helper/pkg/utils/vcenterutils.go @@ -52,6 +52,7 @@ type MigrationParams struct { ArrayCredsMapping string CurrentInstanceID string + ProxyVMName string } // GetMigrationParams is function that returns the migration parameters @@ -100,6 +101,7 @@ func GetMigrationParams(ctx context.Context, client client.Client) (*MigrationPa StorageCopyMethod: string(configMap.Data["STORAGE_COPY_METHOD"]), VendorType: string(configMap.Data["VENDOR_TYPE"]), ArrayCredsMapping: string(configMap.Data["ARRAY_CREDS_MAPPING"]), + ProxyVMName: string(configMap.Data["PROXY_VM_NAME"]), AcknowledgeNetworkConflictRisk: string(configMap.Data["ACKNOWLEDGE_NETWORK_CONFLICT_RISK"]) == constants.TrueString, NetworkOverrides: string(configMap.Data["NETWORK_OVERRIDES"]), CurrentInstanceID: string(configMap.Data["CURRENT_INSTANCE_ID"]),