diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 08fa4b438f63..db8d3bfffe31 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -2041,7 +2041,6 @@ protected StartAnswer execute(StartCommand cmd) { VirtualMachineDefinedProfileSpec diskProfileSpec = null; VirtualMachineDefinedProfileSpec vmProfileSpec = null; - DeployAsIsInfoTO deployAsIsInfo = vmSpec.getDeployAsIsInfo(); boolean deployAsIs = deployAsIsInfo != null; @@ -2085,7 +2084,6 @@ protected StartAnswer execute(StartCommand cmd) { } VirtualMachineDiskInfoBuilder diskInfoBuilder = null; - VirtualDevice[] nicDevices = null; VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName); DiskControllerType systemVmScsiControllerType = DiskControllerType.lsilogic; int firstScsiControllerBusNum = 0; @@ -2102,7 +2100,6 @@ protected StartAnswer execute(StartCommand cmd) { diskDatastores = vmMo.getAllDiskDatastores(); diskInfoBuilder = vmMo.getDiskInfoBuilder(); hasSnapshot = vmMo.hasSnapshot(); - nicDevices = vmMo.getNicDevices(); tearDownVmDevices(vmMo, hasSnapshot, deployAsIs); ensureDiskControllersInternal(vmMo, systemVm, controllerInfo, systemVmScsiControllerType, @@ -2118,17 +2115,20 @@ protected StartAnswer execute(StartCommand cmd) { } takeVmFromOtherHyperHost(hyperHost, vmInternalCSName); + vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName); - if (getVmPowerState(vmMo) != PowerState.PowerOff) - vmMo.safePowerOff(_shutdownWaitMs); + if (vmMo != null) { + if (getVmPowerState(vmMo) != PowerState.PowerOff) + vmMo.safePowerOff(_shutdownWaitMs); - diskInfoBuilder = vmMo.getDiskInfoBuilder(); - hasSnapshot = vmMo.hasSnapshot(); - diskDatastores = vmMo.getAllDiskDatastores(); + diskInfoBuilder = vmMo.getDiskInfoBuilder(); + hasSnapshot = vmMo.hasSnapshot(); + diskDatastores = vmMo.getAllDiskDatastores(); - tearDownVmDevices(vmMo, hasSnapshot, deployAsIs); - ensureDiskControllersInternal(vmMo, systemVm, controllerInfo, systemVmScsiControllerType, - numScsiControllerForSystemVm, firstScsiControllerBusNum, deployAsIs); + tearDownVmDevices(vmMo, hasSnapshot, deployAsIs); + ensureDiskControllersInternal(vmMo, systemVm, controllerInfo, systemVmScsiControllerType, + numScsiControllerForSystemVm, firstScsiControllerBusNum, deployAsIs); + } } else { // If a VM with the same name is found in a different cluster in the DC, unregister the old VM and configure a new VM (cold-migration). VirtualMachineMO existingVmInDc = dcMo.findVm(vmInternalCSName); @@ -2145,7 +2145,7 @@ protected StartAnswer execute(StartCommand cmd) { vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName); if (vmMo == null) { logger.info("Cloned deploy-as-is VM " + vmInternalCSName + " is not in this host, relocating it"); - vmMo = takeVmFromOtherHyperHost(hyperHost, vmInternalCSName); + takeVmFromOtherHyperHost(hyperHost, vmInternalCSName); } } else { DiskTO rootDisk = null; @@ -2255,11 +2255,11 @@ protected StartAnswer execute(StartCommand cmd) { vmConfigSpec.setCpuHotAddEnabled(vmMo.isCpuHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm()); } - if(!vmMo.isMemoryHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm()){ + if (!vmMo.isMemoryHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm()) { logger.warn("hotadd of memory is not supported, dynamic scaling feature can not be applied to vm: " + vmInternalCSName); } - if(!vmMo.isCpuHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm()){ + if (!vmMo.isCpuHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm()) { logger.warn("hotadd of cpu is not supported, dynamic scaling feature can not be applied to vm: " + vmInternalCSName); } @@ -2709,14 +2709,24 @@ protected StartAnswer execute(StartCommand cmd) { } private boolean powerOnVM(final VirtualMachineMO vmMo, final String vmInternalCSName, final String vmNameOnVcenter) throws Exception { - int retry = 20; - while (retry-- > 0) { + final int retry = 20; + int retryAttempt = 0; + while (++retryAttempt <= retry) { try { + logger.debug(String.format("VM %s, powerOn attempt #%d", vmInternalCSName, retryAttempt)); return vmMo.powerOn(); } catch (Exception e) { logger.info(String.format("Got exception while power on VM %s with hostname %s", vmInternalCSName, vmNameOnVcenter), e); - if (e.getMessage() != null && e.getMessage().contains("File system specific implementation of Ioctl[file] failed")) { + if (e.getMessage() != null && + (e.getMessage().contains("File system specific implementation of Ioctl[file] failed") || + e.getMessage().contains("Unable to access file") || + e.getMessage().contains("it is locked"))) { logger.debug(String.format("Failed to power on VM %s with hostname %s. Retrying", vmInternalCSName, vmNameOnVcenter)); + try { + Thread.sleep(1000); + } catch (InterruptedException ie) { + logger.debug(String.format("Waiting to power on VM %s been interrupted: ", vmInternalCSName)); + } } else { throw e; } @@ -3219,7 +3229,7 @@ private void tearDownVm(VirtualMachineMO vmMo) throws Exception { int getReservedMemoryMb(VirtualMachineTO vmSpec) { if (vmSpec.getDetails().get(VMwareGuru.VmwareReserveMemory.key()).equalsIgnoreCase("true")) { - if(vmSpec.getDetails().get(VmDetailConstants.RAM_RESERVATION) != null){ + if (vmSpec.getDetails().get(VmDetailConstants.RAM_RESERVATION) != null) { float reservedMemory = (vmSpec.getMaxRam() * Float.parseFloat(vmSpec.getDetails().get(VmDetailConstants.RAM_RESERVATION))); return (int) (reservedMemory / ResourceType.bytesToMiB); } @@ -4848,7 +4858,7 @@ private Answer migrateVolume(MigrateVolumeCommand cmd) { VmwareHypervisorHost dsHost = hyperHostInTargetCluster == null ? hyperHost : hyperHostInTargetCluster; String targetDsName = cmd.getTargetPool().getUuid(); morDestinationDS = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(dsHost, targetDsName); - if(morDestinationDS == null) { + if (morDestinationDS == null) { String msg = "Unable to find the target datastore: " + targetDsName + " on host: " + dsHost.getHyperHostName(); logger.error(msg); throw new CloudRuntimeException(msg); @@ -5815,6 +5825,11 @@ protected Answer execute(CleanupVMCommand cmd) { logger.debug(msg); return new Answer(cmd, true, msg); } catch (Exception e) { + if (e.getMessage().contains("was not found")) { + String msg = String.format("%s - VM [%s] file(s) not found, cleanup not needed .", e.getMessage(), cmd.getVmName()); + logger.debug(msg); + return new Answer(cmd, true, msg); + } return new Answer(cmd, false, createLogMessageException(e, cmd)); } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index c99d7d4d7075..ea40c606c084 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -814,7 +814,7 @@ public Answer cloneVolumeFromBaseTemplate(CopyCommand cmd) { existingVm.detachAllDisksAndDestroy(); } logger.info("ROOT Volume from deploy-as-is template, cloning template"); - cloneVMFromTemplate(hyperHost, template.getPath(), vmName, primaryStore.getUuid()); + cloneVMFromTemplate(hyperHost, template, volume, vmName, primaryStore.getUuid()); } else { logger.info("ROOT Volume from deploy-as-is template, volume already created at this point"); } @@ -1222,10 +1222,10 @@ private Ternary createTemplateFromVolume(VmwareContext conte // Get VMDK filename String templateVMDKName = ""; File[] files = new File(installFullPath).listFiles(); - if(files != null) { + if (files != null) { for(File file : files) { String fileName = file.getName(); - if(fileName.toLowerCase().startsWith(templateUniqueName) && fileName.toLowerCase().endsWith(".vmdk")) { + if (fileName.toLowerCase().startsWith(templateUniqueName) && fileName.toLowerCase().endsWith(".vmdk")) { templateVMDKName += fileName; break; } @@ -1856,16 +1856,16 @@ public Answer backupSnapshot(CopyCommand cmd) { CopyCmdAnswer answer = null; try { - if(vmName != null) { + if (vmName != null) { vmMo = hyperHost.findVmOnHyperHost(vmName); if (vmMo == null) { - if(logger.isDebugEnabled()) { + if (logger.isDebugEnabled()) { logger.debug("Unable to find owner VM for BackupSnapshotCommand on host " + hyperHost.getHyperHostName() + ", will try within datacenter"); } vmMo = hyperHost.findVmOnPeerHyperHost(vmName); } } - if(vmMo == null) { + if (vmMo == null) { dsMo = new DatastoreMO(hyperHost.getContext(), morDs); workerVMName = hostService.getWorkerName(context, cmd, 0, dsMo); vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, workerVMName, null); @@ -1899,10 +1899,10 @@ public Answer backupSnapshot(CopyCommand cmd) { String secondaryMountPoint = mountService.getMountPoint(secondaryStorageUrl, _nfsVersion); String snapshotDir = destSnapshot.getPath() + "/" + snapshotBackupUuid; File[] files = new File(secondaryMountPoint + "/" + snapshotDir).listFiles(); - if(files != null) { + if (files != null) { for(File file : files) { String fileName = file.getName(); - if(fileName.toLowerCase().startsWith(snapshotBackupUuid) && fileName.toLowerCase().endsWith(".vmdk")) { + if (fileName.toLowerCase().startsWith(snapshotBackupUuid) && fileName.toLowerCase().endsWith(".vmdk")) { physicalSize = new File(secondaryMountPoint + "/" + snapshotDir + "/" + fileName).length(); break; } @@ -3651,7 +3651,7 @@ private Long restoreVolumeFromSecStorage(VmwareHypervisorHost hyperHost, Datasto } workerVm.tagAsWorkerVM(); - if(!primaryDsMo.getDatastoreType().equalsIgnoreCase("VVOL")) { + if (!primaryDsMo.getDatastoreType().equalsIgnoreCase("VVOL")) { HypervisorHostHelper.createBaseFolderInDatastore(primaryDsMo, primaryDsMo.getDataCenterMor()); workerVm.moveAllVmDiskFiles(primaryDsMo, HypervisorHostHelper.VSPHERE_DATASTORE_BASE_FOLDER, false); } @@ -3811,8 +3811,9 @@ public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) { /** * Return the cloned VM from the template */ - public VirtualMachineMO cloneVMFromTemplate(VmwareHypervisorHost hyperHost, String templateName, String cloneName, String templatePrimaryStoreUuid) { + public VirtualMachineMO cloneVMFromTemplate(VmwareHypervisorHost hyperHost, TemplateObjectTO template, VolumeObjectTO volume, String cloneName, String templatePrimaryStoreUuid) { try { + String templateName = template.getPath(); VmwareContext context = hyperHost.getContext(); DatacenterMO dcMo = new DatacenterMO(context, hyperHost.getHyperHostDatacenter()); VirtualMachineMO templateMo = dcMo.findVm(templateName); @@ -3826,6 +3827,9 @@ public VirtualMachineMO cloneVMFromTemplate(VmwareHypervisorHost hyperHost, Stri throw new CloudRuntimeException("Unable to find datastore in vSphere"); } logger.info("Cloning VM " + cloneName + " from template " + templateName + " into datastore " + templatePrimaryStoreUuid); + if (template.getSize() != null) { + _fullCloneFlag = volume.getSize() > template.getSize() ? true : _fullCloneFlag; + } if (!_fullCloneFlag) { createVMLinkedClone(templateMo, dcMo, cloneName, morDatastore, morPool, null); } else { diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 021c6ff62267..4617110cab72 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -8383,6 +8383,7 @@ public Pair doInTransaction(final TransactionStatus status) th getRootVolumeSizeForVmRestore(newVol, template, userVm, diskOffering, details, true); volumeMgr.saveVolumeDetails(newVol.getDiskOfferingId(), newVol.getId()); + newVol = _volsDao.findById(newVol.getId()); // 1. Save usage event and update resource count for user vm volumes try { @@ -8482,7 +8483,6 @@ public Pair doInTransaction(final TransactionStatus status) th Long getRootVolumeSizeForVmRestore(Volume vol, VMTemplateVO template, UserVmVO userVm, DiskOffering diskOffering, Map details, boolean update) { VolumeVO resizedVolume = (VolumeVO) vol; - Long size = null; if (template != null && template.getSize() != null) { UserVmDetailVO vmRootDiskSizeDetail = userVmDetailsDao.findDetail(userVm.getId(), VmDetailConstants.ROOT_DISK_SIZE); diff --git a/test/integration/smoke/test_restore_vm.py b/test/integration/smoke/test_restore_vm.py index aac33460da1d..6ce06c957e68 100644 --- a/test/integration/smoke/test_restore_vm.py +++ b/test/integration/smoke/test_restore_vm.py @@ -148,9 +148,13 @@ def test_03_restore_vm_with_disk_offering_custom_size(self): self.assertEqual(root_vol.state, 'Ready', "Volume should be in Ready state") self.assertEqual(root_vol.size, 16 * 1024 * 1024 * 1024, "Size of volume and custom disk size should match") - old_root_vol = Volume.list(self.apiclient, id=old_root_vol.id)[0] - self.assertEqual(old_root_vol.state, "Destroy", "Old volume should be in Destroy state") - Volume.delete(old_root_vol, self.apiclient) + if self.hypervisor.lower() not in ["vmware"]: + old_root_vol = Volume.list(self.apiclient, id=old_root_vol.id)[0] + self.assertEqual(old_root_vol.state, "Destroy", "Old volume should be in Destroy state") + Volume.delete(old_root_vol, self.apiclient) + else: + old_root_vol = Volume.list(self.apiclient, id=old_root_vol.id) + self.assertEqual(old_root_vol, None, "Old volume should be deleted") @attr(tags=["advanced", "basic"], required_hardware="false") def test_04_restore_vm_allocated_root(self): diff --git a/test/integration/smoke/test_ssvm.py b/test/integration/smoke/test_ssvm.py index ad03c3d46e13..0784bc3820c6 100644 --- a/test/integration/smoke/test_ssvm.py +++ b/test/integration/smoke/test_ssvm.py @@ -988,6 +988,9 @@ def test_08_reboot_cpvm(self): # Private IP Address of System VMs are allowed to change after reboot - CLOUDSTACK-7745 + # Agent in Up state for a while after reboot, wait for the agent to Disconnect and back Up. + time.sleep(60) + # Wait for the agent to be up self.waitForSystemVMAgent(cpvm_response.name) @@ -1103,6 +1106,9 @@ def test_10_reboot_cpvm_forced(self): "Check whether CPVM is running or not" ) + # Agent in Up state for a while after reboot, wait for the agent to Disconnect and back Up. + time.sleep(60) + # Wait for the agent to be up self.waitForSystemVMAgent(cpvm_response.name) diff --git a/tools/marvin/marvin/config/test_data.py b/tools/marvin/marvin/config/test_data.py index bde48c876160..c90037ea22e1 100644 --- a/tools/marvin/marvin/config/test_data.py +++ b/tools/marvin/marvin/config/test_data.py @@ -1080,7 +1080,7 @@ "format": "vhd", "hypervisor": "xenserver", "ostype": "Other Linux (64-bit)", - "url": "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-amd64-azure.vhd.tar.gz", + "url": "https://cloud-images.ubuntu.com/releases/jammy/release/ubuntu-22.04-server-cloudimg-amd64-azure.vhd.tar.gz", "requireshvm": "True", "ispublic": "True", "isextractable": "True" @@ -1091,7 +1091,7 @@ "format": "ova", "hypervisor": "vmware", "ostype": "Other Linux (64-bit)", - "url": "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-amd64.ova", + "url": "https://cloud-images.ubuntu.com/releases/jammy/release/ubuntu-22.04-server-cloudimg-amd64.ova", "requireshvm": "True", "ispublic": "True", "deployasis": "True" diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index 4f3975650a08..a73e7580a709 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -2037,7 +2037,7 @@ public void moveAllVmDiskFiles(DatastoreMO destDsMo, String destDsDir, boolean f vmdkDescriptor = getVmdkFileInfo(fileItem.first()); logger.info("Move VM disk file " + srcFile.getPath() + " to " + destFile.getPath()); - srcDsMo.moveDatastoreFile(fileItem.first(), dcMo.getMor(), destDsMo.getMor(), destFile.getPath(), dcMo.getMor(), true); + moveDatastoreFile(srcDsMo, fileItem.first(), dcMo.getMor(), destDsMo.getMor(), destFile.getPath(), dcMo.getMor(), true); if (vmdkDescriptor != null) { String vmdkBaseFileName = vmdkDescriptor.first().getBaseFileName(); @@ -2045,13 +2045,38 @@ public void moveAllVmDiskFiles(DatastoreMO destDsMo, String destDsDir, boolean f destFile = new DatastoreFile(destDsMo.getName(), destDsDir, vmdkBaseFileName); logger.info("Move VM disk file " + baseFilePath + " to " + destFile.getPath()); - srcDsMo.moveDatastoreFile(baseFilePath, dcMo.getMor(), destDsMo.getMor(), destFile.getPath(), dcMo.getMor(), true); + moveDatastoreFile(srcDsMo, baseFilePath, dcMo.getMor(), destDsMo.getMor(), destFile.getPath(), dcMo.getMor(), true); } } } } } + private boolean moveDatastoreFile(final DatastoreMO dsMo, String srcFilePath, ManagedObjectReference morSrcDc, ManagedObjectReference morDestDs, + String destFilePath, ManagedObjectReference morDestDc, boolean forceOverwrite) throws Exception { + final int retry = 20; + int retryAttempt = 0; + while (++retryAttempt <= retry) { + try { + logger.debug(String.format("Move datastore file %s, attempt #%d", srcFilePath, retryAttempt)); + return dsMo.moveDatastoreFile(srcFilePath, morSrcDc, morDestDs, destFilePath, morDestDc, forceOverwrite); + } catch (Exception e) { + logger.info(String.format("Got exception while moving datastore file %s ", srcFilePath), e); + if (e.getMessage() != null && e.getMessage().contains("Unable to access file")) { + logger.debug(String.format("Failed to move datastore file %s. Retrying", srcFilePath)); + try { + Thread.sleep(1000); + } catch (InterruptedException ie) { + logger.debug(String.format("Waiting to move datastore file %s been interrupted: ", srcFilePath)); + } + } else { + throw e; + } + } + } + return false; + } + protected VirtualSCSIController getScsiController(DiskControllerType type) { switch (type) { case pvscsi: