VMware 80u2 and 80u3 updates/fixes

plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java

@@ -2041,7 +2041,6 @@
         VirtualMachineDefinedProfileSpec diskProfileSpec = null;
         VirtualMachineDefinedProfileSpec vmProfileSpec = null;
 
-
         DeployAsIsInfoTO deployAsIsInfo = vmSpec.getDeployAsIsInfo();
         boolean deployAsIs = deployAsIsInfo != null;
 
@@ -2085,7 +2084,6 @@
             }
 
             VirtualMachineDiskInfoBuilder diskInfoBuilder = null;
-            VirtualDevice[] nicDevices = null;
             VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName);
             DiskControllerType systemVmScsiControllerType = DiskControllerType.lsilogic;
             int firstScsiControllerBusNum = 0;
@@ -2102,7 +2100,6 @@
                 diskDatastores = vmMo.getAllDiskDatastores();
                 diskInfoBuilder = vmMo.getDiskInfoBuilder();
                 hasSnapshot = vmMo.hasSnapshot();
-                nicDevices = vmMo.getNicDevices();
 
                 tearDownVmDevices(vmMo, hasSnapshot, deployAsIs);
                 ensureDiskControllersInternal(vmMo, systemVm, controllerInfo, systemVmScsiControllerType,
@@ -2118,17 +2115,20 @@
                     }
 
                     takeVmFromOtherHyperHost(hyperHost, vmInternalCSName);
+                    vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName);
 
-                    if (getVmPowerState(vmMo) != PowerState.PowerOff)
-                        vmMo.safePowerOff(_shutdownWaitMs);
+                    if (vmMo != null) {
+                        if (getVmPowerState(vmMo) != PowerState.PowerOff)
+                            vmMo.safePowerOff(_shutdownWaitMs);
 
-                    diskInfoBuilder = vmMo.getDiskInfoBuilder();
-                    hasSnapshot = vmMo.hasSnapshot();
-                    diskDatastores = vmMo.getAllDiskDatastores();
+                        diskInfoBuilder = vmMo.getDiskInfoBuilder();
+                        hasSnapshot = vmMo.hasSnapshot();
+                        diskDatastores = vmMo.getAllDiskDatastores();
 
-                    tearDownVmDevices(vmMo, hasSnapshot, deployAsIs);
-                    ensureDiskControllersInternal(vmMo, systemVm, controllerInfo, systemVmScsiControllerType,
-                            numScsiControllerForSystemVm, firstScsiControllerBusNum, deployAsIs);
+                        tearDownVmDevices(vmMo, hasSnapshot, deployAsIs);
+                        ensureDiskControllersInternal(vmMo, systemVm, controllerInfo, systemVmScsiControllerType,
+                                numScsiControllerForSystemVm, firstScsiControllerBusNum, deployAsIs);
+                    }
                 } else {
                     // If a VM with the same name is found in a different cluster in the DC, unregister the old VM and configure a new VM (cold-migration).
                     VirtualMachineMO existingVmInDc = dcMo.findVm(vmInternalCSName);
@@ -2145,7 +2145,7 @@
                         vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName);
                         if (vmMo == null) {
                             logger.info("Cloned deploy-as-is VM " + vmInternalCSName + " is not in this host, relocating it");
-                            vmMo = takeVmFromOtherHyperHost(hyperHost, vmInternalCSName);
+                            takeVmFromOtherHyperHost(hyperHost, vmInternalCSName);
                         }
                     } else {
                         DiskTO rootDisk = null;
@@ -2255,11 +2255,11 @@
                 vmConfigSpec.setCpuHotAddEnabled(vmMo.isCpuHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm());
             }
 
-            if(!vmMo.isMemoryHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm()){
+            if (!vmMo.isMemoryHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm()) {
                 logger.warn("hotadd of memory is not supported, dynamic scaling feature can not be applied to vm: " + vmInternalCSName);
             }
 
-            if(!vmMo.isCpuHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm()){
+            if (!vmMo.isCpuHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm()) {
                 logger.warn("hotadd of cpu is not supported, dynamic scaling feature can not be applied to vm: " + vmInternalCSName);
             }
 
@@ -2709,14 +2709,24 @@
     }
 
     private boolean powerOnVM(final VirtualMachineMO vmMo, final String vmInternalCSName, final String vmNameOnVcenter) throws Exception {
-        int retry = 20;
-        while (retry-- > 0) {
+        final int retry = 20;
+        int retryAttempt = 0;
+        while (++retryAttempt <= retry) {
             try {
+                logger.debug(String.format("VM %s, powerOn attempt #%d", vmInternalCSName, retryAttempt));
                 return vmMo.powerOn();
             } catch (Exception e) {
                 logger.info(String.format("Got exception while power on VM %s with hostname %s", vmInternalCSName, vmNameOnVcenter), e);
-                if (e.getMessage() != null && e.getMessage().contains("File system specific implementation of Ioctl[file] failed")) {
+                if (e.getMessage() != null &&
+                        (e.getMessage().contains("File system specific implementation of Ioctl[file] failed") ||
+                                e.getMessage().contains("Unable to access file") ||
+                                e.getMessage().contains("it is locked"))) {
                     logger.debug(String.format("Failed to power on VM %s with hostname %s. Retrying", vmInternalCSName, vmNameOnVcenter));
+                    try {
+                        Thread.sleep(1000);
+                    } catch (InterruptedException ie) {
+                        logger.debug(String.format("Waiting to power on VM %s been interrupted: ", vmInternalCSName));
+                    }
                 } else {
                     throw e;
                 }
@@ -3219,7 +3229,7 @@
 
     int getReservedMemoryMb(VirtualMachineTO vmSpec) {
         if (vmSpec.getDetails().get(VMwareGuru.VmwareReserveMemory.key()).equalsIgnoreCase("true")) {
-            if(vmSpec.getDetails().get(VmDetailConstants.RAM_RESERVATION) != null){
+            if (vmSpec.getDetails().get(VmDetailConstants.RAM_RESERVATION) != null) {
                 float reservedMemory = (vmSpec.getMaxRam() * Float.parseFloat(vmSpec.getDetails().get(VmDetailConstants.RAM_RESERVATION)));
                 return (int) (reservedMemory / ResourceType.bytesToMiB);
             }
@@ -4848,7 +4858,7 @@
             VmwareHypervisorHost dsHost = hyperHostInTargetCluster == null ? hyperHost : hyperHostInTargetCluster;
             String targetDsName = cmd.getTargetPool().getUuid();
             morDestinationDS = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(dsHost, targetDsName);
-            if(morDestinationDS == null) {
+            if (morDestinationDS == null) {
                 String msg = "Unable to find the target datastore: " + targetDsName + " on host: " + dsHost.getHyperHostName();
                 logger.error(msg);
                 throw new CloudRuntimeException(msg);
@@ -5815,6 +5825,11 @@
             logger.debug(msg);
             return new Answer(cmd, true, msg);
         } catch (Exception e) {
+            if (e.getMessage().contains("was not found")) {
+                String msg = String.format("%s - VM [%s] file(s) not found, cleanup not needed .", e.getMessage(), cmd.getVmName());
+                logger.debug(msg);
+                return new Answer(cmd, true, msg);
+            }
             return new Answer(cmd, false, createLogMessageException(e, cmd));
         }
     }

plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java

@@ -814,7 +814,7 @@
                         existingVm.detachAllDisksAndDestroy();
                     }
                     logger.info("ROOT Volume from deploy-as-is template, cloning template");
-                    cloneVMFromTemplate(hyperHost, template.getPath(), vmName, primaryStore.getUuid());
+                    cloneVMFromTemplate(hyperHost, template, volume, vmName, primaryStore.getUuid());
                 } else {
                     logger.info("ROOT Volume from deploy-as-is template, volume already created at this point");
                 }
@@ -1222,10 +1222,10 @@
             // Get VMDK filename
             String templateVMDKName = "";
             File[] files = new File(installFullPath).listFiles();
-            if(files != null) {
+            if (files != null) {
                 for(File file : files) {
                     String fileName = file.getName();
-                    if(fileName.toLowerCase().startsWith(templateUniqueName) && fileName.toLowerCase().endsWith(".vmdk")) {
+                    if (fileName.toLowerCase().startsWith(templateUniqueName) && fileName.toLowerCase().endsWith(".vmdk")) {
                         templateVMDKName += fileName;
                         break;
                     }
@@ -1856,16 +1856,16 @@
             CopyCmdAnswer answer = null;
 
             try {
-                if(vmName != null) {
+                if (vmName != null) {
                     vmMo = hyperHost.findVmOnHyperHost(vmName);
                     if (vmMo == null) {
-                        if(logger.isDebugEnabled()) {
+                        if (logger.isDebugEnabled()) {
                             logger.debug("Unable to find owner VM for BackupSnapshotCommand on host " + hyperHost.getHyperHostName() + ", will try within datacenter");
                         }
                         vmMo = hyperHost.findVmOnPeerHyperHost(vmName);
                     }
                 }
-                if(vmMo == null) {
+                if (vmMo == null) {
                     dsMo = new DatastoreMO(hyperHost.getContext(), morDs);
                     workerVMName = hostService.getWorkerName(context, cmd, 0, dsMo);
                     vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, workerVMName, null);
@@ -1899,10 +1899,10 @@
                     String secondaryMountPoint = mountService.getMountPoint(secondaryStorageUrl, _nfsVersion);
                     String snapshotDir =  destSnapshot.getPath() + "/" + snapshotBackupUuid;
                     File[] files = new File(secondaryMountPoint + "/" + snapshotDir).listFiles();
-                    if(files != null) {
+                    if (files != null) {
                         for(File file : files) {
                             String fileName = file.getName();
-                            if(fileName.toLowerCase().startsWith(snapshotBackupUuid) && fileName.toLowerCase().endsWith(".vmdk")) {
+                            if (fileName.toLowerCase().startsWith(snapshotBackupUuid) && fileName.toLowerCase().endsWith(".vmdk")) {
                                 physicalSize = new File(secondaryMountPoint + "/" + snapshotDir + "/" + fileName).length();
                                 break;
                             }
@@ -3651,7 +3651,7 @@
             }
             workerVm.tagAsWorkerVM();
 
-            if(!primaryDsMo.getDatastoreType().equalsIgnoreCase("VVOL")) {
+            if (!primaryDsMo.getDatastoreType().equalsIgnoreCase("VVOL")) {
                 HypervisorHostHelper.createBaseFolderInDatastore(primaryDsMo, primaryDsMo.getDataCenterMor());
                 workerVm.moveAllVmDiskFiles(primaryDsMo, HypervisorHostHelper.VSPHERE_DATASTORE_BASE_FOLDER, false);
             }
@@ -3811,8 +3811,9 @@
     /**
      * Return the cloned VM from the template
      */
-    public VirtualMachineMO cloneVMFromTemplate(VmwareHypervisorHost hyperHost, String templateName, String cloneName, String templatePrimaryStoreUuid) {
+    public VirtualMachineMO cloneVMFromTemplate(VmwareHypervisorHost hyperHost, TemplateObjectTO template, VolumeObjectTO volume, String cloneName, String templatePrimaryStoreUuid) {
         try {
+            String templateName = template.getPath();
             VmwareContext context = hyperHost.getContext();
             DatacenterMO dcMo = new DatacenterMO(context, hyperHost.getHyperHostDatacenter());
             VirtualMachineMO templateMo = dcMo.findVm(templateName);
@@ -3826,6 +3827,9 @@
                 throw new CloudRuntimeException("Unable to find datastore in vSphere");
             }
             logger.info("Cloning VM " + cloneName + " from template " + templateName + " into datastore " + templatePrimaryStoreUuid);
+            if (template.getSize() != null) {
+                _fullCloneFlag = volume.getSize() > template.getSize() ? true : _fullCloneFlag;
+            }
             if (!_fullCloneFlag) {
                 createVMLinkedClone(templateMo, dcMo, cloneName, morDatastore, morPool, null);
             } else {

server/src/main/java/com/cloud/vm/UserVmManagerImpl.java

@@ -8383,6 +8383,7 @@
 
                         getRootVolumeSizeForVmRestore(newVol, template, userVm, diskOffering, details, true);
                         volumeMgr.saveVolumeDetails(newVol.getDiskOfferingId(), newVol.getId());
+                        newVol = _volsDao.findById(newVol.getId());
 
                         // 1. Save usage event and update resource count for user vm volumes
                         try {
@@ -8482,7 +8483,6 @@
 
     Long getRootVolumeSizeForVmRestore(Volume vol, VMTemplateVO template, UserVmVO userVm, DiskOffering diskOffering, Map<String, String> details, boolean update) {
         VolumeVO resizedVolume = (VolumeVO) vol;
-
         Long size = null;
         if (template != null && template.getSize() != null) {
             UserVmDetailVO vmRootDiskSizeDetail = userVmDetailsDao.findDetail(userVm.getId(), VmDetailConstants.ROOT_DISK_SIZE);

test/integration/smoke/test_restore_vm.py

@@ -148,9 +148,13 @@ def test_03_restore_vm_with_disk_offering_custom_size(self):
         self.assertEqual(root_vol.state, 'Ready', "Volume should be in Ready state")
         self.assertEqual(root_vol.size, 16 * 1024 * 1024 * 1024, "Size of volume and custom disk size should match")
 
-        old_root_vol = Volume.list(self.apiclient, id=old_root_vol.id)[0]
-        self.assertEqual(old_root_vol.state, "Destroy", "Old volume should be in Destroy state")
-        Volume.delete(old_root_vol, self.apiclient)
+        if self.hypervisor.lower() not in ["vmware"]:
+            old_root_vol = Volume.list(self.apiclient, id=old_root_vol.id)[0]
+            self.assertEqual(old_root_vol.state, "Destroy", "Old volume should be in Destroy state")
+            Volume.delete(old_root_vol, self.apiclient)
+        else:
+            old_root_vol = Volume.list(self.apiclient, id=old_root_vol.id)
+            self.assertEqual(old_root_vol, None, "Old volume should be deleted")
 
     @attr(tags=["advanced", "basic"], required_hardware="false")
     def test_04_restore_vm_allocated_root(self):

test/integration/smoke/test_ssvm.py

@@ -988,6 +988,9 @@ def test_08_reboot_cpvm(self):
 
         # Private IP Address of System VMs are allowed to change after reboot - CLOUDSTACK-7745
 
+        # Agent in Up state for a while after reboot, wait for the agent to Disconnect and back Up.
+        time.sleep(60)
+
         # Wait for the agent to be up
         self.waitForSystemVMAgent(cpvm_response.name)
 
@@ -1103,6 +1106,9 @@ def test_10_reboot_cpvm_forced(self):
             "Check whether CPVM is running or not"
         )
 
+        # Agent in Up state for a while after reboot, wait for the agent to Disconnect and back Up.
+        time.sleep(60)
+
         # Wait for the agent to be up
         self.waitForSystemVMAgent(cpvm_response.name)
 

tools/marvin/marvin/config/test_data.py

@@ -1080,7 +1080,7 @@
             "format": "vhd",
             "hypervisor": "xenserver",
             "ostype": "Other Linux (64-bit)",
-            "url": "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-amd64-azure.vhd.tar.gz",
+            "url": "https://cloud-images.ubuntu.com/releases/jammy/release/ubuntu-22.04-server-cloudimg-amd64-azure.vhd.tar.gz",
             "requireshvm": "True",
             "ispublic": "True",
             "isextractable": "True"
@@ -1091,7 +1091,7 @@
             "format": "ova",
             "hypervisor": "vmware",
             "ostype": "Other Linux (64-bit)",
-            "url": "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-amd64.ova",
+            "url": "https://cloud-images.ubuntu.com/releases/jammy/release/ubuntu-22.04-server-cloudimg-amd64.ova",
             "requireshvm": "True",
             "ispublic": "True",
             "deployasis": "True"

vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java

@@ -2037,21 +2037,46 @@
                     vmdkDescriptor = getVmdkFileInfo(fileItem.first());
 
                     logger.info("Move VM disk file " + srcFile.getPath() + " to " + destFile.getPath());
-                    srcDsMo.moveDatastoreFile(fileItem.first(), dcMo.getMor(), destDsMo.getMor(), destFile.getPath(), dcMo.getMor(), true);
+                    moveDatastoreFile(srcDsMo, fileItem.first(), dcMo.getMor(), destDsMo.getMor(), destFile.getPath(), dcMo.getMor(), true);
 
                     if (vmdkDescriptor != null) {
                         String vmdkBaseFileName = vmdkDescriptor.first().getBaseFileName();
                         String baseFilePath = srcFile.getCompanionPath(vmdkBaseFileName);
                         destFile = new DatastoreFile(destDsMo.getName(), destDsDir, vmdkBaseFileName);
 
                         logger.info("Move VM disk file " + baseFilePath + " to " + destFile.getPath());
-                        srcDsMo.moveDatastoreFile(baseFilePath, dcMo.getMor(), destDsMo.getMor(), destFile.getPath(), dcMo.getMor(), true);
+                        moveDatastoreFile(srcDsMo, baseFilePath, dcMo.getMor(), destDsMo.getMor(), destFile.getPath(), dcMo.getMor(), true);
                     }
                 }
             }
         }
     }
 
+    private boolean moveDatastoreFile(final DatastoreMO dsMo, String srcFilePath, ManagedObjectReference morSrcDc, ManagedObjectReference morDestDs,
+                                   String destFilePath, ManagedObjectReference morDestDc, boolean forceOverwrite) throws Exception {
+        final int retry = 20;
+        int retryAttempt = 0;
+        while (++retryAttempt <= retry) {
+            try {
+                logger.debug(String.format("Move datastore file %s, attempt #%d", srcFilePath, retryAttempt));
+                return dsMo.moveDatastoreFile(srcFilePath, morSrcDc, morDestDs, destFilePath, morDestDc, forceOverwrite);
+            } catch (Exception e) {
+                logger.info(String.format("Got exception while moving datastore file %s ", srcFilePath), e);
+                if (e.getMessage() != null && e.getMessage().contains("Unable to access file")) {
+                    logger.debug(String.format("Failed to move datastore file %s. Retrying", srcFilePath));
+                    try {
+                        Thread.sleep(1000);
+                    } catch (InterruptedException ie) {
+                        logger.debug(String.format("Waiting to move datastore file %s been interrupted: ", srcFilePath));
+                    }
+                } else {
+                    throw e;
+                }
+            }
+        }
+        return false;
+    }
+
     protected VirtualSCSIController getScsiController(DiskControllerType type) {
         switch (type) {
             case pvscsi: