diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 04b9bdca39b5..481dbba922ed 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -2042,7 +2042,6 @@ protected StartAnswer execute(StartCommand cmd) { VirtualMachineDefinedProfileSpec diskProfileSpec = null; VirtualMachineDefinedProfileSpec vmProfileSpec = null; - DeployAsIsInfoTO deployAsIsInfo = vmSpec.getDeployAsIsInfo(); boolean deployAsIs = deployAsIsInfo != null; @@ -2086,7 +2085,6 @@ protected StartAnswer execute(StartCommand cmd) { } VirtualMachineDiskInfoBuilder diskInfoBuilder = null; - VirtualDevice[] nicDevices = null; VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName); DiskControllerType systemVmScsiControllerType = DiskControllerType.lsilogic; int firstScsiControllerBusNum = 0; @@ -2103,7 +2101,6 @@ protected StartAnswer execute(StartCommand cmd) { diskDatastores = vmMo.getAllDiskDatastores(); diskInfoBuilder = vmMo.getDiskInfoBuilder(); hasSnapshot = vmMo.hasSnapshot(); - nicDevices = vmMo.getNicDevices(); tearDownVmDevices(vmMo, hasSnapshot, deployAsIs); ensureDiskControllersInternal(vmMo, systemVm, controllerInfo, systemVmScsiControllerType, @@ -2119,17 +2116,20 @@ protected StartAnswer execute(StartCommand cmd) { } takeVmFromOtherHyperHost(hyperHost, vmInternalCSName); + vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName); - if (getVmPowerState(vmMo) != PowerState.PowerOff) - vmMo.safePowerOff(_shutdownWaitMs); + if (vmMo != null) { + if (getVmPowerState(vmMo) != PowerState.PowerOff) + vmMo.safePowerOff(_shutdownWaitMs); - diskInfoBuilder = vmMo.getDiskInfoBuilder(); - hasSnapshot = vmMo.hasSnapshot(); - diskDatastores = vmMo.getAllDiskDatastores(); + diskInfoBuilder = vmMo.getDiskInfoBuilder(); + hasSnapshot = vmMo.hasSnapshot(); + diskDatastores = vmMo.getAllDiskDatastores(); - tearDownVmDevices(vmMo, hasSnapshot, deployAsIs); - ensureDiskControllersInternal(vmMo, systemVm, controllerInfo, systemVmScsiControllerType, - numScsiControllerForSystemVm, firstScsiControllerBusNum, deployAsIs); + tearDownVmDevices(vmMo, hasSnapshot, deployAsIs); + ensureDiskControllersInternal(vmMo, systemVm, controllerInfo, systemVmScsiControllerType, + numScsiControllerForSystemVm, firstScsiControllerBusNum, deployAsIs); + } } else { // If a VM with the same name is found in a different cluster in the DC, unregister the old VM and configure a new VM (cold-migration). VirtualMachineMO existingVmInDc = dcMo.findVm(vmInternalCSName); @@ -2146,7 +2146,7 @@ protected StartAnswer execute(StartCommand cmd) { vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName); if (vmMo == null) { logger.info("Cloned deploy-as-is VM " + vmInternalCSName + " is not in this host, relocating it"); - vmMo = takeVmFromOtherHyperHost(hyperHost, vmInternalCSName); + takeVmFromOtherHyperHost(hyperHost, vmInternalCSName); } } else { DiskTO rootDisk = null; @@ -2256,11 +2256,11 @@ protected StartAnswer execute(StartCommand cmd) { vmConfigSpec.setCpuHotAddEnabled(vmMo.isCpuHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm()); } - if(!vmMo.isMemoryHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm()){ + if (!vmMo.isMemoryHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm()) { logger.warn("hotadd of memory is not supported, dynamic scaling feature can not be applied to vm: " + vmInternalCSName); } - if(!vmMo.isCpuHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm()){ + if (!vmMo.isCpuHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm()) { logger.warn("hotadd of cpu is not supported, dynamic scaling feature can not be applied to vm: " + vmInternalCSName); } @@ -2593,7 +2593,7 @@ protected StartAnswer execute(StartCommand cmd) { Map> iqnToData = new HashMap<>(); - postDiskConfigBeforeStart(vmMo, vmSpec, sortedDisks, ideControllerKey, scsiControllerKey, iqnToData, hyperHost, context); + postDiskConfigBeforeStart(vmMo, vmSpec, sortedDisks, iqnToData, hyperHost, context); // // Power-on VM @@ -2731,14 +2731,24 @@ private void syncVolumeDatastoreAndPathForDatastoreCluster(DiskTO vol, VirtualMa } private boolean powerOnVM(final VirtualMachineMO vmMo, final String vmInternalCSName, final String vmNameOnVcenter) throws Exception { - int retry = 20; - while (retry-- > 0) { + final int retry = 20; + int retryAttempt = 0; + while (++retryAttempt <= retry) { try { + logger.debug(String.format("VM %s, powerOn attempt #%d", vmInternalCSName, retryAttempt)); return vmMo.powerOn(); } catch (Exception e) { logger.info(String.format("Got exception while power on VM %s with hostname %s", vmInternalCSName, vmNameOnVcenter), e); - if (e.getMessage() != null && e.getMessage().contains("File system specific implementation of Ioctl[file] failed")) { + if (e.getMessage() != null && + (e.getMessage().contains("File system specific implementation of Ioctl[file] failed") || + e.getMessage().contains("Unable to access file") || + e.getMessage().contains("it is locked"))) { logger.debug(String.format("Failed to power on VM %s with hostname %s. Retrying", vmInternalCSName, vmNameOnVcenter)); + try { + Thread.sleep(1000); + } catch (InterruptedException ie) { + logger.debug(String.format("Waiting to power on VM %s been interrupted: ", vmInternalCSName)); + } } else { throw e; } @@ -3292,7 +3302,7 @@ private void tearDownVm(VirtualMachineMO vmMo) throws Exception { int getReservedMemoryMb(VirtualMachineTO vmSpec) { if (vmSpec.getDetails().get(VMwareGuru.VmwareReserveMemory.key()).equalsIgnoreCase("true")) { - if(vmSpec.getDetails().get(VmDetailConstants.RAM_RESERVATION) != null){ + if (vmSpec.getDetails().get(VmDetailConstants.RAM_RESERVATION) != null) { float reservedMemory = (vmSpec.getMaxRam() * Float.parseFloat(vmSpec.getDetails().get(VmDetailConstants.RAM_RESERVATION))); return (int) (reservedMemory / ResourceType.bytesToMiB); } @@ -3630,18 +3640,18 @@ private Pair getVMDiskInfo(String volumePath, boolean isManaged, private VirtualMachineDiskInfo getMatchingExistingDisk(VirtualMachineDiskInfoBuilder diskInfoBuilder, DiskTO vol, VmwareHypervisorHost hyperHost, VmwareContext context) throws Exception { - if (diskInfoBuilder != null) { - VolumeObjectTO volume = (VolumeObjectTO) vol.getData(); - String chainInfo = volume.getChainInfo(); - Map details = vol.getDetails(); - boolean isManaged = details != null && Boolean.parseBoolean(details.get(DiskTO.MANAGED)); - String iScsiName = details.get(DiskTO.IQN); - String datastoreUUID = volume.getDataStore().getUuid(); - - return getMatchingExistingDiskWithVolumeDetails(diskInfoBuilder, volume.getPath(), chainInfo, isManaged, iScsiName, datastoreUUID, hyperHost, context); - } else { + if (diskInfoBuilder == null) { return null; } + + VolumeObjectTO volume = (VolumeObjectTO) vol.getData(); + String chainInfo = volume.getChainInfo(); + Map details = vol.getDetails(); + boolean isManaged = details != null && Boolean.parseBoolean(details.get(DiskTO.MANAGED)); + String iScsiName = details.get(DiskTO.IQN); + String datastoreUUID = volume.getDataStore().getUuid(); + + return getMatchingExistingDiskWithVolumeDetails(diskInfoBuilder, volume.getPath(), chainInfo, isManaged, iScsiName, datastoreUUID, hyperHost, context); } private String getDiskController(VirtualMachineMO vmMo, VirtualMachineDiskInfo matchingExistingDisk, DiskTO vol, Pair controllerInfo, boolean deployAsIs) throws Exception { @@ -3666,34 +3676,36 @@ private String getDiskController(VirtualMachineMO vmMo, VirtualMachineDiskInfo m return VmwareHelper.getControllerBasedOnDiskType(controllerInfo, vol); } - private void postDiskConfigBeforeStart(VirtualMachineMO vmMo, VirtualMachineTO vmSpec, DiskTO[] sortedDisks, int ideControllerKey, - int scsiControllerKey, Map> iqnToData, VmwareHypervisorHost hyperHost, VmwareContext context) throws Exception { + private void postDiskConfigBeforeStart(VirtualMachineMO vmMo, VirtualMachineTO vmSpec, DiskTO[] sortedDisks, + Map> iqnToData, VmwareHypervisorHost hyperHost, VmwareContext context) throws Exception { VirtualMachineDiskInfoBuilder diskInfoBuilder = vmMo.getDiskInfoBuilder(); for (DiskTO vol : sortedDisks) { if (vol.getType() == Volume.Type.ISO) continue; - VolumeObjectTO volumeTO = (VolumeObjectTO) vol.getData(); - VirtualMachineDiskInfo diskInfo = getMatchingExistingDisk(diskInfoBuilder, vol, hyperHost, context); - assert (diskInfo != null); + if (diskInfo == null) { + continue; + } String[] diskChain = diskInfo.getDiskChain(); - assert (diskChain.length > 0); + if (diskChain.length <= 0) { + continue; + } - Map details = vol.getDetails(); - boolean managed = false; + DatastoreFile file = new DatastoreFile(diskChain[0]); + boolean managed = false; + Map details = vol.getDetails(); if (details != null) { managed = Boolean.parseBoolean(details.get(DiskTO.MANAGED)); } - DatastoreFile file = new DatastoreFile(diskChain[0]); + VolumeObjectTO volumeTO = (VolumeObjectTO) vol.getData(); if (managed) { DatastoreFile originalFile = new DatastoreFile(volumeTO.getPath()); - if (!file.getFileBaseName().equalsIgnoreCase(originalFile.getFileBaseName())) { if (logger.isInfoEnabled()) logger.info("Detected disk-chain top file change on volume: " + volumeTO.getId() + " " + volumeTO.getPath() + " -> " + diskChain[0]); @@ -3706,7 +3718,6 @@ private void postDiskConfigBeforeStart(VirtualMachineMO vmMo, VirtualMachineTO v } VolumeObjectTO volInSpec = getVolumeInSpec(vmSpec, volumeTO); - if (volInSpec != null) { if (managed) { Map data = new HashMap<>(); @@ -3871,20 +3882,20 @@ private DatastoreMO getDataStoreWhereDiskExists(VmwareHypervisorHost hyperHost, if (diskInfo != null) { logger.info("Found existing disk info from volume path: " + volume.getPath()); return dsMo; - } else { - String chainInfo = volume.getChainInfo(); - if (chainInfo != null) { - VirtualMachineDiskInfo infoInChain = _gson.fromJson(chainInfo, VirtualMachineDiskInfo.class); - if (infoInChain != null) { - String[] disks = infoInChain.getDiskChain(); - if (disks.length > 0) { - for (String diskPath : disks) { - DatastoreFile file = new DatastoreFile(diskPath); - diskInfo = diskInfoBuilder.getDiskInfoByBackingFileBaseName(file.getFileBaseName(), dsName); - if (diskInfo != null) { - logger.info("Found existing disk from chain info: " + diskPath); - return dsMo; - } + } + + String chainInfo = volume.getChainInfo(); + if (chainInfo != null) { + VirtualMachineDiskInfo infoInChain = _gson.fromJson(chainInfo, VirtualMachineDiskInfo.class); + if (infoInChain != null) { + String[] disks = infoInChain.getDiskChain(); + if (disks.length > 0) { + for (String diskPath : disks) { + DatastoreFile file = new DatastoreFile(diskPath); + diskInfo = diskInfoBuilder.getDiskInfoByBackingFileBaseName(file.getFileBaseName(), dsName); + if (diskInfo != null) { + logger.info("Found existing disk from chain info: " + diskPath); + return dsMo; } } } @@ -4747,7 +4758,7 @@ private Answer migrateAndAnswer(VirtualMachineMO vmMo, String poolUuid, VmwareHy Map volumeDeviceKey = new HashMap<>(); if (cmd instanceof MigrateVolumeCommand) { // Else device keys will be found in relocateVirtualMachine MigrateVolumeCommand mcmd = (MigrateVolumeCommand) cmd; - addVolumeDiskmapping(vmMo, volumeDeviceKey, mcmd.getVolumePath(), mcmd.getVolumeId()); + addVolumeDiskMapping(vmMo, volumeDeviceKey, mcmd.getVolumePath(), mcmd.getVolumeId()); if (logger.isTraceEnabled()) { for (Integer diskId: volumeDeviceKey.keySet()) { logger.trace(String.format("Disk to migrate has disk id %d and volumeId %d", diskId, volumeDeviceKey.get(diskId))); @@ -4765,9 +4776,7 @@ private Answer migrateAndAnswer(VirtualMachineMO vmMo, String poolUuid, VmwareHy Answer createAnswerForCmd(VirtualMachineMO vmMo, List volumeObjectToList, Command cmd, Map volumeDeviceKey) throws Exception { List volumeToList; - VirtualMachineDiskInfoBuilder diskInfoBuilder = vmMo.getDiskInfoBuilder(); VirtualDisk[] disks = vmMo.getAllDiskDevice(); - Answer answer; if (logger.isTraceEnabled()) { logger.trace(String.format("creating answer for %s", cmd.getClass().getSimpleName())); } @@ -4784,7 +4793,7 @@ Answer createAnswerForCmd(VirtualMachineMO vmMo, List volumeObje return new Answer(cmd, false, null); } - private void addVolumeDiskmapping(VirtualMachineMO vmMo, Map volumeDeviceKey, String volumePath, long volumeId) throws Exception { + private void addVolumeDiskMapping(VirtualMachineMO vmMo, Map volumeDeviceKey, String volumePath, long volumeId) throws Exception { if (logger.isDebugEnabled()) { logger.debug(String.format("locating disk for volume (%d) using path %s", volumeId, volumePath)); } @@ -4919,7 +4928,7 @@ private Answer migrateVolume(MigrateVolumeCommand cmd) { VmwareHypervisorHost dsHost = hyperHostInTargetCluster == null ? hyperHost : hyperHostInTargetCluster; String targetDsName = cmd.getTargetPool().getUuid(); morDestinationDS = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(dsHost, targetDsName); - if(morDestinationDS == null) { + if (morDestinationDS == null) { String msg = "Unable to find the target datastore: " + targetDsName + " on host: " + dsHost.getHyperHostName(); logger.error(msg); throw new CloudRuntimeException(msg); @@ -5886,6 +5895,11 @@ protected Answer execute(CleanupVMCommand cmd) { logger.debug(msg); return new Answer(cmd, true, msg); } catch (Exception e) { + if (e.getMessage().contains("was not found")) { + String msg = String.format("%s - VM [%s] file(s) not found, cleanup not needed .", e.getMessage(), cmd.getVmName()); + logger.debug(msg); + return new Answer(cmd, true, msg); + } return new Answer(cmd, false, createLogMessageException(e, cmd)); } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java index ab9754a7c9e0..69572a3cd177 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java @@ -20,6 +20,7 @@ import java.util.Arrays; import java.util.List; +import com.vmware.vim25.ManagedObjectReference; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.logging.log4j.Logger; @@ -193,7 +194,7 @@ public static String syncVolumeToVmDefaultFolder(DatacenterMO dcMo, String vmNam if (ds.fileExists(vmdkFullCloneModeLegacyPair[i])) { LOGGER.info("sync " + vmdkFullCloneModeLegacyPair[i] + "->" + vmdkFullCloneModePair[i]); - ds.moveDatastoreFile(vmdkFullCloneModeLegacyPair[i], dcMo.getMor(), ds.getMor(), vmdkFullCloneModePair[i], dcMo.getMor(), true); + moveDatastoreFile(ds, vmdkFullCloneModeLegacyPair[i], dcMo.getMor(), ds.getMor(), vmdkFullCloneModePair[i], dcMo.getMor(), true); } } @@ -201,13 +202,13 @@ public static String syncVolumeToVmDefaultFolder(DatacenterMO dcMo, String vmNam if (ds.fileExists(vmdkLinkedCloneModeLegacyPair[i])) { LOGGER.info("sync " + vmdkLinkedCloneModeLegacyPair[i] + "->" + vmdkLinkedCloneModePair[i]); - ds.moveDatastoreFile(vmdkLinkedCloneModeLegacyPair[i], dcMo.getMor(), ds.getMor(), vmdkLinkedCloneModePair[i], dcMo.getMor(), true); + moveDatastoreFile(ds, vmdkLinkedCloneModeLegacyPair[i], dcMo.getMor(), ds.getMor(), vmdkLinkedCloneModePair[i], dcMo.getMor(), true); } } if (ds.fileExists(vmdkLinkedCloneModeLegacyPair[0])) { LOGGER.info("sync " + vmdkLinkedCloneModeLegacyPair[0] + "->" + vmdkLinkedCloneModePair[0]); - ds.moveDatastoreFile(vmdkLinkedCloneModeLegacyPair[0], dcMo.getMor(), ds.getMor(), vmdkLinkedCloneModePair[0], dcMo.getMor(), true); + moveDatastoreFile(ds, vmdkLinkedCloneModeLegacyPair[0], dcMo.getMor(), ds.getMor(), vmdkLinkedCloneModePair[0], dcMo.getMor(), true); } // Note: we will always return a path @@ -242,14 +243,14 @@ public static void syncVolumeToRootFolder(DatacenterMO dcMo, DatastoreMO ds, Str String targetPath = getDatastorePathBaseFolderFromVmdkFileName(ds, String.format("%s-%s",vmdkName, linkedCloneExtension)); LOGGER.info("Fixup folder-synchronization. move " + companionFilePath + " -> " + targetPath); - ds.moveDatastoreFile(companionFilePath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true); + moveDatastoreFile(ds, companionFilePath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true); } } // move the identity VMDK file the last String targetPath = getDatastorePathBaseFolderFromVmdkFileName(ds, vmdkName + ".vmdk"); LOGGER.info("Fixup folder-synchronization. move " + fileDsFullPath + " -> " + targetPath); - ds.moveDatastoreFile(fileDsFullPath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true); + moveDatastoreFile(ds, fileDsFullPath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true); try { if (folderName != null) { @@ -287,7 +288,7 @@ public static void moveVolumeToRootFolder(DatacenterMO dcMo, List detach DatastoreFile targetFile = new DatastoreFile(file.getDatastoreName(), HypervisorHostHelper.VSPHERE_DATASTORE_BASE_FOLDER, file.getFileName()); if (!targetFile.getPath().equalsIgnoreCase(file.getPath())) { LOGGER.info("Move " + file.getPath() + " -> " + targetFile.getPath()); - dsMo.moveDatastoreFile(file.getPath(), dcMo.getMor(), dsMo.getMor(), targetFile.getPath(), dcMo.getMor(), true); + moveDatastoreFile(dsMo, file.getPath(), dcMo.getMor(), dsMo.getMor(), targetFile.getPath(), dcMo.getMor(), true); List vSphereFileExtensions = new ArrayList<>(Arrays.asList(VsphereLinkedCloneExtensions.value().trim().split("\\s*,\\s*"))); // add flat file format to the above list @@ -297,7 +298,7 @@ public static void moveVolumeToRootFolder(DatacenterMO dcMo, List detach String pairTargetFilePath = targetFile.getCompanionPath(String.format("%s-%s", file.getFileBaseName(), linkedCloneExtension)); if (dsMo.fileExists(pairSrcFilePath)) { LOGGER.info("Move " + pairSrcFilePath + " -> " + pairTargetFilePath); - dsMo.moveDatastoreFile(pairSrcFilePath, dcMo.getMor(), dsMo.getMor(), pairTargetFilePath, dcMo.getMor(), true); + moveDatastoreFile(dsMo, pairSrcFilePath, dcMo.getMor(), dsMo.getMor(), pairTargetFilePath, dcMo.getMor(), true); } } } @@ -429,6 +430,31 @@ public static String getDatastoreVolumePath(DatastoreMO dsMo, String vmName, Str return dsMo.searchFileInSubFolders(volumePath + ".vmdk", false, null); } + public static boolean moveDatastoreFile(final DatastoreMO dsMo, String srcFilePath, ManagedObjectReference morSrcDc, ManagedObjectReference morDestDs, + String destFilePath, ManagedObjectReference morDestDc, boolean forceOverwrite) throws Exception { + final int retry = 20; + int retryAttempt = 0; + while (++retryAttempt <= retry) { + try { + LOGGER.debug(String.format("Move datastore file %s, attempt #%d", srcFilePath, retryAttempt)); + return dsMo.moveDatastoreFile(srcFilePath, morSrcDc, morDestDs, destFilePath, morDestDc, forceOverwrite); + } catch (Exception e) { + LOGGER.info(String.format("Got exception while moving datastore file %s ", srcFilePath), e); + if (e.getMessage() != null && e.getMessage().contains("Unable to access file")) { + LOGGER.debug(String.format("Failed to move datastore file %s. Retrying", srcFilePath)); + try { + Thread.sleep(1000); + } catch (InterruptedException ie) { + LOGGER.debug(String.format("Waiting to move datastore file %s been interrupted: ", srcFilePath)); + } + } else { + throw e; + } + } + } + return false; + } + @Override public String getConfigComponentName() { return VmwareStorageLayoutHelper.class.getSimpleName(); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index c99d7d4d7075..e4b3282defb6 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -682,9 +682,9 @@ public Answer copyTemplateToPrimaryStorage(CopyCommand cmd) { String[] legacyCloudStackLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairManagedDatastorePath(dsMo, null, managedStoragePoolRootVolumeName, VmwareStorageLayoutType.CLOUDSTACK_LEGACY, false); - dsMo.moveDatastoreFile(vmwareLayoutFilePair[0], dcMo.getMor(), dsMo.getMor(), legacyCloudStackLayoutFilePair[0], dcMo.getMor(), true); + VmwareStorageLayoutHelper.moveDatastoreFile(dsMo, vmwareLayoutFilePair[0], dcMo.getMor(), dsMo.getMor(), legacyCloudStackLayoutFilePair[0], dcMo.getMor(), true); for (int i=1; i createTemplateFromVolume(VmwareContext conte // Get VMDK filename String templateVMDKName = ""; File[] files = new File(installFullPath).listFiles(); - if(files != null) { + if (files != null) { for(File file : files) { String fileName = file.getName(); - if(fileName.toLowerCase().startsWith(templateUniqueName) && fileName.toLowerCase().endsWith(".vmdk")) { + if (fileName.toLowerCase().startsWith(templateUniqueName) && fileName.toLowerCase().endsWith(".vmdk")) { templateVMDKName += fileName; break; } @@ -1856,16 +1856,16 @@ public Answer backupSnapshot(CopyCommand cmd) { CopyCmdAnswer answer = null; try { - if(vmName != null) { + if (vmName != null) { vmMo = hyperHost.findVmOnHyperHost(vmName); if (vmMo == null) { - if(logger.isDebugEnabled()) { + if (logger.isDebugEnabled()) { logger.debug("Unable to find owner VM for BackupSnapshotCommand on host " + hyperHost.getHyperHostName() + ", will try within datacenter"); } vmMo = hyperHost.findVmOnPeerHyperHost(vmName); } } - if(vmMo == null) { + if (vmMo == null) { dsMo = new DatastoreMO(hyperHost.getContext(), morDs); workerVMName = hostService.getWorkerName(context, cmd, 0, dsMo); vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, workerVMName, null); @@ -1899,10 +1899,10 @@ public Answer backupSnapshot(CopyCommand cmd) { String secondaryMountPoint = mountService.getMountPoint(secondaryStorageUrl, _nfsVersion); String snapshotDir = destSnapshot.getPath() + "/" + snapshotBackupUuid; File[] files = new File(secondaryMountPoint + "/" + snapshotDir).listFiles(); - if(files != null) { + if (files != null) { for(File file : files) { String fileName = file.getName(); - if(fileName.toLowerCase().startsWith(snapshotBackupUuid) && fileName.toLowerCase().endsWith(".vmdk")) { + if (fileName.toLowerCase().startsWith(snapshotBackupUuid) && fileName.toLowerCase().endsWith(".vmdk")) { physicalSize = new File(secondaryMountPoint + "/" + snapshotDir + "/" + fileName).length(); break; } @@ -3651,7 +3651,7 @@ private Long restoreVolumeFromSecStorage(VmwareHypervisorHost hyperHost, Datasto } workerVm.tagAsWorkerVM(); - if(!primaryDsMo.getDatastoreType().equalsIgnoreCase("VVOL")) { + if (!primaryDsMo.getDatastoreType().equalsIgnoreCase("VVOL")) { HypervisorHostHelper.createBaseFolderInDatastore(primaryDsMo, primaryDsMo.getDataCenterMor()); workerVm.moveAllVmDiskFiles(primaryDsMo, HypervisorHostHelper.VSPHERE_DATASTORE_BASE_FOLDER, false); } @@ -3811,8 +3811,9 @@ public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) { /** * Return the cloned VM from the template */ - public VirtualMachineMO cloneVMFromTemplate(VmwareHypervisorHost hyperHost, String templateName, String cloneName, String templatePrimaryStoreUuid) { + public VirtualMachineMO cloneVMFromTemplate(VmwareHypervisorHost hyperHost, TemplateObjectTO template, VolumeObjectTO volume, String cloneName, String templatePrimaryStoreUuid) { try { + String templateName = template.getPath(); VmwareContext context = hyperHost.getContext(); DatacenterMO dcMo = new DatacenterMO(context, hyperHost.getHyperHostDatacenter()); VirtualMachineMO templateMo = dcMo.findVm(templateName); @@ -3826,6 +3827,9 @@ public VirtualMachineMO cloneVMFromTemplate(VmwareHypervisorHost hyperHost, Stri throw new CloudRuntimeException("Unable to find datastore in vSphere"); } logger.info("Cloning VM " + cloneName + " from template " + templateName + " into datastore " + templatePrimaryStoreUuid); + if (template.getSize() != null) { + _fullCloneFlag = volume.getSize() > template.getSize() ? true : _fullCloneFlag; + } if (!_fullCloneFlag) { createVMLinkedClone(templateMo, dcMo, cloneName, morDatastore, morPool, null); } else { diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index bb0297c5abbd..69d64a4153cd 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -8394,6 +8394,7 @@ public Pair doInTransaction(final TransactionStatus status) th getRootVolumeSizeForVmRestore(newVol, template, userVm, diskOffering, details, true); volumeMgr.saveVolumeDetails(newVol.getDiskOfferingId(), newVol.getId()); + newVol = _volsDao.findById(newVol.getId()); // 1. Save usage event and update resource count for user vm volumes try { @@ -8493,7 +8494,6 @@ public Pair doInTransaction(final TransactionStatus status) th Long getRootVolumeSizeForVmRestore(Volume vol, VMTemplateVO template, UserVmVO userVm, DiskOffering diskOffering, Map details, boolean update) { VolumeVO resizedVolume = (VolumeVO) vol; - Long size = null; if (template != null && template.getSize() != null) { UserVmDetailVO vmRootDiskSizeDetail = userVmDetailsDao.findDetail(userVm.getId(), VmDetailConstants.ROOT_DISK_SIZE); diff --git a/test/integration/smoke/test_ssvm.py b/test/integration/smoke/test_ssvm.py index ad03c3d46e13..0784bc3820c6 100644 --- a/test/integration/smoke/test_ssvm.py +++ b/test/integration/smoke/test_ssvm.py @@ -988,6 +988,9 @@ def test_08_reboot_cpvm(self): # Private IP Address of System VMs are allowed to change after reboot - CLOUDSTACK-7745 + # Agent in Up state for a while after reboot, wait for the agent to Disconnect and back Up. + time.sleep(60) + # Wait for the agent to be up self.waitForSystemVMAgent(cpvm_response.name) @@ -1103,6 +1106,9 @@ def test_10_reboot_cpvm_forced(self): "Check whether CPVM is running or not" ) + # Agent in Up state for a while after reboot, wait for the agent to Disconnect and back Up. + time.sleep(60) + # Wait for the agent to be up self.waitForSystemVMAgent(cpvm_response.name) diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index 7d6cbbaeaed5..010b29ce378c 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -2332,7 +2332,7 @@ public void moveAllVmDiskFiles(DatastoreMO destDsMo, String destDsDir, boolean f vmdkDescriptor = getVmdkFileInfo(fileItem.first()); logger.info("Move VM disk file " + srcFile.getPath() + " to " + destFile.getPath()); - srcDsMo.moveDatastoreFile(fileItem.first(), dcMo.getMor(), destDsMo.getMor(), destFile.getPath(), dcMo.getMor(), true); + moveDatastoreFile(srcDsMo, fileItem.first(), dcMo.getMor(), destDsMo.getMor(), destFile.getPath(), dcMo.getMor(), true); if (vmdkDescriptor != null) { String vmdkBaseFileName = vmdkDescriptor.first().getBaseFileName(); @@ -2340,13 +2340,38 @@ public void moveAllVmDiskFiles(DatastoreMO destDsMo, String destDsDir, boolean f destFile = new DatastoreFile(destDsMo.getName(), destDsDir, vmdkBaseFileName); logger.info("Move VM disk file " + baseFilePath + " to " + destFile.getPath()); - srcDsMo.moveDatastoreFile(baseFilePath, dcMo.getMor(), destDsMo.getMor(), destFile.getPath(), dcMo.getMor(), true); + moveDatastoreFile(srcDsMo, baseFilePath, dcMo.getMor(), destDsMo.getMor(), destFile.getPath(), dcMo.getMor(), true); } } } } } + private boolean moveDatastoreFile(final DatastoreMO dsMo, String srcFilePath, ManagedObjectReference morSrcDc, ManagedObjectReference morDestDs, + String destFilePath, ManagedObjectReference morDestDc, boolean forceOverwrite) throws Exception { + final int retry = 20; + int retryAttempt = 0; + while (++retryAttempt <= retry) { + try { + logger.debug(String.format("Move datastore file %s, attempt #%d", srcFilePath, retryAttempt)); + return dsMo.moveDatastoreFile(srcFilePath, morSrcDc, morDestDs, destFilePath, morDestDc, forceOverwrite); + } catch (Exception e) { + logger.info(String.format("Got exception while moving datastore file %s ", srcFilePath), e); + if (e.getMessage() != null && e.getMessage().contains("Unable to access file")) { + logger.debug(String.format("Failed to move datastore file %s. Retrying", srcFilePath)); + try { + Thread.sleep(1000); + } catch (InterruptedException ie) { + logger.debug(String.format("Waiting to move datastore file %s been interrupted: ", srcFilePath)); + } + } else { + throw e; + } + } + } + return false; + } + public int getPvScsiDeviceControllerKeyNoException() throws Exception { List devices = (List)_context.getVimClient(). getDynamicProperty(_mor, "config.hardware.device");