From 830b2a1a27a186c856f378ac0fe39cccfdd28bb4 Mon Sep 17 00:00:00 2001 From: Siddhartha Kattoju Date: Mon, 25 Jan 2021 15:33:39 -0500 Subject: [PATCH 01/16] Fix data volume import / export functionality when using swift as secondary storage --- .../java/com/cloud/agent/api/to/DataTO.java | 2 + .../template/SwiftVolumeDownloader.java | 396 ++++++++++++++++++ .../motion/AncientDataMotionStrategy.java | 4 +- .../image/BaseImageStoreDriverImpl.java | 8 + .../storage/volume/VolumeDataFactoryImpl.java | 3 +- .../storage/volume/VolumeObject.java | 2 + .../CloudStackImageStoreDriverImpl.java | 1 + .../driver/SwiftImageStoreDriverImpl.java | 64 ++- .../cloud/storage/VolumeApiServiceImpl.java | 14 + .../storage/download/DownloadMonitorImpl.java | 6 +- .../resource/NfsSecondaryStorageResource.java | 240 ++++++----- .../storage/template/DownloadManager.java | 21 +- .../storage/template/DownloadManagerImpl.java | 160 ++++++- 13 files changed, 757 insertions(+), 164 deletions(-) create mode 100644 core/src/main/java/com/cloud/storage/template/SwiftVolumeDownloader.java diff --git a/api/src/main/java/com/cloud/agent/api/to/DataTO.java b/api/src/main/java/com/cloud/agent/api/to/DataTO.java index af43aa102e84..94fc89da004e 100644 --- a/api/src/main/java/com/cloud/agent/api/to/DataTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/DataTO.java @@ -32,5 +32,7 @@ public interface DataTO { */ String getPath(); + String getName(); + long getId(); } diff --git a/core/src/main/java/com/cloud/storage/template/SwiftVolumeDownloader.java b/core/src/main/java/com/cloud/storage/template/SwiftVolumeDownloader.java new file mode 100644 index 000000000000..01d3be5b99cd --- /dev/null +++ b/core/src/main/java/com/cloud/storage/template/SwiftVolumeDownloader.java @@ -0,0 +1,396 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.storage.template; + +import com.cloud.agent.api.to.SwiftTO; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.storage.command.DownloadCommand; +import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.http.Header; +import org.apache.http.HttpEntityEnclosingRequest; +import org.apache.http.HttpRequest; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpRequestRetryHandler; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.conn.ConnectTimeoutException; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.TrustStrategy; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.protocol.HttpContext; +import org.apache.http.ssl.SSLContextBuilder; +import org.apache.log4j.Logger; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLException; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import java.net.UnknownHostException; +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.util.Date; + +/** + * Download a volume file using HTTP(S) + * + * This class, once instantiated, has the purpose to download a Volume to staging nfs or cache when using as Swift Image Store. + * + * Execution of the instance is started when runInContext() is called. + */ +public class SwiftVolumeDownloader extends ManagedContextRunnable implements TemplateDownloader { + private static final Logger LOGGER = Logger.getLogger(SwiftVolumeDownloader.class.getName()); + private static final int DOWNLOAD_BUFFER_SIZE_BYTES = 1024* 1024; + + private final String downloadUrl; + private final String fileName; + private final String fileExtension; + private final long volumeId; + private final CloseableHttpClient httpClient; + private final HttpGet httpGet; + private final DownloadCompleteCallback downloadCompleteCallback; + private final SwiftTO swiftTO; + private String errorString = ""; + private Status status = Status.NOT_STARTED; + private final ResourceType resourceType = ResourceType.VOLUME; + private long remoteSize; + private String md5sum; + private long downloadTime; + private long totalBytes; + private final long maxVolumeSizeInBytes; + private final String installPathPrefix; + private final String installPath; + private File volumeFile; + private boolean resume = false; + + public SwiftVolumeDownloader(DownloadCommand cmd, DownloadCompleteCallback downloadCompleteCallback, long maxVolumeSizeInBytes, String installPathPrefix) { + this.downloadUrl = cmd.getUrl(); + this.swiftTO = (SwiftTO) cmd.getDataStore(); + this.maxVolumeSizeInBytes = maxVolumeSizeInBytes; + this.httpClient = initializeHttpClient(); + this.downloadCompleteCallback = downloadCompleteCallback; + this.fileName = cmd.getName(); + this.fileExtension = cmd.getFormat().getFileExtension(); + this.volumeId = cmd.getId(); + this.installPathPrefix = installPathPrefix; + this.installPath = cmd.getInstallPath(); + this.httpGet = new HttpGet(downloadUrl); + } + + private CloseableHttpClient initializeHttpClient(){ + + CloseableHttpClient client = null; + try { + //trust all certs + SSLContext sslContext = new SSLContextBuilder() + .loadTrustMaterial(null, (TrustStrategy) (chain, authType) -> true) + .build(); + client = HttpClients.custom().setSSLContext(sslContext) + .setSSLHostnameVerifier(new NoopHostnameVerifier()) + .setRetryHandler(buildRetryHandler(5)) + .build(); + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + } catch (KeyManagementException e) { + e.printStackTrace(); + } catch (KeyStoreException e) { + e.printStackTrace(); + } + + return client; + } + + private HttpRequestRetryHandler buildRetryHandler(int retryCount){ + + HttpRequestRetryHandler customRetryHandler = new HttpRequestRetryHandler() { + @Override + public boolean retryRequest( + IOException exception, + int executionCount, + HttpContext context) { + if (executionCount >= retryCount) { + // Do not retry if over max retry count + return false; + } + if (exception instanceof InterruptedIOException) { + // Timeout + return false; + } + if (exception instanceof UnknownHostException) { + // Unknown host + return false; + } + if (exception instanceof ConnectTimeoutException) { + // Connection refused + return false; + } + if (exception instanceof SSLException) { + // SSL handshake exception + return false; + } + HttpClientContext clientContext = HttpClientContext.adapt(context); + HttpRequest request = clientContext.getRequest(); + boolean idempotent = !(request instanceof HttpEntityEnclosingRequest); + if (idempotent) { + // Retry if the request is considered idempotent + return true; + } + return false; + } + + }; + return customRetryHandler; + } + + @Override + public long download(boolean resume, DownloadCompleteCallback callback) { + if (!status.equals(Status.NOT_STARTED)) { + // Only start downloading if we haven't started yet. + LOGGER.info("Volume download is already started, not starting again. Volume: " + downloadUrl); + return 0; + } + + HttpResponse response = null; + try { + response = httpClient.execute(httpGet); + } catch (IOException e) { + e.printStackTrace(); + errorString = "Exception while executing HttpMethod " + httpGet.getMethod() + " on URL " + downloadUrl + " " + + response.getStatusLine().getStatusCode() + " " + response.getStatusLine().getReasonPhrase(); + LOGGER.error(errorString); + status = Status.UNRECOVERABLE_ERROR; + return 0; + } + + // Headers + long contentLength = response.getEntity().getContentLength(); + Header contentType = response.getEntity().getContentType(); + + // Check the contentLengthHeader and transferEncodingHeader. + if (contentLength <= 0) { + errorString = "The Content Length of " + downloadUrl + " is <= 0 and content Type is "+contentType.toString(); + LOGGER.error(errorString); + status = Status.UNRECOVERABLE_ERROR; + return 0; + } else { + // The ContentLengthHeader is supplied, parse it's value. + remoteSize = contentLength; + } + + if (remoteSize > maxVolumeSizeInBytes) { + errorString = "Remote size is too large for volume " + downloadUrl + " remote size is " + remoteSize + " max allowed is " + maxVolumeSizeInBytes; + LOGGER.error(errorString); + status = Status.UNRECOVERABLE_ERROR; + return 0; + } + + InputStream inputStream; + try { + inputStream = new BufferedInputStream(response.getEntity().getContent()); + } catch (IOException e) { + errorString = "Exception occurred while opening InputStream for volume from " + downloadUrl; + LOGGER.error(errorString); + status = Status.UNRECOVERABLE_ERROR; + return 0; + } + + String filePath = installPathPrefix + File.separator + installPath; + File directory = new File(filePath); + File srcFile = new File(filePath + File.separator + fileName); + try { + if (!directory.exists()) { + LOGGER.info("Creating directories "+filePath); + directory.mkdirs(); + } + if (!srcFile.createNewFile()) { + LOGGER.info("Reusing existing file " + srcFile.getPath()); + } + } catch (IOException e) { + errorString = "Exception occurred while creating temp file " + srcFile.getPath(); + LOGGER.error(errorString); + status = Status.UNRECOVERABLE_ERROR; + return 0; + } + + LOGGER.info("Starting download from " + downloadUrl + " to staging with size " + remoteSize + " bytes to "+filePath); + final Date downloadStart = new Date(); + + try (FileOutputStream fileOutputStream = new FileOutputStream(srcFile);) { + BufferedOutputStream outputStream = new BufferedOutputStream(fileOutputStream,DOWNLOAD_BUFFER_SIZE_BYTES); + byte[] data = new byte[DOWNLOAD_BUFFER_SIZE_BYTES]; + int bufferLength = 0; + while((bufferLength = inputStream.read(data,0,DOWNLOAD_BUFFER_SIZE_BYTES)) >= 0){ + totalBytes += bufferLength; + outputStream.write(data,0,bufferLength); + status = Status.IN_PROGRESS; + LOGGER.trace("Download in progress: " + getDownloadPercent() + "%"); + if(totalBytes >= remoteSize){ + volumeFile = srcFile; + status = Status.DOWNLOAD_FINISHED; + } + } + outputStream.close(); + inputStream.close(); + } catch (IOException e) { + LOGGER.error("Exception when downloading from url to staging nfs:" + e.getMessage(), e); + status = Status.RECOVERABLE_ERROR; + return 0; + } + + downloadTime = new Date().getTime() - downloadStart.getTime(); + + try (FileInputStream fs = new FileInputStream(srcFile)) { + md5sum = DigestUtils.md5Hex(fs); + } catch (IOException e) { + LOGGER.error("Failed to get md5sum: " + srcFile.getAbsoluteFile()); + } + + if (status == Status.DOWNLOAD_FINISHED) { + LOGGER.info("Template download from " + downloadUrl + " to staging nfs, transferred " + totalBytes + " in " + (downloadTime / 1000) + " seconds, completed successfully!"); + } else { + LOGGER.error("Template download from " + downloadUrl + " to staging nfs, transferred " + totalBytes + " in " + (downloadTime / 1000) + " seconds, completed with status " + status.toString()); + } + + // Close http connection + httpGet.releaseConnection(); + + // Call the callback! + if (callback != null) { + callback.downloadComplete(status); + } + + return totalBytes; + } + + public String getDownloadUrl() { + return httpGet.getURI().toString(); + } + + @Override + public Status getStatus() { + return status; + } + + @Override + public long getDownloadTime() { + return downloadTime; + } + + @Override + public long getDownloadedBytes() { + return totalBytes; + } + + @Override + public boolean stopDownload() { + switch (status) { + case IN_PROGRESS: + if (httpGet != null) { + httpGet.abort(); + } + break; + case UNKNOWN: + case NOT_STARTED: + case RECOVERABLE_ERROR: + case UNRECOVERABLE_ERROR: + case ABORTED: + case DOWNLOAD_FINISHED: + // Remove the object if it already has been uploaded. + // SwiftUtil.deleteObject(swiftTO, swiftPath); + break; + default: + break; + } + + status = Status.ABORTED; + return true; + } + + @Override + public int getDownloadPercent() { + if (remoteSize == 0) { + return 0; + } + + return (int) (100.0 * totalBytes / remoteSize); + } + + @Override + protected void runInContext() { + LOGGER.info("Starting download in managed context resume = " + resume + " callback = " + downloadCompleteCallback.toString()); + download(resume, downloadCompleteCallback); + } + + @Override + public void setStatus(Status status) { + this.status = status; + } + + public boolean isResume() { + return resume; + } + + @Override + public String getDownloadError() { + return errorString; + } + + @Override + public String getDownloadLocalPath() { + return installPath; + } + + @Override + public void setResume(boolean resume) { + this.resume = resume; + } + + @Override + public void setDownloadError(String error) { + errorString = error; + } + + @Override + public boolean isInited() { + return true; + } + + public ResourceType getResourceType() { + return resourceType; + } + + public String getFileExtension() { + return fileExtension; + } + + public String getMd5sum() { return md5sum; } + + public File getVolumeFile() { return volumeFile; } + + public long getMaxTemplateSizeInBytes() {return maxVolumeSizeInBytes;} +} \ No newline at end of file diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java index c49ffba0b82b..95ebd5f68c33 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java @@ -146,7 +146,9 @@ private Scope pickCacheScopeForCopy(DataObject srcData, DataObject destData) { } protected Answer copyObject(DataObject srcData, DataObject destData, Host destHost) { - int primaryStorageDownloadWait = StorageManager.PRIMARY_STORAGE_DOWNLOAD_WAIT.value(); + int imgSizeGigs = (int)Math.ceil(srcData.getSize() * 1.0d / (1024 * 1024 * 1024)); + imgSizeGigs++; // add one just in case + int primaryStorageDownloadWait = StorageManager.PRIMARY_STORAGE_DOWNLOAD_WAIT.value()*imgSizeGigs; Answer answer = null; DataObject cacheData = null; DataObject srcForCopy = srcData; diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java index 0c55545d66a8..afe455712ba1 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java @@ -174,6 +174,14 @@ protected Long getMaxTemplateSizeInBytes() { } } + protected Long getMaxVolumeSizeInBytes() { + try { + return Long.parseLong(configDao.getValue("storage.max.volume.upload.size")) * 1024L * 1024L * 1024L; + } catch (NumberFormatException e) { + return null; + } + } + @Override public void createAsync(DataStore dataStore, DataObject data, AsyncCompletionCallback callback) { CreateContext context = new CreateContext(callback, data); diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java index 53fa21f3a794..0ddd40dde9f8 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java @@ -23,6 +23,7 @@ import javax.inject.Inject; +import com.cloud.storage.Volume; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateDao; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; @@ -83,7 +84,7 @@ public VolumeInfo getVolume(long volumeId) { return null; } VolumeObject vol = null; - if (volumeVO.getPoolId() == null) { + if (volumeVO.getPoolId() == null || volumeVO.getState() == Volume.State.Uploaded) { DataStore store = null; VolumeDataStoreVO volumeStore = volumeStoreDao.findByVolume(volumeId); if (volumeStore != null) { diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java index 5ec9cfb27e5f..f3b7b8b0242c 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java @@ -687,6 +687,8 @@ public void processEvent(ObjectInDataStoreStateMachine.Event event, Answer answe VolumeDataStoreVO volStore = volumeStoreDao.findByStoreVolume(dataStore.getId(), getId()); volStore.setInstallPath(dwdAnswer.getInstallPath()); volStore.setChecksum(dwdAnswer.getCheckSum()); + volStore.setSize(dwdAnswer.getTemplateSize()); + volStore.setPhysicalSize(dwdAnswer.getTemplatePhySicalSize()); volumeStoreDao.update(volStore.getId(), volStore); } else if (answer instanceof CopyCmdAnswer) { CopyCmdAnswer cpyAnswer = (CopyCmdAnswer)answer; diff --git a/plugins/storage/image/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java b/plugins/storage/image/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java index 8abf802d9de6..14f18cd84d60 100644 --- a/plugins/storage/image/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java +++ b/plugins/storage/image/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java @@ -73,6 +73,7 @@ public String createEntityExtractUrl(DataStore store, String installPath, ImageF String uuid = UUID.randomUUID().toString() + "." + format.getFileExtension(); CreateEntityDownloadURLCommand cmd = new CreateEntityDownloadURLCommand(((ImageStoreEntity)store).getMountPoint(), path, uuid, dataObject == null ? null: dataObject.getTO()); + cmd.setSecUrl(((ImageStoreEntity) store).getUrl()); Answer ans = null; if (ep == null) { String errMsg = "No remote endpoint to send command, check if host or ssvm is down?"; diff --git a/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java b/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java index 7e1486214bcf..6afb4a052af0 100644 --- a/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java +++ b/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java @@ -18,17 +18,16 @@ */ package org.apache.cloudstack.storage.datastore.driver; -import java.net.URL; -import java.util.Map; -import java.util.UUID; - -import javax.inject.Inject; - +import com.cloud.agent.api.storage.DownloadAnswer; +import com.cloud.agent.api.to.DataObjectType; +import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.agent.api.to.SwiftTO; import com.cloud.configuration.Config; +import com.cloud.storage.RegisterVolumePayload; +import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.download.DownloadMonitor; import com.cloud.utils.SwiftUtil; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.log4j.Logger; - +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; @@ -36,21 +35,22 @@ import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.storage.command.DownloadCommand; import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao; import org.apache.cloudstack.storage.image.BaseImageStoreDriverImpl; import org.apache.cloudstack.storage.image.store.ImageStoreImpl; import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.log4j.Logger; -import com.cloud.agent.api.storage.DownloadAnswer; -import com.cloud.agent.api.to.DataObjectType; -import com.cloud.agent.api.to.DataStoreTO; -import com.cloud.agent.api.to.SwiftTO; -import com.cloud.storage.Storage.ImageFormat; -import com.cloud.template.VirtualMachineTemplate; -import com.cloud.utils.exception.CloudRuntimeException; +import javax.inject.Inject; +import java.net.URL; +import java.util.Map; +import java.util.UUID; public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl { private static final Logger s_logger = Logger.getLogger(SwiftImageStoreDriverImpl.class); @@ -63,6 +63,8 @@ public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl { StorageCacheManager cacheManager; @Inject ConfigurationDao _configDao; + @Inject + private DownloadMonitor _downloadMonitor; @Override public DataStoreTO getStoreTO(DataStore store) { @@ -100,12 +102,28 @@ public String createEntityExtractUrl(DataStore store, String installPath, ImageF @Override public void createAsync(DataStore dataStore, DataObject data, AsyncCompletionCallback callback) { - Long maxTemplateSizeInBytes = getMaxTemplateSizeInBytes(); - VirtualMachineTemplate tmpl = _templateDao.findById(data.getId()); + + DownloadCommand downloadCommand = null; + if (data.getType() == DataObjectType.TEMPLATE) { + Long maxTemplateSizeInBytes = getMaxTemplateSizeInBytes(); + downloadCommand = new DownloadCommand((TemplateObjectTO) (data.getTO()), maxTemplateSizeInBytes); + }else if (data.getType() == DataObjectType.VOLUME){ + Long maxDownloadSizeInBytes = getMaxVolumeSizeInBytes(); + VolumeInfo volumeInfo = (VolumeInfo) data; + RegisterVolumePayload payload = (RegisterVolumePayload) volumeInfo.getpayload(); + ImageFormat format = ImageFormat.valueOf(payload.getFormat()); + downloadCommand = new DownloadCommand((VolumeObjectTO) (data.getTO()), maxDownloadSizeInBytes, payload.getChecksum(), payload.getUrl(), format); + } + + if (downloadCommand == null){ + String errMsg = "Unable to build download command, DataObject is of neither VOLUME or TEMPLATE type"; + s_logger.error(errMsg); + throw new CloudRuntimeException(errMsg); + } + DataStore cacheStore = cacheManager.getCacheStorage(dataStore.getScope()); - DownloadCommand dcmd = new DownloadCommand((TemplateObjectTO)(data.getTO()), maxTemplateSizeInBytes); - dcmd.setCacheStore(cacheStore.getTO()); - dcmd.setProxy(getHttpProxy()); + downloadCommand.setCacheStore(cacheStore.getTO()); + downloadCommand.setProxy(getHttpProxy()); EndPoint ep = _epSelector.select(data); if (ep == null) { @@ -120,11 +138,11 @@ public void createAsync(DataStore dataStore, DataObject data, AsyncCompletionCal if (data.getType() == DataObjectType.TEMPLATE) { caller.setCallback(caller.getTarget().createTemplateAsyncCallback(null, null)); + ep.sendMessageAsync(downloadCommand, caller); } else if (data.getType() == DataObjectType.VOLUME) { caller.setCallback(caller.getTarget().createVolumeAsyncCallback(null, null)); + _downloadMonitor.downloadVolumeToStorage(data,caller); } - ep.sendMessageAsync(dcmd, caller); - } } diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 6f362a7720f1..04d21bfdef51 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -1596,6 +1596,19 @@ private Volume orchestrateAttachVolumeToVM(Long vmId, Long volumeId, Long device StoragePoolVO destPrimaryStorage = null; if (exstingVolumeOfVm != null && !exstingVolumeOfVm.getState().equals(Volume.State.Allocated)) { destPrimaryStorage = _storagePoolDao.findById(exstingVolumeOfVm.getPoolId()); + if(destPrimaryStorage.getPodId() == null) { + destPrimaryStorage.setPodId(vm.getPodIdToDeployIn()); + + } + if(destPrimaryStorage.getClusterId() == null) { + HostVO hostVO = null; + if (vm.getHostId() != null) { + hostVO = _hostDao.findById(vm.getHostId()); + } else { + hostVO = _hostDao.findById(vm.getLastHostId()); + } + destPrimaryStorage.setClusterId(hostVO.getClusterId()); + } } boolean volumeOnSecondary = volumeToAttach.getState() == Volume.State.Uploaded; @@ -3020,6 +3033,7 @@ private String orchestrateExtractVolume(long volumeId, long zoneId) { volumeStoreRef.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); volumeStoreRef.setDownloadPercent(100); volumeStoreRef.setZoneId(zoneId); + volumeStoreRef.setSize(vol.getSize()); _volumeStoreDao.update(volumeStoreRef.getId(), volumeStoreRef); diff --git a/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java b/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java index 1954cdea6879..7f298f260762 100644 --- a/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java +++ b/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java @@ -25,6 +25,7 @@ import javax.inject.Inject; +import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -73,6 +74,8 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor private ConfigurationDao _configDao; @Inject private EndPointSelector _epSelector; + @Inject + private StorageCacheManager cacheManager; private String _copyAuthPasswd; private String _proxy = null; @@ -218,7 +221,8 @@ public void downloadVolumeToStorage(DataObject volume, AsyncCompletionCallback true) + .build(); + client = HttpClients.custom().setSSLContext(sslContext) + .setSSLHostnameVerifier(new NoopHostnameVerifier()) + .build(); + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + } catch (KeyManagementException e) { + e.printStackTrace(); + } catch (KeyStoreException e) { + e.printStackTrace(); + } + try { + HttpGet get = new HttpGet(url); HttpResponse response = client.execute(get); HttpEntity entity = response.getEntity(); if (entity == null) { - s_logger.debug("Faled to get entity"); + s_logger.error("Failed to get entity "+ response.getStatusLine().getStatusCode() + " " + + response.getStatusLine().getReasonPhrase()); throw new CloudRuntimeException("Failed to get url: " + url); } @@ -1137,11 +1167,11 @@ protected File downloadFromUrlToNfs(String url, NfsTO nfs, String path, String n try (FileOutputStream outputStream = new FileOutputStream(destFile);) { entity.writeTo(outputStream); } catch (IOException e) { - s_logger.debug("downloadFromUrlToNfs:Exception:" + e.getMessage(), e); + s_logger.error("downloadFromUrlToNfs:Exception:" + e.getMessage(), e); } return new File(destFile.getAbsolutePath()); } catch (IOException e) { - s_logger.debug("Faild to get url:" + url + ", due to " + e.toString()); + s_logger.error("Faild to get url:" + url + ", due to " + e.toString()); throw new CloudRuntimeException(e); } } @@ -1180,13 +1210,13 @@ protected Answer registerTemplateOnSwift(DownloadCommand cmd) { try (FileInputStream fs = new FileInputStream(file)) { md5sum = DigestUtils.md5Hex(fs); } catch (IOException e) { - s_logger.debug("Failed to get md5sum: " + file.getAbsoluteFile()); + s_logger.error("Failed to get md5sum: " + file.getAbsoluteFile()); } DownloadAnswer answer = new DownloadAnswer(null, 100, null, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, swiftPath, swiftPath, virtualSize, file.length(), md5sum); return answer; } catch (IOException e) { - s_logger.debug("Failed to register template into swift", e); + s_logger.error("Failed to register template into swift", e); return new DownloadAnswer(e.toString(), VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); } finally { if (file != null) { @@ -1199,8 +1229,10 @@ private Answer execute(DownloadCommand cmd) { DataStoreTO dstore = cmd.getDataStore(); if (dstore instanceof NfsTO || dstore instanceof S3TO) { return _dlMgr.handleDownloadCommand(this, cmd); - } else if (dstore instanceof SwiftTO) { + } else if (dstore instanceof SwiftTO && cmd.getResourceType() == DownloadCommand.ResourceType.TEMPLATE) { return registerTemplateOnSwift(cmd); + } else if (dstore instanceof SwiftTO && cmd.getResourceType() == DownloadCommand.ResourceType.VOLUME){ + return _dlMgr.handleDownloadCommand(this, cmd); } else { return new Answer(cmd, false, "Unsupported image data store: " + dstore); } @@ -1268,7 +1300,7 @@ protected long getVirtualSize(File file, ImageFormat format) { processor.configure("template processor", params); return processor.getVirtualSize(file); } catch (Exception e) { - s_logger.warn("Failed to get virtual size of file " + file.getPath() + ", returning file size instead: ", e); + s_logger.error("Failed to get virtual size of file " + file.getPath() + ", returning file size instead: ", e); return file.length(); } @@ -1392,9 +1424,7 @@ protected Answer copyFromNfsToS3(CopyCommand cmd) { try { final String templatePath = determineStorageTemplatePath(srcStore.getUrl(), srcData.getPath(), _nfsVersion); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Found " + srcData.getObjectType() + " from directory " + templatePath + " to upload to S3."); - } + s_logger.info("Found " + srcData.getObjectType() + " from directory " + templatePath + " to upload to S3."); final String bucket = s3.getBucketName(); File srcFile = findFile(templatePath); @@ -1554,7 +1584,7 @@ String swiftDownload(SwiftTO swift, String container, String rfilename, String l String result = command.execute(parser); if (result != null) { String errMsg = "swiftDownload failed err=" + result; - s_logger.warn(errMsg); + s_logger.error(errMsg); return errMsg; } if (parser.getLines() != null) { @@ -1562,7 +1592,7 @@ String swiftDownload(SwiftTO swift, String container, String rfilename, String l for (String line : lines) { if (line.contains("Errno") || line.contains("failed")) { String errMsg = "swiftDownload failed , err=" + parser.getLines(); - s_logger.warn(errMsg); + s_logger.error(errMsg); return errMsg; } } @@ -1580,7 +1610,7 @@ String swiftDownloadContainer(SwiftTO swift, String container, String ldir) { String result = command.execute(parser); if (result != null) { String errMsg = "swiftDownloadContainer failed err=" + result; - s_logger.warn(errMsg); + s_logger.error(errMsg); return errMsg; } if (parser.getLines() != null) { @@ -1588,7 +1618,7 @@ String swiftDownloadContainer(SwiftTO swift, String container, String ldir) { for (String line : lines) { if (line.contains("Errno") || line.contains("failed")) { String errMsg = "swiftDownloadContainer failed , err=" + parser.getLines(); - s_logger.warn(errMsg); + s_logger.error(errMsg); return errMsg; } } @@ -1631,7 +1661,7 @@ String swiftUpload(SwiftTO swift, String container, String lDir, String lFilenam String result = command.execute(parser); if (result != null) { String errMsg = "swiftUpload failed , err=" + result; - s_logger.warn(errMsg); + s_logger.error(errMsg); return errMsg; } if (parser.getLines() != null) { @@ -1639,7 +1669,7 @@ String swiftUpload(SwiftTO swift, String container, String lDir, String lFilenam for (String line : lines) { if (line.contains("Errno") || line.contains("failed")) { String errMsg = "swiftUpload failed , err=" + parser.getLines(); - s_logger.warn(errMsg); + s_logger.error(errMsg); return errMsg; } } @@ -1662,10 +1692,10 @@ String[] swiftList(SwiftTO swift, String container, String rFilename) { } else { if (result != null) { String errMsg = "swiftList failed , err=" + result; - s_logger.warn(errMsg); + s_logger.error(errMsg); } else { String errMsg = "swiftList failed, no lines returns"; - s_logger.warn(errMsg); + s_logger.error(errMsg); } } return null; @@ -1680,7 +1710,7 @@ String swiftDelete(SwiftTO swift, String container, String object) { String result = command.execute(parser); if (result != null) { String errMsg = "swiftDelete failed , err=" + result; - s_logger.warn(errMsg); + s_logger.error(errMsg); return errMsg; } if (parser.getLines() != null) { @@ -1688,7 +1718,7 @@ String swiftDelete(SwiftTO swift, String container, String object) { for (String line : lines) { if (line.contains("Errno") || line.contains("failed")) { String errMsg = "swiftDelete failed , err=" + parser.getLines(); - s_logger.warn(errMsg); + s_logger.error(errMsg); return errMsg; } } @@ -1715,7 +1745,7 @@ public Answer execute(DeleteSnapshotsDirCommand cmd) { String details = null; if (!snapshotDir.exists()) { details = "snapshot directory " + snapshotDir.getName() + " doesn't exist"; - s_logger.debug(details); + s_logger.info(details); return new Answer(cmd, true, details); } // delete all files in the directory @@ -1723,13 +1753,13 @@ public Answer execute(DeleteSnapshotsDirCommand cmd) { String result = deleteLocalFile(lPath); if (result != null) { String errMsg = "failed to delete all snapshots " + lPath + " , err=" + result; - s_logger.warn(errMsg); + s_logger.error(errMsg); return new Answer(cmd, false, errMsg); } // delete the directory if (!snapshotDir.delete()) { details = "Unable to delete directory " + snapshotDir.getName() + " under snapshot path " + relativeSnapshotPath; - s_logger.debug(details); + s_logger.error(details); return new Answer(cmd, false, details); } return new Answer(cmd, true, null); @@ -1761,7 +1791,7 @@ public Answer execute(DeleteSnapshotsDirCommand cmd) { String result = swiftDelete((SwiftTO)dstore, "V-" + volumeId.toString(), ""); if (result != null) { String errMsg = "failed to delete snapshot for volume " + volumeId + " , err=" + result; - s_logger.warn(errMsg); + s_logger.error(errMsg); return new Answer(cmd, false, errMsg); } return new Answer(cmd, true, "Deleted snapshot " + path + " from swift"); @@ -1943,7 +1973,7 @@ public void run() { ch.closeFuture().sync(); } catch (InterruptedException e) { s_logger.info("Failed to start post upload server"); - s_logger.debug("Exception while starting post upload server", e); + s_logger.error("Exception while starting post upload server", e); } finally { bossGroup.shutdownGracefully(); workerGroup.shutdownGracefully(); diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManager.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManager.java index 78190af05989..a39653e2dffb 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManager.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManager.java @@ -16,21 +16,20 @@ // under the License. package org.apache.cloudstack.storage.template; -import java.util.Map; - -import com.cloud.storage.template.Processor; -import org.apache.cloudstack.storage.command.DownloadCommand; -import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; -import org.apache.cloudstack.storage.resource.SecondaryStorageResource; - import com.cloud.agent.api.storage.DownloadAnswer; -import com.cloud.utils.net.Proxy; import com.cloud.agent.api.to.S3TO; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.VMTemplateHostVO; +import com.cloud.storage.template.Processor; import com.cloud.storage.template.TemplateDownloader; import com.cloud.storage.template.TemplateProp; import com.cloud.utils.component.Manager; +import com.cloud.utils.net.Proxy; +import org.apache.cloudstack.storage.command.DownloadCommand; +import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; +import org.apache.cloudstack.storage.resource.SecondaryStorageResource; + +import java.util.Map; public interface DownloadManager extends Manager { @@ -41,8 +40,8 @@ public interface DownloadManager extends Manager { * @param hvm whether the template is a hardware virtual machine * @param accountId the accountId of the iso owner (null if public iso) * @param descr description of the template - * @param user username used for authentication to the server - * @param password password used for authentication to the server + * @param userName username used for authentication to the server + * @param passwd password used for authentication to the server * @param maxDownloadSizeInBytes (optional) max download size for the template, in bytes. * @param resourceType signifying the type of resource like template, volume etc. * @return job-id that can be used to interrogate the status of the download. @@ -53,6 +52,8 @@ public String downloadPublicTemplate(long id, String url, String name, ImageForm public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType); + public String downloadSwiftVolume(DownloadCommand cmd, String installPathPrefix, long maxDownloadSizeInBytes); + Map getProcessors(); /** diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index 7ce252a8ed7c..4e66c5c88048 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@ -65,26 +65,74 @@ import org.apache.log4j.Logger; import com.cloud.agent.api.storage.DownloadAnswer; -import com.cloud.utils.net.Proxy; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.S3TO; +import com.cloud.agent.api.to.SwiftTO; import com.cloud.exception.InternalErrorException; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.StorageLayer; import com.cloud.storage.VMTemplateHostVO; import com.cloud.storage.VMTemplateStorageResourceAssoc; +import com.cloud.storage.template.HttpTemplateDownloader; +import com.cloud.storage.template.IsoProcessor; +import com.cloud.storage.template.LocalTemplateDownloader; +import com.cloud.storage.template.MetalinkTemplateDownloader; +import com.cloud.storage.template.OVAProcessor; +import com.cloud.storage.template.Processor; import com.cloud.storage.template.Processor.FormatInfo; +import com.cloud.storage.template.QCOW2Processor; +import com.cloud.storage.template.RawImageProcessor; +import com.cloud.storage.template.S3TemplateDownloader; +import com.cloud.storage.template.ScpTemplateDownloader; +import com.cloud.storage.template.SwiftVolumeDownloader; +import com.cloud.storage.template.TARProcessor; +import com.cloud.storage.template.TemplateConstants; +import com.cloud.storage.template.TemplateDownloader; import com.cloud.storage.template.TemplateDownloader.DownloadCompleteCallback; import com.cloud.storage.template.TemplateDownloader.Status; +import com.cloud.storage.template.TemplateLocation; +import com.cloud.storage.template.TemplateProp; +import com.cloud.storage.template.VhdProcessor; +import com.cloud.storage.template.VmdkProcessor; import com.cloud.utils.NumbersUtil; import com.cloud.utils.StringUtils; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.Proxy; +import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; import com.cloud.utils.storage.QCOW2Utils; +import org.apache.cloudstack.storage.command.DownloadCommand; +import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; +import org.apache.cloudstack.storage.command.DownloadProgressCommand; +import org.apache.cloudstack.storage.command.DownloadProgressCommand.RequestType; +import org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource; +import org.apache.cloudstack.storage.resource.SecondaryStorageResource; import org.apache.cloudstack.utils.security.ChecksumValue; import org.apache.cloudstack.utils.security.DigestHelper; +import org.apache.log4j.Logger; + +import javax.naming.ConfigurationException; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.NoSuchAlgorithmException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; @@ -143,6 +191,19 @@ public DownloadJob(TemplateDownloader td, String jobId, long id, String tmpltNam this.resourceType = resourceType; } + public DownloadJob(TemplateDownloader td, String jobId, long id, String tmpltName, ImageFormat format, String installPathPrefix) { + super(); + this.td = td; + this.tmpltName = tmpltName; + this.format = format; + this.hvm = false; + this.description = null; + this.installPathPrefix = installPathPrefix; + this.templatesize = 0; + this.id = id; + this.resourceType = null; + } + public String getDescription() { return description; } @@ -264,7 +325,7 @@ public void setStorageLayer(StorageLayer storage) { public void setDownloadStatus(String jobId, Status status) { DownloadJob dj = jobs.get(jobId); if (dj == null) { - LOGGER.warn("setDownloadStatus for jobId: " + jobId + ", status=" + status + " no job found"); + LOGGER.info("setDownloadStatus for jobId: " + jobId + ", status=" + status + " no job found"); return; } TemplateDownloader td = dj.getTemplateDownloader(); @@ -305,8 +366,27 @@ public void setDownloadStatus(String jobId, Status status) { td.setStatus(Status.POST_DOWNLOAD_FINISHED); td.setDownloadError("Install completed successfully at " + new SimpleDateFormat().format(new Date())); } - } - else { + } else if (td instanceof SwiftVolumeDownloader) { + dj.setCheckSum(((SwiftVolumeDownloader) td).getMd5sum()); + if ("vhd".equals(((SwiftVolumeDownloader) td).getFileExtension()) || + "VHD".equals(((SwiftVolumeDownloader) td).getFileExtension())) { + Processor vhdProcessor = _processors.get("VHD Processor"); + long virtualSize = 0; + try { + virtualSize = vhdProcessor.getVirtualSize(((SwiftVolumeDownloader) td).getVolumeFile()); + dj.setTemplatesize(virtualSize); + } catch (IOException e) { + LOGGER.error("Unable to read VHD file", e); + e.printStackTrace(); + } + } else { + dj.setTemplatesize(((SwiftVolumeDownloader) td).getDownloadedBytes()); + } + dj.setTemplatePhysicalSize(((SwiftVolumeDownloader) td).getDownloadedBytes()); + dj.setTmpltPath(((SwiftVolumeDownloader) td).getDownloadLocalPath()); + td.setStatus(Status.POST_DOWNLOAD_FINISHED); + td.setDownloadError("Volume downloaded to swift cache successfully at " + new SimpleDateFormat().format(new Date())); + } else { // For other TemplateDownloaders where files are locally available, // we run the postLocalDownload() method. td.setDownloadError("Download success, starting install "); @@ -497,7 +577,7 @@ private String checkOrCreateTheChecksum(DownloadJob dnld, File targetFile) { } String checksum = newValue.toString(); if (checksum == null) { - LOGGER.warn("Something wrong happened when try to calculate the checksum of downloaded template!"); + LOGGER.error("Something wrong happened when trying to calculate the checksum of downloaded template!"); } dnld.setCheckSum(checksum); return null; @@ -530,7 +610,7 @@ private String postProcessAfterDownloadComplete(DownloadJob dnld, String resourc } if (!loc.save()) { - LOGGER.warn("Cleaning up because we're unable to save the formats"); + LOGGER.info("Cleaning up because we're unable to save the formats"); loc.purge(); } @@ -579,6 +659,36 @@ public String downloadS3Template(S3TO s3, long id, String url, String name, Imag return jobId; } + @Override + public String downloadSwiftVolume(DownloadCommand cmd, String installPathPrefix, long maxDownloadSizeInBytes) { + UUID uuid = UUID.randomUUID(); + String jobId = uuid.toString(); + //TODO get from global config + long maxVolumeSizeInBytes = maxDownloadSizeInBytes; + URI uri = null; + try { + uri = new URI(cmd.getUrl()); + } catch (URISyntaxException e) { + throw new CloudRuntimeException("URI is incorrect: " + cmd.getUrl()); + } + TemplateDownloader td; + if ((uri != null) && (uri.getScheme() != null)) { + if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) { + td = new SwiftVolumeDownloader(cmd, new Completion(jobId), maxVolumeSizeInBytes, installPathPrefix); + } else { + throw new CloudRuntimeException("Scheme is not supported " + cmd.getUrl()); + } + } else { + throw new CloudRuntimeException("Unable to download from URL: " + cmd.getUrl()); + } + DownloadJob dj = new DownloadJob(td, jobId, cmd.getId(), cmd.getName(), cmd.getFormat(), cmd.getInstallPath()); + dj.setTmpltPath(installPathPrefix); + jobs.put(jobId, dj); + threadPool.execute(td); + + return jobId; + } + @Override public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, String templatePath, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) { @@ -589,7 +699,7 @@ public String downloadPublicTemplate(long id, String url, String name, ImageForm try { if (!_storage.mkdirs(tmpDir)) { - LOGGER.warn("Unable to create " + tmpDir); + LOGGER.error("Unable to create " + tmpDir); return "Unable to create " + tmpDir; } // TO DO - define constant for volume properties. @@ -598,12 +708,12 @@ public String downloadPublicTemplate(long id, String url, String name, ImageForm "volume.properties"); if (file.exists()) { if(! file.delete()) { - LOGGER.warn("Deletion of file '" + file.getAbsolutePath() + "' failed."); + LOGGER.error("Deletion of file '" + file.getAbsolutePath() + "' failed."); } } if (!file.createNewFile()) { - LOGGER.warn("Unable to create new file: " + file.getAbsolutePath()); + LOGGER.error("Unable to create new file: " + file.getAbsolutePath()); return "Unable to create new file: " + file.getAbsolutePath(); } @@ -645,7 +755,7 @@ public String downloadPublicTemplate(long id, String url, String name, ImageForm return jobId; } catch (IOException e) { - LOGGER.warn("Unable to download to " + tmpDir, e); + LOGGER.error("Unable to download to " + tmpDir, e); return null; } } @@ -735,7 +845,7 @@ public DownloadAnswer handleDownloadCommand(SecondaryStorageResource resource, D this._processTimeout = timeout; ResourceType resourceType = cmd.getResourceType(); if (cmd instanceof DownloadProgressCommand) { - return handleDownloadProgressCmd(resource, (DownloadProgressCommand)cmd); + return handleDownloadProgressCmd(resource, (DownloadProgressCommand) cmd); } if (cmd.getUrl() == null) { @@ -747,7 +857,7 @@ public DownloadAnswer handleDownloadCommand(SecondaryStorageResource resource, D return new DownloadAnswer("Invalid Name", VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); } - if(! DigestHelper.isAlgorithmSupported(cmd.getChecksum())) { + if (cmd.getChecksum() != null && !DigestHelper.isAlgorithmSupported(cmd.getChecksum())) { return new DownloadAnswer("invalid algorithm: " + cmd.getChecksum(), VMTemplateStorageResourceAssoc.Status.NOT_DOWNLOADED); } @@ -755,7 +865,9 @@ public DownloadAnswer handleDownloadCommand(SecondaryStorageResource resource, D String installPathPrefix = cmd.getInstallPath(); // for NFS, we need to get mounted path if (dstore instanceof NfsTO) { - installPathPrefix = resource.getRootDir(((NfsTO)dstore).getUrl(), _nfsVersion) + File.separator + installPathPrefix; + installPathPrefix = resource.getRootDir(((NfsTO) dstore).getUrl(), _nfsVersion) + File.separator + installPathPrefix; + } else if (dstore instanceof SwiftTO) { + installPathPrefix = resource.getRootDir(cmd.getCacheStore().getUrl(),_nfsVersion); } String user = null; String password = null; @@ -769,8 +881,10 @@ public DownloadAnswer handleDownloadCommand(SecondaryStorageResource resource, D String jobId = null; if (dstore instanceof S3TO) { jobId = - downloadS3Template((S3TO)dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), + downloadS3Template((S3TO) dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType); + } else if (dstore instanceof SwiftTO) { + jobId = downloadSwiftVolume(cmd, installPathPrefix, maxDownloadSizeInBytes); } else { jobId = downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), @@ -898,7 +1012,7 @@ public Map gatherTemplateInfo(String rootDir) { continue; } } catch (IOException e) { - LOGGER.warn("Unable to load template location " + path, e); + LOGGER.error("Unable to load template location " + path, e); continue; } @@ -918,7 +1032,7 @@ public Map gatherTemplateInfo(String rootDir) { } result.put(tInfo.getTemplateName(), tInfo); - LOGGER.debug("Added template name: " + tInfo.getTemplateName() + ", path: " + tmplt); + LOGGER.info("Added template name: " + tInfo.getTemplateName() + ", path: " + tmplt); } return result; } @@ -944,7 +1058,7 @@ public Map gatherVolumeInfo(String rootDir) { continue; } } catch (IOException e) { - LOGGER.warn("Unable to load volume location " + path, e); + LOGGER.error("Unable to load volume location " + path, e); continue; } @@ -964,7 +1078,7 @@ public Map gatherVolumeInfo(String rootDir) { } result.put(vInfo.getId(), vInfo); - LOGGER.debug("Added volume name: " + vInfo.getTemplateName() + ", path: " + vol); + LOGGER.info("Added volume name: " + vInfo.getTemplateName() + ", path: " + vol); } return result; } @@ -1092,7 +1206,7 @@ private void blockOutgoingOnPrivate() { String result = command.execute(); if (result != null) { - LOGGER.warn("Error in blocking outgoing to port 80/443 err=" + result); + LOGGER.error("Error in blocking outgoing to port 80/443 err=" + result); return; } } @@ -1118,7 +1232,7 @@ private void startAdditionalServices() { command.add("apache2"); String result = command.execute(); if (result != null) { - LOGGER.warn("Error in stopping httpd service err=" + result); + LOGGER.error("Error in stopping httpd service err=" + result); } String port = Integer.toString(TemplateConstants.DEFAULT_TMPLT_COPY_PORT); String intf = TemplateConstants.DEFAULT_TMPLT_COPY_INTF; @@ -1130,7 +1244,7 @@ private void startAdditionalServices() { result = command.execute(); if (result != null) { - LOGGER.warn("Error in opening up apache2 port err=" + result); + LOGGER.error("Error in opening up apache2 port err=" + result); return; } @@ -1139,7 +1253,7 @@ private void startAdditionalServices() { command.add("apache2"); result = command.execute(); if (result != null) { - LOGGER.warn("Error in starting apache2 service err=" + result); + LOGGER.error("Error in starting apache2 service err=" + result); return; } @@ -1151,7 +1265,7 @@ private void startAdditionalServices() { command.add("www-data"); result = command.execute(); if (result != null) { - LOGGER.warn("Error in creating directory =" + result); + LOGGER.error("Error in creating directory =" + result); return; } } From 12b7f26273e0f8d1911c64292033370c7b0aea54 Mon Sep 17 00:00:00 2001 From: Sid Kattoju Date: Thu, 25 Mar 2021 19:29:19 -0400 Subject: [PATCH 02/16] added getName to reflect DataTO interface change --- .../apache/cloudstack/diagnostics/to/DiagnosticsDataTO.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/main/java/org/apache/cloudstack/diagnostics/to/DiagnosticsDataTO.java b/server/src/main/java/org/apache/cloudstack/diagnostics/to/DiagnosticsDataTO.java index 115ee718fbed..a063ffa19e34 100644 --- a/server/src/main/java/org/apache/cloudstack/diagnostics/to/DiagnosticsDataTO.java +++ b/server/src/main/java/org/apache/cloudstack/diagnostics/to/DiagnosticsDataTO.java @@ -26,6 +26,7 @@ public class DiagnosticsDataTO implements DataTO { private DataStoreTO dataStoreTO; private Hypervisor.HypervisorType hypervisorType; private String path; + private String name; private long id; public DiagnosticsDataTO(Hypervisor.HypervisorType hypervisorType, DataStoreTO dataStoreTO) { @@ -57,4 +58,7 @@ public String getPath() { public long getId() { return id; } + + @Override + public String getName() { return name; } } From b943108f7844e1a7a3e506b4f9acf25c39618a02 Mon Sep 17 00:00:00 2001 From: Sid Kattoju Date: Thu, 25 Mar 2021 20:22:58 -0400 Subject: [PATCH 03/16] organize imports --- .../storage/template/DownloadManagerImpl.java | 52 +------------------ 1 file changed, 2 insertions(+), 50 deletions(-) diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index 4e66c5c88048..a7f0a7cf67d5 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@ -16,57 +16,10 @@ // under the License. package org.apache.cloudstack.storage.template; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.net.URISyntaxException; -import java.security.NoSuchAlgorithmException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import javax.naming.ConfigurationException; - -import com.cloud.agent.api.to.OVFInformationTO; -import com.cloud.storage.template.Processor; -import com.cloud.storage.template.S3TemplateDownloader; -import com.cloud.storage.template.TemplateDownloader; -import com.cloud.storage.template.TemplateLocation; -import com.cloud.storage.template.MetalinkTemplateDownloader; -import com.cloud.storage.template.HttpTemplateDownloader; -import com.cloud.storage.template.LocalTemplateDownloader; -import com.cloud.storage.template.ScpTemplateDownloader; -import com.cloud.storage.template.TemplateProp; -import com.cloud.storage.template.OVAProcessor; -import com.cloud.storage.template.IsoProcessor; -import com.cloud.storage.template.QCOW2Processor; -import com.cloud.storage.template.VmdkProcessor; -import com.cloud.storage.template.RawImageProcessor; -import com.cloud.storage.template.TARProcessor; -import com.cloud.storage.template.VhdProcessor; -import com.cloud.storage.template.TemplateConstants; -import org.apache.cloudstack.storage.command.DownloadCommand; -import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; -import org.apache.cloudstack.storage.command.DownloadProgressCommand; -import org.apache.cloudstack.storage.command.DownloadProgressCommand.RequestType; -import org.apache.cloudstack.storage.NfsMountManagerImpl.PathParser; -import org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource; -import org.apache.cloudstack.storage.resource.SecondaryStorageResource; -import org.apache.log4j.Logger; - import com.cloud.agent.api.storage.DownloadAnswer; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.NfsTO; +import com.cloud.agent.api.to.OVFInformationTO; import com.cloud.agent.api.to.S3TO; import com.cloud.agent.api.to.SwiftTO; import com.cloud.exception.InternalErrorException; @@ -100,9 +53,9 @@ import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.Proxy; -import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; import com.cloud.utils.storage.QCOW2Utils; +import org.apache.cloudstack.storage.NfsMountManagerImpl.PathParser; import org.apache.cloudstack.storage.command.DownloadCommand; import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; import org.apache.cloudstack.storage.command.DownloadProgressCommand; @@ -114,7 +67,6 @@ import org.apache.log4j.Logger; import javax.naming.ConfigurationException; -import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.IOException; From 65646b7a8a56f3728f1b1a1bd4e034ac5ca48c66 Mon Sep 17 00:00:00 2001 From: Sid Kattoju Date: Fri, 26 Mar 2021 10:31:42 -0400 Subject: [PATCH 04/16] prevent null pointer when cloning template --- .../storage/motion/AncientDataMotionStrategy.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java index 95ebd5f68c33..805255b3215f 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java @@ -146,7 +146,11 @@ private Scope pickCacheScopeForCopy(DataObject srcData, DataObject destData) { } protected Answer copyObject(DataObject srcData, DataObject destData, Host destHost) { - int imgSizeGigs = (int)Math.ceil(srcData.getSize() * 1.0d / (1024 * 1024 * 1024)); + long dataSize = 0; + if(srcData != null){ + dataSize = srcData.getSize(); + } + int imgSizeGigs = (int)Math.ceil(dataSize * 1.0d / (1024 * 1024 * 1024)); imgSizeGigs++; // add one just in case int primaryStorageDownloadWait = StorageManager.PRIMARY_STORAGE_DOWNLOAD_WAIT.value()*imgSizeGigs; Answer answer = null; From 38a9204ebfa806f8cb7d1b79dae3923030d48a6f Mon Sep 17 00:00:00 2001 From: Sid Kattoju Date: Fri, 26 Mar 2021 12:05:50 -0400 Subject: [PATCH 05/16] catch null pointer from src data object get size for now --- .../cloudstack/storage/motion/AncientDataMotionStrategy.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java index 805255b3215f..53b1a11a03a0 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java @@ -147,8 +147,10 @@ private Scope pickCacheScopeForCopy(DataObject srcData, DataObject destData) { protected Answer copyObject(DataObject srcData, DataObject destData, Host destHost) { long dataSize = 0; - if(srcData != null){ + try{ dataSize = srcData.getSize(); + }catch(NullPointerException e){ + s_logger.error("Unable to determine size of src data object with uuid: " + srcData.getUuid(),e); } int imgSizeGigs = (int)Math.ceil(dataSize * 1.0d / (1024 * 1024 * 1024)); imgSizeGigs++; // add one just in case From 8eb2133a70bc451458fa8c0a51c34953334803d7 Mon Sep 17 00:00:00 2001 From: Sid Kattoju Date: Wed, 31 Mar 2021 21:49:54 -0400 Subject: [PATCH 06/16] improved log messages --- .../com/cloud/storage/template/SwiftVolumeDownloader.java | 5 +++-- .../datastore/driver/SwiftImageStoreDriverImpl.java | 2 +- .../storage/resource/NfsSecondaryStorageResource.java | 7 ++++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/com/cloud/storage/template/SwiftVolumeDownloader.java b/core/src/main/java/com/cloud/storage/template/SwiftVolumeDownloader.java index 01d3be5b99cd..47a041b4fad0 100644 --- a/core/src/main/java/com/cloud/storage/template/SwiftVolumeDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/SwiftVolumeDownloader.java @@ -237,7 +237,7 @@ public long download(boolean resume, DownloadCompleteCallback callback) { return 0; } - LOGGER.info("Starting download from " + downloadUrl + " to staging with size " + remoteSize + " bytes to "+filePath); + LOGGER.info("Starting download from " + downloadUrl + " to staging with size " + remoteSize + " bytes to " + srcFile.getPath()); final Date downloadStart = new Date(); try (FileOutputStream fileOutputStream = new FileOutputStream(srcFile);) { @@ -257,7 +257,8 @@ public long download(boolean resume, DownloadCompleteCallback callback) { outputStream.close(); inputStream.close(); } catch (IOException e) { - LOGGER.error("Exception when downloading from url to staging nfs:" + e.getMessage(), e); + LOGGER.error("Exception when downloading from url " + downloadUrl + " with remote size " + remoteSize + + " to staging nfs path " + srcFile.getPath() + " : " + e.getMessage(), e); status = Status.RECOVERABLE_ERROR; return 0; } diff --git a/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java b/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java index 6afb4a052af0..6ee6b27d1cba 100644 --- a/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java +++ b/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java @@ -116,7 +116,7 @@ public void createAsync(DataStore dataStore, DataObject data, AsyncCompletionCal } if (downloadCommand == null){ - String errMsg = "Unable to build download command, DataObject is of neither VOLUME or TEMPLATE type"; + String errMsg = "Unable to build download command, unsupported DataObject is of type " + data.getType().toString(); s_logger.error(errMsg); throw new CloudRuntimeException(errMsg); } diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 218995b3ca81..b4bcc85dca36 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -1167,11 +1167,12 @@ protected File downloadFromUrlToNfs(String url, NfsTO nfs, String path, String n try (FileOutputStream outputStream = new FileOutputStream(destFile);) { entity.writeTo(outputStream); } catch (IOException e) { - s_logger.error("downloadFromUrlToNfs:Exception:" + e.getMessage(), e); + s_logger.error(String.format("Unable to download from url [%s] to nfs path [%s] due to [%s]", + url,destFile.getPath(),e.getMessage()), e); } return new File(destFile.getAbsolutePath()); } catch (IOException e) { - s_logger.error("Faild to get url:" + url + ", due to " + e.toString()); + s_logger.error("Failed to get url:" + url + ", due to " + e.toString(), e); throw new CloudRuntimeException(e); } } @@ -1210,7 +1211,7 @@ protected Answer registerTemplateOnSwift(DownloadCommand cmd) { try (FileInputStream fs = new FileInputStream(file)) { md5sum = DigestUtils.md5Hex(fs); } catch (IOException e) { - s_logger.error("Failed to get md5sum: " + file.getAbsoluteFile()); + s_logger.error("Failed to get md5sum: " + file.getAbsoluteFile(), e); } DownloadAnswer answer = new DownloadAnswer(null, 100, null, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, swiftPath, swiftPath, virtualSize, file.length(), md5sum); From b4f421e235e5acc65f05db2cb9b1166dc6b43b78 Mon Sep 17 00:00:00 2001 From: Sid Kattoju <45833770+skattoju4@users.noreply.github.com> Date: Mon, 26 Apr 2021 14:14:29 -0400 Subject: [PATCH 07/16] fix formatting --- .../java/com/cloud/storage/download/DownloadMonitorImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java b/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java index 7f298f260762..dd5113111cd5 100644 --- a/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java +++ b/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java @@ -222,7 +222,7 @@ public void downloadVolumeToStorage(DataObject volume, AsyncCompletionCallback Date: Tue, 19 Jul 2022 14:39:59 +0530 Subject: [PATCH 08/16] address review comments Signed-off-by: Abhishek Kumar --- .../motion/AncientDataMotionStrategy.java | 11 +++--- .../driver/SwiftImageStoreDriverImpl.java | 34 ++++++++++--------- .../storage/template/DownloadManagerImpl.java | 3 +- 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java index 9a1b21ed4880..cb18459c426b 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java @@ -18,14 +18,14 @@ */ package org.apache.cloudstack.storage.motion; +import static com.cloud.storage.snapshot.SnapshotManager.BackupSnapshotAfterTakingSnapshot; + import java.util.HashMap; import java.util.List; import java.util.Map; import javax.inject.Inject; -import com.cloud.agent.api.to.DiskTO; -import com.cloud.storage.Storage; import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionStrategy; @@ -60,18 +60,19 @@ import com.cloud.agent.api.to.DataObjectType; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.DataTO; +import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.configuration.Config; import com.cloud.host.Host; import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.DataStoreRole; -import com.cloud.storage.StorageManager; +import com.cloud.storage.Storage; import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VolumeDao; -import static com.cloud.storage.snapshot.SnapshotManager.BackupSnapshotAfterTakingSnapshot; import com.cloud.utils.NumbersUtil; import com.cloud.utils.db.DB; import com.cloud.utils.exception.CloudRuntimeException; @@ -159,7 +160,7 @@ protected Answer copyObject(DataObject srcData, DataObject destData, Host destHo long dataSize = 0; try{ dataSize = srcData.getSize(); - }catch(NullPointerException e){ + } catch(NullPointerException e) { s_logger.error("Unable to determine size of src data object with uuid: " + srcData.getUuid(),e); } int imgSizeGigs = (int)Math.ceil(dataSize * 1.0d / (1024 * 1024 * 1024)); diff --git a/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java b/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java index 6ee6b27d1cba..462dd2def693 100644 --- a/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java +++ b/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java @@ -18,16 +18,12 @@ */ package org.apache.cloudstack.storage.datastore.driver; -import com.cloud.agent.api.storage.DownloadAnswer; -import com.cloud.agent.api.to.DataObjectType; -import com.cloud.agent.api.to.DataStoreTO; -import com.cloud.agent.api.to.SwiftTO; -import com.cloud.configuration.Config; -import com.cloud.storage.RegisterVolumePayload; -import com.cloud.storage.Storage.ImageFormat; -import com.cloud.storage.download.DownloadMonitor; -import com.cloud.utils.SwiftUtil; -import com.cloud.utils.exception.CloudRuntimeException; +import java.net.URL; +import java.util.Map; +import java.util.UUID; + +import javax.inject.Inject; + import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; @@ -47,10 +43,16 @@ import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.log4j.Logger; -import javax.inject.Inject; -import java.net.URL; -import java.util.Map; -import java.util.UUID; +import com.cloud.agent.api.storage.DownloadAnswer; +import com.cloud.agent.api.to.DataObjectType; +import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.agent.api.to.SwiftTO; +import com.cloud.configuration.Config; +import com.cloud.storage.RegisterVolumePayload; +import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.download.DownloadMonitor; +import com.cloud.utils.SwiftUtil; +import com.cloud.utils.exception.CloudRuntimeException; public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl { private static final Logger s_logger = Logger.getLogger(SwiftImageStoreDriverImpl.class); @@ -107,7 +109,7 @@ public void createAsync(DataStore dataStore, DataObject data, AsyncCompletionCal if (data.getType() == DataObjectType.TEMPLATE) { Long maxTemplateSizeInBytes = getMaxTemplateSizeInBytes(); downloadCommand = new DownloadCommand((TemplateObjectTO) (data.getTO()), maxTemplateSizeInBytes); - }else if (data.getType() == DataObjectType.VOLUME){ + } else if (data.getType() == DataObjectType.VOLUME) { Long maxDownloadSizeInBytes = getMaxVolumeSizeInBytes(); VolumeInfo volumeInfo = (VolumeInfo) data; RegisterVolumePayload payload = (RegisterVolumePayload) volumeInfo.getpayload(); @@ -115,7 +117,7 @@ public void createAsync(DataStore dataStore, DataObject data, AsyncCompletionCal downloadCommand = new DownloadCommand((VolumeObjectTO) (data.getTO()), maxDownloadSizeInBytes, payload.getChecksum(), payload.getUrl(), format); } - if (downloadCommand == null){ + if (downloadCommand == null) { String errMsg = "Unable to build download command, unsupported DataObject is of type " + data.getType().toString(); s_logger.error(errMsg); throw new CloudRuntimeException(errMsg); diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index cb9a6bc136f5..aef4844d11d8 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@ -321,8 +321,7 @@ public void setDownloadStatus(String jobId, Status status) { } } else if (td instanceof SwiftVolumeDownloader) { dj.setCheckSum(((SwiftVolumeDownloader) td).getMd5sum()); - if ("vhd".equals(((SwiftVolumeDownloader) td).getFileExtension()) || - "VHD".equals(((SwiftVolumeDownloader) td).getFileExtension())) { + if ("vhd".equalsIgnoreCase(((SwiftVolumeDownloader) td).getFileExtension())) { Processor vhdProcessor = _processors.get("VHD Processor"); long virtualSize = 0; try { From dd53dcd6b0929ea0e42f42991a539c2b2ceb81f5 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 21 Jul 2022 13:29:46 +0530 Subject: [PATCH 09/16] address sonarcloud issues Signed-off-by: Abhishek Kumar --- .../template/SwiftVolumeDownloader.java | 153 +++++++----------- .../cloud/storage/template/VhdProcessor.java | 26 +-- .../resource/NfsSecondaryStorageResource.java | 10 +- .../storage/template/DownloadManagerImpl.java | 56 ++++--- 4 files changed, 104 insertions(+), 141 deletions(-) diff --git a/core/src/main/java/com/cloud/storage/template/SwiftVolumeDownloader.java b/core/src/main/java/com/cloud/storage/template/SwiftVolumeDownloader.java index 47a041b4fad0..0e7f8c5caed4 100644 --- a/core/src/main/java/com/cloud/storage/template/SwiftVolumeDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/SwiftVolumeDownloader.java @@ -1,25 +1,40 @@ -// -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// - +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package com.cloud.storage.template; -import com.cloud.agent.api.to.SwiftTO; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import java.net.UnknownHostException; +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.util.Date; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLException; + import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.storage.command.DownloadCommand; import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; @@ -31,31 +46,13 @@ import org.apache.http.client.HttpRequestRetryHandler; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.conn.ConnectTimeoutException; import org.apache.http.conn.ssl.NoopHostnameVerifier; import org.apache.http.conn.ssl.TrustStrategy; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; -import org.apache.http.protocol.HttpContext; import org.apache.http.ssl.SSLContextBuilder; import org.apache.log4j.Logger; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLException; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InterruptedIOException; -import java.net.UnknownHostException; -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.util.Date; - /** * Download a volume file using HTTP(S) * @@ -70,14 +67,11 @@ public class SwiftVolumeDownloader extends ManagedContextRunnable implements Tem private final String downloadUrl; private final String fileName; private final String fileExtension; - private final long volumeId; private final CloseableHttpClient httpClient; private final HttpGet httpGet; private final DownloadCompleteCallback downloadCompleteCallback; - private final SwiftTO swiftTO; private String errorString = ""; private Status status = Status.NOT_STARTED; - private final ResourceType resourceType = ResourceType.VOLUME; private long remoteSize; private String md5sum; private long downloadTime; @@ -90,20 +84,17 @@ public class SwiftVolumeDownloader extends ManagedContextRunnable implements Tem public SwiftVolumeDownloader(DownloadCommand cmd, DownloadCompleteCallback downloadCompleteCallback, long maxVolumeSizeInBytes, String installPathPrefix) { this.downloadUrl = cmd.getUrl(); - this.swiftTO = (SwiftTO) cmd.getDataStore(); this.maxVolumeSizeInBytes = maxVolumeSizeInBytes; this.httpClient = initializeHttpClient(); this.downloadCompleteCallback = downloadCompleteCallback; this.fileName = cmd.getName(); this.fileExtension = cmd.getFormat().getFileExtension(); - this.volumeId = cmd.getId(); this.installPathPrefix = installPathPrefix; this.installPath = cmd.getInstallPath(); this.httpGet = new HttpGet(downloadUrl); } - private CloseableHttpClient initializeHttpClient(){ - + private CloseableHttpClient initializeHttpClient() { CloseableHttpClient client = null; try { //trust all certs @@ -114,62 +105,30 @@ private CloseableHttpClient initializeHttpClient(){ .setSSLHostnameVerifier(new NoopHostnameVerifier()) .setRetryHandler(buildRetryHandler(5)) .build(); - } catch (NoSuchAlgorithmException e) { - e.printStackTrace(); - } catch (KeyManagementException e) { - e.printStackTrace(); - } catch (KeyStoreException e) { - e.printStackTrace(); + } catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException e) { + LOGGER.error(String.format("Unable to initialize HTTP client due to %s", e.getMessage()), e); } - return client; } - private HttpRequestRetryHandler buildRetryHandler(int retryCount){ - - HttpRequestRetryHandler customRetryHandler = new HttpRequestRetryHandler() { - @Override - public boolean retryRequest( - IOException exception, - int executionCount, - HttpContext context) { - if (executionCount >= retryCount) { - // Do not retry if over max retry count - return false; - } - if (exception instanceof InterruptedIOException) { - // Timeout - return false; - } - if (exception instanceof UnknownHostException) { - // Unknown host - return false; - } - if (exception instanceof ConnectTimeoutException) { - // Connection refused - return false; - } - if (exception instanceof SSLException) { - // SSL handshake exception - return false; - } - HttpClientContext clientContext = HttpClientContext.adapt(context); - HttpRequest request = clientContext.getRequest(); - boolean idempotent = !(request instanceof HttpEntityEnclosingRequest); - if (idempotent) { - // Retry if the request is considered idempotent - return true; - } + private HttpRequestRetryHandler buildRetryHandler(int retryCount) { + return (exception, executionCount, context) -> { + if (executionCount >= retryCount) { + // Do not retry if over max retry count return false; } - + if (exception instanceof InterruptedIOException || exception instanceof UnknownHostException || exception instanceof SSLException) { + return false; + } + HttpClientContext clientContext = HttpClientContext.adapt(context); + HttpRequest request = clientContext.getRequest(); + return !(request instanceof HttpEntityEnclosingRequest); }; - return customRetryHandler; } @Override public long download(boolean resume, DownloadCompleteCallback callback) { - if (!status.equals(Status.NOT_STARTED)) { + if (!Status.NOT_STARTED.equals(status)) { // Only start downloading if we haven't started yet. LOGGER.info("Volume download is already started, not starting again. Volume: " + downloadUrl); return 0; @@ -180,9 +139,8 @@ public long download(boolean resume, DownloadCompleteCallback callback) { response = httpClient.execute(httpGet); } catch (IOException e) { e.printStackTrace(); - errorString = "Exception while executing HttpMethod " + httpGet.getMethod() + " on URL " + downloadUrl + " " - + response.getStatusLine().getStatusCode() + " " + response.getStatusLine().getReasonPhrase(); - LOGGER.error(errorString); + errorString = String.format("Exception while executing HttpMethod %s on URL %s", httpGet.getMethod(), downloadUrl); + LOGGER.error(errorString, e); status = Status.UNRECOVERABLE_ERROR; return 0; } @@ -244,12 +202,12 @@ public long download(boolean resume, DownloadCompleteCallback callback) { BufferedOutputStream outputStream = new BufferedOutputStream(fileOutputStream,DOWNLOAD_BUFFER_SIZE_BYTES); byte[] data = new byte[DOWNLOAD_BUFFER_SIZE_BYTES]; int bufferLength = 0; - while((bufferLength = inputStream.read(data,0,DOWNLOAD_BUFFER_SIZE_BYTES)) >= 0){ + while((bufferLength = inputStream.read(data,0,DOWNLOAD_BUFFER_SIZE_BYTES)) >= 0) { totalBytes += bufferLength; outputStream.write(data,0,bufferLength); status = Status.IN_PROGRESS; LOGGER.trace("Download in progress: " + getDownloadPercent() + "%"); - if(totalBytes >= remoteSize){ + if (totalBytes >= remoteSize) { volumeFile = srcFile; status = Status.DOWNLOAD_FINISHED; } @@ -321,8 +279,7 @@ public boolean stopDownload() { case UNRECOVERABLE_ERROR: case ABORTED: case DOWNLOAD_FINISHED: - // Remove the object if it already has been uploaded. - // SwiftUtil.deleteObject(swiftTO, swiftPath); + LOGGER.debug(String.format("Stopping download while status is %s", status)); break; default: break; @@ -382,7 +339,7 @@ public boolean isInited() { } public ResourceType getResourceType() { - return resourceType; + return ResourceType.VOLUME; } public String getFileExtension() { diff --git a/core/src/main/java/com/cloud/storage/template/VhdProcessor.java b/core/src/main/java/com/cloud/storage/template/VhdProcessor.java index baea7bf0db51..1e7f927556e3 100644 --- a/core/src/main/java/com/cloud/storage/template/VhdProcessor.java +++ b/core/src/main/java/com/cloud/storage/template/VhdProcessor.java @@ -19,17 +19,6 @@ package com.cloud.storage.template; -import com.cloud.exception.InternalErrorException; -import com.cloud.storage.Storage.ImageFormat; -import com.cloud.storage.StorageLayer; -import com.cloud.utils.NumbersUtil; -import com.cloud.utils.component.AdapterBase; -import org.apache.commons.compress.compressors.CompressorException; -import org.apache.commons.compress.compressors.CompressorInputStream; -import org.apache.commons.compress.compressors.CompressorStreamFactory; -import org.apache.log4j.Logger; - -import javax.naming.ConfigurationException; import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; @@ -38,6 +27,19 @@ import java.io.InputStream; import java.util.Map; +import javax.naming.ConfigurationException; + +import org.apache.commons.compress.compressors.CompressorException; +import org.apache.commons.compress.compressors.CompressorInputStream; +import org.apache.commons.compress.compressors.CompressorStreamFactory; +import org.apache.log4j.Logger; + +import com.cloud.exception.InternalErrorException; +import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.StorageLayer; +import com.cloud.utils.NumbersUtil; +import com.cloud.utils.component.AdapterBase; + /** * VhdProcessor processes the downloaded template for VHD. It * currently does not handle any type of template conversion @@ -46,6 +48,8 @@ */ public class VhdProcessor extends AdapterBase implements Processor { + public static final String NAME = "VHD Processor"; + private static final Logger s_logger = Logger.getLogger(VhdProcessor.class); StorageLayer _storage; private int vhdFooterSize = 512; diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 15418c401bd6..fb99482cab05 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -1136,12 +1136,10 @@ protected File downloadFromUrlToNfs(String url, NfsTO nfs, String path, String n client = HttpClients.custom().setSSLContext(sslContext) .setSSLHostnameVerifier(new NoopHostnameVerifier()) .build(); - } catch (NoSuchAlgorithmException e) { - e.printStackTrace(); - } catch (KeyManagementException e) { - e.printStackTrace(); - } catch (KeyStoreException e) { - e.printStackTrace(); + } catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException e) { + String msg = String.format("Failed to initialize download url: %s, due to %s", url, e.getMessage()); + s_logger.error(msg, e); + throw new CloudRuntimeException(msg, e); } try { diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index aef4844d11d8..445140087aad 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@ -258,6 +258,27 @@ public void setOvfInformationTO(OVFInformationTO ovfInformationTO) { private String listVolScr; private int installTimeoutPerGig = 180 * 60 * 1000; + private void setSwiftVolumeDownloadStatus(DownloadJob dj, TemplateDownloader td) { + dj.setCheckSum(((SwiftVolumeDownloader) td).getMd5sum()); + if ("vhd".equalsIgnoreCase(((SwiftVolumeDownloader) td).getFileExtension())) { + Processor vhdProcessor = _processors.get("VHD Processor"); + long virtualSize = 0; + try { + virtualSize = vhdProcessor.getVirtualSize(((SwiftVolumeDownloader) td).getVolumeFile()); + dj.setTemplatesize(virtualSize); + } catch (IOException e) { + LOGGER.error("Unable to read VHD file", e); + e.printStackTrace(); + } + } else { + dj.setTemplatesize(td.getDownloadedBytes()); + } + dj.setTemplatePhysicalSize(td.getDownloadedBytes()); + dj.setTmpltPath(td.getDownloadLocalPath()); + td.setStatus(Status.POST_DOWNLOAD_FINISHED); + td.setDownloadError("Volume downloaded to swift cache successfully at " + new SimpleDateFormat().format(new Date())); + } + public void setThreadPool(ExecutorService threadPool) { this.threadPool = threadPool; } @@ -278,7 +299,9 @@ public void setStorageLayer(StorageLayer storage) { public void setDownloadStatus(String jobId, Status status) { DownloadJob dj = jobs.get(jobId); if (dj == null) { - LOGGER.info("setDownloadStatus for jobId: " + jobId + ", status=" + status + " no job found"); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Updating DownloadStatus: %s for jobId: %s failed as no job found", status, jobId)); + } return; } TemplateDownloader td = dj.getTemplateDownloader(); @@ -304,7 +327,7 @@ public void setDownloadStatus(String jobId, Status status) { threadPool.execute(td); break; case DOWNLOAD_FINISHED: - if(td instanceof S3TemplateDownloader) { + if (td instanceof S3TemplateDownloader) { // For S3 and Swift, which are considered "remote", // as in the file cannot be accessed locally, // we run the postRemoteDownload() method. @@ -320,24 +343,7 @@ public void setDownloadStatus(String jobId, Status status) { td.setDownloadError("Install completed successfully at " + new SimpleDateFormat().format(new Date())); } } else if (td instanceof SwiftVolumeDownloader) { - dj.setCheckSum(((SwiftVolumeDownloader) td).getMd5sum()); - if ("vhd".equalsIgnoreCase(((SwiftVolumeDownloader) td).getFileExtension())) { - Processor vhdProcessor = _processors.get("VHD Processor"); - long virtualSize = 0; - try { - virtualSize = vhdProcessor.getVirtualSize(((SwiftVolumeDownloader) td).getVolumeFile()); - dj.setTemplatesize(virtualSize); - } catch (IOException e) { - LOGGER.error("Unable to read VHD file", e); - e.printStackTrace(); - } - } else { - dj.setTemplatesize(((SwiftVolumeDownloader) td).getDownloadedBytes()); - } - dj.setTemplatePhysicalSize(((SwiftVolumeDownloader) td).getDownloadedBytes()); - dj.setTmpltPath(((SwiftVolumeDownloader) td).getDownloadLocalPath()); - td.setStatus(Status.POST_DOWNLOAD_FINISHED); - td.setDownloadError("Volume downloaded to swift cache successfully at " + new SimpleDateFormat().format(new Date())); + setSwiftVolumeDownloadStatus(dj, td); } else { // For other TemplateDownloaders where files are locally available, // we run the postLocalDownload() method. @@ -417,7 +423,7 @@ private String postLocalDownload(String jobId) { ResourceType resourceType = dnld.getResourceType(); File originalTemplate = new File(td.getDownloadLocalPath()); - if(StringUtils.isBlank(dnld.getChecksum())) { + if (StringUtils.isBlank(dnld.getChecksum())) { if (LOGGER.isInfoEnabled()) { LOGGER.info(String.format("No checksum available for '%s'", originalTemplate.getName())); } @@ -594,7 +600,7 @@ public String downloadS3Template(S3TO s3, long id, String url, String name, Imag throw new CloudRuntimeException("URI is incorrect: " + url); } TemplateDownloader td; - if ((uri != null) && (uri.getScheme() != null)) { + if (uri.getScheme() != null) { if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) { td = new S3TemplateDownloader(s3, url, installPathPrefix, new Completion(jobId), maxTemplateSizeInBytes, user, password, proxy, resourceType); } else { @@ -615,8 +621,6 @@ public String downloadS3Template(S3TO s3, long id, String url, String name, Imag public String downloadSwiftVolume(DownloadCommand cmd, String installPathPrefix, long maxDownloadSizeInBytes) { UUID uuid = UUID.randomUUID(); String jobId = uuid.toString(); - //TODO get from global config - long maxVolumeSizeInBytes = maxDownloadSizeInBytes; URI uri = null; try { uri = new URI(cmd.getUrl()); @@ -626,7 +630,7 @@ public String downloadSwiftVolume(DownloadCommand cmd, String installPathPrefix, TemplateDownloader td; if ((uri != null) && (uri.getScheme() != null)) { if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) { - td = new SwiftVolumeDownloader(cmd, new Completion(jobId), maxVolumeSizeInBytes, installPathPrefix); + td = new SwiftVolumeDownloader(cmd, new Completion(jobId), maxDownloadSizeInBytes, installPathPrefix); } else { throw new CloudRuntimeException("Scheme is not supported " + cmd.getUrl()); } @@ -659,7 +663,7 @@ public String downloadPublicTemplate(long id, String url, String name, ImageForm ResourceType.TEMPLATE == resourceType ? _storage.getFile(tmpDir + File.separator + TemplateLocation.Filename) : _storage.getFile(tmpDir + File.separator + "volume.properties"); if (file.exists()) { - if(! file.delete()) { + if (! file.delete()) { LOGGER.error("Deletion of file '" + file.getAbsolutePath() + "' failed."); } } From 2b1eab86e87f7fa486abe1b5d28aa1375123da80 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 21 Jul 2022 13:32:12 +0530 Subject: [PATCH 10/16] fix Signed-off-by: Abhishek Kumar --- .../storage/template/SwiftVolumeDownloader.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/cloud/storage/template/SwiftVolumeDownloader.java b/core/src/main/java/com/cloud/storage/template/SwiftVolumeDownloader.java index 0e7f8c5caed4..89d2f1a71c70 100644 --- a/core/src/main/java/com/cloud/storage/template/SwiftVolumeDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/SwiftVolumeDownloader.java @@ -346,9 +346,15 @@ public String getFileExtension() { return fileExtension; } - public String getMd5sum() { return md5sum; } + public String getMd5sum() { + return md5sum; + } - public File getVolumeFile() { return volumeFile; } + public File getVolumeFile() { + return volumeFile; + } - public long getMaxTemplateSizeInBytes() {return maxVolumeSizeInBytes;} -} \ No newline at end of file + public long getMaxTemplateSizeInBytes() { + return maxVolumeSizeInBytes; + } +} From 0d83ce9337d4d0165a36d91ac23e61016ee8ea1b Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 22 Jul 2022 16:55:48 +0530 Subject: [PATCH 11/16] address more Signed-off-by: Abhishek Kumar --- .../java/com/cloud/storage/template/SwiftVolumeDownloader.java | 1 - .../storage/datastore/driver/SwiftImageStoreDriverImpl.java | 2 -- .../apache/cloudstack/storage/template/DownloadManagerImpl.java | 1 - 3 files changed, 4 deletions(-) diff --git a/core/src/main/java/com/cloud/storage/template/SwiftVolumeDownloader.java b/core/src/main/java/com/cloud/storage/template/SwiftVolumeDownloader.java index 89d2f1a71c70..3005c92c8d7a 100644 --- a/core/src/main/java/com/cloud/storage/template/SwiftVolumeDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/SwiftVolumeDownloader.java @@ -138,7 +138,6 @@ public long download(boolean resume, DownloadCompleteCallback callback) { try { response = httpClient.execute(httpGet); } catch (IOException e) { - e.printStackTrace(); errorString = String.format("Exception while executing HttpMethod %s on URL %s", httpGet.getMethod(), downloadUrl); LOGGER.error(errorString, e); status = Status.UNRECOVERABLE_ERROR; diff --git a/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java b/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java index 462dd2def693..b38b72d44b35 100644 --- a/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java +++ b/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java @@ -65,8 +65,6 @@ public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl { StorageCacheManager cacheManager; @Inject ConfigurationDao _configDao; - @Inject - private DownloadMonitor _downloadMonitor; @Override public DataStoreTO getStoreTO(DataStore store) { diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index 445140087aad..8a4885ed0df4 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@ -268,7 +268,6 @@ private void setSwiftVolumeDownloadStatus(DownloadJob dj, TemplateDownloader td) dj.setTemplatesize(virtualSize); } catch (IOException e) { LOGGER.error("Unable to read VHD file", e); - e.printStackTrace(); } } else { dj.setTemplatesize(td.getDownloadedBytes()); From dc84792fd42440846d3022af5f9e25e6d82293b9 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 22 Jul 2022 16:56:56 +0530 Subject: [PATCH 12/16] fix Signed-off-by: Abhishek Kumar --- .../cloudstack/storage/image/BaseImageStoreDriverImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java index eb5bef52df47..634cbf4d3706 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java @@ -99,7 +99,7 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { @Inject protected VMTemplateDao _templateDao; @Inject - DownloadMonitor _downloadMonitor; + protected DownloadMonitor _downloadMonitor; @Inject VolumeDao volumeDao; @Inject From d7d286cf0b0d0e7d58a7f2172dea3fcd89d4412b Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 29 Jul 2022 13:41:23 +0530 Subject: [PATCH 13/16] fix build failure Signed-off-by: Abhishek Kumar --- .../storage/datastore/driver/SwiftImageStoreDriverImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java b/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java index b38b72d44b35..a6b1422852cc 100644 --- a/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java +++ b/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java @@ -50,7 +50,6 @@ import com.cloud.configuration.Config; import com.cloud.storage.RegisterVolumePayload; import com.cloud.storage.Storage.ImageFormat; -import com.cloud.storage.download.DownloadMonitor; import com.cloud.utils.SwiftUtil; import com.cloud.utils.exception.CloudRuntimeException; From 3b59f4b64db34b862d6634ef61caf6f3838227f8 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 9 Jan 2023 14:52:09 +0530 Subject: [PATCH 14/16] fix import error Signed-off-by: Abhishek Kumar --- .../cloudstack/storage/motion/AncientDataMotionStrategy.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java index cb18459c426b..bd0162355b39 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java @@ -18,8 +18,6 @@ */ package org.apache.cloudstack.storage.motion; -import static com.cloud.storage.snapshot.SnapshotManager.BackupSnapshotAfterTakingSnapshot; - import java.util.HashMap; import java.util.List; import java.util.Map; @@ -73,6 +71,7 @@ import com.cloud.storage.StoragePool; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.snapshot.SnapshotManager; import com.cloud.utils.NumbersUtil; import com.cloud.utils.db.DB; import com.cloud.utils.exception.CloudRuntimeException; @@ -592,7 +591,7 @@ protected Answer copySnapshot(DataObject srcData, DataObject destData) { } Map options = new HashMap(); options.put("fullSnapshot", fullSnapshot.toString()); - options.put(BackupSnapshotAfterTakingSnapshot.key(), String.valueOf(BackupSnapshotAfterTakingSnapshot.value())); + options.put(SnapshotManager.BackupSnapshotAfterTakingSnapshot.key(), String.valueOf(SnapshotManager.BackupSnapshotAfterTakingSnapshot.value())); Answer answer = null; try { if (needCacheStorage(srcData, destData)) { From 2f92a984181781f82401bcb3683d92d09fc4c014 Mon Sep 17 00:00:00 2001 From: Sjnas Date: Tue, 12 Sep 2023 09:25:43 -0400 Subject: [PATCH 15/16] Removed duplicated import --- .../cloudstack/storage/resource/NfsSecondaryStorageResource.java | 1 - 1 file changed, 1 deletion(-) diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 01682a362005..1ae30544ec92 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -43,7 +43,6 @@ import java.nio.file.Path; import java.security.KeyManagementException; import java.security.KeyStoreException; -import java.nio.file.Paths; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Arrays; From 99707042d78dd18c4677f7214ad3eb9271b050e0 Mon Sep 17 00:00:00 2001 From: Sjnas Date: Tue, 3 Oct 2023 09:58:47 -0400 Subject: [PATCH 16/16] Adding missing import --- .../cloudstack/storage/resource/NfsSecondaryStorageResource.java | 1 + 1 file changed, 1 insertion(+) diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index d3515a97c249..b4e88cfffe6c 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -41,6 +41,7 @@ import java.net.UnknownHostException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.security.KeyManagementException; import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException;