diff --git a/api/src/main/java/com/cloud/agent/api/to/DataTO.java b/api/src/main/java/com/cloud/agent/api/to/DataTO.java index af43aa102e84..94fc89da004e 100644 --- a/api/src/main/java/com/cloud/agent/api/to/DataTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/DataTO.java @@ -32,5 +32,7 @@ public interface DataTO { */ String getPath(); + String getName(); + long getId(); } diff --git a/core/src/main/java/com/cloud/storage/template/SwiftVolumeDownloader.java b/core/src/main/java/com/cloud/storage/template/SwiftVolumeDownloader.java new file mode 100644 index 000000000000..3005c92c8d7a --- /dev/null +++ b/core/src/main/java/com/cloud/storage/template/SwiftVolumeDownloader.java @@ -0,0 +1,359 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.storage.template; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import java.net.UnknownHostException; +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.util.Date; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLException; + +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.storage.command.DownloadCommand; +import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.http.Header; +import org.apache.http.HttpEntityEnclosingRequest; +import org.apache.http.HttpRequest; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpRequestRetryHandler; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.TrustStrategy; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.ssl.SSLContextBuilder; +import org.apache.log4j.Logger; + +/** + * Download a volume file using HTTP(S) + * + * This class, once instantiated, has the purpose to download a Volume to staging nfs or cache when using as Swift Image Store. + * + * Execution of the instance is started when runInContext() is called. + */ +public class SwiftVolumeDownloader extends ManagedContextRunnable implements TemplateDownloader { + private static final Logger LOGGER = Logger.getLogger(SwiftVolumeDownloader.class.getName()); + private static final int DOWNLOAD_BUFFER_SIZE_BYTES = 1024* 1024; + + private final String downloadUrl; + private final String fileName; + private final String fileExtension; + private final CloseableHttpClient httpClient; + private final HttpGet httpGet; + private final DownloadCompleteCallback downloadCompleteCallback; + private String errorString = ""; + private Status status = Status.NOT_STARTED; + private long remoteSize; + private String md5sum; + private long downloadTime; + private long totalBytes; + private final long maxVolumeSizeInBytes; + private final String installPathPrefix; + private final String installPath; + private File volumeFile; + private boolean resume = false; + + public SwiftVolumeDownloader(DownloadCommand cmd, DownloadCompleteCallback downloadCompleteCallback, long maxVolumeSizeInBytes, String installPathPrefix) { + this.downloadUrl = cmd.getUrl(); + this.maxVolumeSizeInBytes = maxVolumeSizeInBytes; + this.httpClient = initializeHttpClient(); + this.downloadCompleteCallback = downloadCompleteCallback; + this.fileName = cmd.getName(); + this.fileExtension = cmd.getFormat().getFileExtension(); + this.installPathPrefix = installPathPrefix; + this.installPath = cmd.getInstallPath(); + this.httpGet = new HttpGet(downloadUrl); + } + + private CloseableHttpClient initializeHttpClient() { + CloseableHttpClient client = null; + try { + //trust all certs + SSLContext sslContext = new SSLContextBuilder() + .loadTrustMaterial(null, (TrustStrategy) (chain, authType) -> true) + .build(); + client = HttpClients.custom().setSSLContext(sslContext) + .setSSLHostnameVerifier(new NoopHostnameVerifier()) + .setRetryHandler(buildRetryHandler(5)) + .build(); + } catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException e) { + LOGGER.error(String.format("Unable to initialize HTTP client due to %s", e.getMessage()), e); + } + return client; + } + + private HttpRequestRetryHandler buildRetryHandler(int retryCount) { + return (exception, executionCount, context) -> { + if (executionCount >= retryCount) { + // Do not retry if over max retry count + return false; + } + if (exception instanceof InterruptedIOException || exception instanceof UnknownHostException || exception instanceof SSLException) { + return false; + } + HttpClientContext clientContext = HttpClientContext.adapt(context); + HttpRequest request = clientContext.getRequest(); + return !(request instanceof HttpEntityEnclosingRequest); + }; + } + + @Override + public long download(boolean resume, DownloadCompleteCallback callback) { + if (!Status.NOT_STARTED.equals(status)) { + // Only start downloading if we haven't started yet. + LOGGER.info("Volume download is already started, not starting again. Volume: " + downloadUrl); + return 0; + } + + HttpResponse response = null; + try { + response = httpClient.execute(httpGet); + } catch (IOException e) { + errorString = String.format("Exception while executing HttpMethod %s on URL %s", httpGet.getMethod(), downloadUrl); + LOGGER.error(errorString, e); + status = Status.UNRECOVERABLE_ERROR; + return 0; + } + + // Headers + long contentLength = response.getEntity().getContentLength(); + Header contentType = response.getEntity().getContentType(); + + // Check the contentLengthHeader and transferEncodingHeader. + if (contentLength <= 0) { + errorString = "The Content Length of " + downloadUrl + " is <= 0 and content Type is "+contentType.toString(); + LOGGER.error(errorString); + status = Status.UNRECOVERABLE_ERROR; + return 0; + } else { + // The ContentLengthHeader is supplied, parse it's value. + remoteSize = contentLength; + } + + if (remoteSize > maxVolumeSizeInBytes) { + errorString = "Remote size is too large for volume " + downloadUrl + " remote size is " + remoteSize + " max allowed is " + maxVolumeSizeInBytes; + LOGGER.error(errorString); + status = Status.UNRECOVERABLE_ERROR; + return 0; + } + + InputStream inputStream; + try { + inputStream = new BufferedInputStream(response.getEntity().getContent()); + } catch (IOException e) { + errorString = "Exception occurred while opening InputStream for volume from " + downloadUrl; + LOGGER.error(errorString); + status = Status.UNRECOVERABLE_ERROR; + return 0; + } + + String filePath = installPathPrefix + File.separator + installPath; + File directory = new File(filePath); + File srcFile = new File(filePath + File.separator + fileName); + try { + if (!directory.exists()) { + LOGGER.info("Creating directories "+filePath); + directory.mkdirs(); + } + if (!srcFile.createNewFile()) { + LOGGER.info("Reusing existing file " + srcFile.getPath()); + } + } catch (IOException e) { + errorString = "Exception occurred while creating temp file " + srcFile.getPath(); + LOGGER.error(errorString); + status = Status.UNRECOVERABLE_ERROR; + return 0; + } + + LOGGER.info("Starting download from " + downloadUrl + " to staging with size " + remoteSize + " bytes to " + srcFile.getPath()); + final Date downloadStart = new Date(); + + try (FileOutputStream fileOutputStream = new FileOutputStream(srcFile);) { + BufferedOutputStream outputStream = new BufferedOutputStream(fileOutputStream,DOWNLOAD_BUFFER_SIZE_BYTES); + byte[] data = new byte[DOWNLOAD_BUFFER_SIZE_BYTES]; + int bufferLength = 0; + while((bufferLength = inputStream.read(data,0,DOWNLOAD_BUFFER_SIZE_BYTES)) >= 0) { + totalBytes += bufferLength; + outputStream.write(data,0,bufferLength); + status = Status.IN_PROGRESS; + LOGGER.trace("Download in progress: " + getDownloadPercent() + "%"); + if (totalBytes >= remoteSize) { + volumeFile = srcFile; + status = Status.DOWNLOAD_FINISHED; + } + } + outputStream.close(); + inputStream.close(); + } catch (IOException e) { + LOGGER.error("Exception when downloading from url " + downloadUrl + " with remote size " + remoteSize + + " to staging nfs path " + srcFile.getPath() + " : " + e.getMessage(), e); + status = Status.RECOVERABLE_ERROR; + return 0; + } + + downloadTime = new Date().getTime() - downloadStart.getTime(); + + try (FileInputStream fs = new FileInputStream(srcFile)) { + md5sum = DigestUtils.md5Hex(fs); + } catch (IOException e) { + LOGGER.error("Failed to get md5sum: " + srcFile.getAbsoluteFile()); + } + + if (status == Status.DOWNLOAD_FINISHED) { + LOGGER.info("Template download from " + downloadUrl + " to staging nfs, transferred " + totalBytes + " in " + (downloadTime / 1000) + " seconds, completed successfully!"); + } else { + LOGGER.error("Template download from " + downloadUrl + " to staging nfs, transferred " + totalBytes + " in " + (downloadTime / 1000) + " seconds, completed with status " + status.toString()); + } + + // Close http connection + httpGet.releaseConnection(); + + // Call the callback! + if (callback != null) { + callback.downloadComplete(status); + } + + return totalBytes; + } + + public String getDownloadUrl() { + return httpGet.getURI().toString(); + } + + @Override + public Status getStatus() { + return status; + } + + @Override + public long getDownloadTime() { + return downloadTime; + } + + @Override + public long getDownloadedBytes() { + return totalBytes; + } + + @Override + public boolean stopDownload() { + switch (status) { + case IN_PROGRESS: + if (httpGet != null) { + httpGet.abort(); + } + break; + case UNKNOWN: + case NOT_STARTED: + case RECOVERABLE_ERROR: + case UNRECOVERABLE_ERROR: + case ABORTED: + case DOWNLOAD_FINISHED: + LOGGER.debug(String.format("Stopping download while status is %s", status)); + break; + default: + break; + } + + status = Status.ABORTED; + return true; + } + + @Override + public int getDownloadPercent() { + if (remoteSize == 0) { + return 0; + } + + return (int) (100.0 * totalBytes / remoteSize); + } + + @Override + protected void runInContext() { + LOGGER.info("Starting download in managed context resume = " + resume + " callback = " + downloadCompleteCallback.toString()); + download(resume, downloadCompleteCallback); + } + + @Override + public void setStatus(Status status) { + this.status = status; + } + + public boolean isResume() { + return resume; + } + + @Override + public String getDownloadError() { + return errorString; + } + + @Override + public String getDownloadLocalPath() { + return installPath; + } + + @Override + public void setResume(boolean resume) { + this.resume = resume; + } + + @Override + public void setDownloadError(String error) { + errorString = error; + } + + @Override + public boolean isInited() { + return true; + } + + public ResourceType getResourceType() { + return ResourceType.VOLUME; + } + + public String getFileExtension() { + return fileExtension; + } + + public String getMd5sum() { + return md5sum; + } + + public File getVolumeFile() { + return volumeFile; + } + + public long getMaxTemplateSizeInBytes() { + return maxVolumeSizeInBytes; + } +} diff --git a/core/src/main/java/com/cloud/storage/template/VhdProcessor.java b/core/src/main/java/com/cloud/storage/template/VhdProcessor.java index baea7bf0db51..1e7f927556e3 100644 --- a/core/src/main/java/com/cloud/storage/template/VhdProcessor.java +++ b/core/src/main/java/com/cloud/storage/template/VhdProcessor.java @@ -19,17 +19,6 @@ package com.cloud.storage.template; -import com.cloud.exception.InternalErrorException; -import com.cloud.storage.Storage.ImageFormat; -import com.cloud.storage.StorageLayer; -import com.cloud.utils.NumbersUtil; -import com.cloud.utils.component.AdapterBase; -import org.apache.commons.compress.compressors.CompressorException; -import org.apache.commons.compress.compressors.CompressorInputStream; -import org.apache.commons.compress.compressors.CompressorStreamFactory; -import org.apache.log4j.Logger; - -import javax.naming.ConfigurationException; import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; @@ -38,6 +27,19 @@ import java.io.InputStream; import java.util.Map; +import javax.naming.ConfigurationException; + +import org.apache.commons.compress.compressors.CompressorException; +import org.apache.commons.compress.compressors.CompressorInputStream; +import org.apache.commons.compress.compressors.CompressorStreamFactory; +import org.apache.log4j.Logger; + +import com.cloud.exception.InternalErrorException; +import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.StorageLayer; +import com.cloud.utils.NumbersUtil; +import com.cloud.utils.component.AdapterBase; + /** * VhdProcessor processes the downloaded template for VHD. It * currently does not handle any type of template conversion @@ -46,6 +48,8 @@ */ public class VhdProcessor extends AdapterBase implements Processor { + public static final String NAME = "VHD Processor"; + private static final Logger s_logger = Logger.getLogger(VhdProcessor.class); StorageLayer _storage; private int vhdFooterSize = 512; diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java index 1d463cad7eac..0902779ee1c3 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java @@ -24,8 +24,6 @@ import javax.inject.Inject; -import com.cloud.agent.api.to.DiskTO; -import com.cloud.storage.Storage; import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionStrategy; @@ -60,14 +58,16 @@ import com.cloud.agent.api.to.DataObjectType; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.DataTO; +import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.configuration.Config; import com.cloud.host.Host; import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.DataStoreRole; -import com.cloud.storage.StorageManager; +import com.cloud.storage.Storage; import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VolumeDao; @@ -158,7 +158,15 @@ private Scope pickCacheScopeForCopy(DataObject srcData, DataObject destData) { } protected Answer copyObject(DataObject srcData, DataObject destData, Host destHost) { - int primaryStorageDownloadWait = StorageManager.PRIMARY_STORAGE_DOWNLOAD_WAIT.value(); + long dataSize = 0; + try{ + dataSize = srcData.getSize(); + } catch(NullPointerException e) { + s_logger.error("Unable to determine size of src data object with uuid: " + srcData.getUuid(),e); + } + int imgSizeGigs = (int)Math.ceil(dataSize * 1.0d / (1024 * 1024 * 1024)); + imgSizeGigs++; // add one just in case + int primaryStorageDownloadWait = StorageManager.PRIMARY_STORAGE_DOWNLOAD_WAIT.value()*imgSizeGigs; Answer answer = null; DataObject cacheData = null; DataObject srcForCopy = srcData; diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java index 3ef9fbc4225e..634cbf4d3706 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java @@ -99,7 +99,7 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { @Inject protected VMTemplateDao _templateDao; @Inject - DownloadMonitor _downloadMonitor; + protected DownloadMonitor _downloadMonitor; @Inject VolumeDao volumeDao; @Inject @@ -175,6 +175,14 @@ protected Long getMaxTemplateSizeInBytes() { } } + protected Long getMaxVolumeSizeInBytes() { + try { + return Long.parseLong(configDao.getValue("storage.max.volume.upload.size")) * 1024L * 1024L * 1024L; + } catch (NumberFormatException e) { + return null; + } + } + @Override public void createAsync(DataStore dataStore, DataObject data, AsyncCompletionCallback callback) { CreateContext context = new CreateContext(callback, data); diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java index 53fa21f3a794..0ddd40dde9f8 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java @@ -23,6 +23,7 @@ import javax.inject.Inject; +import com.cloud.storage.Volume; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateDao; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; @@ -83,7 +84,7 @@ public VolumeInfo getVolume(long volumeId) { return null; } VolumeObject vol = null; - if (volumeVO.getPoolId() == null) { + if (volumeVO.getPoolId() == null || volumeVO.getState() == Volume.State.Uploaded) { DataStore store = null; VolumeDataStoreVO volumeStore = volumeStoreDao.findByVolume(volumeId); if (volumeStore != null) { diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java index 5ebee87acd4d..08f2e7804b38 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java @@ -751,6 +751,8 @@ protected void handleProcessEventAnswer(DownloadAnswer downloadAnswer) { volumeDataStoreVo.setInstallPath(downloadAnswer.getInstallPath()); volumeDataStoreVo.setChecksum(downloadAnswer.getCheckSum()); + volumeDataStoreVo.setSize(downloadAnswer.getTemplateSize()); + volumeDataStoreVo.setPhysicalSize(downloadAnswer.getTemplatePhySicalSize()); volumeStoreDao.update(volumeDataStoreVo.getId(), volumeDataStoreVo); String newValues = ReflectionToStringBuilderUtils.reflectOnlySelectedFields(volumeDataStoreVo, "installPath", "checksum"); diff --git a/plugins/storage/image/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java b/plugins/storage/image/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java index 8abf802d9de6..14f18cd84d60 100644 --- a/plugins/storage/image/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java +++ b/plugins/storage/image/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java @@ -73,6 +73,7 @@ public String createEntityExtractUrl(DataStore store, String installPath, ImageF String uuid = UUID.randomUUID().toString() + "." + format.getFileExtension(); CreateEntityDownloadURLCommand cmd = new CreateEntityDownloadURLCommand(((ImageStoreEntity)store).getMountPoint(), path, uuid, dataObject == null ? null: dataObject.getTO()); + cmd.setSecUrl(((ImageStoreEntity) store).getUrl()); Answer ans = null; if (ep == null) { String errMsg = "No remote endpoint to send command, check if host or ssvm is down?"; diff --git a/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java b/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java index 7e1486214bcf..a6b1422852cc 100644 --- a/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java +++ b/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java @@ -24,11 +24,6 @@ import javax.inject.Inject; -import com.cloud.configuration.Config; -import com.cloud.utils.SwiftUtil; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.log4j.Logger; - import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; @@ -36,20 +31,26 @@ import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.storage.command.DownloadCommand; import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao; import org.apache.cloudstack.storage.image.BaseImageStoreDriverImpl; import org.apache.cloudstack.storage.image.store.ImageStoreImpl; import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.log4j.Logger; import com.cloud.agent.api.storage.DownloadAnswer; import com.cloud.agent.api.to.DataObjectType; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.SwiftTO; +import com.cloud.configuration.Config; +import com.cloud.storage.RegisterVolumePayload; import com.cloud.storage.Storage.ImageFormat; -import com.cloud.template.VirtualMachineTemplate; +import com.cloud.utils.SwiftUtil; import com.cloud.utils.exception.CloudRuntimeException; public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl { @@ -100,12 +101,28 @@ public String createEntityExtractUrl(DataStore store, String installPath, ImageF @Override public void createAsync(DataStore dataStore, DataObject data, AsyncCompletionCallback callback) { - Long maxTemplateSizeInBytes = getMaxTemplateSizeInBytes(); - VirtualMachineTemplate tmpl = _templateDao.findById(data.getId()); + + DownloadCommand downloadCommand = null; + if (data.getType() == DataObjectType.TEMPLATE) { + Long maxTemplateSizeInBytes = getMaxTemplateSizeInBytes(); + downloadCommand = new DownloadCommand((TemplateObjectTO) (data.getTO()), maxTemplateSizeInBytes); + } else if (data.getType() == DataObjectType.VOLUME) { + Long maxDownloadSizeInBytes = getMaxVolumeSizeInBytes(); + VolumeInfo volumeInfo = (VolumeInfo) data; + RegisterVolumePayload payload = (RegisterVolumePayload) volumeInfo.getpayload(); + ImageFormat format = ImageFormat.valueOf(payload.getFormat()); + downloadCommand = new DownloadCommand((VolumeObjectTO) (data.getTO()), maxDownloadSizeInBytes, payload.getChecksum(), payload.getUrl(), format); + } + + if (downloadCommand == null) { + String errMsg = "Unable to build download command, unsupported DataObject is of type " + data.getType().toString(); + s_logger.error(errMsg); + throw new CloudRuntimeException(errMsg); + } + DataStore cacheStore = cacheManager.getCacheStorage(dataStore.getScope()); - DownloadCommand dcmd = new DownloadCommand((TemplateObjectTO)(data.getTO()), maxTemplateSizeInBytes); - dcmd.setCacheStore(cacheStore.getTO()); - dcmd.setProxy(getHttpProxy()); + downloadCommand.setCacheStore(cacheStore.getTO()); + downloadCommand.setProxy(getHttpProxy()); EndPoint ep = _epSelector.select(data); if (ep == null) { @@ -120,11 +137,11 @@ public void createAsync(DataStore dataStore, DataObject data, AsyncCompletionCal if (data.getType() == DataObjectType.TEMPLATE) { caller.setCallback(caller.getTarget().createTemplateAsyncCallback(null, null)); + ep.sendMessageAsync(downloadCommand, caller); } else if (data.getType() == DataObjectType.VOLUME) { caller.setCallback(caller.getTarget().createVolumeAsyncCallback(null, null)); + _downloadMonitor.downloadVolumeToStorage(data,caller); } - ep.sendMessageAsync(dcmd, caller); - } } diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 3fba19947235..eae174a880ff 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -2227,6 +2227,18 @@ private Volume orchestrateAttachVolumeToVM(Long vmId, Long volumeId, Long device StoragePoolVO destPrimaryStorage = null; if (existingVolumeOfVm != null && !existingVolumeOfVm.getState().equals(Volume.State.Allocated)) { destPrimaryStorage = _storagePoolDao.findById(existingVolumeOfVm.getPoolId()); + if(destPrimaryStorage.getPodId() == null) { + destPrimaryStorage.setPodId(vm.getPodIdToDeployIn()); + } + if(destPrimaryStorage.getClusterId() == null) { + HostVO hostVO = null; + if (vm.getHostId() != null) { + hostVO = _hostDao.findById(vm.getHostId()); + } else { + hostVO = _hostDao.findById(vm.getLastHostId()); + } + destPrimaryStorage.setClusterId(hostVO.getClusterId()); + } if (s_logger.isTraceEnabled() && destPrimaryStorage != null) { s_logger.trace(String.format("decided on target storage: %s/%s", destPrimaryStorage.getName(), destPrimaryStorage.getUuid())); } @@ -3975,6 +3987,7 @@ private String orchestrateExtractVolume(long volumeId, long zoneId) { volumeStoreRef.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); volumeStoreRef.setDownloadPercent(100); volumeStoreRef.setZoneId(zoneId); + volumeStoreRef.setSize(vol.getSize()); _volumeStoreDao.update(volumeStoreRef.getId(), volumeStoreRef); diff --git a/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java b/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java index 1954cdea6879..dd5113111cd5 100644 --- a/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java +++ b/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java @@ -25,6 +25,7 @@ import javax.inject.Inject; +import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -73,6 +74,8 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor private ConfigurationDao _configDao; @Inject private EndPointSelector _epSelector; + @Inject + private StorageCacheManager cacheManager; private String _copyAuthPasswd; private String _proxy = null; @@ -218,7 +221,8 @@ public void downloadVolumeToStorage(DataObject volume, AsyncCompletionCallback true) + .build(); + client = HttpClients.custom().setSSLContext(sslContext) + .setSSLHostnameVerifier(new NoopHostnameVerifier()) + .build(); + } catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException e) { + String msg = String.format("Failed to initialize download url: %s, due to %s", url, e.getMessage()); + s_logger.error(msg, e); + throw new CloudRuntimeException(msg, e); + } + + try { + HttpGet get = new HttpGet(url); HttpResponse response = client.execute(get); HttpEntity entity = response.getEntity(); if (entity == null) { - s_logger.debug("Faled to get entity"); + s_logger.error("Failed to get entity "+ response.getStatusLine().getStatusCode() + " " + + response.getStatusLine().getReasonPhrase()); throw new CloudRuntimeException("Failed to get url: " + url); } @@ -1198,11 +1229,12 @@ protected File downloadFromUrlToNfs(String url, NfsTO nfs, String path, String n try (FileOutputStream outputStream = new FileOutputStream(destFile);) { entity.writeTo(outputStream); } catch (IOException e) { - s_logger.debug("downloadFromUrlToNfs:Exception:" + e.getMessage(), e); + s_logger.error(String.format("Unable to download from url [%s] to nfs path [%s] due to [%s]", + url,destFile.getPath(),e.getMessage()), e); } return new File(destFile.getAbsolutePath()); } catch (IOException e) { - s_logger.debug("Failed to get url: " + url + ", due to " + e.toString()); + s_logger.error("Failed to get url:" + url + ", due to " + e.toString(), e); throw new CloudRuntimeException(e); } } @@ -1241,13 +1273,13 @@ protected Answer registerTemplateOnSwift(DownloadCommand cmd) { try (FileInputStream fs = new FileInputStream(file)) { md5sum = DigestUtils.md5Hex(fs); } catch (IOException e) { - s_logger.debug("Failed to get md5sum: " + file.getAbsoluteFile()); + s_logger.error("Failed to get md5sum: " + file.getAbsoluteFile(), e); } DownloadAnswer answer = new DownloadAnswer(null, 100, null, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, swiftPath, swiftPath, virtualSize, file.length(), md5sum); return answer; } catch (IOException e) { - s_logger.debug("Failed to register template into swift", e); + s_logger.error("Failed to register template into swift", e); return new DownloadAnswer(e.toString(), VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); } finally { if (file != null) { @@ -1260,8 +1292,10 @@ private Answer execute(DownloadCommand cmd) { DataStoreTO dstore = cmd.getDataStore(); if (dstore instanceof NfsTO || dstore instanceof S3TO) { return _dlMgr.handleDownloadCommand(this, cmd); - } else if (dstore instanceof SwiftTO) { + } else if (dstore instanceof SwiftTO && cmd.getResourceType() == DownloadCommand.ResourceType.TEMPLATE) { return registerTemplateOnSwift(cmd); + } else if (dstore instanceof SwiftTO && cmd.getResourceType() == DownloadCommand.ResourceType.VOLUME){ + return _dlMgr.handleDownloadCommand(this, cmd); } else { return new Answer(cmd, false, "Unsupported image data store: " + dstore); } @@ -1329,7 +1363,7 @@ protected long getVirtualSize(File file, ImageFormat format) { processor.configure("template processor", params); return processor.getVirtualSize(file); } catch (Exception e) { - s_logger.warn("Failed to get virtual size of file " + file.getPath() + ", returning file size instead: ", e); + s_logger.error("Failed to get virtual size of file " + file.getPath() + ", returning file size instead: ", e); return file.length(); } @@ -1453,9 +1487,7 @@ protected Answer copyFromNfsToS3(CopyCommand cmd) { try { final String templatePath = determineStorageTemplatePath(srcStore.getUrl(), srcData.getPath(), _nfsVersion); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Found " + srcData.getObjectType() + " from directory " + templatePath + " to upload to S3."); - } + s_logger.info("Found " + srcData.getObjectType() + " from directory " + templatePath + " to upload to S3."); final String bucket = s3.getBucketName(); File srcFile = findFile(templatePath); @@ -1615,7 +1647,7 @@ String swiftDownload(SwiftTO swift, String container, String rfilename, String l String result = command.execute(parser); if (result != null) { String errMsg = "swiftDownload failed err=" + result; - s_logger.warn(errMsg); + s_logger.error(errMsg); return errMsg; } if (parser.getLines() != null) { @@ -1623,7 +1655,7 @@ String swiftDownload(SwiftTO swift, String container, String rfilename, String l for (String line : lines) { if (line.contains("Errno") || line.contains("failed")) { String errMsg = "swiftDownload failed , err=" + parser.getLines(); - s_logger.warn(errMsg); + s_logger.error(errMsg); return errMsg; } } @@ -1641,7 +1673,7 @@ String swiftDownloadContainer(SwiftTO swift, String container, String ldir) { String result = command.execute(parser); if (result != null) { String errMsg = "swiftDownloadContainer failed err=" + result; - s_logger.warn(errMsg); + s_logger.error(errMsg); return errMsg; } if (parser.getLines() != null) { @@ -1649,7 +1681,7 @@ String swiftDownloadContainer(SwiftTO swift, String container, String ldir) { for (String line : lines) { if (line.contains("Errno") || line.contains("failed")) { String errMsg = "swiftDownloadContainer failed , err=" + parser.getLines(); - s_logger.warn(errMsg); + s_logger.error(errMsg); return errMsg; } } @@ -1692,7 +1724,7 @@ String swiftUpload(SwiftTO swift, String container, String lDir, String lFilenam String result = command.execute(parser); if (result != null) { String errMsg = "swiftUpload failed , err=" + result; - s_logger.warn(errMsg); + s_logger.error(errMsg); return errMsg; } if (parser.getLines() != null) { @@ -1700,7 +1732,7 @@ String swiftUpload(SwiftTO swift, String container, String lDir, String lFilenam for (String line : lines) { if (line.contains("Errno") || line.contains("failed")) { String errMsg = "swiftUpload failed , err=" + parser.getLines(); - s_logger.warn(errMsg); + s_logger.error(errMsg); return errMsg; } } @@ -1723,10 +1755,10 @@ String[] swiftList(SwiftTO swift, String container, String rFilename) { } else { if (result != null) { String errMsg = "swiftList failed , err=" + result; - s_logger.warn(errMsg); + s_logger.error(errMsg); } else { String errMsg = "swiftList failed, no lines returns"; - s_logger.warn(errMsg); + s_logger.error(errMsg); } } return null; @@ -1741,7 +1773,7 @@ String swiftDelete(SwiftTO swift, String container, String object) { String result = command.execute(parser); if (result != null) { String errMsg = "swiftDelete failed , err=" + result; - s_logger.warn(errMsg); + s_logger.error(errMsg); return errMsg; } if (parser.getLines() != null) { @@ -1749,7 +1781,7 @@ String swiftDelete(SwiftTO swift, String container, String object) { for (String line : lines) { if (line.contains("Errno") || line.contains("failed")) { String errMsg = "swiftDelete failed , err=" + parser.getLines(); - s_logger.warn(errMsg); + s_logger.error(errMsg); return errMsg; } } @@ -1776,7 +1808,7 @@ public Answer execute(DeleteSnapshotsDirCommand cmd) { String details = null; if (!snapshotDir.exists()) { details = "snapshot directory " + snapshotDir.getName() + " doesn't exist"; - s_logger.debug(details); + s_logger.info(details); return new Answer(cmd, true, details); } // delete all files in the directory @@ -1784,13 +1816,13 @@ public Answer execute(DeleteSnapshotsDirCommand cmd) { String result = deleteLocalFile(lPath); if (result != null) { String errMsg = "failed to delete all snapshots " + lPath + " , err=" + result; - s_logger.warn(errMsg); + s_logger.error(errMsg); return new Answer(cmd, false, errMsg); } // delete the directory if (!snapshotDir.delete()) { details = "Unable to delete directory " + snapshotDir.getName() + " under snapshot path " + relativeSnapshotPath; - s_logger.debug(details); + s_logger.error(details); return new Answer(cmd, false, details); } return new Answer(cmd, true, null); @@ -1822,7 +1854,7 @@ public Answer execute(DeleteSnapshotsDirCommand cmd) { String result = swiftDelete((SwiftTO)dstore, "V-" + volumeId.toString(), ""); if (result != null) { String errMsg = "failed to delete snapshot for volume " + volumeId + " , err=" + result; - s_logger.warn(errMsg); + s_logger.error(errMsg); return new Answer(cmd, false, errMsg); } return new Answer(cmd, true, "Deleted snapshot " + path + " from swift"); @@ -2004,7 +2036,7 @@ public void run() { ch.closeFuture().sync(); } catch (InterruptedException e) { s_logger.info("Failed to start post upload server"); - s_logger.debug("Exception while starting post upload server", e); + s_logger.error("Exception while starting post upload server", e); } finally { bossGroup.shutdownGracefully(); workerGroup.shutdownGracefully(); diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManager.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManager.java index a04177040284..8f0fe1e19399 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManager.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManager.java @@ -18,19 +18,19 @@ import java.util.Map; -import com.cloud.storage.template.Processor; import org.apache.cloudstack.storage.command.DownloadCommand; import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; import org.apache.cloudstack.storage.resource.SecondaryStorageResource; import com.cloud.agent.api.storage.DownloadAnswer; -import com.cloud.utils.net.Proxy; import com.cloud.agent.api.to.S3TO; import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; +import com.cloud.storage.template.Processor; import com.cloud.storage.template.TemplateDownloader; import com.cloud.storage.template.TemplateProp; -import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; import com.cloud.utils.component.Manager; +import com.cloud.utils.net.Proxy; public interface DownloadManager extends Manager { @@ -41,8 +41,8 @@ public interface DownloadManager extends Manager { * @param hvm whether the template is a hardware virtual machine * @param accountId the accountId of the iso owner (null if public iso) * @param descr description of the template - * @param user username used for authentication to the server - * @param password password used for authentication to the server + * @param userName username used for authentication to the server + * @param passwd password used for authentication to the server * @param maxDownloadSizeInBytes (optional) max download size for the template, in bytes. * @param resourceType signifying the type of resource like template, volume etc. * @return job-id that can be used to interrogate the status of the download. @@ -53,6 +53,8 @@ public String downloadPublicTemplate(long id, String url, String name, ImageForm public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType); + public String downloadSwiftVolume(DownloadCommand cmd, String installPathPrefix, long maxDownloadSizeInBytes); + Map getProcessors(); /** diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index f647b497f586..e0ec00be797d 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@ -16,6 +16,8 @@ // under the License. package org.apache.cloudstack.storage.template; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -37,55 +39,55 @@ import javax.naming.ConfigurationException; -import com.cloud.agent.api.to.OVFInformationTO; -import com.cloud.storage.template.Processor; -import com.cloud.storage.template.S3TemplateDownloader; -import com.cloud.storage.template.TemplateDownloader; -import com.cloud.storage.template.TemplateLocation; -import com.cloud.storage.template.MetalinkTemplateDownloader; -import com.cloud.storage.template.HttpTemplateDownloader; -import com.cloud.storage.template.LocalTemplateDownloader; -import com.cloud.storage.template.ScpTemplateDownloader; -import com.cloud.storage.template.TemplateProp; -import com.cloud.storage.template.OVAProcessor; -import com.cloud.storage.template.IsoProcessor; -import com.cloud.storage.template.QCOW2Processor; -import com.cloud.storage.template.VmdkProcessor; -import com.cloud.storage.template.RawImageProcessor; -import com.cloud.storage.template.TARProcessor; -import com.cloud.storage.template.VhdProcessor; -import com.cloud.storage.template.TemplateConstants; +import org.apache.cloudstack.storage.NfsMountManagerImpl.PathParser; import org.apache.cloudstack.storage.command.DownloadCommand; import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; import org.apache.cloudstack.storage.command.DownloadProgressCommand; import org.apache.cloudstack.storage.command.DownloadProgressCommand.RequestType; -import org.apache.cloudstack.storage.NfsMountManagerImpl.PathParser; import org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource; import org.apache.cloudstack.storage.resource.SecondaryStorageResource; +import org.apache.cloudstack.utils.security.ChecksumValue; +import org.apache.cloudstack.utils.security.DigestHelper; +import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import com.cloud.agent.api.storage.DownloadAnswer; -import com.cloud.utils.net.Proxy; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.NfsTO; +import com.cloud.agent.api.to.OVFInformationTO; import com.cloud.agent.api.to.S3TO; +import com.cloud.agent.api.to.SwiftTO; import com.cloud.exception.InternalErrorException; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.StorageLayer; import com.cloud.storage.VMTemplateStorageResourceAssoc; +import com.cloud.storage.template.HttpTemplateDownloader; +import com.cloud.storage.template.IsoProcessor; +import com.cloud.storage.template.LocalTemplateDownloader; +import com.cloud.storage.template.MetalinkTemplateDownloader; +import com.cloud.storage.template.OVAProcessor; +import com.cloud.storage.template.Processor; import com.cloud.storage.template.Processor.FormatInfo; +import com.cloud.storage.template.QCOW2Processor; +import com.cloud.storage.template.RawImageProcessor; +import com.cloud.storage.template.S3TemplateDownloader; +import com.cloud.storage.template.ScpTemplateDownloader; +import com.cloud.storage.template.SwiftVolumeDownloader; +import com.cloud.storage.template.TARProcessor; +import com.cloud.storage.template.TemplateConstants; +import com.cloud.storage.template.TemplateDownloader; import com.cloud.storage.template.TemplateDownloader.DownloadCompleteCallback; import com.cloud.storage.template.TemplateDownloader.Status; +import com.cloud.storage.template.TemplateLocation; +import com.cloud.storage.template.TemplateProp; +import com.cloud.storage.template.VhdProcessor; +import com.cloud.storage.template.VmdkProcessor; import com.cloud.utils.NumbersUtil; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.Proxy; import com.cloud.utils.script.Script; import com.cloud.utils.storage.QCOW2Utils; -import org.apache.cloudstack.utils.security.ChecksumValue; -import org.apache.cloudstack.utils.security.DigestHelper; -import org.apache.commons.lang3.StringUtils; - -import static com.cloud.utils.NumbersUtil.toHumanReadableSize; public class DownloadManagerImpl extends ManagerBase implements DownloadManager { private String _name; @@ -142,6 +144,19 @@ public DownloadJob(TemplateDownloader td, String jobId, long id, String tmpltNam this.resourceType = resourceType; } + public DownloadJob(TemplateDownloader td, String jobId, long id, String tmpltName, ImageFormat format, String installPathPrefix) { + super(); + this.td = td; + this.tmpltName = tmpltName; + this.format = format; + this.hvm = false; + this.description = null; + this.installPathPrefix = installPathPrefix; + this.templatesize = 0; + this.id = id; + this.resourceType = null; + } + public String getDescription() { return description; } @@ -243,6 +258,26 @@ public void setOvfInformationTO(OVFInformationTO ovfInformationTO) { private String listVolScr; private int installTimeoutPerGig = 180 * 60 * 1000; + private void setSwiftVolumeDownloadStatus(DownloadJob dj, TemplateDownloader td) { + dj.setCheckSum(((SwiftVolumeDownloader) td).getMd5sum()); + if ("vhd".equalsIgnoreCase(((SwiftVolumeDownloader) td).getFileExtension())) { + Processor vhdProcessor = _processors.get("VHD Processor"); + long virtualSize = 0; + try { + virtualSize = vhdProcessor.getVirtualSize(((SwiftVolumeDownloader) td).getVolumeFile()); + dj.setTemplatesize(virtualSize); + } catch (IOException e) { + LOGGER.error("Unable to read VHD file", e); + } + } else { + dj.setTemplatesize(td.getDownloadedBytes()); + } + dj.setTemplatePhysicalSize(td.getDownloadedBytes()); + dj.setTmpltPath(td.getDownloadLocalPath()); + td.setStatus(Status.POST_DOWNLOAD_FINISHED); + td.setDownloadError("Volume downloaded to swift cache successfully at " + new SimpleDateFormat().format(new Date())); + } + public void setThreadPool(ExecutorService threadPool) { this.threadPool = threadPool; } @@ -263,7 +298,9 @@ public void setStorageLayer(StorageLayer storage) { public void setDownloadStatus(String jobId, Status status) { DownloadJob dj = jobs.get(jobId); if (dj == null) { - LOGGER.warn("setDownloadStatus for jobId: " + jobId + ", status=" + status + " no job found"); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Updating DownloadStatus: %s for jobId: %s failed as no job found", status, jobId)); + } return; } TemplateDownloader td = dj.getTemplateDownloader(); @@ -289,7 +326,7 @@ public void setDownloadStatus(String jobId, Status status) { threadPool.execute(td); break; case DOWNLOAD_FINISHED: - if(td instanceof S3TemplateDownloader) { + if (td instanceof S3TemplateDownloader) { // For S3 and Swift, which are considered "remote", // as in the file cannot be accessed locally, // we run the postRemoteDownload() method. @@ -304,8 +341,9 @@ public void setDownloadStatus(String jobId, Status status) { td.setStatus(Status.POST_DOWNLOAD_FINISHED); td.setDownloadError("Install completed successfully at " + new SimpleDateFormat().format(new Date())); } - } - else { + } else if (td instanceof SwiftVolumeDownloader) { + setSwiftVolumeDownloadStatus(dj, td); + } else { // For other TemplateDownloaders where files are locally available, // we run the postLocalDownload() method. td.setDownloadError("Download success, starting install "); @@ -384,7 +422,7 @@ private String postLocalDownload(String jobId) { ResourceType resourceType = dnld.getResourceType(); File originalTemplate = new File(td.getDownloadLocalPath()); - if(StringUtils.isBlank(dnld.getChecksum())) { + if (StringUtils.isBlank(dnld.getChecksum())) { if (LOGGER.isInfoEnabled()) { LOGGER.info(String.format("No checksum available for '%s'", originalTemplate.getName())); } @@ -496,7 +534,7 @@ private String checkOrCreateTheChecksum(DownloadJob dnld, File targetFile) { } String checksum = newValue.toString(); if (checksum == null) { - LOGGER.warn("Something wrong happened when try to calculate the checksum of downloaded template!"); + LOGGER.error("Something wrong happened when trying to calculate the checksum of downloaded template!"); } dnld.setCheckSum(checksum); return null; @@ -529,7 +567,7 @@ private String postProcessAfterDownloadComplete(DownloadJob dnld, String resourc } if (!loc.save()) { - LOGGER.warn("Cleaning up because we're unable to save the formats"); + LOGGER.info("Cleaning up because we're unable to save the formats"); loc.purge(); } @@ -561,7 +599,7 @@ public String downloadS3Template(S3TO s3, long id, String url, String name, Imag throw new CloudRuntimeException("URI is incorrect: " + url); } TemplateDownloader td; - if ((uri != null) && (uri.getScheme() != null)) { + if (uri.getScheme() != null) { if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) { td = new S3TemplateDownloader(s3, url, installPathPrefix, new Completion(jobId), maxTemplateSizeInBytes, user, password, proxy, resourceType); } else { @@ -578,6 +616,34 @@ public String downloadS3Template(S3TO s3, long id, String url, String name, Imag return jobId; } + @Override + public String downloadSwiftVolume(DownloadCommand cmd, String installPathPrefix, long maxDownloadSizeInBytes) { + UUID uuid = UUID.randomUUID(); + String jobId = uuid.toString(); + URI uri = null; + try { + uri = new URI(cmd.getUrl()); + } catch (URISyntaxException e) { + throw new CloudRuntimeException("URI is incorrect: " + cmd.getUrl()); + } + TemplateDownloader td; + if ((uri != null) && (uri.getScheme() != null)) { + if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) { + td = new SwiftVolumeDownloader(cmd, new Completion(jobId), maxDownloadSizeInBytes, installPathPrefix); + } else { + throw new CloudRuntimeException("Scheme is not supported " + cmd.getUrl()); + } + } else { + throw new CloudRuntimeException("Unable to download from URL: " + cmd.getUrl()); + } + DownloadJob dj = new DownloadJob(td, jobId, cmd.getId(), cmd.getName(), cmd.getFormat(), cmd.getInstallPath()); + dj.setTmpltPath(installPathPrefix); + jobs.put(jobId, dj); + threadPool.execute(td); + + return jobId; + } + @Override public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, String templatePath, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) { @@ -588,7 +654,7 @@ public String downloadPublicTemplate(long id, String url, String name, ImageForm try { if (!_storage.mkdirs(tmpDir)) { - LOGGER.warn("Unable to create " + tmpDir); + LOGGER.error("Unable to create " + tmpDir); return "Unable to create " + tmpDir; } // TO DO - define constant for volume properties. @@ -596,13 +662,13 @@ public String downloadPublicTemplate(long id, String url, String name, ImageForm ResourceType.TEMPLATE == resourceType ? _storage.getFile(tmpDir + File.separator + TemplateLocation.Filename) : _storage.getFile(tmpDir + File.separator + "volume.properties"); if (file.exists()) { - if(! file.delete()) { - LOGGER.warn("Deletion of file '" + file.getAbsolutePath() + "' failed."); + if (! file.delete()) { + LOGGER.error("Deletion of file '" + file.getAbsolutePath() + "' failed."); } } if (!file.createNewFile()) { - LOGGER.warn("Unable to create new file: " + file.getAbsolutePath()); + LOGGER.error("Unable to create new file: " + file.getAbsolutePath()); return "Unable to create new file: " + file.getAbsolutePath(); } @@ -644,7 +710,7 @@ public String downloadPublicTemplate(long id, String url, String name, ImageForm return jobId; } catch (IOException e) { - LOGGER.warn("Unable to download to " + tmpDir, e); + LOGGER.error("Unable to download to " + tmpDir, e); return null; } } @@ -734,7 +800,7 @@ public DownloadAnswer handleDownloadCommand(SecondaryStorageResource resource, D this._processTimeout = timeout; ResourceType resourceType = cmd.getResourceType(); if (cmd instanceof DownloadProgressCommand) { - return handleDownloadProgressCmd(resource, (DownloadProgressCommand)cmd); + return handleDownloadProgressCmd(resource, (DownloadProgressCommand) cmd); } if (cmd.getUrl() == null) { @@ -746,7 +812,7 @@ public DownloadAnswer handleDownloadCommand(SecondaryStorageResource resource, D return new DownloadAnswer("Invalid Name", VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); } - if(! DigestHelper.isAlgorithmSupported(cmd.getChecksum())) { + if (cmd.getChecksum() != null && !DigestHelper.isAlgorithmSupported(cmd.getChecksum())) { return new DownloadAnswer("invalid algorithm: " + cmd.getChecksum(), VMTemplateStorageResourceAssoc.Status.NOT_DOWNLOADED); } @@ -754,7 +820,9 @@ public DownloadAnswer handleDownloadCommand(SecondaryStorageResource resource, D String installPathPrefix = cmd.getInstallPath(); // for NFS, we need to get mounted path if (dstore instanceof NfsTO) { - installPathPrefix = resource.getRootDir(((NfsTO)dstore).getUrl(), _nfsVersion) + File.separator + installPathPrefix; + installPathPrefix = resource.getRootDir(((NfsTO) dstore).getUrl(), _nfsVersion) + File.separator + installPathPrefix; + } else if (dstore instanceof SwiftTO) { + installPathPrefix = resource.getRootDir(cmd.getCacheStore().getUrl(),_nfsVersion); } String user = null; String password = null; @@ -768,8 +836,10 @@ public DownloadAnswer handleDownloadCommand(SecondaryStorageResource resource, D String jobId = null; if (dstore instanceof S3TO) { jobId = - downloadS3Template((S3TO)dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), + downloadS3Template((S3TO) dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType); + } else if (dstore instanceof SwiftTO) { + jobId = downloadSwiftVolume(cmd, installPathPrefix, maxDownloadSizeInBytes); } else { jobId = downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), @@ -897,7 +967,7 @@ public Map gatherTemplateInfo(String rootDir) { continue; } } catch (IOException e) { - LOGGER.warn("Unable to load template location " + path, e); + LOGGER.error("Unable to load template location " + path, e); continue; } @@ -917,7 +987,7 @@ public Map gatherTemplateInfo(String rootDir) { } result.put(tInfo.getTemplateName(), tInfo); - LOGGER.debug("Added template name: " + tInfo.getTemplateName() + ", path: " + tmplt); + LOGGER.info("Added template name: " + tInfo.getTemplateName() + ", path: " + tmplt); } return result; } @@ -943,7 +1013,7 @@ public Map gatherVolumeInfo(String rootDir) { continue; } } catch (IOException e) { - LOGGER.warn("Unable to load volume location " + path, e); + LOGGER.error("Unable to load volume location " + path, e); continue; } @@ -963,7 +1033,7 @@ public Map gatherVolumeInfo(String rootDir) { } result.put(vInfo.getId(), vInfo); - LOGGER.debug("Added volume name: " + vInfo.getTemplateName() + ", path: " + vol); + LOGGER.info("Added volume name: " + vInfo.getTemplateName() + ", path: " + vol); } return result; } @@ -989,11 +1059,7 @@ public boolean configure(String name, Map params) throws Configu try { clazz = (Class)Class.forName(value); _storage = clazz.newInstance(); - } catch (ClassNotFoundException e) { - throw new ConfigurationException("Unable to instantiate " + value); - } catch (InstantiationException e) { - throw new ConfigurationException("Unable to instantiate " + value); - } catch (IllegalAccessException e) { + } catch (ClassNotFoundException | InstantiationException | IllegalAccessException e) { throw new ConfigurationException("Unable to instantiate " + value); } } @@ -1091,7 +1157,7 @@ private void blockOutgoingOnPrivate() { String result = command.execute(); if (result != null) { - LOGGER.warn("Error in blocking outgoing to port 80/443 err=" + result); + LOGGER.error("Error in blocking outgoing to port 80/443 err=" + result); return; } } @@ -1117,7 +1183,7 @@ private void startAdditionalServices() { command.add("apache2"); String result = command.execute(); if (result != null) { - LOGGER.warn("Error in stopping httpd service err=" + result); + LOGGER.error("Error in stopping httpd service err=" + result); } String port = Integer.toString(TemplateConstants.DEFAULT_TMPLT_COPY_PORT); String intf = TemplateConstants.DEFAULT_TMPLT_COPY_INTF; @@ -1129,7 +1195,7 @@ private void startAdditionalServices() { result = command.execute(); if (result != null) { - LOGGER.warn("Error in opening up apache2 port err=" + result); + LOGGER.error("Error in opening up apache2 port err=" + result); return; } @@ -1138,7 +1204,7 @@ private void startAdditionalServices() { command.add("apache2"); result = command.execute(); if (result != null) { - LOGGER.warn("Error in starting apache2 service err=" + result); + LOGGER.error("Error in starting apache2 service err=" + result); return; } @@ -1150,7 +1216,7 @@ private void startAdditionalServices() { command.add("www-data"); result = command.execute(); if (result != null) { - LOGGER.warn("Error in creating directory =" + result); + LOGGER.error("Error in creating directory =" + result); return; } }