Skip to content

Commit 969660a

Browse files
shwstpprshwstppr
committed
change for downloading templates if allowed
Signed-off-by: Abhishek Kumar <[email protected]>
1 parent bed9666 commit 969660a

File tree

3 files changed

+90
-59
lines changed

3 files changed

+90
-59
lines changed

Diff for: ‎engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java

+31-52
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
import java.util.List;
3434
import java.util.Locale;
3535
import java.util.Map;
36-
import java.util.Set;
36+
import java.util.Objects;
3737
import java.util.UUID;
3838
import java.util.stream.Collectors;
3939

@@ -84,54 +84,14 @@
8484
import com.cloud.utils.db.TransactionCallbackNoReturn;
8585
import com.cloud.utils.db.TransactionStatus;
8686
import com.cloud.utils.exception.CloudRuntimeException;
87+
import com.cloud.utils.net.NetUtils;
8788
import com.cloud.utils.script.Script;
8889
import com.cloud.vm.dao.VMInstanceDao;
8990
import com.cloud.vm.dao.VMInstanceDaoImpl;
90-
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
91-
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
92-
import org.apache.cloudstack.framework.config.dao.ConfigurationDaoImpl;
93-
import org.apache.cloudstack.framework.config.impl.ConfigurationVO;
94-
import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
95-
import org.apache.cloudstack.storage.datastore.db.ImageStoreDaoImpl;
96-
import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao;
97-
import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDaoImpl;
98-
import org.apache.cloudstack.storage.datastore.db.ImageStoreVO;
99-
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
100-
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
101-
import org.apache.cloudstack.utils.security.DigestHelper;
102-
import org.apache.commons.lang3.StringUtils;
103-
import org.apache.logging.log4j.Logger;
104-
import org.apache.logging.log4j.LogManager;
105-
import org.ini4j.Ini;
106-
107-
import javax.inject.Inject;
108-
import java.io.BufferedReader;
109-
import java.io.File;
110-
import java.io.FileReader;
111-
import java.io.IOException;
112-
import java.net.URI;
113-
import java.nio.file.Files;
114-
import java.nio.file.Path;
115-
import java.nio.file.Paths;
116-
import java.sql.Connection;
117-
import java.sql.Date;
118-
import java.util.ArrayList;
119-
import java.util.Arrays;
120-
import java.util.Collection;
121-
import java.util.Collections;
122-
import java.util.HashMap;
123-
import java.util.HashSet;
124-
import java.util.List;
125-
import java.util.Locale;
126-
import java.util.Map;
127-
import java.util.Objects;
128-
import java.util.Set;
129-
import java.util.UUID;
130-
import java.util.stream.Collectors;
13191

13292
public class SystemVmTemplateRegistration {
13393
protected static Logger LOGGER = LogManager.getLogger(SystemVmTemplateRegistration.class);
134-
private static final String MOUNT_COMMAND = "sudo mount -t nfs %s %s";
94+
private static final String MOUNT_COMMAND_BASE = "sudo mount -t nfs";
13595
private static final String UMOUNT_COMMAND = "sudo umount %s";
13696
private static final String RELATIVE_TEMPLATE_PATH = "./engine/schema/dist/systemvm-templates/";
13797
private static final String ABSOLUTE_TEMPLATE_PATH = "/usr/share/cloudstack-management/templates/systemvm/";
@@ -146,6 +106,9 @@ public class SystemVmTemplateRegistration {
146106
private static final Integer LINUX_7_ID = 183;
147107
private static final Integer SCRIPT_TIMEOUT = 1800000;
148108
private static final Integer LOCK_WAIT_TIMEOUT = 1200;
109+
private static final List<String> DOWNLOADABLE_TEMPLATE_ARCH_TYPES = Arrays.asList(
110+
CPU.archARM64Identifier
111+
);
149112

150113

151114
public static String CS_MAJOR_VERSION = null;
@@ -193,7 +156,7 @@ public SystemVmTemplateRegistration(String systemVmTemplateVersion) {
193156
}
194157

195158
public static String getMountCommand(String nfsVersion, String device, String dir) {
196-
String cmd = "sudo mount -t nfs";
159+
String cmd = MOUNT_COMMAND_BASE;
197160
if (StringUtils.isNotBlank(nfsVersion)) {
198161
cmd = String.format("%s -o vers=%s", cmd, nfsVersion);
199162
}
@@ -789,7 +752,7 @@ public static String parseMetadataFile() {
789752
Ini.Section section = ini.get(key);
790753
NewTemplateMap.put(key, new MetadataTemplateDetails(hypervisorType.first(), section.get("templatename"),
791754
section.get("filename"), section.get("downloadurl"), section.get("checksum"),
792-
section.get("arch")));
755+
hypervisorType.second()));
793756
}
794757
Ini.Section section = ini.get("default");
795758
return section.get("version");
@@ -809,6 +772,21 @@ private static void cleanupStore(Long templateId, String filePath) {
809772
}
810773
}
811774

775+
protected File getTemplateFile(MetadataTemplateDetails templateDetails) {
776+
final String filePath = TEMPLATES_PATH + templateDetails.getFilename();
777+
File tempFile = new File(filePath);
778+
if (!tempFile.exists() && DOWNLOADABLE_TEMPLATE_ARCH_TYPES.contains(templateDetails.getArch()) &&
779+
StringUtils.isNotBlank(templateDetails.getUrl())) {
780+
LOGGER.debug("Downloading the template file {} for hypervisor {} and arch {} as it is not present",
781+
templateDetails.getUrl(), templateDetails.getHypervisorType().name(), templateDetails.getArch());
782+
if (!NetUtils.downloadFileWithProgress(templateDetails.getUrl(), filePath, LOGGER)) {
783+
return null;
784+
}
785+
return new File(filePath);
786+
}
787+
return tempFile;
788+
}
789+
812790
private void validateTemplates(List<Pair<Hypervisor.HypervisorType, String>> hypervisorsArchInUse) {
813791
boolean templatesFound = true;
814792
for (Pair<Hypervisor.HypervisorType, String> hypervisorArch : hypervisorsArchInUse) {
@@ -822,11 +800,12 @@ private void validateTemplates(List<Pair<Hypervisor.HypervisorType, String>> hyp
822800
templatesFound = false;
823801
break;
824802
}
825-
if (CPU.archARM64Identifier.equals(matchedTemplate.getArch())) {
826-
LOGGER.debug("Skipping checksum comparison for the template file and metadata as the arch for template is {}",
827-
matchedTemplate.getArch());
803+
File tempFile = getTemplateFile(matchedTemplate);
804+
if (tempFile == null) {
805+
LOGGER.warn("Failed to download template for hypervisor {} and arch {}, moving ahead",
806+
matchedTemplate.getHypervisorType().name(), matchedTemplate.getArch());
807+
continue;
828808
}
829-
File tempFile = new File(TEMPLATES_PATH + matchedTemplate.getFilename());
830809
String templateChecksum = DigestHelper.calculateChecksum(tempFile);
831810
if (!templateChecksum.equals(matchedTemplate.getChecksum())) {
832811
LOGGER.error("Checksum {} for file {} does not match checksum {} from metadata",
@@ -842,7 +821,7 @@ private void validateTemplates(List<Pair<Hypervisor.HypervisorType, String>> hyp
842821
}
843822
}
844823

845-
protected void registerTemplatesForZone(long zoneId) {
824+
protected void registerTemplatesForZone(long zoneId, String filePath) {
846825
Pair<String, Long> storeUrlAndId = getNfsStoreInZone(zoneId);
847826
String nfsVersion = getNfsVersion(storeUrlAndId.second());
848827
mountStore(storeUrlAndId.first(), filePath, nfsVersion);
@@ -856,7 +835,7 @@ protected void registerTemplatesForZone(long zoneId) {
856835
TemplateDataStoreVO templateDataStoreVO = templateDataStoreDao.findByStoreTemplate(storeUrlAndId.second(), templateId);
857836
if (templateDataStoreVO != null) {
858837
String installPath = templateDataStoreVO.getInstallPath();
859-
if (validateIfSeeded(storeUrlAndId.first(), installPath, nfsVersion)) {
838+
if (validateIfSeeded(templateDataStoreVO, storeUrlAndId.first(), installPath, nfsVersion)) {
860839
continue;
861840
}
862841
}
@@ -891,7 +870,7 @@ public void doInTransactionWithoutResult(final TransactionStatus status) {
891870
if (filePath == null) {
892871
throw new CloudRuntimeException("Failed to create temporary file path to mount the store");
893872
}
894-
registerTemplatesForZone(zoneId);
873+
registerTemplatesForZone(zoneId, filePath);
895874
unmountStore(filePath);
896875
} catch (Exception e) {
897876
unmountStore(filePath);

Diff for: ‎engine/schema/templateConfig.sh

+7-6
Original file line numberDiff line numberDiff line change
@@ -71,12 +71,13 @@ function createMetadataFile() {
7171

7272
declare -a templates
7373
getTemplateVersion $1
74-
templates=( "kvm:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-x86_64-kvm.qcow2.bz2"
75-
"vmware:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-x86_64-vmware.ova"
76-
"xenserver:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-xen.vhd.bz2"
77-
"hyperv:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-hyperv.vhd.zip"
78-
"lxc:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-kvm.qcow2.bz2"
79-
"ovm3:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-ovm.raw.bz2" )
74+
templates=( "kvm-x86_64:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-x86_64-kvm.qcow2.bz2"
75+
"kvm-aarch64:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-aarch64-kvm.qcow2.bz2"
76+
"vmware-x86_64:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-x86_64-vmware.ova"
77+
"xenserver-x86_64:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-xen.vhd.bz2"
78+
"hyperv-x86_64:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-hyperv.vhd.zip"
79+
"lxc-x86_64:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-kvm.qcow2.bz2"
80+
"ovm3-x86_64:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-ovm.raw.bz2" )
8081

8182
PARENTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )/dist/systemvm-templates/"
8283
mkdir -p $PARENTPATH

Diff for: ‎utils/src/main/java/com/cloud/utils/net/NetUtils.java

+52-1
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,12 @@
2020
package com.cloud.utils.net;
2121

2222
import java.io.BufferedReader;
23+
import java.io.FileOutputStream;
2324
import java.io.IOException;
25+
import java.io.InputStream;
2426
import java.io.InputStreamReader;
2527
import java.math.BigInteger;
28+
import java.net.HttpURLConnection;
2629
import java.net.Inet4Address;
2730
import java.net.Inet6Address;
2831
import java.net.InetAddress;
@@ -31,6 +34,7 @@
3134
import java.net.SocketException;
3235
import java.net.URI;
3336
import java.net.URISyntaxException;
37+
import java.net.URL;
3438
import java.net.UnknownHostException;
3539
import java.util.ArrayList;
3640
import java.util.Arrays;
@@ -51,8 +55,8 @@
5155
import org.apache.commons.net.util.SubnetUtils;
5256
import org.apache.commons.validator.routines.InetAddressValidator;
5357
import org.apache.commons.validator.routines.RegexValidator;
54-
import org.apache.logging.log4j.Logger;
5558
import org.apache.logging.log4j.LogManager;
59+
import org.apache.logging.log4j.Logger;
5660

5761
import com.cloud.utils.IteratorUtil;
5862
import com.cloud.utils.Pair;
@@ -1874,4 +1878,51 @@ public static String transformCidr(final String cidr) {
18741878
final long start = (ip & startNetMask);
18751879
return String.format("%s/%s", long2Ip(start), size);
18761880
}
1881+
1882+
public static boolean downloadFileWithProgress(final String fileURL, final String savePath, final Logger logger) {
1883+
HttpURLConnection httpConn = null;
1884+
try {
1885+
URL url = new URL(fileURL);
1886+
httpConn = (HttpURLConnection) url.openConnection();
1887+
int responseCode = httpConn.getResponseCode();
1888+
if (responseCode == HttpURLConnection.HTTP_OK) {
1889+
int contentLength = httpConn.getContentLength();
1890+
if (contentLength < 0) {
1891+
logger.warn("Content length not provided for {}, progress updates may not be accurate",
1892+
fileURL);
1893+
}
1894+
try (InputStream inputStream = httpConn.getInputStream();
1895+
FileOutputStream outputStream = new FileOutputStream(savePath)) {
1896+
byte[] buffer = new byte[4096];
1897+
int bytesRead;
1898+
int downloaded = 0;
1899+
int lastReportedPercent = 0;
1900+
while ((bytesRead = inputStream.read(buffer)) != -1) {
1901+
outputStream.write(buffer, 0, bytesRead);
1902+
downloaded += bytesRead;
1903+
if (contentLength > 0) {
1904+
int percentDownloaded = (int) ((downloaded / (double) contentLength) * 100);
1905+
// Update every 5 percent or on completion
1906+
if (percentDownloaded - lastReportedPercent >= 5 || percentDownloaded == 100) {
1907+
logger.debug("Downloaded {}% from {}", downloaded, fileURL);
1908+
lastReportedPercent = percentDownloaded;
1909+
}
1910+
}
1911+
}
1912+
}
1913+
logger.info("File {} downloaded successfully using {}.", fileURL, savePath);
1914+
} else {
1915+
logger.error("No file to download {}. Server replied with code: {}", fileURL, responseCode);
1916+
return false;
1917+
}
1918+
} catch (IOException ex) {
1919+
logger.error("Failed to download {} due to: {}", fileURL, ex.getMessage(), ex);
1920+
return false;
1921+
} finally {
1922+
if (httpConn != null) {
1923+
httpConn.disconnect();
1924+
}
1925+
}
1926+
return true;
1927+
}
18771928
}

0 commit comments

Comments
 (0)