Force update script to download the image and calculate SHA512

Previously the precomputed SHA256 was used for most images.
Since we want to check the hash against the openstack backend,
we need to have the SHA512. Sadly, most images creators do not
provide us with that hash pre-computed.

The only solution is to download the images and computing
the hash at runtime.

Signed-off-by: Gondermann <[email protected]>

Author: gndrmnn

openstack_image_manager/update.py

@@ -7,8 +7,9 @@
 import shutil
 import sys
 import time
+import hashlib
+import math
 from urllib.parse import urlparse
-from urllib.request import urlopen
 
 from loguru import logger
 from minio import Minio
@@ -72,6 +73,11 @@ def mirror_image(
         client.fput_object(minio_bucket, os.path.join(dirname, new_filename), filename)
         os.remove(filename)
 
+def size_clean(size):
+        size_name = ("B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB")
+        i = int(math.floor(math.log(size, 1024)))
+        s = size / 1024 ** i
+        return f"{s:.2f} {size_name[i]}"
 
 def update_image(image, minio_server, minio_bucket, minio_access_key, minio_secret_key):
     name = image["name"]
@@ -87,54 +93,54 @@ def update_image(image, minio_server, minio_bucket, minio_access_key, minio_secr
     logger.info(f"Getting checksums from {latest_checksum_url}")
 
     result = requests.get(latest_checksum_url)
-    checksums = {}
 
-    checksum_type = "sha256"
+    hash_obj = hashlib.new("sha512")
+    file_headers = None
+    with requests.get(url=latest_url, stream=True, timeout=30) as response:
+        if response.status_code != 200:
+            logger.error(f"Downloading image '{name}' failed with error code {response.status_code}")
+            return None
+
+        file_headers = response.headers
+        file_size = int(file_headers["Content-Length"])
+        logger.info(f"Image size {size_clean(file_size)}")
+
+        downloadedBytes = 0
+        lastProgress = 0
+        for chunk in response.iter_content(chunk_size=8192):
+            downloadedBytes += 8192
+            progressPercent = (downloadedBytes / file_size) * 100
+            progress = round(min(max(progressPercent, 0), 100))
+            if progress - lastProgress >= 5:
+                logger.info(f"Downloading image: {progress}%")
+                lastProgress = progress
+
+            hash_obj.update(chunk)
+
+    sha512_value = hash_obj.hexdigest()
+
     filename_pattern = None
 
     if image["shortname"] in ["centos-stream-8", "centos-stream-9", "centos-7"]:
         filename_pattern = latest_filename.replace("HEREBE", "")
         filename_pattern = filename_pattern.replace("DRAGONS", "")
-    elif image["shortname"] in ["debian-10", "debian-11", "debian-12"]:
-        checksum_type = "sha512"
+        new_latest_filename_list = []
 
     for line in result.text.split("\n"):
-        if image["shortname"] in ["rocky-8", "rocky-9"]:
-            splitted_line = re.split("\s+", line)  # noqa W605
-            if splitted_line[0] == "SHA256":
-                checksums[latest_filename] = splitted_line[3]
-        elif image["shortname"] in [
-            "ubuntu-14.04",
-            "ubuntu-16.04",
-            "ubuntu-16.04-minimal",
-            "ubuntu-18.04",
-            "ubuntu-18.04-minimal",
-            "ubuntu-20.04",
-            "ubuntu-20.04-minimal",
-            "ubuntu-22.04",
-            "ubuntu-22.04-minimal",
-        ]:
-            splitted_line = re.split("\s+", line)  # noqa W605
-            if len(splitted_line) == 2:
-                checksums[splitted_line[1][1:]] = splitted_line[0]
-        elif image["shortname"] in ["centos-7"]:
+        if image["shortname"] in ["centos-7"]:
             splitted_line = re.split("\s+", line)  # noqa W605
             if len(splitted_line) == 2:
                 if re.search(filename_pattern, splitted_line[1]):
-                    checksums[splitted_line[1]] = splitted_line[0]
+                    new_latest_filename_list.append(splitted_line[1])
         elif image["shortname"] in ["centos-stream-8", "centos-stream-9"]:
             splitted_line = re.split("\s+", line)  # noqa W605
             if splitted_line[0] == "SHA256" and re.search(
                 filename_pattern, splitted_line[1][1:-1]
             ):
-                checksums[splitted_line[1][1:-1]] = splitted_line[3]
-        else:
-            splitted_line = re.split("\s+", line)  # noqa W605
-            if len(splitted_line) == 2:
-                checksums[splitted_line[1]] = splitted_line[0]
+                new_latest_filename_list.append(splitted_line[1][1:-1])
 
     if filename_pattern:
-        new_latest_filename = natsorted(checksums.keys())[-1]
+        new_latest_filename = natsorted(new_latest_filename_list)[-1]
         new_latest_url = latest_url.replace(latest_filename, new_latest_filename)
 
         logger.info(f"Latest URL is now {new_latest_url}")
@@ -143,7 +149,7 @@ def update_image(image, minio_server, minio_bucket, minio_access_key, minio_secr
         latest_filename = new_latest_filename
         latest_url = new_latest_url
 
-    current_checksum = f"{checksum_type}:{checksums[latest_filename]}"
+    current_checksum = f"sha512:{sha512_value}"
     logger.info(f"Checksum of current {latest_filename} is {current_checksum}")
 
     try:
@@ -165,9 +171,8 @@ def update_image(image, minio_server, minio_bucket, minio_access_key, minio_secr
     if latest_checksum != current_checksum:
         logger.info(f"Checking {latest_url}")
 
-        conn = urlopen(latest_url, timeout=30)
         struct = time.strptime(
-            conn.headers["last-modified"], "%a, %d %b %Y %H:%M:%S %Z"
+            file_headers["last-modified"], "%a, %d %b %Y %H:%M:%S %Z"
         )
         dt = datetime.fromtimestamp(time.mktime(struct))
 
@@ -250,7 +255,8 @@ def main(
                     minio_access_key,
                     minio_secret_key,
                 )
-                data["images"][index] = updated_image
+                if updated_image:
+                    data["images"][index] = updated_image
 
         with open(p, "w+") as fp:
             ryaml = ruamel.yaml.YAML()