Allow image size specification using any SI or IEC unit

Currently, oz only allows image sizes to be specified as integer
amounts of gibibytes or tebibytes (that's IEC power-of-two units).
This is unfortunately inflexible. Consider that storage media are
typically specified in SI power-of-ten sizes, so a USB stick may
be 16 gigabytes (SI power-of-ten GB) in size - that's
16,000,000,000 bytes. Let's say we want to build an image that
will fit on such a USB stick. oz will only allow us to build an
image of 15,032,385,536 bytes (14 gibibytes) or 16,106,127,360
bytes (15 gibibytes). So we're either slightly too big, or leaving
nearly a gigabyte on the table.

This allows the image size to be specified in the TDL with most
any IEC or SI unit suffix, from B (bytes) all the way up to YiB
(yobibytes). A size with no suffix or the suffix "G" is taken as
gibibytes and a size with the suffix "T" is taken as tebibytes,
as before, but other ambiguous suffixes are not accepted. All
casing is accepted. Behind the scenes, we convert the size to
bytes and specify it that way in the libvirt XML when creating
the image in _internal_generate_diskimage.

This does change the interface of generate_diskimage(), by making
the unit for the size argument bytes instead of gibibytes. I can't
see a clean way to avoid this while allowing flexibility. I have
checked, and AFAICT, among active projects, only oz itself and
the ImageFactory TinMan plugin call this function. The TinMan
plugin will need a trivial change to its fallback default value.

Signed-off-by: Adam Williamson <[email protected]>

Author: AdamWill

oz/Fedora.py

@@ -298,11 +298,11 @@ def _modify_iso(self):
         initrdline += " inst.nosave=output_ks"
         self._modify_isolinux(initrdline)
 
-    def generate_diskimage(self, size=10, force=False):
+    def generate_diskimage(self, size=10*1024*1024*1024, force=False):
         """
         Method to generate a diskimage.  By default, a blank diskimage of
-        10GB will be created; the caller can override this with the size
-        parameter, specified in GB.  If force is False (the default), then
+        10 GiB will be created; the caller can override this with the size
+        parameter, specified in bytes.  If force is False (the default), then
         a diskimage will not be created if a cached JEOS is found.  If
         force is True, a diskimage will be created regardless of whether a
         cached JEOS exists.  See the oz-install man page for more

oz/Guest.py

@@ -571,7 +571,7 @@ def _generate_xml(self, bootdev, installdev, kernel=None, initrd=None,
 
         return xml
 
-    def _internal_generate_diskimage(self, size=10, force=False,
+    def _internal_generate_diskimage(self, size=10*1024*1024*1024, force=False,
                                      create_partition=False,
                                      image_filename=None,
                                      backing_filename=None):
@@ -587,7 +587,7 @@ def _internal_generate_diskimage(self, size=10, force=False,
             # we'll copy the JEOS itself later on
             return
 
-        self.log.info("Generating %dGB diskimage for %s", size, self.tdl.name)
+        self.log.info("Generating %s diskimage for %s", oz.ozutil.sizeof_fmt(int(size)), self.tdl.name)
 
         diskimage = self.diskimage
         if image_filename:
@@ -628,17 +628,17 @@ def _internal_generate_diskimage(self, size=10, force=False,
             # allows creation without an explicit capacity element.
             qcow_size = oz.ozutil.check_qcow_size(backing_filename)
             if qcow_size:
-                capacity = qcow_size / 1024 / 1024 / 1024
+                capacity = qcow_size
                 backing_format = 'qcow2'
             else:
-                capacity = os.path.getsize(backing_filename) / 1024 / 1024 / 1024
+                capacity = os.path.getsize(backing_filename)
                 backing_format = 'raw'
             backing = oz.ozutil.lxml_subelement(vol, "backingStore")
             oz.ozutil.lxml_subelement(backing, "path", backing_filename)
             oz.ozutil.lxml_subelement(backing, "format", None,
                                       {"type": backing_format})
 
-        oz.ozutil.lxml_subelement(vol, "capacity", str(int(capacity)), {'unit': 'G'})
+        oz.ozutil.lxml_subelement(vol, "capacity", str(int(capacity)), {'unit': 'B'})
         vol_xml = lxml.etree.tostring(vol, pretty_print=True, encoding="unicode")
 
         # sigh.  Yes, this is racy; if a pool is defined during this loop, we
@@ -718,11 +718,11 @@ def _vol_create_cb(args):
                 g_handle.create_msdos_partition_table()
                 g_handle.cleanup()
 
-    def generate_diskimage(self, size=10, force=False):
+    def generate_diskimage(self, size=10*1024*1024*1024, force=False):
         """
         Method to generate a diskimage.  By default, a blank diskimage of
-        10GB will be created; the caller can override this with the size
-        parameter, specified in GB.  If force is False (the default), then
+        10 GiB will be created; the caller can override this with the size
+        parameter, specified in bytes.  If force is False (the default), then
         a diskimage will not be created if a cached JEOS is found.  If
         force is True, a diskimage will be created regardless of whether a
         cached JEOS exists.  See the oz-install man page for more

oz/TDL.py

@@ -327,20 +327,39 @@ def _parse_disksize(self):
             # a sensible default
             return None
 
-        match = re.match(r'([0-9]*) *([GT]?)$', size)
-        if not match or len(match.groups()) != 2:
-            raise oz.OzException.OzException("Invalid disk size; it must be specified as a size in gigabytes, optionally suffixed with 'G' or 'T'")
-
-        number = match.group(1)
-        suffix = match.group(2)
-
-        if not suffix or suffix == 'G':
-            # for backwards compatibility, we assume G when there is no suffix
-            size = number
-        elif suffix == 'T':
-            size = str(int(number) * 1024)
-
-        return size
+        # drop spaces and downcase
+        size = size.replace(" ", "").lower()
+        # isolate digits
+        number = ""
+        suffix = ""
+        for (idx, char) in enumerate(size):
+            if char.isdigit():
+                number += char
+            else:
+                suffix = size[idx:]
+                break
+
+        if not suffix:
+            # for backwards compatibility, we assume GiB when there is no suffix
+            suffix = "gib"
+
+        # also for backwards compatibility with an earlier attempt to support
+        # suffixes, treat "T" and "G" as "TiB" and "GiB"
+        units = {"b": 1, "g": 2**30, "t": 2**40}
+        tenscale = 3
+        twoscale = 10
+        for (i, pref) in enumerate(("k", "m", "g", "t", "p", "e", "z", "y"), start=1):
+            # this is giving us {"gib": 2 ** 30, "gb": 10 ** 9}, etc
+            units[pref + "b"] = (10 ** (i*tenscale))
+            units[pref + "ib"] = (2 ** (i*twoscale))
+
+        factor = units.get(suffix)
+        if not number or not factor:
+            raise oz.OzException.OzException(
+                "Invalid disk size; it must be specified as an integer size with an optional SI or IEC unit suffix, e.g. '10TB' or '16GiB'"
+            )
+
+        return str(int(number) * factor)
 
     def _parse_commands(self, xpath):
         """

oz/Windows.py

@@ -78,12 +78,12 @@ def _generate_new_iso(self):
                                            self.iso_contents],
                                           printfn=self.log.debug)
 
-    def generate_diskimage(self, size=10, force=False):
+    def generate_diskimage(self, size=10*1024*1024*1024, force=False):
         """
         Method to generate a diskimage.  By default, a blank diskimage of
-        10GB will be created; the caller can override this with the size
-        parameter, specified in GB.  If force is False (the default), then
-        a diskimage will not be created if a cached JEOS is found.  If
+        10 GiB will be created; the caller can override this with the size
+        parameter, specified in bytes.  If force is False (the default),
+        then a diskimage will not be created if a cached JEOS is found.  If
         force is True, a diskimage will be created regardless of whether a
         cached JEOS exists.  See the oz-install man page for more
         information about JEOS caching.

oz/ozutil.py

@@ -1178,3 +1178,20 @@ def get_free_port():
     sock.close()
 
     return listen_port
+
+
+def sizeof_fmt(num, suffix="B"):
+    """
+    Give a convenient human-readable representation of a large size in
+    bytes. Initially by Fred Cirera:
+    https://web.archive.org/web/20111010015624/http://blogmag.net/blog/read/38/Print_human_readable_file_size
+    edited by multiple contributors at:
+    https://stackoverflow.com/questions/1094841
+    Per Richard Fontana this is too trivial to be copyrightable, so
+    there are no licensing concerns
+    """
+    for unit in ("", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"):
+        if abs(num) < 1024.0:
+            return "%3.1f%s%s" % (num, unit, suffix)
+        num /= 1024.0
+    return "%.1f%s%s" % (num, 'Yi', suffix)

tests/guest/test_guest.py

@@ -352,7 +352,8 @@ def test_geteltorito_bogus_bootp(tmpdir):
 def test_init_guest():
     guest = setup_guest(tdlxml2)
 
-    assert guest.disksize == 20
+    # size without units is taken to be GiB
+    assert guest.disksize == 20*(2**30)
     assert guest.image_name() == 'tester'
     assert guest.output_image_path() in (
         # user's image storage

tests/tdl/test-58-disk-size-terabyte.tdl tests/tdl/test-58-disk-size-tebibyte-compat.tdl

@@ -0,0 +1,15 @@
+<template>
+  <name>help</name>
+  <os>
+    <name>Fedora</name>
+    <version>12</version>
+    <arch>i386</arch>
+    <install type='url'>
+      <url>http://download.fedoraproject.org/pub/fedora/linux/releases/12/Fedora/x86_64/os/</url>
+    </install>
+  </os>
+  <description>My Fedora 12 JEOS image</description>
+  <disk>
+    <size>2EiB</size>
+  </disk>
+</template>

tests/tdl/test-63-disk-size-exbibyte.tdl

@@ -0,0 +1,15 @@
+<template>
+  <name>help</name>
+  <os>
+    <name>Fedora</name>
+    <version>12</version>
+    <arch>i386</arch>
+    <install type='url'>
+      <url>http://download.fedoraproject.org/pub/fedora/linux/releases/12/Fedora/x86_64/os/</url>
+    </install>
+  </os>
+  <description>My Fedora 12 JEOS image</description>
+  <disk>
+    <size>10ZB</size>
+  </disk>
+</template>

tests/tdl/test-64-disk-size-zettabyte.tdl

@@ -0,0 +1,15 @@
+<template>
+  <name>help</name>
+  <os>
+    <name>Fedora</name>
+    <version>12</version>
+    <arch>i386</arch>
+    <install type='url'>
+      <url>http://download.fedoraproject.org/pub/fedora/linux/releases/12/Fedora/x86_64/os/</url>
+    </install>
+  </os>
+  <description>My Fedora 12 JEOS image</description>
+  <disk>
+    <size>10000000 B</size>
+  </disk>
+</template>

tests/tdl/test-65-disk-size-byte.tdl

@@ -92,8 +92,11 @@
     "test-55-files-http-url.tdl": True,
     "test-56-invalid-disk-size.tdl": False,
     "test-57-invalid-disk-size.tdl": False,
-    "test-58-disk-size-terabyte.tdl": True,
+    "test-58-disk-size-tebibyte-compat.tdl": True,
     "test-59-command-sorting.tdl": True,
+    "test-63-disk-size-exbibyte.tdl": True,
+    "test-64-disk-size-zettabyte.tdl": True,
+    "test-65-disk-size-byte.tdl": True,
 }
 
 # Test that iterates over all .tdl files