Merge pull request #310 from AdamWill/image-size-bytes

Allow image size specification using any SI or IEC unit

Author: nullr0ute

oz/Fedora.py

@@ -298,11 +298,11 @@ def _modify_iso(self):
         initrdline += " inst.nosave=output_ks"
         self._modify_isolinux(initrdline)
 
-    def generate_diskimage(self, size=10, force=False):
+    def generate_diskimage(self, size=10*1024*1024*1024, force=False):
         """
         Method to generate a diskimage.  By default, a blank diskimage of
-        10GB will be created; the caller can override this with the size
-        parameter, specified in GB.  If force is False (the default), then
+        10 GiB will be created; the caller can override this with the size
+        parameter, specified in bytes.  If force is False (the default), then
         a diskimage will not be created if a cached JEOS is found.  If
         force is True, a diskimage will be created regardless of whether a
         cached JEOS exists.  See the oz-install man page for more

oz/Guest.py

@@ -572,7 +572,7 @@ def _generate_xml(self, bootdev, installdev, kernel=None, initrd=None,
 
         return xml
 
-    def _internal_generate_diskimage(self, size=10, force=False,
+    def _internal_generate_diskimage(self, size=10*1024*1024*1024, force=False,
                                      create_partition=False,
                                      image_filename=None,
                                      backing_filename=None):
@@ -588,7 +588,7 @@ def _internal_generate_diskimage(self, size=10, force=False,
             # we'll copy the JEOS itself later on
             return
 
-        self.log.info("Generating %dGB diskimage for %s", size, self.tdl.name)
+        self.log.info("Generating %s diskimage for %s", oz.ozutil.sizeof_fmt(int(size)), self.tdl.name)
 
         diskimage = self.diskimage
         if image_filename:
@@ -629,17 +629,17 @@ def _internal_generate_diskimage(self, size=10, force=False,
             # allows creation without an explicit capacity element.
             qcow_size = oz.ozutil.check_qcow_size(backing_filename)
             if qcow_size:
-                capacity = qcow_size / 1024 / 1024 / 1024
+                capacity = qcow_size
                 backing_format = 'qcow2'
             else:
-                capacity = os.path.getsize(backing_filename) / 1024 / 1024 / 1024
+                capacity = os.path.getsize(backing_filename)
                 backing_format = 'raw'
             backing = oz.ozutil.lxml_subelement(vol, "backingStore")
             oz.ozutil.lxml_subelement(backing, "path", backing_filename)
             oz.ozutil.lxml_subelement(backing, "format", None,
                                       {"type": backing_format})
 
-        oz.ozutil.lxml_subelement(vol, "capacity", str(int(capacity)), {'unit': 'G'})
+        oz.ozutil.lxml_subelement(vol, "capacity", str(int(capacity)), {'unit': 'B'})
         vol_xml = lxml.etree.tostring(vol, pretty_print=True, encoding="unicode")
 
         # sigh.  Yes, this is racy; if a pool is defined during this loop, we
@@ -719,11 +719,11 @@ def _vol_create_cb(args):
                 g_handle.create_msdos_partition_table()
                 g_handle.cleanup()
 
-    def generate_diskimage(self, size=10, force=False):
+    def generate_diskimage(self, size=10*1024*1024*1024, force=False):
         """
         Method to generate a diskimage.  By default, a blank diskimage of
-        10GB will be created; the caller can override this with the size
-        parameter, specified in GB.  If force is False (the default), then
+        10 GiB will be created; the caller can override this with the size
+        parameter, specified in bytes.  If force is False (the default), then
         a diskimage will not be created if a cached JEOS is found.  If
         force is True, a diskimage will be created regardless of whether a
         cached JEOS exists.  See the oz-install man page for more

oz/TDL.py

@@ -327,20 +327,39 @@ def _parse_disksize(self):
             # a sensible default
             return None
 
-        match = re.match(r'([0-9]*) *([GT]?)$', size)
-        if not match or len(match.groups()) != 2:
-            raise oz.OzException.OzException("Invalid disk size; it must be specified as a size in gigabytes, optionally suffixed with 'G' or 'T'")
-
-        number = match.group(1)
-        suffix = match.group(2)
-
-        if not suffix or suffix == 'G':
-            # for backwards compatibility, we assume G when there is no suffix
-            size = number
-        elif suffix == 'T':
-            size = str(int(number) * 1024)
-
-        return size
+        # drop spaces and downcase
+        size = size.replace(" ", "").lower()
+        # isolate digits
+        number = ""
+        suffix = ""
+        for (idx, char) in enumerate(size):
+            if char.isdigit():
+                number += char
+            else:
+                suffix = size[idx:]
+                break
+
+        if not suffix:
+            # for backwards compatibility, we assume GiB when there is no suffix
+            suffix = "gib"
+
+        # also for backwards compatibility with an earlier attempt to support
+        # suffixes, treat "T" and "G" as "TiB" and "GiB"
+        units = {"b": 1, "g": 2**30, "t": 2**40}
+        tenscale = 3
+        twoscale = 10
+        for (i, pref) in enumerate(("k", "m", "g", "t", "p", "e", "z", "y"), start=1):
+            # this is giving us {"gib": 2 ** 30, "gb": 10 ** 9}, etc
+            units[pref + "b"] = (10 ** (i*tenscale))
+            units[pref + "ib"] = (2 ** (i*twoscale))
+
+        factor = units.get(suffix)
+        if not number or not factor:
+            raise oz.OzException.OzException(
+                "Invalid disk size; it must be specified as an integer size with an optional SI or IEC unit suffix, e.g. '10TB' or '16GiB'"
+            )
+
+        return str(int(number) * factor)
 
     def _parse_commands(self, xpath):
         """

oz/Windows.py

@@ -78,12 +78,12 @@ def _generate_new_iso(self):
                                            self.iso_contents],
                                           printfn=self.log.debug)
 
-    def generate_diskimage(self, size=10, force=False):
+    def generate_diskimage(self, size=10*1024*1024*1024, force=False):
         """
         Method to generate a diskimage.  By default, a blank diskimage of
-        10GB will be created; the caller can override this with the size
-        parameter, specified in GB.  If force is False (the default), then
-        a diskimage will not be created if a cached JEOS is found.  If
+        10 GiB will be created; the caller can override this with the size
+        parameter, specified in bytes.  If force is False (the default),
+        then a diskimage will not be created if a cached JEOS is found.  If
         force is True, a diskimage will be created regardless of whether a
         cached JEOS exists.  See the oz-install man page for more
         information about JEOS caching.

oz/ozutil.py

@@ -1178,3 +1178,20 @@ def get_free_port():
     sock.close()
 
     return listen_port
+
+
+def sizeof_fmt(num, suffix="B"):
+    """
+    Give a convenient human-readable representation of a large size in
+    bytes. Initially by Fred Cirera:
+    https://web.archive.org/web/20111010015624/http://blogmag.net/blog/read/38/Print_human_readable_file_size
+    edited by multiple contributors at:
+    https://stackoverflow.com/questions/1094841
+    Per Richard Fontana this is too trivial to be copyrightable, so
+    there are no licensing concerns
+    """
+    for unit in ("", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"):
+        if abs(num) < 1024.0:
+            return "%3.1f%s%s" % (num, unit, suffix)
+        num /= 1024.0
+    return "%.1f%s%s" % (num, 'Yi', suffix)

oz/tdl.rng

@@ -325,7 +325,7 @@
 
   <define name='disk_size'>
     <data type="string">
-      <param name="pattern">([0-9]*) *([GT]?)</param>
+      <param name="pattern">([0-9]*) *([kKmMgGtTpPeEzZyY]?[iI]?[bB]?)</param>
     </data>
   </define>
 

tests/guest/test_guest.py

@@ -352,7 +352,8 @@ def test_geteltorito_bogus_bootp(tmpdir):
 def test_init_guest():
     guest = setup_guest(tdlxml2)
 
-    assert guest.disksize == 20
+    # size without units is taken to be GiB
+    assert guest.disksize == 20*(2**30)
     assert guest.image_name() == 'tester'
     assert guest.output_image_path() in (
         # user's image storage

tests/tdl/test-58-disk-size-terabyte.tdl tests/tdl/test-58-disk-size-tebibyte-compat.tdl

@@ -0,0 +1,15 @@
+<template>
+  <name>help</name>
+  <os>
+    <name>Fedora</name>
+    <version>12</version>
+    <arch>i386</arch>
+    <install type='url'>
+      <url>http://download.fedoraproject.org/pub/fedora/linux/releases/12/Fedora/x86_64/os/</url>
+    </install>
+  </os>
+  <description>My Fedora 12 JEOS image</description>
+  <disk>
+    <size>2EiB</size>
+  </disk>
+</template>

tests/tdl/test-63-disk-size-exbibyte.tdl

@@ -0,0 +1,15 @@
+<template>
+  <name>help</name>
+  <os>
+    <name>Fedora</name>
+    <version>12</version>
+    <arch>i386</arch>
+    <install type='url'>
+      <url>http://download.fedoraproject.org/pub/fedora/linux/releases/12/Fedora/x86_64/os/</url>
+    </install>
+  </os>
+  <description>My Fedora 12 JEOS image</description>
+  <disk>
+    <size>10ZB</size>
+  </disk>
+</template>

tests/tdl/test-64-disk-size-zettabyte.tdl

@@ -0,0 +1,15 @@
+<template>
+  <name>help</name>
+  <os>
+    <name>Fedora</name>
+    <version>12</version>
+    <arch>i386</arch>
+    <install type='url'>
+      <url>http://download.fedoraproject.org/pub/fedora/linux/releases/12/Fedora/x86_64/os/</url>
+    </install>
+  </os>
+  <description>My Fedora 12 JEOS image</description>
+  <disk>
+    <size>10000000 B</size>
+  </disk>
+</template>

tests/tdl/test-65-disk-size-byte.tdl

@@ -92,8 +92,11 @@
     "test-55-files-http-url.tdl": True,
     "test-56-invalid-disk-size.tdl": False,
     "test-57-invalid-disk-size.tdl": False,
-    "test-58-disk-size-terabyte.tdl": True,
+    "test-58-disk-size-tebibyte-compat.tdl": True,
     "test-59-command-sorting.tdl": True,
+    "test-63-disk-size-exbibyte.tdl": True,
+    "test-64-disk-size-zettabyte.tdl": True,
+    "test-65-disk-size-byte.tdl": True,
 }
 
 # Test that iterates over all .tdl files