refactor: use VHD header to fetch block size

libs/sm/VDI.py

@@ -101,6 +101,7 @@ def __init__(self, sr, uuid):
         self.description = ''
         self.vbds = []
         self.size = 0
+        self._block_size = -1
         self.utilisation = 0
         self.vdi_type = ''
         self.has_child = 0
@@ -120,6 +121,12 @@ def __init__(self, sr, uuid):
 
         self.load(uuid)
 
+    @property
+    def block_size(self):
+        if self._block_size < 0:
+            self._block_size = vhdutil.getBlockSize(self.path)
+        return self._block_size
+
     @staticmethod
     def from_uuid(session, vdi_uuid):
 

libs/sm/cleanup.py

@@ -535,12 +535,19 @@ def __init__(self, sr, uuid, raw):
         self.sizeVirt = -1
         self._sizeVHD = -1
         self._sizeAllocated = -1
+        self._block_size = -1
         self._hidden = False
         self.parent = None
         self.children = []
         self._vdiRef = None
         self._clearRef()
 
+    @property
+    def block_size(self):
+        if self._block_size < 0:
+            self._block_size = vhdutil.getBlockSize(self.path)
+        return self._block_size
+
     @staticmethod
     def extractUuid(path):
         raise NotImplementedError("Implement in sub class")
@@ -1075,14 +1082,16 @@ def _getCoalescedSizeData(self):
         blocksParent = self.parent.getVHDBlocks()
         numBlocks = Util.countBits(blocksChild, blocksParent)
         Util.log("Num combined blocks = %d" % numBlocks)
-        sizeData = numBlocks * vhdutil.VHD_BLOCK_SIZE
+        sizeData = numBlocks * self.block_size
         assert(sizeData <= self.sizeVirt)
         return sizeData
 
     def _calcExtraSpaceForCoalescing(self):
         sizeData = self._getCoalescedSizeData()
-        sizeCoalesced = sizeData + vhdutil.calcOverheadBitmap(sizeData) + \
-                vhdutil.calcOverheadEmpty(self.sizeVirt)
+        sizeCoalesced = sizeData + vhdutil.calcOverheadBitmap(
+            sizeData,
+            self.block_size
+        ) + vhdutil.calcOverheadEmpty(self.sizeVirt)
         Util.log("Coalesced size = %s" % Util.num2str(sizeCoalesced))
         return sizeCoalesced - self.parent.getSizeVHD()
 
@@ -1238,7 +1247,7 @@ def deflate(self):
         self._sizeAllocated = -1
 
     def inflateFully(self):
-        self.inflate(lvhdutil.calcSizeVHDLV(self.sizeVirt))
+        self.inflate(lvhdutil.calcSizeVHDLV(self.sizeVirt, self.block_size))
 
     def inflateParentForCoalesce(self):
         """Inflate the parent only as much as needed for the purposes of
@@ -1461,7 +1470,10 @@ def _queryVHDBlocks(self):
     def _calcExtraSpaceForCoalescing(self):
         if self.parent.raw:
             return 0  # raw parents are never deflated in the first place
-        sizeCoalesced = lvhdutil.calcSizeVHDLV(self._getCoalescedSizeData())
+        sizeCoalesced = lvhdutil.calcSizeVHDLV(
+            self._getCoalescedSizeData(),
+            self.block_size
+        )
         Util.log("Coalesced size = %s" % Util.num2str(sizeCoalesced))
         return sizeCoalesced - self.parent.sizeLV
 
@@ -2775,7 +2787,8 @@ def _finishCoalesceLeaf(self, parent):
             parent.deflate()
 
     def _calcExtraSpaceNeeded(self, child, parent):
-        return lvhdutil.calcSizeVHDLV(parent.sizeVirt) - parent.sizeLV
+        return lvhdutil.calcSizeVHDLV(parent.sizeVirt, parent.block_size) - \
+            parent.sizeLV
 
     def _handleInterruptedCoalesceLeaf(self):
         entries = self.journaler.getAll(VDI.JRN_LEAF)

libs/sm/drivers/FileSR.py

@@ -561,7 +561,10 @@ def create(self, sr_uuid, vdi_uuid, size):
 
         if self.vdi_type == vhdutil.VDI_TYPE_VHD:
             try:
-                size = vhdutil.validate_and_round_vhd_size(int(size))
+                size = vhdutil.validate_and_round_vhd_size(
+                    int(size),
+                    vhdutil.VHD_BLOCK_SIZE
+                )
                 mb = 1024 * 1024
                 size_mb = size // mb
                 util.ioretry(lambda: self._create(str(size_mb), self.path))
@@ -656,7 +659,7 @@ def resize(self, sr_uuid, vdi_uuid, size):
             return VDI.VDI.get_params(self)
 
         # We already checked it is a VDI_TYPE_VHD
-        size = vhdutil.validate_and_round_vhd_size(int(size))
+        size = vhdutil.validate_and_round_vhd_size(int(size), self.block_size)
 
         jFile = JOURNAL_FILE_PREFIX + self.uuid
         try:

libs/sm/drivers/LVHDSR.py

@@ -716,7 +716,10 @@ def scan(self, uuid):
                                     util.roundup(lvutil.LVM_SIZE_INCREMENT,
                                       vhdutil.calcOverheadEmpty(lvhdutil.MSIZE))
                             else:
-                                utilisation = lvhdutil.calcSizeVHDLV(int(size))
+                                utilisation = lvhdutil.calcSizeVHDLV(
+                                    int(size),
+                                    vhdutil.getBlockSize(lvPath)
+                                )
 
                         vdi_ref = self.session.xenapi.VDI.db_introduce(
                                         vdi_uuid,
@@ -984,7 +987,10 @@ def _undoCloneOp(self, lvs, origUuid, baseUuid, clonUuid):
 
         # inflate the parent to fully-allocated size
         if base.vdiType == vhdutil.VDI_TYPE_VHD:
-            fullSize = lvhdutil.calcSizeVHDLV(vhdInfo.sizeVirt)
+            fullSize = lvhdutil.calcSizeVHDLV(
+                vhdInfo.sizeVirt,
+                vhdutil.getBlockSize(basePath)
+            )
             lvhdutil.inflate(self.journaler, self.uuid, baseUuid, fullSize)
 
         # rename back
@@ -1173,7 +1179,7 @@ def _undoAllVHDJournals(self):
             util.SMlog("Found VHD journal %s, reverting %s" % (uuid, vdi.path))
             self.lvActivator.activate(uuid, vdi.lvname, False)
             self.lvmCache.activateNoRefcount(jlvName)
-            fullSize = lvhdutil.calcSizeVHDLV(vdi.size)
+            fullSize = lvhdutil.calcSizeVHDLV(vdi.size, vdi.block_size)
             lvhdutil.inflate(self.journaler, self.uuid, vdi.uuid, fullSize)
             try:
                 jFile = os.path.join(self.path, jlvName)
@@ -1184,7 +1190,7 @@ def _undoAllVHDJournals(self):
                 util.SMlog("VHD revert failed but VHD ok: removing journal")
             # Attempt to reclaim unused space
             vhdInfo = vhdutil.getVHDInfo(vdi.path, lvhdutil.extractUuid, False)
-            NewSize = lvhdutil.calcSizeVHDLV(vhdInfo.sizeVirt)
+            NewSize = lvhdutil.calcSizeVHDLV(vhdInfo.sizeVirt, vdi.block_size)
             if NewSize < fullSize:
                 lvhdutil.deflate(self.lvmCache, vdi.lvname, int(NewSize))
             lvhdutil.lvRefreshOnAllSlaves(self.session, self.uuid,
@@ -1343,7 +1349,10 @@ def create(self, sr_uuid, vdi_uuid, size):
         if self.exists:
             raise xs_errors.XenError('VDIExists')
 
-        size = vhdutil.validate_and_round_vhd_size(int(size))
+        size = vhdutil.validate_and_round_vhd_size(
+            int(size),
+            vhdutil.VHD_BLOCK_SIZE
+        )
 
         util.SMlog("LVHDVDI.create: type = %s, %s (size=%s)" % \
                 (self.vdi_type, self.path, size))
@@ -1356,7 +1365,10 @@ def create(self, sr_uuid, vdi_uuid, size):
                 lvSize = util.roundup(lvutil.LVM_SIZE_INCREMENT,
                         vhdutil.calcOverheadEmpty(lvhdutil.MSIZE))
             elif self.sr.provision == "thick":
-                lvSize = lvhdutil.calcSizeVHDLV(int(size))
+                lvSize = lvhdutil.calcSizeVHDLV(
+                    int(size),
+                    vhdutil.VHD_BLOCK_SIZE
+                )
 
         self.sr._ensureSpaceAvailable(lvSize)
 
@@ -1459,7 +1471,10 @@ def attach(self, sr_uuid, vdi_uuid):
             needInflate = False
         else:
             self._loadThis()
-            if self.utilisation >= lvhdutil.calcSizeVHDLV(self.size):
+            if (
+                self.utilisation >=
+                lvhdutil.calcSizeVHDLV(self.size, self.block_size)
+            ):
                 needInflate = False
 
         if needInflate:
@@ -1479,7 +1494,7 @@ def detach(self, sr_uuid, vdi_uuid):
         util.SMlog("LVHDVDI.detach for %s" % self.uuid)
         self._loadThis()
         already_deflated = (self.utilisation < \
-                lvhdutil.calcSizeVHDLV(self.size))
+                lvhdutil.calcSizeVHDLV(self.size, self.block_size))
         needDeflate = True
         if self.vdi_type == vhdutil.VDI_TYPE_RAW or already_deflated:
             needDeflate = False
@@ -1520,7 +1535,7 @@ def resize(self, sr_uuid, vdi_uuid, size):
                     '(current size: %d, new size: %d)' % (self.size, size))
             raise xs_errors.XenError('VDISize', opterr='shrinking not allowed')
 
-        size = vhdutil.validate_and_round_vhd_size(int(size))
+        size = vhdutil.validate_and_round_vhd_size(int(size), self.block_size)
 
         if size == self.size:
             return VDI.VDI.get_params(self)
@@ -1530,7 +1545,7 @@ def resize(self, sr_uuid, vdi_uuid, size):
             lvSizeNew = util.roundup(lvutil.LVM_SIZE_INCREMENT, size)
         else:
             lvSizeOld = self.utilisation
-            lvSizeNew = lvhdutil.calcSizeVHDLV(size)
+            lvSizeNew = lvhdutil.calcSizeVHDLV(size, self.block_size)
             if self.sr.provision == "thin":
                 # VDI is currently deflated, so keep it deflated
                 lvSizeNew = lvSizeOld
@@ -1696,7 +1711,7 @@ def _snapshot(self, snapType, cloneOp=False, cbtlog=None, cbt_consistency=None):
         self.issnap = self.session.xenapi.VDI.get_is_a_snapshot( \
                                                 self.sr.srcmd.params['vdi_ref'])
 
-        fullpr = lvhdutil.calcSizeVHDLV(self.size)
+        fullpr = lvhdutil.calcSizeVHDLV(self.size, self.block_size)
         thinpr = util.roundup(lvutil.LVM_SIZE_INCREMENT, \
                 vhdutil.calcOverheadEmpty(lvhdutil.MSIZE))
         lvSizeOrig = thinpr

libs/sm/lvhdutil.py

@@ -95,11 +95,11 @@ def calcSizeLV(sizeVHD):
     return util.roundup(LVM_SIZE_INCREMENT, sizeVHD)
 
 
-def calcSizeVHDLV(sizeVirt):
+def calcSizeVHDLV(sizeVirt, block_size):
     # all LVHD VDIs have the metadata area preallocated for the maximum
     # possible virtual size (for fast online VDI.resize)
     metaOverhead = vhdutil.calcOverheadEmpty(MSIZE)
-    bitmapOverhead = vhdutil.calcOverheadBitmap(sizeVirt)
+    bitmapOverhead = vhdutil.calcOverheadBitmap(sizeVirt, block_size)
     return calcSizeLV(sizeVirt + metaOverhead + bitmapOverhead)
 
 
@@ -208,7 +208,12 @@ def setSizeVirt(journaler, srUuid, vdiUuid, size, jFile):
     lvName = LV_PREFIX[vhdutil.VDI_TYPE_VHD] + vdiUuid
     vgName = VG_PREFIX + srUuid
     path = os.path.join(VG_LOCATION, vgName, lvName)
-    inflate(journaler, srUuid, vdiUuid, calcSizeVHDLV(size))
+    inflate(
+        journaler,
+        srUuid,
+        vdiUuid,
+        calcSizeVHDLV(size, vhdutil.getBlockSize(path))
+    )
     vhdutil.setSizeVirt(path, size, jFile)
 
 
@@ -233,7 +238,8 @@ def attachThin(journaler, srUuid, vdiUuid):
     _tryAcquire(lock)
     lvmCache.refresh()
     vhdInfo = vhdutil.getVHDInfoLVM(lvName, extractUuid, vgName)
-    newSize = calcSizeVHDLV(vhdInfo.sizeVirt)
+    path = os.path.join(VG_LOCATION, vgName, lvName)
+    newSize = calcSizeVHDLV(vhdInfo.sizeVirt, vhdutil.getBlockSize(path))
     currSizeLV = lvmCache.getSize(lvName)
     if newSize <= currSizeLV:
         return

libs/sm/vhdutil.py

@@ -82,9 +82,9 @@ def calcOverheadEmpty(virtual_size):
     return overhead
 
 
-def calcOverheadBitmap(virtual_size):
-    num_blocks = virtual_size // VHD_BLOCK_SIZE
-    if virtual_size % VHD_BLOCK_SIZE:
+def calcOverheadBitmap(virtual_size, block_size):
+    num_blocks = virtual_size // block_size
+    if virtual_size % block_size:
         num_blocks += 1
     return num_blocks * 4096
 
@@ -93,10 +93,29 @@ def ioretry(cmd, text=True):
     return util.ioretry(lambda: util.pread2(cmd, text=text),
                         errlist=[errno.EIO, errno.EAGAIN])
 
+def getBlockSize(path):
+    cmd = [VHD_UTIL, "read", "-pn", path]
+    try:
+        ret = ioretry(cmd)
+    except util.CommandException as e:
+        util.SMlog("WARN: unable to fetch block size: {}".format(e))
+        return VHD_BLOCK_SIZE
+    if isinstance(ret, bytes):
+        import locale
+        ret = ret.decode(
+            encoding = locale.getpreferredencoding(False),
+            errors="replace"
+        )
+    fields = ret.strip().split('\n')
+    for field in fields:
+        field = field.strip()
+        if not field.startswith("Block size"): continue
+        return int(field.split(':')[1].strip().split(' ')[0])
+    return VHD_BLOCK_SIZE
+
 
-def convertAllocatedSizeToBytes(size):
-    # Assume we have standard 2MB allocation blocks
-    return size * 2 * 1024 * 1024
+def convertAllocatedSizeToBytes(size, block_size):
+    return size * block_size
 
 
 def getVHDInfo(path, extractUuidFunction, includeParent=True):
@@ -120,7 +139,10 @@ def getVHDInfo(path, extractUuidFunction, includeParent=True):
             vhdInfo.parentUuid = extractUuidFunction(fields[nextIndex])
         nextIndex += 1
     vhdInfo.hidden = int(fields[nextIndex].replace("hidden: ", ""))
-    vhdInfo.sizeAllocated = convertAllocatedSizeToBytes(int(fields[nextIndex+1]))
+    vhdInfo.sizeAllocated = convertAllocatedSizeToBytes(
+        int(fields[nextIndex+1]),
+        getBlockSize(path)
+    )
     vhdInfo.path = path
     return vhdInfo
 
@@ -279,7 +301,7 @@ def setSizePhys(path, size, debug=True):
 def getAllocatedSize(path):
     cmd = [VHD_UTIL, "query", OPT_LOG_ERR, '-a', '-n', path]
     ret = ioretry(cmd)
-    return convertAllocatedSizeToBytes(int(ret))
+    return convertAllocatedSizeToBytes(int(ret), getBlockSize(path))
 
 def killData(path):
     "zero out the disk (kill all data inside the VHD file)"
@@ -406,7 +428,7 @@ def repair(path):
     ioretry([VHD_UTIL, 'repair', '-n', path])
 
 
-def validate_and_round_vhd_size(size):
+def validate_and_round_vhd_size(size, block_size):
     """ Take the supplied vhd size, in bytes, and check it is positive and less
     that the maximum supported size, rounding up to the next block boundary
     """
@@ -419,7 +441,7 @@ def validate_and_round_vhd_size(size):
     if size < MIN_VHD_SIZE:
         size = MIN_VHD_SIZE
 
-    size = util.roundup(VHD_BLOCK_SIZE, size)
+    size = util.roundup(block_size, size)
 
     return size
 

tests/test_FileSR.py

@@ -327,6 +327,7 @@ def test_create_vdi_vhd(self, mock_vhdutil):
         vdi = FakeFileVDI(sr, vdi_uuid)
         vdi.vdi_type = vhdutil.VDI_TYPE_VHD
         mock_vhdutil.validate_and_round_vhd_size.side_effect = vhdutil.validate_and_round_vhd_size
+        mock_vhdutil.VHD_BLOCK_SIZE = 2 * 1024 * 1024
 
         # Act
         vdi.create(sr_uuid, vdi_uuid, 20 * 1024 * 1024)