Fix race condition that was causing dangling loop devices

Apparently it is wrong to assume that `DeleteVolume` gets called
only after `UnstageVolume` returns success. This was causing the
disk image file to be deleted while the volume was still mounted.
This would prevent the loop device from getting detached and in
turn disk space from getting reclaimed.

Author: semekh

consts.py

@@ -5,3 +5,4 @@
 DATA_DIR = "/data"
 CONFIG = {}
 RESOURCE_EXHAUSTED_EXIT_CODE = 101
+VOLUME_IN_USE_EXIT_CODE = 102

rawfile_servicer.py

@@ -5,7 +5,12 @@
 from google.protobuf.wrappers_pb2 import BoolValue
 
 import rawfile_util
-from consts import PROVISIONER_VERSION, PROVISIONER_NAME, RESOURCE_EXHAUSTED_EXIT_CODE
+from consts import (
+    PROVISIONER_VERSION,
+    PROVISIONER_NAME,
+    RESOURCE_EXHAUSTED_EXIT_CODE,
+    VOLUME_IN_USE_EXIT_CODE,
+)
 from csi import csi_pb2, csi_pb2_grpc
 from declarative import be_symlink, be_absent
 from fs_util import device_stats, mountpoint_to_dev
@@ -208,7 +213,13 @@ def CreateVolume(self, request, context):
 
     @log_grpc_request
     def DeleteVolume(self, request, context):
-        scrub(volume_id=request.volume_id)
+        try:
+            scrub(volume_id=request.volume_id)
+        except CalledProcessError as exc:
+            if exc.returncode == VOLUME_IN_USE_EXIT_CODE:
+                context.abort(grpc.StatusCode.FAILED_PRECONDITION, "Volume in use")
+            else:
+                raise exc
         return csi_pb2.DeleteVolumeResponse()
 
     @log_grpc_request

remote.py

@@ -3,12 +3,20 @@
 
 def scrub(volume_id):
     import time
+    from subprocess import CalledProcessError
+
     import rawfile_util
+    from consts import VOLUME_IN_USE_EXIT_CODE
 
     img_dir = rawfile_util.img_dir(volume_id)
     if not img_dir.exists():
         return
 
+    img_file = rawfile_util.img_file(volume_id)
+    loops = rawfile_util.attached_loops(img_file)
+    if len(loops) > 0:
+        raise CalledProcessError(returncode=VOLUME_IN_USE_EXIT_CODE, cmd="")
+
     now = time.time()
     deleted_at = now
     gc_at = now  # TODO: GC sensitive PVCs later