Update parallelization and log

Author: Fabien Servant

python/reducer.py

@@ -50,7 +50,7 @@ def reduce_samples(inputSfMData, imagePairsList, samplesFolder, featuresFolder,
 
     # Retrieve list of images pairs to process
     plist = avmic.PairSet()
-    if not avmic.loadPairsFromFile(imagePairsList, plist, 0, -1, False):
+    if not avmic.loadPairsFromFile(imagePairsList, plist, False):
         raise RuntimeError("Error in image pairs list loading")
 
     # build a list of image pairs indexed by their reference images
@@ -72,38 +72,64 @@ def reduce_samples(inputSfMData, imagePairsList, samplesFolder, featuresFolder,
     for referenceId, pairs in plistByRef.items():
         logging.info(f"Processing reference #{referenceId}", flush=True)
 
+        # Output features file for the reference image
         path_coords = os.path.join(samplesFolder, str(referenceId) + ".npy")
 
+        # Load file with coordinates for a given reference view
+        # This is an array of size [n, 3], where n is the number of coordinates used in this reference view
+        # first column is the x coordinate
+        # second column is the y coordinate
+        # third column is the 1
+        # fourth column is the scale
         try:
             coords_A_B = np.load(path_coords)
         except:
             coords_A_B = np.array(())
 
+        # Transform this array to a set of feature points for the reference view
+        # referenceId may already exists in regionMap, because the reference view may have been used as
+        # a matching view. We therefore return the offset in the list of features to be able to compute
+        # indices for matches
         refOffset = export_features(regionsMap, referenceId, coords_A_B)
 
+        # For all pairs with the current reference view
         for item in pairs:
 
             otherId = item[1]
 
+            # Load file with precomputed matches for a given reference view
+            # This is an array of size [n, 3], where n is the number of matches
+            # first column is the x coordinates in the other image
+            # second column is the y coordinates in the other image
+            # third column is the confidence score
+            # fourth column is the scale
             path_samples = os.path.join(samplesFolder, str(referenceId) + "_" + str(otherId) + ".npy")
 
             try:
                 match_A_B = np.load(path_samples)
             except:
                 continue
 
+            # Transform this array to a set of feature points for the other view
+            # otherId may already exists in regionMap, because the view may have been used as
+            # another matching view. We therefore return the offset in the list of features to be able to compute
+            # indices for matches
             otherOffset = export_features(regionsMap, otherId, match_A_B)
 
+            # Build a list of matches using offset and indices
             pos = 0
             matches = avmatch.IndMatches() 
             for rowId in range(0, match_A_B.shape[0]):
                 if match_A_B[rowId, 2] > 1e-6:
                     matches.append(avmatch.IndMatch(refOffset + rowId, otherOffset + pos))
                     pos = pos + 1
 
+            # Save matches for pair per desc.
+            # Obviously, here we only have on descriptor type
             perdesc = avmatch.MatchesPerDescType()
             perdesc[avmatch.EImageDescriberType_SIFT] = matches
 
+            # Save matches for pair per desc in a global container
             pair = avmatch.Pair(referenceId, otherId)
             global_matches[pair] = perdesc
 
@@ -115,6 +141,8 @@ def reduce_samples(inputSfMData, imagePairsList, samplesFolder, featuresFolder,
 if __name__ == '__main__':
     import argparse
 
+    logger = logging.getLogger().setLevel(logging.INFO)
+
     # create the top-level parser
     parser = argparse.ArgumentParser(prog='romaProcessor')
 

python/sampler.py

@@ -254,7 +254,8 @@ def compute_samples(inputSfMData, imagePairsList, warpFolder, certaintyFolder, s
         rangeIteration: if of chunk to compute between [0; rangeBlocksCount[
         rangeBlocksCount: count of chunks blocks
     """
-
+    from pyalicevision import system as avsys
+    
     # First of all, load the optional filters
     filters = load_filters(filtersFolder)
 
@@ -263,7 +264,7 @@ def compute_samples(inputSfMData, imagePairsList, warpFolder, certaintyFolder, s
 
     # Retrieve list of images pairs to process
     plist = avmic.PairSet()
-    if not avmic.loadPairsFromFile(imagePairsList, plist, 0, -1, False):
+    if not avmic.loadPairsFromFile(imagePairsList, plist, False):
         raise RuntimeError("Error in image pairs list loading")
 
     # build a list of image pairs indexed by their reference images
@@ -276,24 +277,29 @@ def compute_samples(inputSfMData, imagePairsList, warpFolder, certaintyFolder, s
             plistByRef[ref] = [item]
     refsToProcess = list(plistByRef)
 
-    #Compute parallelization
-    blockSize = int(len(refsToProcess) / rangeBlocksCount)
-    rangeStart = rangeIteration * blockSize
-    rangeEnd = rangeStart + blockSize
-    if rangeIteration + 1 == rangeBlocksCount:
-        rangeEnd = len(refsToProcess)
+    # Parallelization is done by splitting pairs based on their reference image
+    # We want to have access to all the pairs from the same reference
+
+    # Computeing parallelization parameters
+    (valid, rangeStart, rangeEnd) = avsys.rangeComputation(rangeIteration, rangeBlocksCount, len(refsToProcess))
+    if not valid:
+        logging.error("Range is out of bounds.")
+        return
+        
     refsToProcess = refsToProcess[rangeStart:rangeEnd]
 
      #Loop over all reference images
     for referenceId in refsToProcess:
 
+        # Retrieve all pairs for this reference image
         pairs = plistByRef[referenceId]
 
         logging.info(f"Processing reference #{referenceId}", flush=True)
 
-        # Load uncertainties
+        # Load uncertainties and store them using pair as key
         uncertaintiesByPair = build_uncertainties(iinfos, warpFolder, certaintyFolder, pairs, filters, minCertainty)
         if len(uncertaintiesByPair) == 0:
+            logging.info("")
             continue
 
         #If groupUncertainties, we sum the certainties together for the same reference image
@@ -358,6 +364,8 @@ def compute_samples(inputSfMData, imagePairsList, warpFolder, certaintyFolder, s
 if __name__ == '__main__':
     import argparse
 
+    logger = logging.getLogger().setLevel(logging.INFO)
+
     # create the top-level parser
     parser = argparse.ArgumentParser(prog='romaProcessor')