fix: las_merge: handle filenames with different prefix

Author: gliegard

CHANGELOG.md

@@ -1,3 +1,5 @@
+- [fix] las_merge: handle filenames with different prefix
+
 # 1.15.7
 - [update] Update las_comparison with tolerance 
 - [update] Colorisation of Las with stream or files

pdaltools/las_merge.py

@@ -6,83 +6,118 @@
 from pdaltools.las_info import parse_filename
 
 
-def create_filenames(file: str, tile_width: int = 1000, tile_coord_scale: int = 1000):
+def create_filenames_suffixes(file: str, tile_width: int = 1000, tile_coord_scale: int = 1000):
     """Generate the name of the tiles around the input LIDAR tile
     It supposes that the file names are formatted as {prefix1}_{prefix2}_{coordx}_{coordy}_{suffix}
     with coordx and coordy having at least 4 digits
 
     For example Semis_2021_0000_1111_LA93_IGN69.las
 
+    Generates only the suffix part of the filename, for example, for file like above, it will generate:
+    _0000_1112_LA93_IGN69.las
+    _0001_1112_LA93_IGN69.las
+    ...
+
     Args:
         file(str): name of LIDAR file
         tile width (int): width of tiles in meters (usually 1000m)
         tile_coord_scale (int) : scale used in the filename to describe coordinates in meters
                 (usually 1000m)
     Returns:
-        list_input(list): List of LIDAR's name
+        list_input(list): List of LIDAR's filename suffix.
     """
 
     # Create name of LIDAR tiles who cercle the tile
     # # Parameters
     _prefix, coord_x, coord_y, _suffix = parse_filename(file)
     offset = int(tile_width / tile_coord_scale)
     # On left
-    _tile_hl = f"{_prefix}_{(coord_x - offset):04d}_{(coord_y + offset):04d}_{_suffix}"
-    _tile_ml = f"{_prefix}_{(coord_x - offset):04d}_{coord_y:04d}_{_suffix}"
-    _tile_bl = f"{_prefix}_{(coord_x - offset):04d}_{(coord_y - offset):04d}_{_suffix}"
+    _tile_hl = f"_{(coord_x - offset):04d}_{(coord_y + offset):04d}_{_suffix}"
+    _tile_ml = f"_{(coord_x - offset):04d}_{coord_y:04d}_{_suffix}"
+    _tile_bl = f"_{(coord_x - offset):04d}_{(coord_y - offset):04d}_{_suffix}"
     # On Right
-    _tile_hr = f"{_prefix}_{(coord_x + offset):04d}_{(coord_y + offset):04d}_{_suffix}"
-    _tile_mr = f"{_prefix}_{(coord_x + offset):04d}_{coord_y:04d}_{_suffix}"
-    _tile_br = f"{_prefix}_{(coord_x + offset):04d}_{(coord_y - offset):04d}_{_suffix}"
+    _tile_hr = f"_{(coord_x + offset):04d}_{(coord_y + offset):04d}_{_suffix}"
+    _tile_mr = f"_{(coord_x + offset):04d}_{coord_y:04d}_{_suffix}"
+    _tile_br = f"_{(coord_x + offset):04d}_{(coord_y - offset):04d}_{_suffix}"
     # Above
-    _tile_a = f"{_prefix}_{coord_x:04d}_{(coord_y + offset):04d}_{_suffix}"
+    _tile_a = f"_{coord_x:04d}_{(coord_y + offset):04d}_{_suffix}"
     # Below
-    _tile_b = f"{_prefix}_{coord_x:04d}_{(coord_y - offset):04d}_{_suffix}"
+    _tile_b = f"_{coord_x:04d}_{(coord_y - offset):04d}_{_suffix}"
     # Return the severals tile's names
     return _tile_hl, _tile_ml, _tile_bl, _tile_a, _tile_b, _tile_hr, _tile_mr, _tile_br
 
 
-def check_tiles_exist(list_las: list):
-    """Check if pointclouds exist
+def match_suffix_with_filenames(suffix_list: list, all_files: list, las_dir: str):
+    """Match suffix list with real filenames
     Args:
-        list_las (list): Filenames of the tiles around the LIDAR tile
+        suffix_list (list): List of suffix patterns to match
+        all_files (list): List of all files in las_dir
+        las_dir (str): Directory of pointclouds
 
     Returns:
-        li(List): Pruned list of filenames with only existing files
+        las_list(List): List of matched files
     """
-    li = []
-    for i in list_las:
-        if not os.path.exists(i):
-            logging.info(f"NOK : {i}")
-            pass
+    las_list = []
+    for suffix in suffix_list:
+        matches = [filename for filename in all_files if filename.endswith(suffix)]
+        if len(matches) == 0:
+            logging.info(f"NOK : {suffix}")
         else:
-            li.append(i)
-    return li
+            # in case of multiple matches, select the most recent year (ex: Semis_2021_ before Semis_2020_ )
+            matches.sort(reverse=True)
+            selected = matches[0]
+            if len(matches) > 1:
+                logging.warning(f"Multiple matches for {suffix} : {matches} ; taking {selected}")
 
+            # Append full path
+            las_list.append(os.path.join(las_dir, selected))
+    return las_list
 
-def create_list(las_dir, input_file, tile_width=1000, tile_coord_scale=1000):
+
+def create_tiles_list(all_files, las_dir, input_file, tile_width=1000, tile_coord_scale=1000):
     """Return the paths of 8 tiles around the tile + the input tile
     Args:
+        all_files (list): list of all files in las_dir
         las_dir (str): directory of pointclouds
         input_file (str): path to queried LIDAR tile
         tile_width (int): Width of a tile(in the reference unit: 1m)
         tile_coord_scale (int): Scale used in filename to describe coordinates (usually kilometers)
         1000 * 1m (with 1m being the reference)
 
     Returns:
-        list_files(li): list of tiles
+        list_files: list of tiles
     """
 
-    # Return list 8 tiles around the tile
-    list_input = create_filenames(os.path.basename(input_file), tile_width, tile_coord_scale)
-    # List pointclouds
-    li = [os.path.join(las_dir, e) for e in list_input]
-    # Keep only existing files
-    li = check_tiles_exist(li)
+    # Return list 8 tiles around the tile, but only the suffix part of the name.
+    suffix_list = create_filenames_suffixes(os.path.basename(input_file), tile_width, tile_coord_scale)
+
+    # Match suffix patterns with real files
+    list_files = match_suffix_with_filenames(suffix_list, all_files, las_dir)
+
     # Appending queried tile to list
-    li.append(input_file)
+    list_files.append(input_file)
+
+    return list_files
+
+
+def create_list(las_dir, input_file, tile_width=1000, tile_coord_scale=1000):
+    """Return the paths of 8 tiles around the tile + the input tile
+    Args:
+        las_dir (str): directory of pointclouds
+        input_file (str): path to queried LIDAR tile
+        tile_width (int): Width of a tile(in the reference unit: 1m)
+        tile_coord_scale (int): Scale used in filename to describe coordinates (usually kilometers)
+        1000 * 1m (with 1m being the reference)
+
+    Returns:
+        list_files: list of tiles
+    """
+
+    # list files on the disk
+    all_files = os.listdir(las_dir)
 
-    return li
+    # call the function with the list of files
+    return create_tiles_list(all_files, las_dir, input_file, tile_width, tile_coord_scale)
 
 
 def las_merge(las_dir, input_file, merge_file, tile_width=1000, tile_coord_scale=1000):

test/test_las_merge.py

@@ -1,11 +1,12 @@
 import logging
 import os
+from pathlib import Path
 from test.utils import assert_header_info_are_similar
 
 import laspy
 import numpy as np
 
-from pdaltools.las_merge import las_merge
+from pdaltools.las_merge import create_tiles_list, las_merge
 
 TEST_PATH = os.path.dirname(os.path.abspath(__file__))
 TMP_PATH = os.path.join(TEST_PATH, "tmp")
@@ -74,6 +75,25 @@ def test_las_merge():
     assert_header_info_are_similar(output_file, input_file)
 
 
+def test_create_tiles_list_with_different_prefix():
+    files = [f"Semis_2021_0770_{6200 + i}_LA93_IGN69.las" for i in range(1, 10)]
+
+    # add another tile with different year in prefix
+    files.append("Semis_2018_0770_6200_LA93_IGN69.las")
+    files.append("Semis_2020_0770_6200_LA93_IGN69.las")
+    files.append("Semis_2019_0770_6200_LA93_IGN69.las")
+
+    result = create_tiles_list(files, "las_dir", "las_dir/Semis_2021_0770_6201_LA93_IGN69.las")
+    result_path = [Path(p) for p in result]
+
+    expected_path = [
+        Path("las_dir") / Path("Semis_2021_0770_6202_LA93_IGN69.las"),
+        Path("las_dir") / Path("Semis_2020_0770_6200_LA93_IGN69.las"),
+        Path("las_dir") / Path("Semis_2021_0770_6201_LA93_IGN69.las"),
+    ]
+    assert result_path == expected_path
+
+
 if __name__ == "__main__":
     logging.basicConfig(level=logging.INFO)
     test_las_merge()