Merge pull request #21 from IGNF/fix_different_las

Fix different las

Author: alavenant

.gitignore

@@ -53,6 +53,8 @@ coverage.xml
 .hypothesis/
 .pytest_cache/
 cover/
+.idea/
+.vscode/
 
 # Translations
 *.mo

CHANGELOG.md

@@ -1,5 +1,8 @@
 # CHANGELOG
 
+## dev
+- fix lorsque les las donneurs ne sont pas de même version (1.2 et 1.4): on ne garde que les attributs communs
+
 ## 1.4.0
 - **Changement de comportement** :
     Le champ "DONOR_CLASS_TRANSLATION" décrit maintenant l'association entre les classes du fichier donneur et les classes correspondantes dans le fichier de sortie. Au lieu de choisir entre ajouter une colonne et modifier les classes entre le fichier de donneur et le fichier de sortie, on applique maintenant les 2 traitements :

patchwork/patchwork.py

@@ -95,6 +95,22 @@ def get_type(new_column_size: int):
             raise ValueError(f"{new_column_size} is not a correct value for NEW_COLUMN_SIZE")
 
 
+def get_common_las_columns(las_files: List[str]) -> List[str]:
+    """Return the columns common to all las files (lowercase)."""
+    with laspy.open(las_files[0]) as las_file:
+        common_columns = set(get_field_from_header(las_file))
+    for las_file_path in las_files[1:]:
+        with laspy.open(las_file_path) as las_file:
+            common_columns = common_columns & set(get_field_from_header(las_file))
+    return list(common_columns)
+
+
+def get_common_donor_columns(df_donor_info: gpd.GeoDataFrame) -> List[str]:
+    """Return the columns common to all donor files (lowercase)."""
+    las_files = [row["full_path"] for _, row in df_donor_info.iterrows()]
+    return get_common_las_columns(las_files)
+
+
 def get_complementary_points(
     df_donor_info: gpd.GeoDataFrame, recipient_file_path: str, tile_origin: Tuple[int, int], config: DictConfig
 ) -> pd.DataFrame:
@@ -120,31 +136,31 @@ def get_complementary_points(
     )
 
     dfs_donor_points = []
-
-    for index, row in df_donor_info.iterrows():
-        with laspy.open(row["full_path"]) as donor_file:
-            raw_donor_points = donor_file.read().points
-            points_loc_gdf = gpd.GeoDataFrame(
-                geometry=gpd.points_from_xy(raw_donor_points.x, raw_donor_points.y, raw_donor_points.z, crs=config.CRS)
-            )
-            footprint_gdf = gpd.GeoDataFrame(geometry=[row["geometry"]], crs=config.CRS)
-            points_in_footprint_gdf = points_loc_gdf.sjoin(footprint_gdf, how="inner", predicate="intersects")
-            donor_points = raw_donor_points[points_in_footprint_gdf.index.values]
-
-            donor_columns = get_field_from_header(donor_file)
-            dfs_donor_points.append(
-                get_selected_classes_points(
-                    tile_origin,
-                    donor_points,
-                    config.DONOR_CLASS_LIST,
-                    config.DONOR_USE_SYNTHETIC_POINTS,
-                    donor_columns,
-                    patch_size=config.PATCH_SIZE,
-                    tile_size=config.TILE_SIZE,
+    
+    if len(df_donor_info.index):
+        donor_common_columns = get_common_donor_columns(df_donor_info)
+        for index, row in df_donor_info.iterrows():
+            with laspy.open(row["full_path"]) as donor_file:
+                raw_donor_points = donor_file.read().points
+                points_loc_gdf = gpd.GeoDataFrame(
+                    geometry=gpd.points_from_xy(raw_donor_points.x, raw_donor_points.y, raw_donor_points.z, crs=config.CRS)
+                )
+                footprint_gdf = gpd.GeoDataFrame(geometry=[row["geometry"]], crs=config.CRS)
+                points_in_footprint_gdf = points_loc_gdf.sjoin(footprint_gdf, how="inner", predicate="intersects")
+                donor_points = raw_donor_points[points_in_footprint_gdf.index.values]
+
+                dfs_donor_points.append(
+                    get_selected_classes_points(
+                        tile_origin,
+                        donor_points,
+                        config.DONOR_CLASS_LIST,
+                        config.DONOR_USE_SYNTHETIC_POINTS,
+                        donor_common_columns,
+                        patch_size=config.PATCH_SIZE,
+                        tile_size=config.TILE_SIZE,
+                    )
                 )
-            )
 
-    if len(df_donor_info.index):
         df_donor_points = pd.concat(dfs_donor_points)
 
     else:

test/data/Semis_2021_0963_6543_LA93_IGN69_decimate.laz

@@ -0,0 +1,9 @@
+{
+"type": "FeatureCollection",
+"name": "zones",
+"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::2154" } },
+"features": [
+{ "type": "Feature", "properties": { "NAME": "Object", "ALTITUDE": 0.0, "x": "0963", "Format_nua": "fbi", "Code": "QK", "id": 213, "y": 6543, "Nuage_LHD": "./test/data/grand_geneve/grand_geneve_BD/data", "nom_coord": "oui", "nuage_mixa": "./test/data/grand_geneve/grand_geneve_BD/data", "Chant_DGPR": "MC-Est", "attributio": 2 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 963451.432107000029646, 6542722.056231999769807, 1.0 ], [ 963439.130614000023343, 6542718.448168000206351, 1.0 ], [ 963380.6346, 6542807.320198000408709, 1.0 ], [ 963350.571229, 6542900.249169999733567, 1.0 ], [ 963349.298465, 6543000.002213999629021, 1.0 ], [ 963409.089970999979414, 6543000.002213999629021, 1.0 ], [ 963401.601116999983788, 6542938.244648000225425, 1.0 ], [ 963401.538614999968559, 6542936.540051000192761, 1.0 ], [ 963401.771577000035904, 6542934.85249900072813, 1.0 ], [ 963402.282954999944195, 6542933.233131000772119, 1.0 ], [ 963448.545659999945201, 6542820.928568, 1.0 ], [ 963449.875244, 6542818.627360999584198, 1.0 ], [ 963513.411176000023261, 6542736.283939000219107, 1.0 ], [ 963511.070198999950662, 6542735.244135000742972, 1.0 ], [ 963500.882404000032693, 6542733.880456999875605, 1.0 ], [ 963500.780127999954857, 6542733.857729, 1.0 ], [ 963451.432107000029646, 6542722.056231999769807, 1.0 ] ] ] ] } },
+{ "type": "Feature", "properties": { "NAME": "Object", "ALTITUDE": 0.0, "x": "0963", "Format_nua": "laz", "Code": "QK", "id": 213, "y": 6543, "Nuage_LHD": "./test/data/grand_geneve/grand_geneve_BD/data2", "nom_coord": "oui", "nuage_mixa": "./test/data/grand_geneve/grand_geneve_BD/data2", "Chant_DGPR": "MC-Est", "attributio": 2 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 963999.999143, 6542545.442888, 1.0 ], [ 963999.999143, 6542259.206880999729037, 1.0 ], [ 963985.385082999942824, 6542263.888841999694705, 1.0 ], [ 963978.788301000022329, 6542266.985527000389993, 1.0 ], [ 963959.759339999989606, 6542278.679065999574959, 1.0 ], [ 963943.798650000011548, 6542294.304543000645936, 1.0 ], [ 963938.497360000037588, 6542300.804740999825299, 1.0 ], [ 963930.548266, 6542312.060766000300646, 1.0 ], [ 963924.349223, 6542322.259940999560058, 1.0 ], [ 963916.502405000035651, 6542338.203609, 1.0 ], [ 963911.604536000057124, 6542350.902861, 1.0 ], [ 963909.257877000025474, 6542358.800828999839723, 1.0 ], [ 963903.502027999958955, 6542362.397529999725521, 1.0 ], [ 963902.286084000021219, 6542363.170280000194907, 1.0 ], [ 963874.387547999969684, 6542381.170830000191927, 1.0 ], [ 963871.927248999942094, 6542382.807243000715971, 1.0 ], [ 963853.028974000015296, 6542395.807640000246465, 1.0 ], [ 963847.898143999977037, 6542399.796397998929024, 1.0 ], [ 963847.312899000011384, 6542399.665711999870837, 1.0 ], [ 963836.812595000024885, 6542397.665651000104845, 1.0 ], [ 963819.516639000037685, 6542395.909916000440717, 1.0 ], [ 963802.175227000028826, 6542397.177000000141561, 1.0 ], [ 963799.078546999953687, 6542397.677015, 1.0 ], [ 963794.396593000041321, 6542398.54636, 1.0 ], [ 963788.69756400003098, 6542399.745260000228882, 1.0 ], [ 963768.14015, 6542406.467055999673903, 1.0 ], [ 963749.503246000036597, 6542417.45034599956125, 1.0 ], [ 963746.048600999987684, 6542420.024287999607623, 1.0 ], [ 963740.11092899995856, 6542420.4106639996171, 1.0 ], [ 963717.439816999947652, 6542424.547152999788523, 1.0 ], [ 963709.235034000012092, 6542427.047229999676347, 1.0 ], [ 963708.655471, 6542427.229053, 1.0 ], [ 963688.757166999974288, 6542433.428105999715626, 1.0 ], [ 963673.393086, 6542439.649887000210583, 1.0 ], [ 963654.989142999984324, 6542448.951306998729706, 1.0 ], [ 963639.556878000032157, 6542458.610692999325693, 1.0 ], [ 963628.187230000039563, 6542467.258684, 1.0 ], [ 963608.771895000012591, 6542478.543119000270963, 1.0 ], [ 963578.003956999978982, 6542493.748129, 1.0 ], [ 963573.009494, 6542496.390255000442266, 1.0 ], [ 963555.7135390000185, 6542506.191689999774098, 1.0 ], [ 963546.258719000034034, 6542512.271421001292765, 1.0 ], [ 963527.661589000024833, 6542525.771832999773324, 1.0 ], [ 963518.848833999945782, 6542532.965234998613596, 1.0 ], [ 963502.149485999951139, 6542548.266838001087308, 1.0 ], [ 963497.04706500004977, 6542553.284036999568343, 1.0 ], [ 963477.847644999972545, 6542573.585792999714613, 1.0 ], [ 963474.500956999952905, 6542577.307497001253068, 1.0 ], [ 963467.097334000049159, 6542585.961170000955462, 1.0 ], [ 963460.949428000021726, 6542589.648783000186086, 1.0 ], [ 963455.994738999987021, 6542592.825016000308096, 1.0 ], [ 963437.698754000011832, 6542605.325396999716759, 1.0 ], [ 963428.8178149999585, 6542612.149468999356031, 1.0 ], [ 963413.794652999960817, 6542625.098728000186384, 1.0 ], [ 963393.475882000057027, 6542642.445848000235856, 1.0 ], [ 963378.55499500001315, 6542658.292923000641167, 1.0 ], [ 963364.452312999987043, 6542676.998039000667632, 1.0 ], [ 963357.935078999958932, 6542686.788111000321805, 1.0 ], [ 963345.735862000030465, 6542707.686475999653339, 1.0 ], [ 963344.099450999987312, 6542710.607019000686705, 1.0 ], [ 963333.90029100002721, 6542729.505323000252247, 1.0 ], [ 963329.604712, 6542738.516962001100183, 1.0 ], [ 963318.303247999981977, 6542765.614380000159144, 1.0 ], [ 963313.024685999960639, 6542782.205795000307262, 1.0 ], [ 963311.433731000055559, 6542789.30260300077498, 1.0 ], [ 963305.388100999989547, 6542798.814256999641657, 1.0 ], [ 963297.814018000033684, 6542809.820274000987411, 1.0 ], [ 963289.643326999968849, 6542824.076391000300646, 1.0 ], [ 963284.44431299995631, 6542835.179003000259399, 1.0 ], [ 963276.915685000014491, 6542858.134248999878764, 1.0 ], [ 963274.313336999970488, 6542871.231239000335336, 1.0 ], [ 963273.068983, 6542879.140572000294924, 1.0 ], [ 963270.370040999958292, 6542902.34014300070703, 1.0 ], [ 963269.756386999972165, 6542910.579031, 1.0 ], [ 963268.756357999984175, 6542940.682222000323236, 1.0 ], [ 963268.722266, 6542941.767483000643551, 1.0 ], [ 963268.125658, 6542968.671712999232113, 1.0 ], [ 963268.108612000010908, 6542972.046815999783576, 1.0 ], [ 963268.307480999967083, 6542989.450756000354886, 1.0 ], [ 963269.409786000032909, 6543000.002213999629021, 1.0 ], [ 963349.298465, 6543000.002213999629021, 1.0 ], [ 963350.571229, 6542900.249169999733567, 1.0 ], [ 963380.6346, 6542807.320198000408709, 1.0 ], [ 963439.130614000023343, 6542718.448168000206351, 1.0 ], [ 963451.432107000029646, 6542722.056231999769807, 1.0 ], [ 963500.780127999954857, 6542733.857729, 1.0 ], [ 963500.882404000032693, 6542733.880456999875605, 1.0 ], [ 963511.070198999950662, 6542735.244135000742972, 1.0 ], [ 963513.411176000023261, 6542736.283939000219107, 1.0 ], [ 963449.875244, 6542818.627360999584198, 1.0 ], [ 963448.545659999945201, 6542820.928568, 1.0 ], [ 963402.282954999944195, 6542933.233131000772119, 1.0 ], [ 963401.771577000035904, 6542934.85249900072813, 1.0 ], [ 963401.538614999968559, 6542936.540051000192761, 1.0 ], [ 963401.601116999983788, 6542938.244648000225425, 1.0 ], [ 963409.089970999979414, 6543000.002213999629021, 1.0 ], [ 963486.950181999942288, 6543000.002213999629021, 1.0 ], [ 963486.660400999942794, 6542989.814403999596834, 1.0 ], [ 963485.20013100001961, 6542975.433283000253141, 1.0 ], [ 963482.495507000014186, 6542960.029404000379145, 1.0 ], [ 963479.648834, 6542948.131312999874353, 1.0 ], [ 963479.398827000055462, 6542947.307424999773502, 1.0 ], [ 963479.398827000055462, 6542944.926669999025762, 1.0 ], [ 963479.853385000023991, 6542936.648008000105619, 1.0 ], [ 963479.893158999970183, 6542935.772982000373304, 1.0 ], [ 963480.495449000038207, 6542922.675990998744965, 1.0 ], [ 963480.50681299995631, 6542922.414619000628591, 1.0 ], [ 963480.688637000042945, 6542918.198582001030445, 1.0 ], [ 963482.489825000055134, 6542915.243946000933647, 1.0 ], [ 963483.671678000013344, 6542913.255248999223113, 1.0 ], [ 963492.171924000023864, 6542898.555936999619007, 1.0 ], [ 963497.450486000045203, 6542888.038569998927414, 1.0 ], [ 963502.251762000028975, 6542876.890502998605371, 1.0 ], [ 963509.013321000034921, 6542864.282164, 1.0 ], [ 963512.041818000026979, 6542858.145612999796867, 1.0 ], [ 963519.945456000044942, 6542840.645079, 1.0 ], [ 963521.53073, 6542836.934737998992205, 1.0 ], [ 963527.92296, 6542821.099027999676764, 1.0 ], [ 963530.729859999963082, 6542814.252228000201285, 1.0 ], [ 963532.894695000024512, 6542810.8941710004583, 1.0 ], [ 963541.900638, 6542798.547203000634909, 1.0 ], [ 963550.895217000041157, 6542788.183249999769032, 1.0 ], [ 963559.543195000034757, 6542779.381844999268651, 1.0 ], [ 963565.549051000038162, 6542775.086260000243783, 1.0 ], [ 963582.754094999982044, 6542764.034785998985171, 1.0 ], [ 963587.049674, 6542761.108560999855399, 1.0 ], [ 963600.436425999971107, 6542751.494630999863148, 1.0 ], [ 963610.823089999961667, 6542744.073950001038611, 1.0 ], [ 963621.016567999962717, 6542735.727103999815881, 1.0 ], [ 963633.414653999963775, 6542724.124477, 1.0 ], [ 963641.267155, 6542715.896953999064863, 1.0 ], [ 963652.665212000021711, 6542702.498816999606788, 1.0 ], [ 963654.210712000029162, 6542700.640805999748409, 1.0 ], [ 963663.824627000023611, 6542688.765444000251591, 1.0 ], [ 963668.415669, 6542683.168681999668479, 1.0 ], [ 963677.950036, 6542678.986735999584198, 1.0 ], [ 963681.563776999944821, 6542677.316231000237167, 1.0 ], [ 963711.763516, 6542662.611236999742687, 1.0 ], [ 963720.831960999988951, 6542657.605402000248432, 1.0 ], [ 963738.832482, 6542646.406196999363601, 1.0 ], [ 963741.349600999965332, 6542644.786829001270235, 1.0 ], [ 963753.30449300003238, 6542636.843405, 1.0 ], [ 963763.390011999988928, 6542631.405739, 1.0 ], [ 963773.270981000037864, 6542627.149926998652518, 1.0 ], [ 963778.918871999951079, 6542626.916965001262724, 1.0 ], [ 963784.953136999974959, 6542626.485133999027312, 1.0 ], [ 963799.652427, 6542624.985088, 1.0 ], [ 963807.28333, 6542623.905509999021888, 1.0 ], [ 963815.584706999943592, 6542622.405464, 1.0 ], [ 963835.443237000028603, 6542616.643925, 1.0 ], [ 963836.886460999958217, 6542615.876856000162661, 1.0 ], [ 963841.369545000023209, 6542616.490511000156403, 1.0 ], [ 963855.364269000012428, 6542617.592817000113428, 1.0 ], [ 963868.870342000038363, 6542617.740549, 1.0 ], [ 963877.671733999974094, 6542617.240534001030028, 1.0 ], [ 963894.905187999946065, 6542614.740458000451326, 1.0 ], [ 963906.803259999956936, 6542611.939236000180244, 1.0 ], [ 963925.19583900005091, 6542605.677681, 1.0 ], [ 963942.025872000027448, 6542595.972839000634849, 1.0 ], [ 963947.179430000018328, 6542592.290908999741077, 1.0 ], [ 963954.622828, 6542587.648722000420094, 1.0 ], [ 963969.532351, 6542576.279056999832392, 1.0 ], [ 963982.05544100003317, 6542562.324085, 1.0 ], [ 963987.532873000018299, 6542554.931814000941813, 1.0 ], [ 963988.578357000020333, 6542553.744277999736369, 1.0 ], [ 963995.033090000040829, 6542548.352067999541759, 1.0 ], [ 963999.027524000033736, 6542545.584937999024987, 1.0 ], [ 963999.999143, 6542545.442888, 1.0 ] ] ] ] } }
+]
+}

test/data/grand_geneve/geometry_GrandGeneve/zones.geojson

@@ -0,0 +1,27 @@
+<!DOCTYPE qgis PUBLIC 'http://mrcc.com/qgis.dtd' 'SYSTEM'>
+<qgis version="3.34.3-Prizren">
+  <identifier></identifier>
+  <parentidentifier></parentidentifier>
+  <language></language>
+  <type>dataset</type>
+  <title></title>
+  <abstract></abstract>
+  <links/>
+  <dates/>
+  <fees></fees>
+  <encoding></encoding>
+  <crs>
+    <spatialrefsys nativeFormat="Wkt">
+      <wkt>PROJCRS["RGF93 / Lambert-93",BASEGEOGCRS["RGF93",DATUM["Reseau Geodesique Francais 1993",ELLIPSOID["GRS 1980",6378137,298.257222101,LENGTHUNIT["metre",1]]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433]],ID["EPSG",4171]],CONVERSION["Lambert-93",METHOD["Lambert Conic Conformal (2SP)",ID["EPSG",9802]],PARAMETER["Latitude of false origin",46.5,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8821]],PARAMETER["Longitude of false origin",3,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8822]],PARAMETER["Latitude of 1st standard parallel",49,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8823]],PARAMETER["Latitude of 2nd standard parallel",44,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8824]],PARAMETER["Easting at false origin",700000,LENGTHUNIT["metre",1],ID["EPSG",8826]],PARAMETER["Northing at false origin",6600000,LENGTHUNIT["metre",1],ID["EPSG",8827]]],CS[Cartesian,2],AXIS["easting (X)",east,ORDER[1],LENGTHUNIT["metre",1]],AXIS["northing (Y)",north,ORDER[2],LENGTHUNIT["metre",1]],USAGE[SCOPE["Engineering survey, topographic mapping."],AREA["France - onshore and offshore, mainland and Corsica."],BBOX[41.15,-9.86,51.56,10.38]],ID["EPSG",2154]]</wkt>
+      <proj4>+proj=lcc +lat_0=46.5 +lon_0=3 +lat_1=49 +lat_2=44 +x_0=700000 +y_0=6600000 +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs</proj4>
+      <srsid>145</srsid>
+      <srid>2154</srid>
+      <authid>EPSG:2154</authid>
+      <description>RGF93 / Lambert-93</description>
+      <projectionacronym>lcc</projectionacronym>
+      <ellipsoidacronym>EPSG:7019</ellipsoidacronym>
+      <geographicflag>false</geographicflag>
+    </spatialrefsys>
+  </crs>
+  <extent/>
+</qgis>