Merge pull request #107 from IGNF/dev

dev dans master pour v1.10.0

Author: gliegard

.github/workflows/cicd_deploy.yml

@@ -27,8 +27,12 @@ jobs:
       id: build
       uses: docker/build-push-action@v5
       with:
+        file: ./Dockerfile.pdal
         load: true
         tags: ${{ env.TEST_TAG }}
+        build-args: |
+          GITHUB_REPOSITORY=alavenant/PDAL
+          GITHUB_SHA=master_07_05_25
 
     # run the test on the docker image
     - name: Run tests in docker image
@@ -64,11 +68,16 @@ jobs:
       uses: docker/build-push-action@v5
       with:
         context: .
+        file: ./Dockerfile.pdal
         push: true
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
+        build-args: |
+          GITHUB_REPOSITORY=alavenant/PDAL
+          GITHUB_SHA=master_07_05_25
 
   deploy-pypi:
+
     runs-on: ubuntu-latest
 
     environment:
@@ -97,11 +106,11 @@ jobs:
 
       - name: Run tests with pytest
         shell: micromamba-shell {0}
-        run: python -m pytest ./test -s --log-cli-level DEBUG
+        run: python -m pytest ./test -s -m "not pdal_custom" --log-cli-level DEBUG
 
       - name: Build pip package
         shell: micromamba-shell {0}
         run: make build
 
       - name: pypi-publish
-        uses: pypa/gh-action-pypi-publish@v1.12.4
+        uses: pypa/gh-action-pypi-publish@v1.12.4

.github/workflows/cicd_full.yml

@@ -32,8 +32,13 @@ jobs:
       id: build
       uses: docker/build-push-action@v5
       with:
+        context: .
+        file: ./Dockerfile.pdal
         load: true
         tags: ${{ env.TEST_TAG }}
+        build-args: |
+          GITHUB_REPOSITORY=alavenant/PDAL
+          GITHUB_SHA=master_07_05_25
 
     # run the test on the docker image
     - name: Run tests in docker image
@@ -66,4 +71,4 @@ jobs:
 
       - name: Run tests with pytest
         shell: micromamba-shell {0}
-        run: python -m pytest ./test -s --log-cli-level DEBUG
+        run: python -m pytest ./test -s -m "not pdal_custom" --log-cli-level DEBUG

.github/workflows/cicd_light.yml

@@ -34,5 +34,5 @@ jobs:
 
       - name: Run tests with pytest
         shell: micromamba-shell {0}
-        run: python -m pytest ./test -s --log-cli-level DEBUG -m "not geopf"
+        run: python -m pytest ./test -s --log-cli-level DEBUG -m "not geopf and not pdal_custom"
 

CHANGELOG.md

@@ -1,3 +1,13 @@
+# 1.10.0
+- custom PDAL: fix CI for cicd_full (build docker image with custom PDAL, and skip custom PDAL test for local pytest)
+- las_rename_dimension: new tool to rename one or many dimensions
+
+# 1.9.1
+- las_add_points_to_pointcloud: Fix add points to LAS (use PDAL instead of Laspy)
+
+# 1.9.0
+- custom PDAL: in the docker image, compile custom PDAL (waiting for PDAL 2.9)
+
 # 1.8.1
 - add_points_in_pointcloud: fix case when there is no points to add in the las file extent (copy input file to the output)
 - color: temporarily disable tests on no_data values in downloaded images

Dockerfile.pdal

@@ -0,0 +1,100 @@
+# code from https://github.com/PDAL/PDAL/blob/master/scripts/docker/ubuntu/Dockerfile
+FROM condaforge/mambaforge:latest AS mamba_pdal
+
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
+
+RUN conda create -n pdaltools -y
+ARG GITHUB_SHA
+ARG GITHUB_REPOSITORY="PDAL/PDAL"
+ARG GITHUB_SERVER_URL="https://github.com"
+
+SHELL ["conda", "run", "-n", "pdaltools", "/bin/bash", "-c"]
+
+RUN mamba install -c conda-forge git compilers conda-pack cmake make ninja sysroot_linux-64=2.17 && \
+    mamba install --yes -c conda-forge pdal --only-deps
+
+RUN git clone "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" pdal && \
+    cd pdal ; \
+    git checkout ${GITHUB_SHA}
+
+RUN mkdir -p pdal/build && \
+    cd pdal/build  && \
+    CXXFLAGS="-Werror=strict-aliasing" LDFLAGS="-Wl,-rpath-link,$CONDA_PREFIX/lib" cmake -G Ninja  \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_LIBRARY_PATH:FILEPATH="$CONDA_PREFIX/lib" \
+        -DCMAKE_INCLUDE_PATH:FILEPATH="$CONDA_PREFIX/include" \
+        -DCMAKE_INSTALL_PREFIX="$CONDA_PREFIX" \
+        -DBUILD_PLUGIN_CPD=OFF \
+        -DBUILD_PLUGIN_PGPOINTCLOUD=ON \
+        -DBUILD_PLUGIN_NITF=ON \
+        -DBUILD_PLUGIN_ICEBRIDGE=ON \
+        -DBUILD_PLUGIN_HDF=ON \
+        -DBUILD_PLUGIN_TILEDB=ON \
+        -DBUILD_PLUGIN_E57=ON \
+        -DBUILD_PGPOINTCLOUD_TESTS=OFF \
+        -DWITH_ZSTD=ON \
+        ..
+
+RUN cd pdal/build  && \
+    ninja
+
+RUN cd pdal/build  && \
+    ctest -V
+
+RUN cd pdal/build  && \
+    ninja install
+
+RUN git clone https://github.com/PDAL/python.git
+
+RUN mamba install --yes -c conda-forge pybind11
+
+RUN mkdir -p python/build && \
+    cd python/build  && \
+    CXXFLAGS="-Werror=strict-aliasing" LDFLAGS="-Wl,-rpath-link,$CONDA_PREFIX/lib" cmake -G Ninja  \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_LIBRARY_PATH:FILEPATH="$CONDA_PREFIX/lib" \
+        -DCMAKE_INCLUDE_PATH:FILEPATH="$CONDA_PREFIX/include" \
+        -DCMAKE_INSTALL_PREFIX="$CONDA_PREFIX/lib/python3.13/site-packages/" \
+        ..
+
+RUN cd python/build  && ninja
+
+RUN cd python/build  && ctest -V
+
+RUN cd python/build  && ninja install
+
+
+RUN git clone https://github.com/PDAL/python-plugins.git pdal-python && \
+    cd pdal-python && git checkout 1.6.2  && \
+    pip install -vv . --no-deps
+
+RUN conda-pack -n pdaltools --dest-prefix=/opt/conda/envs/pdaltools -o  /tmp/env.tar && \
+     mkdir /venv && cd /venv && tar xf /tmp/env.tar  && \
+     rm /tmp/env.tar
+
+
+# Add our environment
+RUN mamba install numpy requests gdal lastools geopandas pytest requests-mock tqdm pip
+# Install laspy with laszip laz compression (as laszip is an implementation of the laz 1.4 standard)
+RUN pip install laspy[lazrs]
+
+
+FROM debian:bullseye-slim
+
+# install PDAL
+COPY --from=mamba_pdal /opt/conda/envs/pdaltools/bin/pdal /opt/conda/envs/pdaltools/bin/pdal
+COPY --from=mamba_pdal /opt/conda/envs/pdaltools/bin/python /opt/conda/envs/pdaltools/bin/python
+COPY --from=mamba_pdal /opt/conda/envs/pdaltools/bin/las2las /opt/conda/envs/pdaltools/bin/las2las
+COPY --from=mamba_pdal /opt/conda/envs/pdaltools/bin/lasinfo /opt/conda/envs/pdaltools/bin/lasinfo
+COPY --from=mamba_pdal /opt/conda/envs/pdaltools/lib/ /opt/conda/envs/pdaltools/lib/
+COPY --from=mamba_pdal /opt/conda/envs/pdaltools/ssl /opt/conda/envs/pdaltools/ssl
+COPY --from=mamba_pdal /opt/conda/envs/pdaltools/share/proj/proj.db /opt/conda/envs/pdaltools/share/proj/proj.db
+
+ENV PATH=$PATH:/opt/conda/envs/pdaltools/bin/
+ENV PROJ_LIB=/opt/conda/envs/pdaltools/share/proj/
+ENV GDAL_DATA=/opt/conda/envs/pdaltools/share/gdal
+
+WORKDIR /pdaltools
+RUN mkdir tmp
+COPY pdaltools pdaltools
+COPY test test

Makefile

@@ -19,7 +19,7 @@ install:
 ##############################
 
 testing:
-	python -m pytest ./test -s --log-cli-level DEBUG -m "not geopf"
+	python -m pytest ./test -s --log-cli-level DEBUG -m "not geopf and not pdal_custom"
 
 testing_full:
 	python -m pytest ./test -s --log-cli-level DEBUG
@@ -62,6 +62,12 @@ FULL_IMAGE_NAME=${REGISTRY}/${NAMESPACE}/${IMAGE_NAME}:${VERSION}
 docker-build: clean
 	docker build --no-cache -t ${IMAGE_NAME}:${VERSION} -f Dockerfile .
 
+docker-build-pdal: clean
+	docker build --build-arg GITHUB_REPOSITORY=alavenant/PDAL --build-arg GITHUB_SHA=master_07_05_25 -t ${IMAGE_NAME}:${VERSION} -f Dockerfile.pdal .
+
+docker-test-pdal: clean
+	docker run --rm  -t ${IMAGE_NAME}:${VERSION} python -m pytest -m "not geopf" --log-cli-level=debug
+
 docker-test:
 	docker run --rm -it ${IMAGE_NAME}:${VERSION} python -m pytest -s
 

pdaltools/_version.py

@@ -1,4 +1,4 @@
-__version__ = "1.8.1"
+__version__ = "1.10.0"
 
 
 if __name__ == "__main__":

pdaltools/add_points_in_pointcloud.py

@@ -1,5 +1,6 @@
 import argparse
 from shutil import copy2
+import tempfile
 
 import geopandas as gpd
 import laspy
@@ -10,6 +11,8 @@
 
 from pdaltools.las_info import get_epsg_from_las, get_tile_bbox
 
+import pdal
+
 
 def parse_args(argv=None):
     parser = argparse.ArgumentParser("Add points from GeoJSON in LIDAR tile")
@@ -127,13 +130,12 @@ def add_points_to_las(
         crs (str): CRS of the data.
         virtual_points_classes (int): The classification value to assign to those virtual points (default: 66).
     """
-    # Copy data pointcloud
-    copy2(input_las, output_las)
 
     if input_points_with_z.empty:
         print(
             "No points to add. All points of the geojson file are outside the tile. Copying the input file to output"
         )
+        copy2(input_las, output_las)
         return
 
     # Extract XYZ coordinates and additional attribute (classification)
@@ -148,24 +150,30 @@ def add_points_to_las(
         header = las.header
         if not header:
             header = laspy.LasHeader(point_format=8, version="1.4")
+
+    new_points = laspy.ScaleAwarePointRecord.zeros(nb_points, header=header)  # use header for input_las
+    # then fill in the gaps (X, Y, Z an classification)
+    new_points.x = x_coords.astype(new_points.x.dtype)
+    new_points.y = y_coords.astype(new_points.y.dtype)
+    new_points.z = z_coords.astype(new_points.z.dtype)
+    new_points.classification = classes.astype(new_points.classification.dtype)
+
+    with tempfile.NamedTemporaryFile(suffix="_new_points.las") as tmp:
+        with laspy.open(tmp.name, mode="w", header=header) as las_file:
+            las_file.write_points(new_points)
+
         if crs:
-            try:
-                crs_obj = CRS.from_user_input(crs)  # Convert to a pyproj.CRS object
-            except CRSError:
-                raise ValueError(f"Invalid CRS: {crs}")
-            header.add_crs(crs_obj)
-
-    # Add the new points with 3D points
-    with laspy.open(output_las, mode="a", header=header) as output_las:  # mode `a` for adding points
-        # create nb_points points with "0" everywhere
-        new_points = laspy.ScaleAwarePointRecord.zeros(nb_points, header=header)  # use header for input_las
-        # then fill in the gaps (X, Y, Z an classification)
-        new_points.x = x_coords.astype(new_points.x.dtype)
-        new_points.y = y_coords.astype(new_points.y.dtype)
-        new_points.z = z_coords.astype(new_points.z.dtype)
-        new_points.classification = classes.astype(new_points.classification.dtype)
-
-        output_las.append_points(new_points)
+            a_srs = crs
+        else:
+            a_srs = get_epsg_from_las(input_las)
+
+        # Use pdal to merge the new points with the existing points
+        pipeline = pdal.Pipeline()
+        pipeline |= pdal.Reader.las(filename=input_las)
+        pipeline |= pdal.Reader.las(filename=tmp.name)
+        pipeline |= pdal.Filter.merge()
+        pipeline |= pdal.Writer.las(filename=output_las, forward="all", a_srs=a_srs)
+        pipeline.execute()
 
 
 def line_to_multipoint(line, spacing: float, z_value: float = None):

pdaltools/count_occurences/count_occurences_for_attribute.py

@@ -1,5 +1,5 @@
 """Count occurences of each value of a given attribute in a set of pointclouds.
-Eg. to count points of each class in classified point clouds """
+Eg. to count points of each class in classified point clouds"""
 
 import argparse
 import json

pdaltools/las_add_buffer.py

@@ -158,7 +158,6 @@ def remove_points_from_buffer(input_file: str, output_file: str):
         pipeline |= pdal.Filter.range(limits=f"{ORIGINAL_TILE_TAG}[1:1]")
         pipeline |= pdal.Writer.las(filename=tmp_las.name, forward="all", extra_dims="all")
         pipeline.execute()
-
         remove_dimensions_from_las(tmp_las.name, dimensions=[ORIGINAL_TILE_TAG], output_las=output_file)