From f6166e52b23d76fd3d4e057e8ff6bc1c80e6a45b Mon Sep 17 00:00:00 2001 From: Michael Kalbermatten Date: Fri, 4 Dec 2015 18:55:24 +0100 Subject: [PATCH 1/7] Create a recipe to get laspy from SITN fork --- buildout.cfg | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/buildout.cfg b/buildout.cfg index c2264d7..28b5348 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -15,7 +15,7 @@ parts = modwsgi template modwsgi-patch - liblas + laspy develop = . @@ -92,8 +92,10 @@ cmds = >>> print line >>> fileinput.close() -[liblas] +[laspy] recipe = collective.recipe.cmd on_install = true on_update = true -cmds = ${buildout:directory}\buildout\bin\pip.exe install ${buildout:directory}\wheels\${vars:liblas} +cmds = ${buildout:directory}\buildout\bin\pip.exe install https://github.com/sitn/laspy/archive/e2a1a3307d4c252f80855c3e80ec373fdfc7265b.zip + + From 37a9866fa856203ae7b0c4d1a85fa366af74d5b7 Mon Sep 17 00:00:00 2001 From: Michael Kalbermatten Date: Fri, 4 Dec 2015 18:56:40 +0100 Subject: [PATCH 2/7] Switch to laspy --- las_extractor/util/point_cloud_profiler.py | 48 +++++++++++++--------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/las_extractor/util/point_cloud_profiler.py b/las_extractor/util/point_cloud_profiler.py index 13d38ca..642e15c 100644 --- a/las_extractor/util/point_cloud_profiler.py +++ b/las_extractor/util/point_cloud_profiler.py @@ -6,7 +6,8 @@ import numpy as np from shapely.geometry import LineString import uuid -from liblas import file + +from laspy import file from datetime import datetime try: @@ -36,7 +37,6 @@ def generate_tile_list(line, bufferSizeMeter, outputDir, fileList, dataDir): for row in intersectResult: checkEmpty += 1 tileList.append(dataDir + str(row.file.strip() + '.las')) - return polygon, checkEmpty, tileList # Read the numpy data and append them to json-serializable list @@ -105,24 +105,32 @@ def pointCloudExtractorV2(coordinates, bufferSizeMeter, outputDir, dataDir, json startIterateTile = datetime.now() for tile in tileList: cloud = file.File(tile, mode = 'r') - # iterate over cloud's points - for p in cloud: - # Needs enhancements... - if p.x <= max(seg['x1'] + bufferSizeMeter, seg['x2'] + bufferSizeMeter) \ - and p.x >= min(seg['x1'] - bufferSizeMeter, seg['x2'] - bufferSizeMeter) \ - and p.y <= max(seg['y1'] + bufferSizeMeter, seg['y2'] + bufferSizeMeter) \ - and p.y >= min(seg['y1'] - bufferSizeMeter, seg['y2'] - bufferSizeMeter): - xOB = p.x - seg['x1'] - yOB = p.y - seg['y1'] - hypo = math.sqrt(xOB * xOB + yOB * yOB) - cosAlpha = (xOA * xOB + yOA * yOB)/(math.sqrt(xOA * xOA + yOA * yOA) * hypo) - alpha = math.acos(cosAlpha) - normalPointToLineDistance = math.sin(alpha) * hypo - # Filter for normal distance smaller or equal to buffer size - if normalPointToLineDistance <= bufferSizeMeter: - exctractedPoints.append({'x': p.x, 'y': p.y, 'z': p.z, 'classification': p.classification}) - lineList = [p.x, p.y, p.z, cosAlpha, p.classification] - table.append(lineList) + + + for x, y, z, classification in np.nditer((cloud.x, cloud.y, cloud.z, cloud.classification)): + x = x.item() + y = y.item() + if x <= max(seg['x1'] + bufferSizeMeter, seg['x2'] + bufferSizeMeter) \ + and x >= min(seg['x1'] - bufferSizeMeter, seg['x2'] - bufferSizeMeter) \ + and y <= max(seg['y1'] + bufferSizeMeter, seg['y2'] + bufferSizeMeter) \ + and y >= min(seg['y1'] - bufferSizeMeter, seg['y2'] - bufferSizeMeter): + xOB = x - seg['x1'] + yOB = y - seg['y1'] + hypo = math.sqrt(xOB * xOB + yOB * yOB) + cosAlpha = (xOA * xOB + yOA * yOB)/(math.sqrt(xOA * xOA + yOA * yOA) * hypo) + if cosAlpha > 1: + cosAlpha = 1 + + alpha = math.acos(cosAlpha) + normalPointToLineDistance = math.sin(alpha) * hypo + # Filter for normal distance smaller or equal to buffer size + if normalPointToLineDistance <= bufferSizeMeter: + z = z.item() + classification = classification.item() + exctractedPoints.append({'x': x, 'y': y, 'z': z, 'classification': classification}) + lineList = [x, y, z, cosAlpha, classification] + table.append(lineList) + cloud.close() stopIterateTile = datetime.now() From 1956176b6351de7b307825d92b2dd4b095171622 Mon Sep 17 00:00:00 2001 From: Michael Kalbermatten Date: Fri, 4 Dec 2015 19:03:21 +0100 Subject: [PATCH 3/7] Add Numpy as a local dependency to make laspy happy... --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 2794b85..0baaf52 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ 'numpy', 'pyyaml', 'pip', + 'numpy' ], packages=find_packages(exclude=['ez_setup']), include_package_data=True, From ce16f00f057ce0ec7c86de222b59e9474824e68f Mon Sep 17 00:00:00 2001 From: Michael Kalbermatten Date: Mon, 7 Dec 2015 09:37:47 +0100 Subject: [PATCH 4/7] Get rid of .exe --- buildout.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildout.cfg b/buildout.cfg index 28b5348..4faa944 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -96,6 +96,6 @@ cmds = recipe = collective.recipe.cmd on_install = true on_update = true -cmds = ${buildout:directory}\buildout\bin\pip.exe install https://github.com/sitn/laspy/archive/e2a1a3307d4c252f80855c3e80ec373fdfc7265b.zip +cmds = ${buildout:directory}\buildout\bin\pip install https://github.com/sitn/laspy/archive/e2a1a3307d4c252f80855c3e80ec373fdfc7265b.zip From 69475372d104b45c3916345fcff71fbf3a2c5bcd Mon Sep 17 00:00:00 2001 From: Michael Kalbermatten Date: Mon, 7 Dec 2015 15:56:33 +0100 Subject: [PATCH 5/7] Use laspy and remove old conf --- buildout.cfg | 15 ++------------- development.ini.in | 3 +-- production.ini.in | 4 ++-- setup.py | 6 +----- versions.cfg | 1 + 5 files changed, 7 insertions(+), 22 deletions(-) diff --git a/buildout.cfg b/buildout.cfg index 4faa944..2b4c7df 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -15,7 +15,6 @@ parts = modwsgi template modwsgi-patch - laspy develop = . @@ -48,14 +47,12 @@ dbport = overwriteme db = overwrite_me # LIDAR tool configuration -lidar_fusion_cmd = overwriteme -lidar_lastool_cmd = overwriteme lidar_data = overwriteme lidar_data_normalized = overwriteme -intranet_code = overwriteme +debug_log = False -liblas = libLAS-1.8.0-cp27-none-win32.whl +intranet_code = overwriteme [pyramid] recipe = zc.recipe.egg @@ -91,11 +88,3 @@ cmds = >>> else: >>> print line >>> fileinput.close() - -[laspy] -recipe = collective.recipe.cmd -on_install = true -on_update = true -cmds = ${buildout:directory}\buildout\bin\pip install https://github.com/sitn/laspy/archive/e2a1a3307d4c252f80855c3e80ec373fdfc7265b.zip - - diff --git a/development.ini.in b/development.ini.in index f930e35..28dc4e0 100644 --- a/development.ini.in +++ b/development.ini.in @@ -14,11 +14,10 @@ pyramid.includes = sqlalchemy.url = postgresql://${vars:dbuser}:${vars:dbpassword}@${vars:dbhost}:${vars:dbport}/${vars:db} # LIDAR tool configuration -lidar_fusion_cmd = ${vars:lidar_fusion_cmd} -lidar_lastool_cmd = ${vars:lidar_lastool_cmd} lidar_data = ${vars:lidar_data} lidar_data_normalized = ${vars:lidar_data_normalized} lidar_output_dir = %(here)s/data/ +debug_log = ${vars:debug_log} app.cfg = %(here)s/config.yaml diff --git a/production.ini.in b/production.ini.in index a4de61b..527d195 100644 --- a/production.ini.in +++ b/production.ini.in @@ -13,12 +13,12 @@ pyramid.includes = sqlalchemy.url = postgresql://${vars:dbuser}:${vars:dbpassword}@${vars:dbhost}:${vars:dbport}/${vars:db} # LIDAR tool configuration -lidar_fusion_cmd = ${vars:lidar_fusion_cmd} -lidar_lastool_cmd = ${vars:lidar_lastool_cmd} lidar_data = ${vars:lidar_data} lidar_data_normalized = ${vars:lidar_data_normalized} lidar_output_dir = %(here)s/data/ +debug_log = ${vars:debug_log} + app.cfg = %(here)s/config.yaml [server:main] diff --git a/setup.py b/setup.py index 0baaf52..acbab74 100644 --- a/setup.py +++ b/setup.py @@ -27,14 +27,10 @@ 'numpy', 'pyyaml', 'pip', - 'numpy' + 'laspy', ], packages=find_packages(exclude=['ez_setup']), include_package_data=True, - message_extractors={'las_extractor': [ - ('static/**', 'ignore', None), - ('**.py', 'python', None), - ('templates/**', 'mako', {'input_encoding': 'utf-8'})]}, zip_safe=False, entry_points={ 'paste.app_factory': [ diff --git a/versions.cfg b/versions.cfg index fc27812..b5717e0 100644 --- a/versions.cfg +++ b/versions.cfg @@ -10,6 +10,7 @@ collective.recipe.modwsgi = 2.1 GeoAlchemy2 = 0.2.4 geoalchemy2 = 0.2.4 geojson = 1.0.9 +laspy = 1.3.0 Mako = 1.0.1 mako = 1.0.1 markupsafe = 0.23 From a3987f00f2878cc25268ffe82814639ea14851ff Mon Sep 17 00:00:00 2001 From: Michael Kalbermatten Date: Mon, 7 Dec 2015 15:57:13 +0100 Subject: [PATCH 6/7] Add logging --- las_extractor/util/point_cloud_profiler.py | 19 ++++++++----- las_extractor/views/lidar_profile.py | 31 +++++++++++++++------- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/las_extractor/util/point_cloud_profiler.py b/las_extractor/util/point_cloud_profiler.py index 642e15c..cd9b841 100644 --- a/las_extractor/util/point_cloud_profiler.py +++ b/las_extractor/util/point_cloud_profiler.py @@ -10,6 +10,10 @@ from laspy import file from datetime import datetime +import logging + +log = logging.getLogger(__name__) + try: import osgeo.ogr as ogr import osgeo.osr as osr @@ -63,7 +67,7 @@ def generate_json(profile, jsonOutput, csvOut, classesList, classesNames): 'y': round(row[3]*100)/100 }) -def pointCloudExtractorV2(coordinates, bufferSizeMeter, outputDir, dataDir, jsonOutput, csvOut, classesList, classesNames, perfLogStr): +def pointCloudExtractorV2(coordinates, bufferSizeMeter, outputDir, dataDir, jsonOutput, csvOut, classesList, classesNames, debug_log): distanceFromOrigin = 0 zMin = [] @@ -90,8 +94,9 @@ def pointCloudExtractorV2(coordinates, bufferSizeMeter, outputDir, dataDir, json beforeRequest = datetime.now() polygon, checkEmpty, tileList = generate_tile_list(segment, bufferSizeMeter, outputDir, fileList, dataDir) afterRequest = datetime.now() - perfLogStr += '***********PG REQUEST TIME*************\n' - perfLogStr += str(afterRequest - beforeRequest) + '\n' + + if debug_log == True: + log.warning("PG REQUEST TIME: "+str(afterRequest - beforeRequest)) # Point Cloud extractor V2 seg = {'y1': xyStart[1], 'x1': xyStart[0], 'y2': xyEnd[1], 'x2': xyEnd[0]} @@ -134,8 +139,10 @@ def pointCloudExtractorV2(coordinates, bufferSizeMeter, outputDir, dataDir, json cloud.close() stopIterateTile = datetime.now() - perfLogStr += '*********ITERATE OVER TILE AND POINTS TIME*************\n' - perfLogStr += str(stopIterateTile - startIterateTile) + '\n' + + if debug_log == True: + log.warning("ITERATE OVER TILE AND POINTS TIME: "+str(stopIterateTile - startIterateTile)) + log.warning('Found '+str(len(table))+" points") # Convert the list into numpy array for fast sorting data = np.array(table) @@ -166,7 +173,7 @@ def pointCloudExtractorV2(coordinates, bufferSizeMeter, outputDir, dataDir, json # Read the numpy data and append them to json-serializable list generate_json(profile, jsonOutput, csvOut, classesList, classesNames) - return jsonOutput, zMin, zMax, checkEmpty, perfLogStr + return jsonOutput, zMin, zMax, checkEmpty # Export csv output file to google kml 3D def csv2kml(csvFile, markerUrl, outputKml, classesNames, kmlColors): diff --git a/las_extractor/views/lidar_profile.py b/las_extractor/views/lidar_profile.py index 2d38bd2..e61a4eb 100644 --- a/las_extractor/views/lidar_profile.py +++ b/las_extractor/views/lidar_profile.py @@ -21,6 +21,10 @@ import sys +import logging + +log = logging.getLogger(__name__) + @view_config(route_name='lidar_profile', renderer='jsonp') def lidar_profile(request): """ @@ -40,8 +44,17 @@ def lidar_profile(request): startProcess = datetime.now() - perfLogStr = "Las extractor performance log \n" - perfLogStr += 'Request: ' + str(uuid.uuid4()) + '\n' + + debug_log = request.registry.settings['debug_log'] + + if debug_log == 'True': + debug_log = True + else: + debug_log = False + print debug_log + if debug_log == True: + log.warning("Request: "+str(uuid.uuid4())) + # Get resolution settings resolution = request.registry.settings['resolution'] @@ -62,8 +75,6 @@ def lidar_profile(request): classesList = [] jsonOutput=[] - performanceLog = open(outputDir + 'PerformanceLog.txt', 'w+') - # Create the csv output file for later shp/kml/csv file download (on user demand) csvOut = open(outputDir + outputCsv, 'w') csvOut.write('distance,altitude,x,y,class\n') # csv file header @@ -117,9 +128,9 @@ def lidar_profile(request): errorMsg += str(math.ceil(fullLine.length * 1000) / 1000) + 'm ' +_('long') +', ' errorMsg += _('max allowed length is') +': ' + str(maxLineDistance) + 'm

' return {'Warning': errorMsg} - + # ***Point cloud extractor, V2*** - jsonOutput, zMin, zMax, checkEmpty, perfLogStr = pointCloudExtractorV2(geom.coordinates, bufferSizeMeter, outputDir, dataDir, jsonOutput, csvOut, classesList, classesNames, perfLogStr) + jsonOutput, zMin, zMax, checkEmpty = pointCloudExtractorV2(geom.coordinates, bufferSizeMeter, outputDir, dataDir, jsonOutput, csvOut, classesList, classesNames, debug_log) # If no tile is found in the area intersected by the segment, return error message if checkEmpty == 0: @@ -134,10 +145,10 @@ def lidar_profile(request): # Close IO stream csvOut.close() endProcess = datetime.now() - perfLogStr += '*********TOTAL TIME***********\n' - perfLogStr += str(endProcess - startProcess) + '\n' - performanceLog.write(perfLogStr) - performanceLog.close() + + if debug_log == True: + log.warning("TOTAL TIME: "+str(endProcess - startProcess)) + return { 'profile': jsonOutput, 'series':classesList, From 91dc63145e856862e174103eb457380e8d37cd78 Mon Sep 17 00:00:00 2001 From: Michael Kalbermatten Date: Mon, 7 Dec 2015 15:58:25 +0100 Subject: [PATCH 7/7] Remove print --- las_extractor/views/lidar_profile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/las_extractor/views/lidar_profile.py b/las_extractor/views/lidar_profile.py index e61a4eb..ba4080a 100644 --- a/las_extractor/views/lidar_profile.py +++ b/las_extractor/views/lidar_profile.py @@ -51,7 +51,7 @@ def lidar_profile(request): debug_log = True else: debug_log = False - print debug_log + if debug_log == True: log.warning("Request: "+str(uuid.uuid4()))