attempt to relax open limits before forking

Author: ramSeraph

topo_map_processor/tools/retile.py

@@ -26,6 +26,8 @@
 
 from pmtiles_mosaic.tile_sources import create_source_from_paths, MissingTileError
 
+from .utils import relax_open_file_limit
+
 def run_external(cmd):
     print(f'running cmd - {cmd}')
     start = time.time()
@@ -445,6 +447,8 @@ def retile_main(args):
     if tile_extension == 'jpg':
         tile_extension = 'jpeg'
 
+    relax_open_file_limit()
+
     os.environ['GDAL_CACHEMAX'] = '2048'
     os.environ['GDAL_MAX_DATASET_POOL_SIZE'] = '5000'
     os.environ['GDAL_DISABLE_READDIR_ON_OPEN'] = 'TRUE'

topo_map_processor/tools/tile.py

@@ -13,6 +13,8 @@
 
 from osgeo_utils.gdal2tiles import submain as gdal2tiles_main
 
+from .utils import relax_open_file_limit
+
 SUPPORTED_FORMATS = ['webp', 'jpeg', 'png']
 
 def run_external(cmd):
@@ -95,6 +97,7 @@ def cli():
     else:
         attribution_text = args.attribution
 
+    relax_open_file_limit()
 
     tiles_dir = Path(args.tiles_dir)
     tiles_dir.mkdir(parents=True, exist_ok=True)

topo_map_processor/tools/utils.py

@@ -0,0 +1,31 @@
+
+import resource
+
+
+def relax_open_file_limit():
+
+    # Get the current soft and hard limits for open files
+    current_soft, current_hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+    print(f"Current open file limits: Soft={current_soft}, Hard={current_hard}")
+    
+    # Set new soft and hard limits for open files
+    new_soft = resource.RLIM_INFINITY  # Set to unlimited
+    new_hard = resource.RLIM_INFINITY  # Set to unlimited
+
+    try:
+        resource.setrlimit(resource.RLIMIT_NOFILE, (new_soft, new_hard))
+        print(f"New open file limits set: Soft={new_soft}, Hard={new_hard}")
+    except Exception as e:
+        print(f"Error setting resource limit to unlimited: {e}")
+        print('Increasing soft limit to match the hard limit instead')
+        new_soft = current_hard
+        new_hard = current_hard
+        try:
+            resource.setrlimit(resource.RLIMIT_NOFILE, (new_soft, new_hard))
+            print(f"New open file limits set: Soft={new_soft}, Hard={new_hard}")
+        except Exception as e:
+            print(f"Error setting soft resource limit to match hard limit: {e}")
+    
+    # Verify the new limits
+    updated_soft, updated_hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+    print(f"Verified open file limits: Soft={updated_soft}, Hard={updated_hard}")