update registry to be a separate dir

Author: avsm

geotessera/core.py

@@ -143,7 +143,7 @@ def _load_tiles_registry_index(self):
 
             # Download the master tiles registry index
             tiles_index_file = pooch.retrieve(
-                url=f"{TESSERA_BASE_URL}/{self.version}/global_0.1_degree_tiff_all/registry_index.txt",
+                url=f"{TESSERA_BASE_URL}/{self.version}/registry/tiles_registry_index.txt",
                 known_hash=None,
                 fname="tiles_registry_index.txt",
                 path=cache_path,
@@ -165,7 +165,7 @@ def _load_registry_index(self):
 
         # Download the master registry index
         registry_index_file = pooch.retrieve(
-            url=f"{TESSERA_BASE_URL}/{self.version}/global_0.1_degree_representation/registry_index.txt",
+            url=f"{TESSERA_BASE_URL}/{self.version}/registry/registry_index.txt",
             known_hash=None,
             fname="registry_index.txt",
             path=cache_path,
@@ -192,7 +192,7 @@ def _ensure_block_loaded(self, year: int, lon: float, lat: float):
         registry_filename = get_block_registry_filename(str(year), block_lon, block_lat)
 
         # Download the specific block registry file
-        registry_url = f"{TESSERA_BASE_URL}/{self.version}/global_0.1_degree_representation/{registry_filename}"
+        registry_url = f"{TESSERA_BASE_URL}/{self.version}/registry/{registry_filename}"
         registry_file = pooch.retrieve(
             url=registry_url,
             known_hash=None,
@@ -227,7 +227,7 @@ def _ensure_tile_block_loaded(self, lon: float, lat: float):
         registry_filename = get_tiles_registry_filename(block_lon, block_lat)
 
         # Download the specific tile block registry file
-        registry_url = f"{TESSERA_BASE_URL}/{self.version}/global_0.1_degree_tiff_all/{registry_filename}"
+        registry_url = f"{TESSERA_BASE_URL}/{self.version}/registry/{registry_filename}"
         registry_file = pooch.retrieve(
             url=registry_url,
             known_hash=None,

geotessera/registry_cli.py

@@ -121,11 +121,13 @@ def update_representations_command(args):
         print(f"Error: Directory {base_dir} does not exist")
         return
 
-    # Determine registry output directory
+    # Determine registry output directory - go up to parent of base_dir
     if hasattr(args, 'registry_dir') and args.registry_dir:
-        registry_dir = os.path.join(os.path.abspath(args.registry_dir), "embeddings")
+        registry_dir = os.path.join(os.path.abspath(args.registry_dir), "registry")
     else:
-        registry_dir = base_dir
+        # Place registry at same level as global_0.1_degree_representation
+        parent_dir = os.path.dirname(base_dir)
+        registry_dir = os.path.join(parent_dir, "registry")
 
     # Ensure registry directory exists
     os.makedirs(registry_dir, exist_ok=True)
@@ -240,6 +242,27 @@ def update_representations_command(args):
 
         print(f"Master index written with {len(all_registry_files)} registry files")
         print(f"Total blocks processed: {total_blocks}")
+        
+        # Generate master registry.txt that includes all individual registries
+        master_registry_path = os.path.join(registry_dir, "registry.txt")
+        print(f"Generating master registry.txt: {master_registry_path}")
+        
+        with open(master_registry_path, 'w') as f:
+            f.write("# GeoTessera Master Registry\n")
+            f.write("# This file combines all block registries for embeddings\n")
+            f.write(f"# Contains entries from {len(all_registry_files)} block registry files\n\n")
+            
+            total_entries = 0
+            for registry_file in sorted(all_registry_files):
+                if os.path.exists(registry_file):
+                    with open(registry_file, 'r') as reg_f:
+                        for line in reg_f:
+                            line = line.strip()
+                            if line and not line.startswith('#'):
+                                f.write(f"{line}\n")
+                                total_entries += 1
+        
+        print(f"Master registry.txt written with {total_entries} total entries")
 
 
 
@@ -250,11 +273,13 @@ def update_tiles_command(args):
         print(f"Error: Directory {base_dir} does not exist")
         return
 
-    # Determine registry output directory
+    # Determine registry output directory - go up to parent of base_dir
     if hasattr(args, 'registry_dir') and args.registry_dir:
-        registry_dir = os.path.join(os.path.abspath(args.registry_dir), "tiles")
+        registry_dir = os.path.join(os.path.abspath(args.registry_dir), "registry")
     else:
-        registry_dir = base_dir
+        # Place registry at same level as global_0.1_degree_tiff_all
+        parent_dir = os.path.dirname(base_dir)
+        registry_dir = os.path.join(parent_dir, "registry")
 
     # Ensure registry directory exists
     os.makedirs(registry_dir, exist_ok=True)
@@ -348,7 +373,7 @@ def update_tiles_command(args):
 
     # Generate master index of all tile registry files
     if all_registry_files:
-        master_index_path = os.path.join(registry_dir, "registry_index.txt")
+        master_index_path = os.path.join(registry_dir, "tiles_registry_index.txt")
         print(f"\nGenerating master tiles registry index: {master_index_path}")
 
         with open(master_index_path, 'w') as f:

geotessera/registry_utils.py

@@ -82,7 +82,7 @@ def get_registry_path_for_tile(registry_base_dir: str, year: str, lon: float, la
     Get the full path to the registry file containing a specific tile.
     
     Args:
-        registry_base_dir: Base directory for registry files
+        registry_base_dir: Base directory for registry files (should be the parent directory)
         year: Year string (e.g., "2024")
         lon: Longitude in decimal degrees
         lat: Latitude in decimal degrees
@@ -93,7 +93,7 @@ def get_registry_path_for_tile(registry_base_dir: str, year: str, lon: float, la
     import os
     block_lon, block_lat = get_block_coordinates(lon, lat)
     registry_filename = get_block_registry_filename(year, block_lon, block_lat)
-    return os.path.join(registry_base_dir, "embeddings", registry_filename)
+    return os.path.join(registry_base_dir, "registry", registry_filename)
 
 
 def get_tiles_registry_filename(block_lon: int, block_lat: int) -> str:
@@ -118,7 +118,7 @@ def get_registry_path_for_tiles(registry_base_dir: str, lon: float, lat: float)
     Get the full path to the tiles registry file containing a specific coordinate.
     
     Args:
-        registry_base_dir: Base directory for registry files
+        registry_base_dir: Base directory for registry files (should be the parent directory)
         lon: Longitude in decimal degrees
         lat: Latitude in decimal degrees
         
@@ -128,7 +128,7 @@ def get_registry_path_for_tiles(registry_base_dir: str, lon: float, lat: float)
     import os
     block_lon, block_lat = get_block_coordinates(lon, lat)
     registry_filename = get_tiles_registry_filename(block_lon, block_lat)
-    return os.path.join(registry_base_dir, "tiles", registry_filename)
+    return os.path.join(registry_base_dir, "registry", registry_filename)
 
 
 def get_all_blocks_in_range(min_lon: float, max_lon: float,