Merge remote-tracking branch 'origin/main'

* origin/main:
  Format code and sort imports
  s3 bucket uploads .gdb's as .zip's to retain data integrity
  Format code and sort imports
  optimizing map ingestion and s3 management workflow
  testing prod rollout
  testing v3 automated deployment
  testing legacy tileserver automated tag deployment
  added CRUD permissions to the macrostrat user for all tables in the macrostrat schema
  updated makefile prefix to match api v3 format
  updating v3 tag automation
  updating version format so tag automation works
  upating makefile and testing staging legacy tileserver release
  testing legacy tileserver workflow
  testing prod workflow
  updating makeful
  fixed version reference
  updated tileserver image tag automation

Author: davenquinn

.github/workflows/build-api-dev.yaml

@@ -28,6 +28,9 @@ jobs:
             type=ref,event=tag,suffix=-{{date 'YYYYMMDDHHmmss'}}
             type=raw,value=latest-itb-{{date 'YYYYMMDDHHmmss'}}
             type=raw,value=sha-{{sha}}
+            
+            # When a STAGING prerelease tag (vX.Y.Z-*) is pushed → also tag dev
+            type=raw,value=main-{{date 'YYYYMMDDHHmmss'}},enable=${{ startsWith(github.ref, 'refs/tags/v') && contains(github.ref, '-') }}
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       - name: Login to OSG DockerHub

.github/workflows/build-api-prod.yaml

@@ -28,6 +28,14 @@ jobs:
             type=semver,pattern={{version}}
             type=raw,value=latest,enable={{is_default_branch}}
             type=raw,value=sha-{{sha}}
+            
+            # When a PRODUCTION tag (vX.Y.Z) is pushed → also tag staging and dev
+            type=raw,value=main-{{date 'YYYYMMDDHHmmss'}},enable=${{ startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-') }}
+            type=semver,pattern={{version}},suffix=-beta.{{date 'YYYYMMDDHHmmss'}},enable=${{ startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-') }}
+            type=semver,pattern={{version}},suffix=-beta,enable=${{ startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-') }}
+
+            # When a STAGING prerelease tag (vX.Y.Z-*) is pushed → also tag dev
+            type=raw,value=main-{{date 'YYYYMMDDHHmmss'}},enable=${{ startsWith(github.ref, 'refs/tags/v') && contains(github.ref, '-') }}
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       - name: Login to OSG DockerHub

.github/workflows/build-legacy-tileserver.yaml

@@ -45,6 +45,16 @@ jobs:
             type=ref,event=branch,suffix=-{{date 'YYYYMMDDHHmmss'}}
             type=raw,value=latest,enable={{is_default_branch}}
             type=raw,value=sha-{{sha}}
+            
+            
+            # When a PRODUCTION tag (legacy-tileserver-vX.Y.Z) → also tag staging and dev
+            type=raw,value=main-{{date 'YYYYMMDDHHmmss'}},enable=${{ startsWith(github.ref, 'refs/tags/legacy-tileserver-v') && !contains(env.VERSION, '-') }}
+            type=raw,value=${{ env.VERSION }}-beta.{{date 'YYYYMMDDHHmmss'}},enable=${{ startsWith(github.ref, 'refs/tags/legacy-tileserver-v') && !contains(env.VERSION, '-') }}
+            type=raw,value=${{ env.VERSION }}-beta,enable=${{ startsWith(github.ref, 'refs/tags/legacy-tileserver-v') && !contains(env.VERSION, '-') }}
+
+            # When a STAGING prerelease tag (legacy-tileserver-vX.Y.Z-*) → also tag dev
+            type=raw,value=main-{{date 'YYYYMMDDHHmmss'}},enable=${{ startsWith(github.ref, 'refs/tags/legacy-tileserver-v') && contains(env.VERSION, '-') }}
+
       - name: Set up Docker BuildX
         uses: docker/setup-buildx-action@v2
       - name: Login to OSG DockerHub

.github/workflows/build-tileserver.yaml

@@ -45,6 +45,16 @@ jobs:
             type=raw,value=${{ env.VERSION }},enable=${{ env.VERSION != '' }}
             type=raw,value=latest,enable={{is_default_branch}}
             type=raw,value=sha-{{sha}}
+      
+            # When a PRODUCTION tag (tileserver-vX.Y.Z) → also tag staging and dev
+            type=raw,value=main-{{date 'YYYYMMDDHHmmss'}},enable=${{ startsWith(github.ref, 'refs/tags/tileserver-v') && !contains(env.VERSION, '-') }}
+            type=raw,value=${{ env.VERSION }}-beta.{{date 'YYYYMMDDHHmmss'}},enable=${{ startsWith(github.ref, 'refs/tags/tileserver-v') && !contains(env.VERSION, '-') }}
+            type=raw,value=${{ env.VERSION }}-beta,enable=${{ startsWith(github.ref, 'refs/tags/tileserver-v') && !contains(env.VERSION, '-') }}
+      
+            # When a STAGING prerelease tag (tileserver-vX.Y.Z-*) → also tag dev
+            type=raw,value=main-{{date 'YYYYMMDDHHmmss'}},enable=${{ startsWith(github.ref, 'refs/tags/tileserver-v') && contains(env.VERSION, '-') }}
+                  
+
       - name: Set up Docker BuildX
         uses: docker/setup-buildx-action@v2
       - name: Login to OSG DockerHub

cli/macrostrat/cli/database/mariadb/postgresql_migration/__init__.py

@@ -94,7 +94,7 @@ def migrate_mariadb_to_postgresql(
                 "This will overwrite the macrostrat schema. Use --force to proceed."
             )
         get_db().run_sql(
-            f"ALTER SCHEMA {final_schema} RENAME TO macrostratbak3",
+            f"ALTER SCHEMA {final_schema} RENAME TO macrostratbak2",
             dict(
                 final_schema=Identifier(final_schema),
             ),

cli/macrostrat/cli/database/migrations/baseline/01-roles.sql

@@ -27,13 +27,11 @@ GRANT SELECT ON ALL TABLES IN SCHEMA macrostrat TO "rockd-reader";
 GRANT SELECT ON ALL TABLES IN SCHEMA public TO "rockd-reader";
 GRANT SELECT ON ALL TABLES IN SCHEMA topology TO "rockd-reader";
 
--- 1) Schema access
 GRANT USAGE ON SCHEMA
   carto, carto_new, geologic_boundaries, hexgrids, lines,
   macrostrat, maps, points, sources, topology, public
 TO macrostrat;
 
--- 2) Read all existing objects (tables/views + sequences)
 GRANT SELECT ON ALL TABLES    IN SCHEMA
   carto, carto_new, geologic_boundaries, hexgrids, lines,
   macrostrat, maps, points, sources, topology, public
@@ -44,7 +42,6 @@ GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA
   macrostrat, maps, points, sources, topology, public
 TO macrostrat;
 
--- 3) Make future objects readable by default (run as each object-creating owner)
 ALTER DEFAULT PRIVILEGES IN SCHEMA
   carto, carto_new, geologic_boundaries, hexgrids, lines,
   macrostrat, maps, points, sources, topology, public
@@ -55,15 +52,19 @@ ALTER DEFAULT PRIVILEGES IN SCHEMA
   macrostrat, maps, points, sources, topology, public
 GRANT USAGE, SELECT ON SEQUENCES TO macrostrat;
 
--- 4) Enums used in tables under public
 GRANT USAGE ON TYPE
   public.measurement_class,
   public.measurement_class_new,
   public.measurement_type,
   public.measurement_type_new
 TO macrostrat;
 
--- 5) Functions (your helper + PostGIS etc.) – safe & useful for read paths
+
+GRANT USAGE ON SCHEMA macrostrat TO "macrostrat";
+GRANT CREATE ON SCHEMA macrostrat TO "macrostrat";
+GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA macrostrat TO "macrostrat";
+GRANT USAGE ON ALL SEQUENCES IN SCHEMA macrostrat TO "macrostrat";
+
 GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public, topology TO macrostrat;
 
 

map-integration/macrostrat/map_integration/__init__.py

@@ -19,7 +19,7 @@
 from macrostrat.map_integration.process.geometry import create_rgeom, create_webgeom
 from macrostrat.map_integration.utils.file_discovery import find_gis_files
 from macrostrat.map_integration.utils.map_info import get_map_info
-from macrostrat.map_integration.utils.staging_upload_dir import *
+from macrostrat.map_integration.utils.s3_file_management import *
 
 from . import pipeline
 from .commands.copy_sources import copy_macrostrat_sources
@@ -85,27 +85,35 @@ def set_active_map(map: MapInfo = None):
 
 @sources.command(name="delete")
 def delete_sources(
-    slugs: list[str],
+    slug: list[str] = Option(
+        ...,
+        help="BULK delete = filename.txt [every line lists the slug_name to delete. no whitespaces.]\n "
+        + "SINGLE delete = 'slug_name' [list the slug_name in quotes]",
+    ),
+    file_name: str = Option(
+        None, help="deletes a specified file within the slug's directory."
+    ),
     dry_run: bool = Option(False, "--dry-run"),
     all_data: bool = Option(False, "--all-data"),
 ):
     """Delete sources from the map ingestion database."""
     db = get_database()
 
-    if not stdin.isatty() and len(slugs) == 1 and slugs[0] == "-":
-        slugs = [line.strip() for line in stdin]
-    elif len(slugs) == 1 and os.path.isfile(slugs[0]):
-        with open(slugs[0]) as file:
-            slugs = [line.strip() for line in file if line.strip()]
+    if not stdin.isatty() and len(slug) == 1 and slug[0] == "-":
+        slug = [line.strip() for line in stdin]
+    elif len(slug) == 1 and os.path.isfile(slug[0]):
+        with open(slug[0]) as file:
+            slug = [line.strip() for line in file if line.strip()]
 
     if dry_run:
         print("Deleting maps:")
-        print("  " + "\n  ".join(slugs))
+        print("  " + "\n  ".join(slug))
 
         print("\nDry run; not actually deleting anything")
         return
 
-    for slug in slugs:
+    for slug in slug:
+        cmd_delete_dir(slug, file_name)
         print(f"Deleting map {slug}")
         print(slug)
         tables = db.run_query(
@@ -260,11 +268,8 @@ def staging(
     slug: str,
     data_path: str,
     name: str,
-    meta_path: str = Option(
-        None, help="metadata URL to merge into the sources polygons/lines/points table"
-    ),
     merge_key: str = Option(
-        None,
+        "mapunit",
         help="primary key to left join the metadata into the sources polygons/lines/points table",
     ),
     meta_table: str = Option(
@@ -277,9 +282,13 @@ def staging(
     Ingest a map, update metadata, prepare fields, and build geometries.
     """
     db = get_database()
+    data_path_ext = Path(data_path)
+    ext = data_path_ext.suffix.lower()
+    # upload to the s3 bucket!
+    cmd_upload_dir(slug, data_path_ext, ext)
     print(f"Ingesting {slug} from {data_path}")
 
-    gis_files, excluded_files = find_gis_files(Path(data_path), filter=filter)
+    gis_files, excluded_files = find_gis_files(data_path_ext, filter=filter)
     if not gis_files:
         raise ValueError(f"No GIS files found in {data_path}")
 
@@ -296,7 +305,7 @@ def staging(
         slug,
         gis_files,
         if_exists="replace",
-        meta_path=meta_path,
+        meta_path=data_path,
         merge_key=merge_key,
         meta_table=meta_table,
     )
@@ -322,8 +331,8 @@ def staging(
     # add map_url later
     db.run_sql(
         """
-        INSERT INTO maps_metadata.ingest_process (state, source_id, object_group_id, ingested_by, ingest_pipeline, comments)
-        VALUES (:state, :source_id, :object_group_id, :ingested_by, :ingest_pipeline, :comments);
+        INSERT INTO maps_metadata.ingest_process (state, source_id, object_group_id, ingested_by, ingest_pipeline, comments, slug)
+        VALUES (:state, :source_id, :object_group_id, :ingested_by, :ingest_pipeline, :comments, :slug);
         """,
         dict(
             state=state,
@@ -332,6 +341,7 @@ def staging(
             ingested_by="macrostrat-admin",
             ingest_pipeline=ingest_pipeline,
             comments=comments,
+            slug=slug,
         ),
     )
 
@@ -378,29 +388,30 @@ def staging(
     if any(val is None for val in [row, ingest_process, rgeom, web_geom]):
         raise RuntimeError("Staging failed: Some expected records were not inserted.")
 
-    print(
-        f"\nFinished staging setup for {slug}. View map here: https://dev.macrostrat.org/maps/ingestion/{source_id}/ \n"
+    console.print(
+        f"[green] \n Finished staging setup for {slug}. "
+        f"View map here: https://dev.macrostrat.org/maps/ingestion/{source_id}/ [\green] \n"
     )
 
 
 staging_cli.add_command(staging, name="ingest")
+staging_cli.command("delete")(delete_sources)
 
 # ------------------------------------------
 # commands nested under 'macrostrat maps staging...'
 
 
-@staging_cli.command("upload-dir")
-def cmd_upload_dir(
-    slug: str = ...,
-    data_path: Path = ...,
-):
+@staging_cli.command("s3-upload-dir")
+def cmd_upload_dir(slug: str = ..., data_path: Path = ..., ext: str = Option("")):
     """Upload a local directory to the staging bucket under SLUG/."""
-    res = staging_upload_dir(slug, data_path)
+    res = staging_upload_dir(slug, data_path, ext)
     pretty_res = json.dumps(res, indent=2)
-    console.print(f"[green] Upload successful! \n {pretty_res} [/green]")
+    console.print(
+        f"[green] Upload to s3 bucket was successful! \n {pretty_res} [/green]"
+    )
 
 
-@staging_cli.command("delete-dir")
+@staging_cli.command("s3-delete-dir")
 def cmd_delete_dir(
     slug: str = ...,
     file_name: str = Option(
@@ -410,63 +421,72 @@ def cmd_delete_dir(
     """Delete all objects under SLUG/ in the staging bucket."""
     staging_delete_dir(slug, file_name)
     console.print(
-        f"[green] Delete successful! \n Deleted objects under slug: {slug} [/green]"
+        f"[green] Successfully deleted objects within the s3 bucket under slug: {slug} [/green]"
     )
 
 
-@staging_cli.command("list")
+@staging_cli.command("s3-list")
 def cmd_list_dir(
-    slug: str = ...,
-    page_token: int = Option(0, "--page-token", "-t", help="Offset to start from"),
-    page_size: int = Option(10, "--page-size", "-s", help="Items per page"),
-    more: bool = Option(
-        False, "--more", "-m", help="Interactively page through results"
+    slug: str = Option(
+        ...,
+        help="lists all files within a slug directory. Input 'all' to list all the slug directories.",
     ),
+    page_token: int = Option(0, "--page-token", "-t", help="Offset to start from"),
+    page_size: int = Option(20, "--page-size", "-s", help="Items per page"),
 ):
     """List paginated files under SLUG."""
-    if not more:
-        page = staging_list_dir(slug, page_token=page_token, page_size=page_size)
-        files = json.dumps(page, indent=2)
-        console.print(f"[green] {files} [/green]")
-        return
 
     token = page_token
+    count = 0
     while True:
         page = staging_list_dir(slug, page_token=token, page_size=page_size)
         for f in page["files"]:
-            console.print(f"[green]{f}[/green]")
+            console.print(f"[blue]{f}[/blue]")
+            count += 1
+
         if page["next_page_token"] is None:
+            console.print(f"[green]Total files: {count}[/green]")
             print("\n-- End of list --")
             break
-        resp = (
-            input("\nPress Enter for next page, or type 'exit' to stop: ")
-            .strip()
-            .lower()
-        )
+        console.print(f"[green]Scrolled through: {count} files[/green]")
+
+        resp = input("\nPress 'enter' for next page, or 'q' to exit: ").strip().lower()
         if resp in ("exit", "quit", "q"):
             break
         token = page["next_page_token"]
 
 
+@staging_cli.command("s3-download-dir")
+def cmd_download_dir(
+    slug: str = ...,
+    dest_path: pathlib.Path = Option(
+        ..., help="Local destination path to save slug directory to."
+    ),
+):
+    """Download a staging prefix to a local directory."""
+    res = staging_download_dir(slug=slug, dest_path=dest_path)
+    console.print(f"[green] Download successful![/green]")
+    console.print(json.dumps(res, indent=2))
+
+
 # ----------------------------------------------------------------------------------------------------------------------
 
 
-@cli.command(name="bulk-staging")
+@staging_cli.command("bulk-ingest")
 def staging_bulk(
     # required
     meta_parent_path: str = Option(
         ..., help="Parent directory containing region subfolders"
     ),
     # required
     prefix: str = Option(..., help="Slug prefix to avoid collisions"),
-    # required
     merge_key: str = Option(
-        ...,
+        "mapunit",
         help="primary key to left join the metadata into the sources polygons/lines/points table",
     ),
     meta_table: str = Option(
         "polygons",
-        help="Options: polygons, lines, or points. specifies the table in which the legend metadata is merged into. It defaults to sources polygons",
+        help="Options: polygons, lines, or points. specifies the table in which the metadata is merged into. It defaults to sources polygons",
     ),
     filter: str = Option(None, help="Filter applied to GIS file selection"),
 ):
@@ -488,9 +508,13 @@ def staging_bulk(
         clean_stem = re.sub(r"[^a-z0-9_]", "", clean_stem)
         slug = f"{prefix}_{clean_stem}"
         name = region_path.stem
+        ext = region_path.suffix.lower()
         meta_path = region_path
         staged_slugs.append(slug)
 
+        # upload to the s3 bucket!
+        cmd_upload_dir(slug, region_path, ext)
+
         print(f"Ingesting {slug} from {meta_path}")
         gis_files, excluded_files = find_gis_files(Path(meta_path), filter=filter)
         if not gis_files:

map-integration/macrostrat/map_integration/commands/ingest.py

@@ -225,9 +225,6 @@ def ingest_map(
         if feature_suffix == "linestrings":
             feature_suffix = "lines"
         # applies legend merge only to the whatever the legend_table is specified as
-        # TODO add flag in db for cross sections. Remove cross section layer from Arizona maps
-        # TODO add metadata page to the sources table. delete all of the old maps
-        # TODO add maps to s3 bucket.
         if meta_path and meta_table == feature_suffix:
             df.columns = df.columns.str.lower()
             df, ingest_pipeline, comments, state = preprocess_dataframe(