Merge branch 'main' of github.com:UW-Macrostrat/macrostrat

Author: amyfromandi

py-modules/map-integration/macrostrat/map_integration/__init__.py

@@ -15,7 +15,8 @@
 from macrostrat.core import app
 from macrostrat.database import Database
 from macrostrat.map_integration.commands.prepare_fields import _prepare_fields
-#from macrostrat.map_integration.pipeline import ingest_map
+
+# from macrostrat.map_integration.pipeline import ingest_map
 from macrostrat.map_integration.process.geometry import create_rgeom, create_webgeom
 from macrostrat.map_integration.utils.ingestion_utils import (
     find_gis_files,
@@ -145,7 +146,6 @@ def delete_sources(
             dict(slug=s),
         ).scalar()
 
-
         # Delete ALL ingest-related rows for this source
         db.run_sql(
             """
@@ -409,7 +409,12 @@ def staging(
 
 
 @staging_cli.command("s3-upload")
-def cmd_upload_dir(slug: str = ..., data_path: Path = ..., ext: str = Option(".gdb", help="extension of the data path"), ingest_process_id: int = Option(None)):
+def cmd_upload_dir(
+    slug: str = ...,
+    data_path: Path = ...,
+    ext: str = Option(".gdb", help="extension of the data path"),
+    ingest_process_id: int = Option(None),
+):
     """Upload a local directory to the staging bucket under SLUG/."""
     db = get_database()
     source_id = db.run_query(
@@ -525,6 +530,7 @@ def list_layers(e00_path: Path) -> set[str]:
     def run(cmd):
         p = subprocess.run(cmd, capture_output=True, text=True)
         return p.returncode, p.stdout, p.stderr
+
     created = False
     for f in e00_files:
         base = f.stem
@@ -542,9 +548,13 @@ def run(cmd):
                 # create/overwrite first successful write
                 cmd += ["-overwrite"]
             cmd += [
-                str(out_gpkg), str(f), lyr,
-                "-nln", f"{base}_lines",
-                "-nlt", "LINESTRING",
+                str(out_gpkg),
+                str(f),
+                lyr,
+                "-nln",
+                f"{base}_lines",
+                "-nlt",
+                "LINESTRING",
             ]
             rc, _, err = run(cmd)
             if rc == 0:
@@ -557,9 +567,13 @@ def run(cmd):
             else:
                 cmd = ["ogr2ogr", "-f", "GPKG", "-update", "-append"]
             cmd += [
-                str(out_gpkg), str(f), lyr,
-                "-nln", f"{base}_points",
-                "-nlt", "POINT",
+                str(out_gpkg),
+                str(f),
+                lyr,
+                "-nln",
+                f"{base}_points",
+                "-nlt",
+                "POINT",
             ]
             rc, _, _ = run(cmd)
             if rc == 0:
@@ -572,9 +586,13 @@ def run(cmd):
             else:
                 cmd = ["ogr2ogr", "-f", "GPKG", "-update", "-append"]
             cmd += [
-                str(out_gpkg), str(f), lyr,
-                "-nln", f"{base}_polygons",
-                "-nlt", "POLYGON",
+                str(out_gpkg),
+                str(f),
+                lyr,
+                "-nln",
+                f"{base}_polygons",
+                "-nlt",
+                "POLYGON",
             ]
             rc, _, _ = run(cmd)
             if rc == 0:
@@ -584,6 +602,7 @@ def run(cmd):
 
     print(f"Done: {out_gpkg}")
 
+
 # ----------------------------------------------------------------------------------------------------------------------
 
 
@@ -715,10 +734,8 @@ def staging_bulk(
             ),
         )
 
-
         cmd_upload_dir(slug=slug, data_path=region_path, ext=ext)
 
-
         map_info = get_map_info(db, slug)
         _prepare_fields(map_info)
         create_rgeom(map_info)
@@ -730,4 +747,4 @@ def staging_bulk(
     slug_list_path = parent / f"staged_slugs.txt"
     with open(slug_list_path, "w") as file:
         for slug in staged_slugs:
-            file.write(slug + "\n")
+            file.write(slug + "\n")

py-modules/map-integration/macrostrat/map_integration/commands/ingest.py

@@ -45,7 +45,11 @@ def merge_metadata_polygons(polygon_df, meta_df, join_col) -> G.GeoDataFrame:
 
 
 def preprocess_dataframe(
-    poly_line_pt_df: G.GeoDataFrame, meta_path: Path, join_col: str, feature_suffix: str, pipeline: str
+    poly_line_pt_df: G.GeoDataFrame,
+    meta_path: Path,
+    join_col: str,
+    feature_suffix: str,
+    pipeline: str,
 ) -> Tuple[G.GeoDataFrame, str, str, str]:
     """
     Preprocess a GeoDataFrame by merging in metadata from a local .tsv,
@@ -291,7 +295,7 @@ def ingest_map(
                 meta_path=meta_path,
                 join_col=join_col.lower(),
                 feature_suffix=feature_suffix,
-                pipeline=pipeline
+                pipeline=pipeline,
             )
             if feature_suffix == "polygons":
                 ingest_results["ingest_pipeline"] = ingest_pipeline

py-modules/map-integration/macrostrat/map_integration/utils/ingestion_utils.py

@@ -107,7 +107,9 @@ def get_strat_names_df() -> pd.DataFrame:
 
 
 # standard map age function. User gets to input their column 1 and a column 2 data to map to our ages.
-def map_t_b_standard(meta_df: G.GeoDataFrame, col_one: str, col_two: str) -> G.GeoDataFrame:
+def map_t_b_standard(
+    meta_df: G.GeoDataFrame, col_one: str, col_two: str
+) -> G.GeoDataFrame:
     interval_df = get_age_interval_df().reset_index(drop=True)
     interval_lookup = {
         str(row["interval_name"]).strip().lower(): int(row["id"])
@@ -132,7 +134,7 @@ def map_t_b_standard(meta_df: G.GeoDataFrame, col_one: str, col_two: str) -> G.G
         meta_df["b_interval"] = mapped_col_one
         meta_df["t_interval"] = mapped_col_one
 
-    #fallback to map col_two if col_one row is empty
+    # fallback to map col_two if col_one row is empty
     if col_two in meta_df.columns:
         needs_fill = meta_df["b_interval"].isna() | meta_df["t_interval"].isna()
         if needs_fill.any():