Merge pull request #127 from UW-Macrostrat/bugfix/ingestion-errors

Starting point to fix ingestion errors

Author: davenquinn

cli/macrostrat/cli/cache.py

@@ -6,7 +6,7 @@
 
 from macrostrat.core.database import get_database
 
-cli = Typer(name="cache")
+cli = Typer(name="cache", help="Tile cache management")
 
 
 @cli.command(name="clear")

local-root/docker-compose.yaml

@@ -85,7 +85,8 @@ services:
       - PGRST_DB_MAX_ROWS=5000
       - PGRST_SERVER_HOST=
       - PGRST_SERVER_TIMING_ENABLED=true
-      # This is sketchy
+      # This is sketchy; we really only want this for a few (auth-gated) routes
+      # Perhaps we can put this config in the database?
       - PGRST_DB_AGGREGATES_ENABLED=true
   # Tileserver
   tileserver:
@@ -94,6 +95,8 @@ services:
       context: ${MACROSTRAT_TILESERVER_SRC:-""}
     environment:
       - DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@database:5432/${POSTGRES_DB}
+      # Ideally the system would fail gracefully if we didn't provide the Rockd database URL but that's not the case.
+      - ROCKD_DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@database:5432/rockd
     # The `--proxy-headers` flag is necessary for the tileserver to work behind a reverse proxy
     # and correctly notify the app that it should trust the X-Forwarded-Proto header.
     # This is necessary for the tileserver to correctly generate URLs for tilejson files.

map-integration/macrostrat/map_integration/pipeline.py

@@ -8,6 +8,7 @@
 import csv
 import datetime
 import hashlib
+import importlib
 import os
 import pathlib
 import re
@@ -19,7 +20,6 @@
 from contextlib import contextmanager
 from typing import Annotated, Any, NoReturn, Optional
 
-import magic
 import requests  # type: ignore[import-untyped]
 from rich.console import Console
 from sqlalchemy import and_, insert, select, update
@@ -42,9 +42,12 @@
 from macrostrat.map_integration.errors import IngestError
 from macrostrat.map_integration.process.geometry import create_rgeom, create_webgeom
 from macrostrat.map_integration.utils.map_info import MapInfo, get_map_info
-
 from .config import get_minio_client
 
+# Do this with importlib so we control the order
+for mod in ["pylibmagic", "magic"]:
+    importlib.import_module("magic")
+
 # The list of arguments to upload_file that ingest_csv will look
 # for in the CSV file given to it.
 FIELDS = [
@@ -345,6 +348,7 @@ def get_source_by_slug(slug: str) -> Optional[Sources]:
 
 def create_source(**data) -> Sources:
     data = truncate_source_metadata(data)
+    print(data)
     with get_db_session() as session:
         new_source = session.scalar(
             insert(Sources).values(**data).returning(Sources),
@@ -451,6 +455,7 @@ def create_slug(
         metadata["raster_url"] = raster_url
 
     if source := get_source_by_slug(slug):
+        console.print(f"Found existing source ID {source.source_id} for slug {slug}")
         source = update_source(source.source_id, **metadata)
     else:
         source = create_source(**metadata)
@@ -644,6 +649,8 @@ def upload_file(
         raster_url=raster_url,
     )
 
+    console.print(f"Created record for map {slug}")
+
     ## Collect metadata for the archive file.
 
     mime_type = magic.Magic(mime=True).from_file(local_file)

map-integration/pyproject.toml

@@ -22,6 +22,10 @@ typer = "^0.12.0"
 
 minio = "^7.2.5"
 python-dotenv = "^1"
+
+# Python-Magic does not ship shared library, but Pylibmagic packages
+# it to fill the gap.
+pylibmagic = "^0.5.0"
 python-magic = "^0.4.27"
 pyyaml = "^6.0"
 shapely = "^2.0.1"