Upload map files

cli/macrostrat/cli/database/__init__.py

@@ -23,7 +23,7 @@
 # First, register all migrations
 # NOTE: right now, this is quite implicit.
 from .migrations import load_migrations
-from .utils import engine_for_db_name
+from .utils import engine_for_db_name, setup_postgrest_access
 
 log = get_logger(__name__)
 
@@ -398,6 +398,19 @@ def run_scripts(migration: str = Argument(None)):
 db_app.command(name="migrations", rich_help_panel="Schema management")(run_migrations)
 
 
+def update_permissions():
+    """Setup permissions for the PostgREST API.
+
+    NOTE: This is a stopgap until we have a better permssions system.
+    """
+    db = get_db()
+    setup_postgrest_access("macrostrat_api")(db)
+    db.run_sql("NOTIFY pgrst, 'reload schema';")
+
+
+db_app.command(name="permissions", rich_help_panel="Helpers")(update_permissions)
+
+
 ### Helpers
 
 

cli/macrostrat/cli/database/migrations/maps_ingest_state_custom_type/__init__.py

@@ -1,6 +1,5 @@
-from macrostrat.database import Database, Identifier
-
 from macrostrat.core.migrations import Migration, _not, custom_type_exists
+from macrostrat.database import Database, Identifier
 
 
 def ingest_type_exists_in_wrong_schema(db: Database) -> bool:

cli/macrostrat/cli/database/utils.py

@@ -336,3 +336,13 @@ def grant_schema_usage(
         """,
             params,
         )
+
+
+def setup_postgrest_access(schema: str):
+    """Run basic grant statements to allow PostgREST to access the schema"""
+
+    def run_updates(db):
+        grant_schema_usage(db, schema, "web_anon")
+        grant_schema_usage(db, schema, "web_user", tables=False, sequences=True)
+
+    return run_updates

cli/macrostrat/cli/subsystems/macrostrat_api/__init__.py

@@ -10,22 +10,11 @@
 from macrostrat.core.migrations import Migration, view_exists
 
 from ...database import SubsystemSchemaDefinition, get_db
-from ...database.utils import grant_schema_usage
+from ...database.utils import setup_postgrest_access
 
 __here__ = Path(__file__).parent
 fixtures_dir = __here__ / "schema"
 
-
-def setup_postgrest_access(schema: str):
-    """Run basic grant statements to allow PostgREST to access the schema"""
-
-    def run_updates(db):
-        grant_schema_usage(db, schema, "web_anon")
-        grant_schema_usage(db, schema, "web_user", tables=False, sequences=True)
-
-    return run_updates
-
-
 macrostrat_api = SubsystemSchemaDefinition(
     name="macrostrat-api",
     fixtures=[fixtures_dir, setup_postgrest_access("macrostrat_api")],

core/macrostrat/core/config.py

@@ -1,15 +1,21 @@
 from os import environ
 from pathlib import Path
+from typing import Optional
 
+from dotenv import load_dotenv
 from dynaconf import Dynaconf, Validator
+from pydantic import BaseModel
 from sqlalchemy.engine import make_url
 from sqlalchemy.engine.url import URL
 from toml import load as load_toml
 
 from macrostrat.app_frame.control_command import BackendType
+from macrostrat.utils import get_logger
 
 from .utils import find_macrostrat_config
 
+log = get_logger(__name__)
+
 
 class MacrostratConfig(Dynaconf):
     """Macrostrat config manager that reads from a TOML file"""
@@ -27,6 +33,7 @@ def __init__(self, *args, **kwargs):
             environments=True,
             env_switcher="MACROSTRAT_ENV",
             settings_files=settings,
+            # We load dotenv files on our own
             load_dotenv=False,
         )
 
@@ -47,17 +54,40 @@ def all_environments(self):
 
 settings.validators.register(
     # `must_exist` is causing huge problems
-    # Validator("COMPOSE_ROOT", "CORELLE_SRC", must_exist=False, cast=Path),
-    Validator("COMPOSE_ROOT", "CORELLE_SRC", cast=Path),
+    Validator("COMPOSE_ROOT", cast=Path),
+    Validator("env_files", cast=list[Path]),
     Validator("pg_database", must_exist=True),
     # Backend information. We could potentially infer this from other environment variables
     Validator("backend", default="kubernetes", cast=BackendType),
 )
 
 macrostrat_env = getattr(settings, "env", "default")
 
+if env_files := getattr(settings, "env_files", None):
+    for env in env_files:
+        e = Path(env)
+        if not e.is_absolute():
+            # Resolve relative to config file
+            e = settings.config_file.parent / e
+
+        if not e.exists():
+            raise FileNotFoundError(f"Environment file {e} not found")
+
+        log.info(f"Loading environment variables from {e}")
+        load_dotenv(env)
+
 settings.validators.validate()
 
+# Settings for storage, if provided
+if storage := getattr(settings, "storage", None):
+    access_key = storage.get("access_key", None)
+    secret_key = storage.get("secret_key", None)
+    if access_key is None or secret_key is None:
+        raise ValueError("Access key and secret key must be provided for storage")
+
+    environ["STORAGE_ACCESS_KEY"] = access_key
+    environ["STORAGE_SECRET_KEY"] = secret_key
+
 # A database connection string for PostgreSQL
 PG_DATABASE = settings.pg_database
 # environ.get("MACROSTRAT_PG_DATABASE", None)
@@ -121,34 +151,54 @@ def all_environments(self):
 # This should eventually become optional if it isn't already
 MYSQL_DATABASE = getattr(settings, "mysql_database", None)
 
+if mapbox_token := getattr(settings, "mapbox_token", None):
+    environ["MAPBOX_TOKEN"] = mapbox_token
 
-# environ.get("MACROSTRAT_MYSQL_DATABASE", None)
+if secret_key := getattr(settings, "secret_key", None):
+    environ["SECRET_KEY"] = secret_key
 
+# Path to the root of the Macrostrat repository
+settings.srcroot = Path(__file__).parent.parent.parent.parent
 
-# REDIS_PORT = environ.get("REDIS_PORT", None)
+environ["MACROSTRAT_ROOT"] = str(settings.srcroot)
 
-# Tile caching
-# CACHE_PATH = environ.get("TILE_CACHE_PATH", "./tiles/burwell")
-# CACHE_PATH_VECTOR = environ.get("TILE_CACHE_PATH_VECTOR", CACHE_PATH)
 
-# TILESERVER_SECRET = environ.get("TILESERVER_SECRET", None)
-# MBTILES_PATH = environ.get("MBTILES_PATH", None)
+# Setup source roots for application components
+class Sources(BaseModel):
+    api: Optional[Path] = None
+    api_v3: Optional[Path] = None
+    tileserver: Optional[Path] = None
+    corelle: Optional[Path] = None
+    web: Optional[Path] = None
 
-# Path to the root of the Macrostrat repository
-settings.srcroot = Path(__file__).parent.parent.parent.parent
 
-environ["MACROSTRAT_ROOT"] = str(settings.srcroot)
+def get_source(key: str) -> Optional[Path]:
+    sources = getattr(settings, "sources", None)
+    if sources is None:
+        return None
+    src = getattr(sources, key, None)
+    if src is not None:
+        return Path(src)
+    return None
 
-# Settings for local installation
 
-# Used for local running of Macrostrat
-environ["MACROSTRAT_DB_PORT"] = str(url.port)
+def setup_environment(sources: Sources):
+    for k, v in sources.dict().items():
+        if v is not None:
+            environ[f"MACROSTRAT_{k.upper()}_SRC"] = str(v)
+
 
-if srcroot := getattr(settings, "api_srcroot", None):
-    environ["MACROSTRAT_API_SRC"] = srcroot
+settings.sources = Sources(
+    api=get_source("api"),
+    api_v3=get_source("api_v3"),
+    tileserver=get_source("tileserver"),
+    corelle=get_source("corelle"),
+    web=get_source("web"),
+)
+
+setup_environment(settings.sources)
 
-if srcroot := getattr(settings, "tileserver_srcroot", None):
-    environ["MACROSTRAT_TILESERVER_SRC"] = srcroot
+# Settings for local installation
 
-if srcroot := getattr(settings, "api_v3_srcroot", None):
-    environ["MACROSTRAT_API_V3_SRC"] = srcroot
+# Used for local running of Macrostrat
+environ["MACROSTRAT_DB_PORT"] = str(url.port)

core/macrostrat/core/database.py → core/macrostrat/core/database/__init__.py

@@ -2,7 +2,7 @@
 
 from macrostrat.database import Database
 
-from .config import PG_DATABASE
+from ..config import PG_DATABASE
 
 db_ctx: ContextVar[Database | None] = ContextVar("db_ctx", default=None)
 

local-root/Caddyfile

@@ -1,15 +1,34 @@
-http://localhost:80
+# Macrostrat.local domain is served by orbstack
+localhost, macrostrat.local {
+    tls internal
 
-handle_path /api/v2/* {
-	reverse_proxy api:5000
-}
+    handle_path /api/v2/* {
+        reverse_proxy api_v2:5000
+    }
+
+    handle_path /api/v3/* {
+        reverse_proxy api_v3:80
+    }
+
+    handle_path /api/pg/* {
+        reverse_proxy postgrest:3000
+    }
 
-handle_path /api/pg/* {
-    reverse_proxy postgrest:3000
+    handle_path /tiles/* {
+        reverse_proxy tileserver:8000
+    }
+
+    handle_path /* {
+        reverse_proxy web:3000
+    }
 }
 
-handle_path /tiles/* {
-    reverse_proxy tileserver:8000
+storage.macrostrat.local {
+    tls internal
+    reverse_proxy storage:9000
 }
 
-respond / "Welcome to your local installation of Macrostrat"
+storage-ui.macrostrat.local {
+    tls internal
+    reverse_proxy storage:9001
+}

local-root/docker-compose.yaml

@@ -5,6 +5,8 @@ services:
       - "8080:80"
     volumes:
       - ./Caddyfile:/etc/caddy/Caddyfile
+    labels:
+      - dev.orbstack.domains=macrostrat.local,storage.macrostrat.local,storage-ui.macrostrat.local
   database:
     # PostgreSQL 13 is needed in order to allow force-dropping the database
     # (in testing mode)
@@ -47,7 +49,7 @@ services:
     restart: always
     profiles:
       - legacy
-  api:
+  api_v2:
     image: hub.opensciencegrid.org/macrostrat/macrostrat-api:main
     build: ${MACROSTRAT_API_SRC:-""}
     environment:
@@ -57,26 +59,53 @@ services:
     build: ${MACROSTRAT_API_V3_SRC:-""}
     environment:
       - DB_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@database:5432/${POSTGRES_DB}
-    ports:
-      - "5000:5000"
+      - REDIRECT_URI=https://macrostrat.local/api/v3/security/callback
+      - SECRET_KEY=${SECRET_KEY}
+      - JWT_ENCRYPTION_ALGORITHM=HS256
+      - OAUTH_TOKEN_URL=${OAUTH_TOKEN_URL:-https://orcid.org/oauth/token}
+      - OAUTH_CLIENT_ID
+      - OAUTH_CLIENT_SECRET
+      - OAUTH_USERINFO_URL=${OAUTH_USERINFO_URL:-https://orcid.org/oauth/userinfo}
+      - OAUTH_AUTHORIZATION_URL=${OAUTH_AUTHORIZATION_URL:-https://orcid.org/oauth/authorize}
   postgrest:
     image: postgrest/postgrest:v12.0.2
     environment:
       - PGRST_DB_URI=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@database:5432/${POSTGRES_DB}
       - PGRST_DB_SCHEMA=macrostrat_api
       - PGRST_DB_ANON_ROLE=web_anon
-      #- PGRST_SERVER_PROXY_URI=http://database:5432
       - PGRST_SERVER_PORT=3000
       - PGRST_SERVER_HOST=
   # Tileserver
   tileserver:
-    image: hub.opensciencegrid.org/macrostrat/tileserver:latest-itb
+    image: hub.opensciencegrid.org/macrostrat/tileserver:main
     #profiles: ['tileserver']
     build:
       context: ${MACROSTRAT_TILESERVER_SRC:-""}
     environment:
       - DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@database:5432/${POSTGRES_DB}
-
+  # Storage
+  storage:
+    image: minio/minio:RELEASE.2024-12-18T13-15-44Z-cpuv1
+    environment:
+      - MINIO_ROOT_USER=${STORAGE_ACCESS_KEY}
+      - MINIO_ROOT_PASSWORD=${STORAGE_SECRET_KEY}
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    volumes:
+      - minio_data:/data
+    command: server --address 0.0.0.0:9000 --console-address 0.0.0.0:9001 /data
+  web:
+    image: hub.opensciencegrid.org/macrostrat/macrostrat-web:main
+    build: ${MACROSTRAT_WEB_SRC:-""}
+    environment:
+      - VITE_MAPBOX_API_TOKEN=${MAPBOX_TOKEN}
+      - VITE_MACROSTRAT_TILESERVER_DOMAIN=https://macrostrat.local/tiles
+      - VITE_MACROSTRAT_API_DOMAIN=https://macrostrat.local
+      # Needed for server-side rendering requests to not fail on self-signed certs (which OrbStack provides)
+      - NODE_TLS_REJECT_UNAUTHORIZED=0
+      # Secret key must be shared with the API that mints the JWT (in this case, the Macrostrat dev API)
+      - SECRET_KEY=${SECRET_KEY}
   # Schema-only Test DB instance for applying migrations
   migrations-test-db:
     image: postgis-with-audit
@@ -93,3 +122,4 @@ services:
 volumes:
   db_cluster:
   mysql_cluster:
+  minio_data: