|
1 | | -"""Deployment-level S3 endpoint configuration. |
| 1 | +"""Deployment-level S3 configuration: the default backend and the source registry. |
2 | 2 |
|
3 | | -Single source of truth for the default S3 backend (#268/#271): one env surface |
4 | | -(S3_DEFAULT_ENDPOINT / S3_DEFAULT_URL_STYLE / S3_DEFAULT_USE_SSL) drives both |
5 | | -DuckDB connection factories — the per-request query engine |
6 | | -(server.get_isolated_db) and the persistent tile connection |
7 | | -(tiles.db.build_tile_connection) — so repointing a deployment at another |
8 | | -backend (MinIO, source.coop, ...) via env covers query AND hex tiles. |
| 3 | +Single source of truth (#268/#271/#264) for everything that maps STAC hrefs to |
| 4 | +DuckDB S3 access, consumed by the query engine (server.get_isolated_db), the |
| 5 | +tile subsystem (tiles.db.build_tile_connection), and the STAC renderer (stac.py): |
9 | 6 |
|
10 | | -Precursor to the data-driven source registry (#264): when routing becomes |
11 | | -registry-driven, this module is where the registry lives. |
| 7 | +- **Default backend** — one env surface (S3_DEFAULT_ENDPOINT / S3_DEFAULT_URL_STYLE / |
| 8 | + S3_DEFAULT_USE_SSL) for the unscoped `s3` secret, so repointing a deployment at |
| 9 | + another backend (MinIO, source.coop, ...) covers query AND hex tiles. |
| 10 | +
|
| 11 | +- **Source registry** — data-driven routing for every *known* non-default source |
| 12 | + (#264). Each entry can carry: |
| 13 | + name registry key; sanitized into the DuckDB secret name |
| 14 | + https_prefix hrefs starting with this are rewritten to s3_prefix |
| 15 | + s3_prefix replacement prefix (s3:// form DuckDB can glob) |
| 16 | + metadata_url_prefix when set, STAC-JSON fetches for https_prefix are rerouted |
| 17 | + here instead (e.g. the NRP in-cluster fast path) |
| 18 | + secret when set, a prefix-scoped DuckDB secret is created on |
| 19 | + every connection: {endpoint, scope, region?, url_style?, |
| 20 | + use_ssl?, key_id?, secret?} |
| 21 | + Built-ins cover the NRP Ceph endpoint (rewrite + in-cluster metadata reroute; |
| 22 | + no secret of its own — its paths route via the default `s3` secret) and the |
| 23 | + anonymous source.coop mirror (gateway rewrite + scoped `source_coop` secret). |
| 24 | +
|
| 25 | + Deployments extend/override via the S3_SOURCES env var: a JSON list of entries, |
| 26 | + merged over the built-ins by name ("disabled": true removes one). Example: |
| 27 | + S3_SOURCES='[{"name": "campus_minio", |
| 28 | + "https_prefix": "https://minio.example.edu/", |
| 29 | + "s3_prefix": "s3://", |
| 30 | + "secret": {"endpoint": "minio.example.edu", |
| 31 | + "scope": "s3://mirror-"}}]' |
| 32 | +
|
| 33 | +- **Route hints** — for hrefs on hosts NOT in the registry, `route_hint()` derives |
| 34 | + the (s3 path, endpoint, scope) a caller can pass per-request to `query` |
| 35 | + (s3_endpoint/s3_scope), so "carry the links" works for endpoints nobody |
| 36 | + pre-configured. Derivation, never a rewrite: unknown hrefs still pass through |
| 37 | + unchanged; the hint is surfaced alongside them by the STAC tools. |
| 38 | +
|
| 39 | +The fully STAC-native end state carries endpoint/routing on the asset itself |
| 40 | +(storage extension); this registry is the server-side stand-in until then. |
12 | 41 | """ |
| 42 | +import json |
13 | 43 | import os |
| 44 | +import re |
| 45 | +import sys |
14 | 46 |
|
15 | 47 |
|
16 | 48 | def sql_quote(value: str) -> str: |
@@ -53,3 +85,169 @@ def default_s3_secret_sql(name: str = "s3") -> str: |
53 | 85 | f"ENDPOINT '{sql_quote(endpoint)}', URL_STYLE '{sql_quote(url_style)}', " |
54 | 86 | f"USE_SSL '{sql_quote(use_ssl)}')" |
55 | 87 | ) |
| 88 | + |
| 89 | + |
| 90 | +# ------------------------------------------------------------------------- |
| 91 | +# Source registry |
| 92 | +# ------------------------------------------------------------------------- |
| 93 | +def _builtin_sources() -> list[dict]: |
| 94 | + # Computed per call (not module-level) so S3_ENDPOINT_URL is honored when it |
| 95 | + # changes between calls — e.g. running stac.py locally against the external |
| 96 | + # endpoint, or monkeypatching in tests. |
| 97 | + return [ |
| 98 | + { |
| 99 | + # Primary NRP Ceph. The STAC catalog publishes external SSL hrefs; |
| 100 | + # rewriting strips the host so DuckDB can glob, and paths then route |
| 101 | + # via the deployment-default `s3` secret (internal endpoint in-cluster) |
| 102 | + # — which is why this entry has no secret of its own. Metadata (STAC |
| 103 | + # JSON) fetches are invisibly rerouted to the in-cluster endpoint for |
| 104 | + # speed; override with S3_ENDPOINT_URL outside the cluster. |
| 105 | + "name": "nrp_ceph", |
| 106 | + "https_prefix": "https://s3-west.nrp-nautilus.io/", |
| 107 | + "s3_prefix": "s3://", |
| 108 | + "metadata_url_prefix": ( |
| 109 | + os.environ.get("S3_ENDPOINT_URL", "http://rook-ceph-rgw-nautiluss3.rook").rstrip("/") |
| 110 | + + "/" |
| 111 | + ), |
| 112 | + }, |
| 113 | + { |
| 114 | + # Anonymous source.coop mirror (#260). STAC entries publish assets |
| 115 | + # under the data.source.coop HTTPS gateway, which cannot list/glob; |
| 116 | + # the AWS-bucket form can. The bucket name is endpoint-unique, so the |
| 117 | + # prefix-scoped secret routes it unambiguously alongside Ceph paths. |
| 118 | + "name": "source_coop", |
| 119 | + "https_prefix": "https://data.source.coop/", |
| 120 | + "s3_prefix": "s3://us-west-2.opendata.source.coop/", |
| 121 | + "secret": { |
| 122 | + "endpoint": "s3.us-west-2.amazonaws.com", |
| 123 | + "region": "us-west-2", |
| 124 | + "scope": "s3://us-west-2.opendata.source.coop", |
| 125 | + }, |
| 126 | + }, |
| 127 | + ] |
| 128 | + |
| 129 | + |
| 130 | +def get_sources() -> list[dict]: |
| 131 | + """The active source registry: built-ins merged with the S3_SOURCES env JSON. |
| 132 | +
|
| 133 | + Env entries merge over built-ins by name (top-level keys replace; new names |
| 134 | + append; {"disabled": true} removes). A malformed S3_SOURCES is reported and |
| 135 | + ignored rather than taking the server down. Parsed per call — the inputs are |
| 136 | + tiny and this keeps env changes and tests cheap and correct. |
| 137 | + """ |
| 138 | + sources = {s["name"]: s for s in _builtin_sources()} |
| 139 | + raw = os.environ.get("S3_SOURCES", "").strip() |
| 140 | + if raw: |
| 141 | + try: |
| 142 | + for entry in json.loads(raw): |
| 143 | + name = entry.get("name") |
| 144 | + if not name: |
| 145 | + print(f"⚠️ S3_SOURCES entry without a name skipped: {entry!r}", file=sys.stderr) |
| 146 | + continue |
| 147 | + if entry.get("disabled"): |
| 148 | + sources.pop(name, None) |
| 149 | + continue |
| 150 | + sources[name] = {**sources.get(name, {}), **entry} |
| 151 | + except Exception as e: |
| 152 | + print(f"⚠️ S3_SOURCES is not valid JSON — ignored: {e}", file=sys.stderr) |
| 153 | + return list(sources.values()) |
| 154 | + |
| 155 | + |
| 156 | +def rewrite_href(href: str) -> str: |
| 157 | + """Registry-driven https→s3 prefix rewrite for data access (read_parquet). |
| 158 | +
|
| 159 | + Returns the href unchanged when no source matches — unknown hosts pass |
| 160 | + through, they are never guessed at (see route_hint for the advisory path). |
| 161 | + """ |
| 162 | + for src in get_sources(): |
| 163 | + hp, sp = src.get("https_prefix"), src.get("s3_prefix") |
| 164 | + if hp and sp and href.startswith(hp): |
| 165 | + return sp + href[len(hp):] |
| 166 | + return href |
| 167 | + |
| 168 | + |
| 169 | +def metadata_href(href: str) -> str: |
| 170 | + """Registry-driven reroute for STAC-metadata (JSON) fetches. |
| 171 | +
|
| 172 | + Distinct from rewrite_href: data reads want s3:// paths for DuckDB, while |
| 173 | + metadata reads stay HTTP — just pointed at a faster/internal host when the |
| 174 | + registry knows one (NRP in-cluster fast path). |
| 175 | + """ |
| 176 | + for src in get_sources(): |
| 177 | + hp, mp = src.get("https_prefix"), src.get("metadata_url_prefix") |
| 178 | + if hp and mp and href.startswith(hp): |
| 179 | + return mp + href[len(hp):] |
| 180 | + return href |
| 181 | + |
| 182 | + |
| 183 | +# Virtual-hosted AWS S3: BUCKET.s3.amazonaws.com or BUCKET.s3.REGION.amazonaws.com. |
| 184 | +_VHOST_S3 = re.compile(r"^(?P<bucket>[^/]+?)\.(?P<endpoint>s3(?:[.-][a-z0-9-]+)?\.amazonaws\.com)$") |
| 185 | + |
| 186 | + |
| 187 | +def route_hint(href: str) -> dict | None: |
| 188 | + """Derive per-request routing for an https href on a host outside the registry. |
| 189 | +
|
| 190 | + Assumes path-style S3 (https://HOST/BUCKET/KEY → s3://BUCKET/KEY @ HOST), |
| 191 | + plus the documented virtual-hosted AWS form. Returns |
| 192 | + {"path", "endpoint", "scope"} — the s3:// path and the s3_endpoint/s3_scope |
| 193 | + a caller passes to `query` — or None when the href isn't https or has no |
| 194 | + bucket/key to split. A hint, not a rewrite: callers surface it alongside the |
| 195 | + original href, they don't substitute silently. |
| 196 | + """ |
| 197 | + m = re.match(r"^https://([^/]+)/(.+)$", href) |
| 198 | + if not m: |
| 199 | + return None |
| 200 | + host, rest = m.groups() |
| 201 | + vhost = _VHOST_S3.match(host) |
| 202 | + if vhost: |
| 203 | + bucket, key = vhost.group("bucket"), rest |
| 204 | + endpoint = vhost.group("endpoint") |
| 205 | + else: |
| 206 | + bucket, _, key = rest.partition("/") |
| 207 | + endpoint = host |
| 208 | + if not bucket or not key: |
| 209 | + return None |
| 210 | + return { |
| 211 | + "path": f"s3://{bucket}/{key}", |
| 212 | + "endpoint": endpoint, |
| 213 | + "scope": f"s3://{bucket}", |
| 214 | + } |
| 215 | + |
| 216 | + |
| 217 | +def _secret_identifier(name: str) -> str: |
| 218 | + """Sanitize a registry name into a DuckDB identifier for CREATE SECRET.""" |
| 219 | + ident = re.sub(r"[^A-Za-z0-9_]", "_", name) |
| 220 | + return ident if not ident[:1].isdigit() else f"s_{ident}" |
| 221 | + |
| 222 | + |
| 223 | +def source_secret_sql() -> list[str]: |
| 224 | + """CREATE SECRET statements for every registry source that declares one. |
| 225 | +
|
| 226 | + All values are quote-escaped; USE_SSL is inferred from the endpoint unless |
| 227 | + the entry says otherwise. Run on every DuckDB connection (query and tiles) |
| 228 | + so scoped routing is identical everywhere. |
| 229 | + """ |
| 230 | + stmts = [] |
| 231 | + for src in get_sources(): |
| 232 | + sec = src.get("secret") |
| 233 | + if not sec or not sec.get("endpoint"): |
| 234 | + continue |
| 235 | + use_ssl = str(sec.get("use_ssl", infer_use_ssl(sec["endpoint"]))).strip().lower() |
| 236 | + parts = [ |
| 237 | + "TYPE S3", |
| 238 | + f"KEY_ID '{sql_quote(sec.get('key_id', ''))}'", |
| 239 | + f"SECRET '{sql_quote(sec.get('secret', ''))}'", |
| 240 | + f"ENDPOINT '{sql_quote(sec['endpoint'])}'", |
| 241 | + f"URL_STYLE '{sql_quote(sec.get('url_style', 'path'))}'", |
| 242 | + f"USE_SSL '{sql_quote(use_ssl)}'", |
| 243 | + ] |
| 244 | + if sec.get("region"): |
| 245 | + parts.append(f"REGION '{sql_quote(sec['region'])}'") |
| 246 | + if sec.get("scope"): |
| 247 | + parts.append(f"SCOPE '{sql_quote(sec['scope'])}'") |
| 248 | + stmts.append( |
| 249 | + f"CREATE OR REPLACE SECRET {_secret_identifier(src['name'])} (" |
| 250 | + + ", ".join(parts) |
| 251 | + + ")" |
| 252 | + ) |
| 253 | + return stmts |
0 commit comments