Merge branch 'main' into column-tags

Author: charlie-haley

go.mod

@@ -25,7 +25,7 @@ require (
 	github.com/coreos/go-oidc/v3 v3.17.0
 	github.com/duckdb/duckdb-go/v2 v2.10501.0
 	github.com/elastic/go-elasticsearch/v8 v8.19.3
-	github.com/go-git/go-git/v5 v5.19.0
+	github.com/go-git/go-git/v5 v5.19.1
 	github.com/go-jose/go-jose/v4 v4.1.4
 	github.com/go-playground/validator/v10 v10.30.1
 	github.com/go-sql-driver/mysql v1.9.3

go.sum

@@ -367,8 +367,8 @@ github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmm
 github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw=
 github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
 github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
-github.com/go-git/go-git/v5 v5.19.0 h1:+WkVUQZSy/F1Gb13udrMKjIM2PrzsNfDKFSfo5tkMtc=
-github.com/go-git/go-git/v5 v5.19.0/go.mod h1:Pb1v0c7/g8aGQJwx9Us09W85yGoyvSwuhEGMH7zjDKQ=
+github.com/go-git/go-git/v5 v5.19.1 h1:nX27AnaU43/K5bKktKwgBmR9lawoYVe1Ckg0rgzzN00=
+github.com/go-git/go-git/v5 v5.19.1/go.mod h1:Pb1v0c7/g8aGQJwx9Us09W85yGoyvSwuhEGMH7zjDKQ=
 github.com/go-jose/go-jose/v3 v3.0.5 h1:BLLJWbC4nMZOfuPVxoZIxeYsn6Nl2r1fITaJ78UQlVQ=
 github.com/go-jose/go-jose/v3 v3.0.5/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ=
 github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=

internal/api/v1/assets/search.go

@@ -216,7 +216,7 @@ func (h *Handler) lookupAsset(w http.ResponseWriter, r *http.Request) {
 	if err != nil {
 		switch err {
 		case asset.ErrAssetNotFound:
-			http.NotFound(w, r)
+			common.RespondError(w, http.StatusNotFound, "asset not found")
 		default:
 			log.Error().
 				Err(err).

sdk/python/examples/claude_agent_marmot/main.py

@@ -0,0 +1,82 @@
+"""Runnable example: a Claude Agent SDK agent against the Marmot MCP server.
+
+Auto-registers the agent as a Marmot Agent asset and writes lineage edges for
+every catalog tool the agent calls. Mirrors `sdk/ts/examples/claude-agent-marmot`.
+
+Usage:
+    pip install marmot-sdk[claude-agent]
+    python main.py [prompt...]
+
+The Marmot host + credential is resolved via the SDK's standard chain:
+explicit kwargs → MARMOT_HOST/MARMOT_API_KEY/MARMOT_TOKEN env vars →
+~/.config/marmot/credentials.json (populated by `marmot login`) → workload
+identity.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import sys
+
+from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient
+
+from marmot import Client, resolve
+from marmot.integrations.claude_agent import MarmotAgentTracker
+
+DEFAULT_PROMPT = (
+    "Use the Marmot catalog tools to find a postgres table related to orders. "
+    "Reply with one sentence summarising the top hit and quoting its MRN."
+)
+
+
+async def main() -> None:
+    prompt = " ".join(sys.argv[1:]) or DEFAULT_PROMPT
+
+    base_url, credential = resolve(base_url=None)
+    client = Client(base_url=base_url, credential=credential)
+    tracker = MarmotAgentTracker(
+        client,
+        name="catalog-explorer-claude-py",
+        model="claude-sonnet-4-5",
+        owner="data-eng",
+    )
+
+    mcp_headers = (
+        {"Authorization": f"Bearer {credential.token}"}
+        if credential.scheme == "Bearer"
+        else {"X-API-Key": credential.token}
+    )
+    options = ClaudeAgentOptions(
+        mcp_servers={
+            "marmot": {
+                "type": "http",
+                "url": f"{base_url}/api/v1/mcp",
+                "headers": mcp_headers,
+            }
+        },
+        hooks=tracker.hooks(),
+        permission_mode="bypassPermissions",
+        allowed_tools=[
+            "mcp__marmot__discover_data",
+            "mcp__marmot__find_ownership",
+            "mcp__marmot__lookup_term",
+        ],
+    )
+
+    print(f"Marmot host: {base_url} (auth via {credential.source})")
+    print(f"Prompt: {prompt}\n")
+
+    async with ClaudeSDKClient(options=options) as agent:
+        await agent.query(prompt)
+        async for msg in agent.receive_response():
+            blocks = getattr(getattr(msg, "message", None), "content", None) or []
+            for block in blocks:
+                text = getattr(block, "text", None)
+                if isinstance(text, str) and text:
+                    print(text)
+
+    print("\nagent registered as:", tracker.agent_mrn or "(not yet registered)")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

sdk/python/pyproject.toml

@@ -46,6 +46,9 @@ dev = [
 langchain = [
     "langchain-core>=0.3",
 ]
+claude-agent = [
+    "claude-agent-sdk>=0.1",
+]
 
 [tool.hatch.build.targets.wheel]
 packages = ["src/marmot"]

sdk/python/src/marmot/__init__.py

@@ -40,6 +40,7 @@
 from marmot._gen.models.update_term_request import UpdateTermRequest
 from marmot._gen.models.user import User
 from marmot._gen.types import UNSET, Unset
+from marmot.auth import Credential, resolve
 from marmot.client import Client, connect
 from marmot.errors import (
     AuthError,
@@ -69,6 +70,7 @@
     "CreateAPIKeyRequest",
     "CreateAssetRequest",
     "CreateTermRequest",
+    "Credential",
     "GlossaryListResult",
     "GlossaryTerm",
     "LineageEdge",
@@ -104,6 +106,7 @@
     "connect",
     "is_not_found",
     "is_rate_limit",
+    "resolve",
 ]
 
 __version__ = "0.3.0"

sdk/python/src/marmot/integrations/_shared.py

@@ -0,0 +1,71 @@
+"""Helpers used by more than one integration. Not part of the public surface."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from typing import Any
+
+# MRN schemes Marmot is known to emit. Conservative — matching ``<scheme>://``
+# unconstrained would also catch arbitrary URLs (HTTP, etc.); we only mine for
+# schemes we control or that land in the catalog.
+MRN_PATTERN = re.compile(
+    r"\b(?:mrn|postgres|postgresql|mysql|kafka|s3|gcs|bigquery|snowflake|redis|"
+    r"clickhouse|elasticsearch|opensearch|mongodb|dynamodb|airflow|"
+    r"dbt|marmot)://[^\s\"'`<>,)\]}]+"
+)
+
+
+def extract_mrns(output: Any) -> set[str]:
+    """Walk an arbitrary tool output for asset MRNs.
+
+    Recognises:
+    - dicts with an ``mrn`` key,
+    - dicts/lists containing such dicts (e.g. ``{"results": [{"mrn": ...}]}``),
+    - JSON-encoded strings of any of the above,
+    - free text mentioning ``<scheme>://...`` for known catalog schemes
+      (markdown bodies returned by the Marmot MCP server fall here),
+    - lists of content blocks shaped like ``{"type": "text", "text": ...}``
+      (the MCP tool-response envelope).
+    """
+    found: set[str] = set()
+    _walk(output, found, depth=0)
+    return found
+
+
+def _walk(value: Any, out: set[str], *, depth: int) -> None:
+    if depth > 5:
+        return
+    if value is None:
+        return
+    if isinstance(value, str):
+        _walk_string(value, out)
+        return
+    if isinstance(value, dict):
+        mrn = value.get("mrn")
+        if isinstance(mrn, str) and mrn:
+            out.add(mrn)
+        for v in value.values():
+            _walk(v, out, depth=depth + 1)
+        return
+    if isinstance(value, list):
+        for v in value:
+            _walk(v, out, depth=depth + 1)
+
+
+def _walk_string(s: str, out: set[str]) -> None:
+    """Try to JSON-decode first (most structured tool outputs come back as
+    JSON-encoded strings); fall back to regex scanning the raw text."""
+    try:
+        parsed = json.loads(s)
+    except (ValueError, TypeError):
+        parsed = None
+    if isinstance(parsed, (dict, list)):
+        _walk(parsed, out, depth=0)
+    for match in MRN_PATTERN.findall(s):
+        out.add(match)
+
+
+def sha256_hex(value: str) -> str:
+    return hashlib.sha256(value.encode()).hexdigest()

sdk/python/src/marmot/integrations/claude_agent/__init__.py

@@ -0,0 +1,16 @@
+"""Claude Agent SDK integration for the Marmot SDK.
+
+Exposes :class:`MarmotAgentTracker`, which auto-registers a Claude Agent SDK
+agent as a Marmot ``Agent`` asset and writes lineage edges for every tool
+call. Pair with the Marmot MCP server (built into your Marmot instance at
+``/api/v1/mcp``) to give the agent catalog-aware tools.
+
+Requires ``claude-agent-sdk``. Install via
+``pip install marmot-sdk[claude-agent]``.
+"""
+
+from __future__ import annotations
+
+from marmot.integrations.claude_agent._tracker import MarmotAgentTracker
+
+__all__ = ["MarmotAgentTracker"]