Skip to content

Commit cbb08f8

Browse files
LeoRoccoBreedtLeoRoccoBreedt
authored
Merge branch 'main' into lrb/tracing-finetuned-models-ci
2 parents 518326a + d7ad69a commit cbb08f8

5 files changed

Lines changed: 133 additions & 84 deletions

File tree

‎use-cases/governance_observability/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
governance_extract_*.json

‎use-cases/governance_observability/agent_tracing.py‎

Lines changed: 47 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -34,18 +34,23 @@
3434
import opik
3535
from opik import opik_context
3636

37-
PROJECT_NAME = os.environ.get("OPIK_PROJECT_NAME", "governance-data-demo")
37+
WORKSPACE = os.environ.get("OPIK_WORKSPACE")
38+
PROJECT_NAME = os.environ.get("OPIK_PROJECT_NAME", "governance-data-demo")
39+
40+
# No Opik credentials -> describe what would be traced and exit without calling Opik.
41+
DRY_RUN = not (os.environ.get("OPIK_API_KEY") and WORKSPACE)
3842

3943
# Tag applied to every trace. The oversight/reporting team filters on this tag
4044
# to identify which traces belong to the governance programme.
4145
# Replace with whatever tag your organisation uses.
42-
GOVERNANCE_TAG = "governance"
46+
GOVERNANCE_TAG = "governance"
4347

4448

4549
# ---------------------------------------------------------------------------
4650
# Agent implementation
4751
# ---------------------------------------------------------------------------
4852

53+
4954
@opik.track(name="retrieve_context", type="tool")
5055
def retrieve_context(query: str) -> list[str]:
5156
opik_context.update_current_span(metadata={"retriever": "vector-index-v3", "top_k": 5})
@@ -93,26 +98,26 @@ def run_agent(
9398
metadata={
9499
# Governance fields — the oversight team filters and slices on all of these.
95100
# Adapt field names and values to match your organisation's schema.
96-
"env": "prod",
97-
"region": "us-east",
98-
"use_case_id": "loan-approval",
99-
"use_case_version": "2.1.0",
100-
"team": "risk-analytics",
101-
"business_unit": business_unit,
102-
"model_name": model,
103-
"model_version": "2024-11-20",
104-
"risk_tier": risk_tier,
101+
"env": "prod",
102+
"region": "us-east",
103+
"use_case_id": "loan-approval",
104+
"use_case_version": "2.1.0",
105+
"team": "risk-analytics",
106+
"business_unit": business_unit,
107+
"model_name": model,
108+
"model_version": "2024-11-20",
109+
"risk_tier": risk_tier,
105110
"data_classification": "confidential",
106-
"regulatory_scope": "internal",
111+
"regulatory_scope": "internal",
107112
# Call-level runtime fields
108-
"request_id": request_id,
109-
"channel": "api",
113+
"request_id": request_id,
114+
"channel": "api",
110115
}
111116
)
112117

113118
context_docs = retrieve_context(query)
114-
answer = call_llm(query, context_docs, model)
115-
119+
answer = call_llm(query, context_docs, model)
120+
116121
return {"answer": answer, "sources": context_docs}
117122

118123

@@ -122,35 +127,42 @@ def run_agent(
122127

123128
SAMPLE_RUNS = [
124129
{
125-
"query": "Assess the risk for a $20,000 personal loan application.",
126-
"request_id": "req-001",
127-
"business_unit": "retail",
128-
"risk_tier": "high",
130+
"query": "Assess the risk for a $20,000 personal loan application.",
131+
"request_id": "req-001",
132+
"business_unit": "retail",
133+
"risk_tier": "high",
129134
"hallucination_rate": 0.03,
130-
"response_quality": 0.91,
131-
"cost_usd": 0.0042,
135+
"response_quality": 0.91,
136+
"cost_usd": 0.0042,
132137
},
133138
{
134-
"query": "Evaluate eligibility for a $500,000 business loan.",
135-
"request_id": "req-002",
136-
"business_unit": "commercial",
137-
"risk_tier": "medium",
139+
"query": "Evaluate eligibility for a $500,000 business loan.",
140+
"request_id": "req-002",
141+
"business_unit": "commercial",
142+
"risk_tier": "medium",
138143
"hallucination_rate": 0.07,
139-
"response_quality": 0.84,
140-
"cost_usd": 0.0061,
144+
"response_quality": 0.84,
145+
"cost_usd": 0.0061,
141146
},
142147
{
143-
"query": "Review a credit limit increase request from $10,000 to $25,000.",
144-
"request_id": "req-003",
145-
"business_unit": "wealth",
146-
"risk_tier": "low",
148+
"query": "Review a credit limit increase request from $10,000 to $25,000.",
149+
"request_id": "req-003",
150+
"business_unit": "wealth",
151+
"risk_tier": "low",
147152
"hallucination_rate": 0.01,
148-
"response_quality": 0.97,
149-
"cost_usd": 0.0038,
153+
"response_quality": 0.97,
154+
"cost_usd": 0.0038,
150155
},
151156
]
152157

153158
if __name__ == "__main__":
159+
if DRY_RUN:
160+
print(
161+
"[DRY RUN] Opik creds not set — would trace 3 governance-tagged "
162+
f"loan-approval agent runs to project '{PROJECT_NAME}'."
163+
)
164+
raise SystemExit(0)
165+
154166
print(f"Project : {PROJECT_NAME}")
155167
print(f"Tag : {GOVERNANCE_TAG}\n")
156168

@@ -161,4 +173,4 @@ def run_agent(
161173

162174
opik.flush_tracker()
163175
print("Done. Traces are visible in the Opik UI under the project:")
164-
print(f" https://www.comet.com/opik/{os.environ['OPIK_WORKSPACE']}/{PROJECT_NAME}/traces")
176+
print(f" https://www.comet.com/opik/{WORKSPACE}/{PROJECT_NAME}/traces")

‎use-cases/governance_observability/data_governance_team.py‎

Lines changed: 32 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -35,16 +35,19 @@
3535
# Config
3636
# ---------------------------------------------------------------------------
3737

38-
WORKSPACE = os.environ["OPIK_WORKSPACE"]
39-
OPIK_BASE_URL = os.environ.get("OPIK_URL_OVERRIDE", "https://www.comet.com/opik/api")
40-
GOVERNANCE_TAG = "governance" # must match the tag used in agent_tracing.py
38+
WORKSPACE = os.environ.get("OPIK_WORKSPACE")
39+
OPIK_BASE_URL = os.environ.get("OPIK_URL_OVERRIDE", "https://www.comet.com/opik/api")
40+
GOVERNANCE_TAG = "governance" # must match the tag used in agent_tracing.py
41+
42+
# No Opik credentials -> describe the extraction and exit without calling the Opik API.
43+
DRY_RUN = not (os.environ.get("OPIK_API_KEY") and WORKSPACE)
4144

4245
_now = datetime.now(UTC)
4346

4447
# Metric types to extract. Each maps to one get_project_metrics() call.
4548
# See the full list of available values in the SDK docs linked above.
4649
METRIC_TYPES = [
47-
"FEEDBACK_SCORES", # average per named feedback score
50+
"FEEDBACK_SCORES", # average per named feedback score
4851
]
4952

5053
# Interval for aggregation. Choose one: "HOURLY" | "DAILY" | "WEEKLY" | "TOTAL"
@@ -66,6 +69,7 @@ def build_client() -> OpikApi:
6669
# Project enumeration
6770
# ---------------------------------------------------------------------------
6871

72+
6973
def list_all_projects(client: OpikApi) -> list[dict]:
7074
"""Page through find_projects() and return [{"id": ..., "name": ...}, ...]."""
7175
projects = []
@@ -96,6 +100,7 @@ def list_all_projects(client: OpikApi) -> list[dict]:
96100
# TraceFilterPublic(field="metadata", key="business_unit", operator="=", value="retail")
97101
# ---------------------------------------------------------------------------
98102

103+
99104
def _governance_filters(metadata_slice: dict[str, str] | None = None) -> list[TraceFilterPublic]:
100105
"""
101106
Build the filter list for a governance extraction.
@@ -105,9 +110,7 @@ def _governance_filters(metadata_slice: dict[str, str] | None = None) -> list[Tr
105110
TraceFilterPublic(field="tags", operator="contains", value=GOVERNANCE_TAG),
106111
]
107112
for key, value in (metadata_slice or {}).items():
108-
filters.append(
109-
TraceFilterPublic(field="metadata", key=key, operator="=", value=value)
110-
)
113+
filters.append(TraceFilterPublic(field="metadata", key=key, operator="=", value=value))
111114
return filters
112115

113116

@@ -138,6 +141,7 @@ def _governance_filters(metadata_slice: dict[str, str] | None = None) -> list[Tr
138141
# Metrics extraction
139142
# ---------------------------------------------------------------------------
140143

144+
141145
def fetch_metrics_for_project(
142146
client: OpikApi,
143147
project_id: str,
@@ -153,7 +157,7 @@ def fetch_metrics_for_project(
153157
result.name — score name (e.g. "composite_risk_score")
154158
result.data — list of DataPointNumberPublic (time, value) data points
155159
"""
156-
trace_filters = _governance_filters(metadata_slice)
160+
trace_filters = _governance_filters(metadata_slice)
157161
interval_start = _now - timedelta(days=LOOKBACK_DAYS)
158162
req_opts: RequestOptions = {"timeout_in_seconds": 60}
159163
metrics: dict = {}
@@ -192,15 +196,16 @@ def fetch_metrics_for_project(
192196
# Main pipeline
193197
# ---------------------------------------------------------------------------
194198

199+
195200
def run_extraction() -> list[dict]:
196-
print(f"\n{'='*60}")
201+
print(f"\n{'=' * 60}")
197202
print(f"Governance Metrics Extraction {_now.strftime('%Y-%m-%d %H:%M UTC')}")
198203
print(f"Workspace : {WORKSPACE}")
199204
print(f"Tag : {GOVERNANCE_TAG}")
200205
print(f"Interval : {INTERVAL} | Look-back: {LOOKBACK_DAYS} days")
201-
print(f"{'='*60}\n")
206+
print(f"{'=' * 60}\n")
202207

203-
client = build_client()
208+
client = build_client()
204209
projects = list_all_projects(client)
205210
payloads = []
206211

@@ -220,11 +225,7 @@ def run_extraction() -> list[dict]:
220225
trace_filters=_governance_filters(),
221226
request_options={"timeout_in_seconds": 60},
222227
)
223-
probe_has_data = any(
224-
point.value
225-
for result in (probe.results or [])
226-
for point in (result.data or [])
227-
)
228+
probe_has_data = any(point.value for result in (probe.results or []) for point in (result.data or []))
228229
if not probe_has_data:
229230
print(" No governance-tagged traces — skipping.\n")
230231
continue
@@ -239,17 +240,17 @@ def run_extraction() -> list[dict]:
239240
for label, metadata_slice in slices_to_run:
240241
sliced_metrics[label] = {
241242
"slice_filter": metadata_slice,
242-
"metrics": fetch_metrics_for_project(client, project["id"], metadata_slice),
243+
"metrics": fetch_metrics_for_project(client, project["id"], metadata_slice),
243244
}
244245

245246
payload = {
246-
"schema_version": "2.0",
247-
"extracted_at": _now.isoformat(),
248-
"workspace": WORKSPACE,
249-
"project_id": project["id"],
250-
"project_name": project["name"],
251-
"governance_tag": GOVERNANCE_TAG,
252-
"slices": sliced_metrics,
247+
"schema_version": "2.0",
248+
"extracted_at": _now.isoformat(),
249+
"workspace": WORKSPACE,
250+
"project_id": project["id"],
251+
"project_name": project["name"],
252+
"governance_tag": GOVERNANCE_TAG,
253+
"slices": sliced_metrics,
253254
}
254255
payloads.append(payload)
255256

@@ -282,4 +283,11 @@ def _push_to_reporting_endpoint(payloads: list[dict]) -> None:
282283

283284

284285
if __name__ == "__main__":
286+
if DRY_RUN:
287+
print(
288+
"[DRY RUN] Opik creds not set — would extract governance metrics across "
289+
f"all projects for '{GOVERNANCE_TAG}'-tagged traces and build the reporting payload."
290+
)
291+
raise SystemExit(0)
292+
285293
run_extraction()

‎use-cases/governance_observability/run.sh‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
4+
export OPIK_PROJECT_NAME="governance-observability"
5+
6+
uv sync
7+
8+
echo "--- Step 1: agent tracing ---"
9+
uv run python agent_tracing.py
10+
11+
echo "--- Step 2: use case team ---"
12+
uv run python use_case_team.py
13+
14+
echo "--- Step 3: data governance team ---"
15+
uv run python data_governance_team.py

0 commit comments

Comments
 (0)