Skip to content

Commit d27b1e2

Browse files
gsvigruhaclaude
andcommitted
refactor(llm-obs): match MCP shape for experiments events submit
Replace the `--file` body with inline `--metrics` (JSON array) and `--tags` flags so the command mirrors the submit_llmobs_experiment_events MCP tool's argument shape, letting skills map between pup and the MCP toolset. Remains a raw pass-through: metrics is parsed only to embed it as JSON and forwarded to the same ingest-events endpoint; the server validates the payload. - experiment_id stays positional - --metrics required, --tags optional (comma-separated) - update tests + add invalid-JSON negative test Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 440c8cb commit d27b1e2

2 files changed

Lines changed: 53 additions & 24 deletions

File tree

src/commands/llm_obs.rs

Lines changed: 41 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -282,16 +282,21 @@ pub async fn experiments_events_get(
282282
pub async fn experiments_events_submit(
283283
cfg: &Config,
284284
experiment_id: &str,
285-
file: &str,
285+
metrics: &str,
286+
tags: Option<Vec<String>>,
286287
) -> Result<()> {
287-
let mut body: serde_json::Value = util::read_json_file(file)?;
288-
if !body.is_object() {
289-
return Err(anyhow::anyhow!(
290-
"events file must contain a JSON object with a \"metrics\" array (and optional \"tags\")"
291-
));
288+
// Mirrors the submit_llmobs_experiment_events MCP tool: experiment_id + metrics
289+
// (array) + optional tags. metrics is passed through as-is; the server validates it.
290+
let metrics: serde_json::Value = serde_json::from_str(metrics).map_err(|e| {
291+
anyhow::anyhow!("--metrics must be a JSON array of eval-metric events: {e}")
292+
})?;
293+
let mut body = serde_json::json!({
294+
"experiment_id": experiment_id,
295+
"metrics": metrics,
296+
});
297+
if let Some(t) = tags {
298+
body["tags"] = serde_json::json!(t);
292299
}
293-
// The experiment_id is taken from the positional arg; it overrides any value in the file.
294-
body["experiment_id"] = serde_json::json!(experiment_id);
295300
let resp = raw_client::raw_post(
296301
cfg,
297302
"/api/unstable/llm-obs-mcp/v1/experiment/ingest-events",
@@ -3058,10 +3063,6 @@ mod tests {
30583063
let mut server = mockito::Server::new_async().await;
30593064
let cfg = test_config(&server.url());
30603065

3061-
let tmp = write_temp_json(
3062-
"pup_test_exp_ingest_events.json",
3063-
r#"{"metrics":[{"label":"accuracy","metric_type":"score","score_value":0.9}],"tags":["run:1"]}"#,
3064-
);
30653066
let resp_body = r#"{"status":"success","data":{"accepted":1}}"#;
30663067
let _mock = mock_post(
30673068
&mut server,
@@ -3071,13 +3072,18 @@ mod tests {
30713072
)
30723073
.await;
30733074

3074-
let result = super::experiments_events_submit(&cfg, "exp-1", tmp.to_str().unwrap()).await;
3075+
let result = super::experiments_events_submit(
3076+
&cfg,
3077+
"exp-1",
3078+
r#"[{"label":"accuracy","metric_type":"score","score_value":0.9}]"#,
3079+
Some(vec!["run:1".to_string()]),
3080+
)
3081+
.await;
30753082
assert!(
30763083
result.is_ok(),
30773084
"experiments_events_submit failed: {:?}",
30783085
result.err()
30793086
);
3080-
let _ = std::fs::remove_file(tmp);
30813087
cleanup_env();
30823088
std::env::remove_var("DD_TOKEN_STORAGE");
30833089
}
@@ -3089,10 +3095,6 @@ mod tests {
30893095
let mut server = mockito::Server::new_async().await;
30903096
let cfg = test_config(&server.url());
30913097

3092-
let tmp = write_temp_json(
3093-
"pup_test_exp_ingest_events_400.json",
3094-
r#"{"metrics":[{"label":"accuracy","metric_type":"score","score_value":0.9}]}"#,
3095-
);
30963098
let _mock = server
30973099
.mock("POST", mockito::Matcher::Any)
30983100
.match_query(mockito::Matcher::Any)
@@ -3102,9 +3104,28 @@ mod tests {
31023104
.create_async()
31033105
.await;
31043106

3105-
let result = super::experiments_events_submit(&cfg, "exp-1", tmp.to_str().unwrap()).await;
3107+
let result = super::experiments_events_submit(
3108+
&cfg,
3109+
"exp-1",
3110+
r#"[{"label":"accuracy","metric_type":"score","score_value":0.9}]"#,
3111+
None,
3112+
)
3113+
.await;
31063114
assert!(result.is_err(), "should fail on 400");
3107-
let _ = std::fs::remove_file(tmp);
3115+
cleanup_env();
3116+
std::env::remove_var("DD_TOKEN_STORAGE");
3117+
}
3118+
3119+
#[tokio::test]
3120+
async fn test_llm_obs_experiments_events_submit_invalid_json() {
3121+
let _lock = lock_env().await;
3122+
std::env::set_var("DD_TOKEN_STORAGE", "file");
3123+
let server = mockito::Server::new_async().await;
3124+
let cfg = test_config(&server.url());
3125+
3126+
// Malformed --metrics should fail locally before any request is made.
3127+
let result = super::experiments_events_submit(&cfg, "exp-1", "not-json", None).await;
3128+
assert!(result.is_err(), "should fail on invalid metrics JSON");
31083129
cleanup_env();
31093130
std::env::remove_var("DD_TOKEN_STORAGE");
31103131
}

src/main.rs

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9172,9 +9172,15 @@ enum LlmObsExperimentsEventsActions {
91729172
experiment_id: String,
91739173
#[arg(
91749174
long,
9175-
help = "JSON file with the events body: {\"metrics\": [...], \"tags\": [...]} (required)"
9175+
help = "JSON array of eval-metric events, e.g. '[{\"label\":\"accuracy\",\"metric_type\":\"score\",\"score_value\":0.9}]' (required)"
91769176
)]
9177-
file: String,
9177+
metrics: String,
9178+
#[arg(
9179+
long,
9180+
value_delimiter = ',',
9181+
help = "Optional \"key:value\" tags applied to every submitted metric (comma-separated)"
9182+
)]
9183+
tags: Option<Vec<String>>,
91789184
},
91799185
}
91809186

@@ -16180,12 +16186,14 @@ async fn main_inner() -> anyhow::Result<()> {
1618016186
}
1618116187
LlmObsExperimentsEventsActions::Submit {
1618216188
experiment_id,
16183-
file,
16189+
metrics,
16190+
tags,
1618416191
} => {
1618516192
commands::llm_obs::experiments_events_submit(
1618616193
&cfg,
1618716194
&experiment_id,
16188-
&file,
16195+
&metrics,
16196+
tags,
1618916197
)
1619016198
.await?;
1619116199
}

0 commit comments

Comments
 (0)