Merge pull request #424 from datadog-labs/feat/llm-obs-eval-config

feat(llm-obs): add custom evaluator config get/update/delete

Author: platinummonkey

src/client.rs

@@ -264,7 +264,7 @@ static UNSTABLE_OPS: &[&str] = &[
     "v2.delete_aws_cloud_auth_persona_mapping",
     "v2.get_aws_cloud_auth_persona_mapping",
     "v2.list_aws_cloud_auth_persona_mappings",
-    // LLM Observability (15)
+    // LLM Observability (18)
     "v2.create_llm_obs_project",
     "v2.list_llm_obs_projects",
     "v2.create_llm_obs_experiment",
@@ -280,6 +280,9 @@ static UNSTABLE_OPS: &[&str] = &[
     "v2.create_llm_obs_annotation_queue_interactions",
     "v2.delete_llm_obs_annotation_queue_interactions",
     "v2.get_llm_obs_annotated_interactions",
+    "v2.get_llm_obs_custom_eval_config",
+    "v2.update_llm_obs_custom_eval_config",
+    "v2.delete_llm_obs_custom_eval_config",
     // Logs Restriction Queries (9)
     "v2.list_restriction_queries",
     "v2.get_restriction_query",
@@ -1018,7 +1021,7 @@ mod tests {
 
     #[test]
     fn test_unstable_ops_count() {
-        assert_eq!(UNSTABLE_OPS.len(), 149);
+        assert_eq!(UNSTABLE_OPS.len(), 152);
     }
 
     #[test]

src/commands/llm_obs.rs

@@ -4,7 +4,7 @@ use datadog_api_client::datadogV2::api_llm_observability::{
 };
 use datadog_api_client::datadogV2::model::{
     LLMObsAnnotationQueueInteractionsRequest, LLMObsAnnotationQueueRequest,
-    LLMObsAnnotationQueueUpdateRequest, LLMObsDatasetRequest,
+    LLMObsAnnotationQueueUpdateRequest, LLMObsCustomEvalConfigUpdateRequest, LLMObsDatasetRequest,
     LLMObsDeleteAnnotationQueueInteractionsRequest, LLMObsDeleteExperimentsRequest,
     LLMObsExperimentRequest, LLMObsExperimentUpdateRequest, LLMObsProjectRequest,
 };
@@ -298,6 +298,36 @@ pub async fn annotation_queue_interactions_list(cfg: &Config, queue_id: &str) ->
     formatter::output(cfg, &resp)
 }
 
+// ---- Custom Evaluator Configs ----
+
+pub async fn eval_config_get(cfg: &Config, eval_name: &str) -> Result<()> {
+    let api = make_api(cfg);
+    let resp = api
+        .get_llm_obs_custom_eval_config(eval_name.to_string())
+        .await
+        .map_err(|e| anyhow::anyhow!("failed to get LLM obs custom eval config: {e:?}"))?;
+    formatter::output(cfg, &resp)
+}
+
+pub async fn eval_config_update(cfg: &Config, eval_name: &str, file: &str) -> Result<()> {
+    let body: LLMObsCustomEvalConfigUpdateRequest = util::read_json_file(file)?;
+    let api = make_api(cfg);
+    api.update_llm_obs_custom_eval_config(eval_name.to_string(), body)
+        .await
+        .map_err(|e| anyhow::anyhow!("failed to update LLM obs custom eval config: {e:?}"))?;
+    eprintln!("LLM obs custom eval config '{eval_name}' updated.");
+    Ok(())
+}
+
+pub async fn eval_config_delete(cfg: &Config, eval_name: &str) -> Result<()> {
+    let api = make_api(cfg);
+    api.delete_llm_obs_custom_eval_config(eval_name.to_string())
+        .await
+        .map_err(|e| anyhow::anyhow!("failed to delete LLM obs custom eval config: {e:?}"))?;
+    eprintln!("LLM obs custom eval config '{eval_name}' deleted.");
+    Ok(())
+}
+
 // ---- Spans (no typed equivalent — unstable MCP endpoint) ----
 
 #[allow(clippy::too_many_arguments)]
@@ -1184,6 +1214,129 @@ mod tests {
         cleanup_env();
     }
 
+    #[tokio::test]
+    async fn test_llm_obs_eval_config_get() {
+        let _lock = lock_env().await;
+        std::env::set_var("DD_TOKEN_STORAGE", "file");
+        let mut server = mockito::Server::new_async().await;
+        let cfg = test_config(&server.url());
+        let body = r#"{"data":{"id":"toxicity","type":"evaluator_config","attributes":{"eval_name":"toxicity","created_at":"2024-01-01T00:00:00Z","updated_at":"2024-01-01T00:00:00Z"}}}"#;
+        let _mock = mock_any(&mut server, "GET", body).await;
+        let result = super::eval_config_get(&cfg, "toxicity").await;
+        assert!(result.is_ok(), "eval_config_get failed: {:?}", result.err());
+        cleanup_env();
+        std::env::remove_var("DD_TOKEN_STORAGE");
+    }
+
+    #[tokio::test]
+    async fn test_llm_obs_eval_config_get_404() {
+        let _lock = lock_env().await;
+        std::env::set_var("DD_TOKEN_STORAGE", "file");
+        let mut server = mockito::Server::new_async().await;
+        let cfg = test_config(&server.url());
+        let _mock = server
+            .mock("GET", mockito::Matcher::Any)
+            .with_status(404)
+            .with_header("content-type", "application/json")
+            .with_body(r#"{"errors":["not found"]}"#)
+            .create_async()
+            .await;
+        let result = super::eval_config_get(&cfg, "missing").await;
+        assert!(result.is_err(), "expected 404 error");
+        cleanup_env();
+        std::env::remove_var("DD_TOKEN_STORAGE");
+    }
+
+    #[tokio::test]
+    async fn test_llm_obs_eval_config_update() {
+        let _lock = lock_env().await;
+        std::env::set_var("DD_TOKEN_STORAGE", "file");
+        let mut server = mockito::Server::new_async().await;
+        let cfg = test_config(&server.url());
+        let tmp = write_temp_json(
+            "pup_test_eval_config_update.json",
+            r#"{"data":{"type":"evaluator_config","attributes":{"target":{"application_name":"my-app","enabled":true}}}}"#,
+        );
+        let _mock = server
+            .mock("PUT", mockito::Matcher::Any)
+            .with_status(204)
+            .create_async()
+            .await;
+        let result = super::eval_config_update(&cfg, "toxicity", tmp.to_str().unwrap()).await;
+        assert!(
+            result.is_ok(),
+            "eval_config_update failed: {:?}",
+            result.err()
+        );
+        let _ = std::fs::remove_file(tmp);
+        cleanup_env();
+        std::env::remove_var("DD_TOKEN_STORAGE");
+    }
+
+    #[tokio::test]
+    async fn test_llm_obs_eval_config_update_400() {
+        let _lock = lock_env().await;
+        std::env::set_var("DD_TOKEN_STORAGE", "file");
+        let mut server = mockito::Server::new_async().await;
+        let cfg = test_config(&server.url());
+        let tmp = write_temp_json(
+            "pup_test_eval_config_update_400.json",
+            r#"{"data":{"type":"evaluator_config","attributes":{"target":{"application_name":"my-app","enabled":true}}}}"#,
+        );
+        let _mock = server
+            .mock("PUT", mockito::Matcher::Any)
+            .with_status(400)
+            .with_header("content-type", "application/json")
+            .with_body(r#"{"errors":["bad request"]}"#)
+            .create_async()
+            .await;
+        let result = super::eval_config_update(&cfg, "toxicity", tmp.to_str().unwrap()).await;
+        assert!(result.is_err(), "expected 400 error");
+        let _ = std::fs::remove_file(tmp);
+        cleanup_env();
+        std::env::remove_var("DD_TOKEN_STORAGE");
+    }
+
+    #[tokio::test]
+    async fn test_llm_obs_eval_config_delete() {
+        let _lock = lock_env().await;
+        std::env::set_var("DD_TOKEN_STORAGE", "file");
+        let mut server = mockito::Server::new_async().await;
+        let cfg = test_config(&server.url());
+        let _mock = server
+            .mock("DELETE", mockito::Matcher::Any)
+            .with_status(204)
+            .create_async()
+            .await;
+        let result = super::eval_config_delete(&cfg, "toxicity").await;
+        assert!(
+            result.is_ok(),
+            "eval_config_delete failed: {:?}",
+            result.err()
+        );
+        cleanup_env();
+        std::env::remove_var("DD_TOKEN_STORAGE");
+    }
+
+    #[tokio::test]
+    async fn test_llm_obs_eval_config_delete_404() {
+        let _lock = lock_env().await;
+        std::env::set_var("DD_TOKEN_STORAGE", "file");
+        let mut server = mockito::Server::new_async().await;
+        let cfg = test_config(&server.url());
+        let _mock = server
+            .mock("DELETE", mockito::Matcher::Any)
+            .with_status(404)
+            .with_header("content-type", "application/json")
+            .with_body(r#"{"errors":["not found"]}"#)
+            .create_async()
+            .await;
+        let result = super::eval_config_delete(&cfg, "missing").await;
+        assert!(result.is_err(), "expected 404 error");
+        cleanup_env();
+        std::env::remove_var("DD_TOKEN_STORAGE");
+    }
+
     #[tokio::test]
     async fn test_llm_obs_spans_search() {
         let _lock = lock_env().await;

src/main.rs

@@ -7881,6 +7881,12 @@ enum LlmObsActions {
         #[command(subcommand)]
         action: LlmObsAnnotationQueuesActions,
     },
+    /// Manage LLM Observability custom evaluator configs
+    #[command(name = "eval-config")]
+    EvalConfig {
+        #[command(subcommand)]
+        action: LlmObsEvalConfigActions,
+    },
 }
 
 #[derive(Subcommand)]
@@ -8092,6 +8098,27 @@ enum LlmObsAnnotationQueueInteractionsActions {
     },
 }
 
+#[derive(Subcommand)]
+enum LlmObsEvalConfigActions {
+    /// Get a custom evaluator config by name
+    Get {
+        #[arg(help = "Evaluator name")]
+        eval_name: String,
+    },
+    /// Create or update a custom evaluator config by name
+    Update {
+        #[arg(help = "Evaluator name")]
+        eval_name: String,
+        #[arg(long, help = "JSON file with evaluator config body (required)")]
+        file: String,
+    },
+    /// Delete a custom evaluator config by name
+    Delete {
+        #[arg(help = "Evaluator name")]
+        eval_name: String,
+    },
+}
+
 // ---- Reference Tables ----
 #[derive(Subcommand)]
 enum ReferenceTablesActions {
@@ -13362,6 +13389,17 @@ async fn main_inner() -> anyhow::Result<()> {
                         }
                     },
                 },
+                LlmObsActions::EvalConfig { action } => match action {
+                    LlmObsEvalConfigActions::Get { eval_name } => {
+                        commands::llm_obs::eval_config_get(&cfg, &eval_name).await?;
+                    }
+                    LlmObsEvalConfigActions::Update { eval_name, file } => {
+                        commands::llm_obs::eval_config_update(&cfg, &eval_name, &file).await?;
+                    }
+                    LlmObsEvalConfigActions::Delete { eval_name } => {
+                        commands::llm_obs::eval_config_delete(&cfg, &eval_name).await?;
+                    }
+                },
             }
         }
         // --- Profiling ---