refactored server.py for better coherence; improved system prompts

Author: Alexander Fischer

litellm_config.yaml

@@ -3,7 +3,7 @@ model_list:
     litellm_params:
       model: openai/gpt-5.6-luna
       api_key: "os.environ/CLIMATECLAW_OPENAI_API_KEY"
-      reasoning_effort: "low"
+      reasoning_effort: "medium"
   - model_name: gpt-4.1
     litellm_params:
       model: openai/gpt-4.1

scripts/dev_evaluate_tool.py

@@ -8,6 +8,7 @@
 import time
 from collections import defaultdict
 from dataclasses import dataclass
+from datetime import datetime
 from pathlib import Path
 
 from matplotlib import pyplot as plt
@@ -63,16 +64,17 @@
 SILENCE_LOGGING = True
 
 # run evaluation of tool calls (TP, FP, FN, accuracy, precision, recall)
-EVAL_TOOL_CALLS = True
+EVAL_TOOL = True
 PLOT_METRICS = True  # plot metrics as bar charts at the end of the benchmark
 
-## evaluation suite, consisting of direct questions about a plugin + indirect ones that are more widely phrased
+## evaluation suite: direct questions about a plugin + indirect ones (more widely phrased)
 path_to_prompts = Path(__file__).parent / "evaluation" / "benchmark_prompts.json"
 with open(path_to_prompts, "r", encoding="utf-8") as f:
     BENCHMARK = {k.lower(): v for k, v in json.load(f).items()}
 # BENCHMARK = {
 #     "leadtimeselektor": [
 #     "How does the 'leadtimeselektor' plugin work from a high-level perspective?",
+#     "How can I best extract lead times from decadal climate predictions?",
 #     "How are you?",
 #     ]
 # }
@@ -264,7 +266,7 @@ async def run_benchmark_for_prompt(
     tool_args = [r.tool_args for r in results]
     tool_outputs = [r.tool_output for r in results]
 
-    if EVAL_TOOL_CALLS:
+    if EVAL_TOOL:
         metrics_dict = _evaluate_tool_call_results(
             tool_names, tool_args, tool_outputs, plugin
         )
@@ -275,8 +277,10 @@ async def run_benchmark_for_prompt(
 
 
 def plot_metrics(avg_metrics: dict[str, float], save_dir: Path) -> None:
+    current_date = datetime.today().strftime("%Y-%m-%d")
     # pie chart for tool/plugin TP, FP, FN
     fig, axs = plt.subplots(1, 2, figsize=(12, 6))
+    metrics = ["TP", "FP", "FN"]
     tool_metrics = {
         "True Positives": avg_metrics.get("true_pos_tool", 0),
         "False Positives": avg_metrics.get("false_pos_tool", 0),
@@ -287,16 +291,22 @@ def plot_metrics(avg_metrics: dict[str, float], save_dir: Path) -> None:
         "False Positives": avg_metrics.get("false_pos_plugin", 0),
         "False Negatives": avg_metrics.get("false_neg_plugin", 0),
     }
-    axs[0].pie(tool_metrics.values(), labels=tool_metrics.keys(), autopct="%1.1f%%")
-    axs[0].set_title("Tool Call Metrics")
-    axs[1].pie(plugin_metrics.values(), labels=plugin_metrics.keys(), autopct="%1.1f%%")
-    axs[1].set_title("Plugin Call Metrics")
+    charts = [tool_metrics, plugin_metrics]
+    titles = ["Tool Call", "Plugin Call"]
+    for ax, chart, title in zip(axs, charts, titles):
+        pie = ax.pie(
+            chart.values(),
+            autopct="%1.1f%%",
+            textprops=dict(fontsize=14, color="white"),
+        )
+        ax.legend(pie.wedges, metrics, title="Metrics", loc="best", fontsize=14)
+        ax.set_title(title, fontsize=16)
     plt.suptitle(
         f"Evaluation Metrics for 'plugin_code_search' tool (Model: {MODEL}, Runs/Query: 30)"
     )
     plt.tight_layout()
-    plt.subplots_adjust(wspace=0.4)
-    plt.savefig(save_dir / "tool_evaluation_pie_charts.png")
+    plt.subplots_adjust(wspace=0.2)
+    plt.savefig(save_dir / f"tool_evaluation_pie_charts_{current_date}.png", dpi=400)
 
     # bar chart for tool/plugin accuracy, precision, recall
     fig, axs = plt.subplots(1, 2, figsize=(12, 6))
@@ -322,7 +332,7 @@ def plot_metrics(avg_metrics: dict[str, float], save_dir: Path) -> None:
     )
     plt.tight_layout()
     plt.subplots_adjust(wspace=0.3)
-    plt.savefig(save_dir / "tool_evaluation_metrics.png")
+    plt.savefig(save_dir / f"tool_evaluation_metrics_{current_date}.png", dpi=400)
 
 
 async def main() -> None:
@@ -340,16 +350,16 @@ async def main() -> None:
                 progress.update(RUNS)
 
     # compute average metrics across all prompts
-    if EVAL_TOOL_CALLS:
+    if EVAL_TOOL:
         avg_metrics = {k: sum(v) / len(v) for k, v in eval_dict.items()}
         print(
             f"\n=== Average metrics [in %] across all prompts ({RUNS} runs/prompt) ==="
         )
         for k, v in avg_metrics.items():
-            print(f"{k}: {v:.1%}")
+            print(f"  {k:<17}: {v:.1%}")
 
     # plot metrics as bar charts (one for tool, one for plugin)
-    if PLOT_METRICS and EVAL_TOOL_CALLS:
+    if PLOT_METRICS and EVAL_TOOL:
         eval_dir = Path(__file__).parent / "evaluation"
         eval_dir.mkdir(parents=True, exist_ok=True)
         plot_metrics(avg_metrics, eval_dir)

scripts/evaluation/tool_evaluation_metrics.png

@@ -1,4 +1,4 @@
-# ClimateClaw Starting Prompt
+# ClimateClaw System Prompt
 
 ## A. IDENTITY & SCOPE
 
@@ -178,15 +178,16 @@ Users may provide paths such as: `/work/bm1159/XCES/xces-work/k204225/MYWORK`. T
 
 ## I. FORMATTING
 
-- For equations, use Markdown math:
-  - in-line: $E = mc^2$ or \\(E = mc^2\\) (double backslash to properly escape)
+- For equations, use Markdown math delimiters with an additional backslash to escape:
+  - in-line: \$E = mc^2\$ or \\(E = mc^2\\)
   - as math block: $$\nabla \cdot \vec{u} = 0$$
 - For code, use Markdown code formatting:
   - in-line: `print("hello world")`
   - as code block:
 
   ```python
   import xarray as xr
+
   ds = xr.open_dataset("file.nc")
   ```
 

scripts/evaluation/tool_evaluation_metrics_2026-08-13.png

@@ -5,7 +5,7 @@
 - Do not invent facts, paths, dataset availability, tool outputs, URLs, or analysis results. Base conclusions on user-provided information, tool output, or loaded data and metadata. State missing evidence plainly and make the smallest useful discovery call or ask for focused clarification when required.
 - For requests requiring data access, numerical analysis, visualization, or file generation, give a short numbered plan and immediately call the appropriate tool. For conceptual questions that need no tools, answer directly. After a tool call, let its output determine the next action and do not claim success without confirmation.
 - Use `code_interpreter` for all Python execution: Freva databrowser queries, data loading, calculations, plotting, and saving files. Import required libraries explicitly. For Freva, include `import freva_client`, use `host='nextgems.dkrz.de'`, inspect metadata before computation, and discover facets before making uncertain dataset or CMOR-variable selections. Use ERA5 reanalysis by default when the dataset is unspecified.
-- Use the relevant stages only: *discover/load → inspect metadata → compute → validate → plot/save*. Report selections, aggregation, and units when they materially affect the interpretation. Prefer scoped operations and real user-provided or Freva-discovered data over synthetic data.
+- Use the relevant stages only: *discover/load → inspect metadata → compute → validate → plot*. Report selections, aggregation, and units when they materially affect the interpretation. Prefer scoped operations and real user-provided or Freva-discovered data over synthetic data.
 - For DKRZ/HPC, Slurm, or ICON documentation, use `web_search` and prefer official sources. Include inline citations with the URLs used.
 - Use `plugin_code_search` when the user asks about a Freva plugin's logic, usage, configuration, or adaptation, and proactively for complex regional, decadal, or extreme-event analyses where plugin-encoded methodology may apply. Skip it for simple generic analysis. Ground explanations only in returned code context; when relevant, identify modules, classes, and functions and link repository paths on the `levante` branch.
 - When a tool action fails, make one focused correction and retry when clear. For an interpreter timeout, reduce scope before retrying; use HPC/Slurm documentation only when an infrastructure, scheduler, or resource problem is indicated.

scripts/evaluation/tool_evaluation_pie_charts.png

@@ -227,8 +227,8 @@ async def select_relevant_files(
             f"Task: You are selecting the most relevant files from the '{plugin}' Freva plugin repository.\n\n"
             "Selection rules:\n"
             "- Prioritize files that seem most relevant to answer the query intent.\n"
-            "- For high level usage/configuration questions, prioritize wrapper/config files and README/docs.\n"
-            "- For questions about implementation logic, prioritize core source code modules.\n"
+            "- For high level usage/configuration questions, prioritize wrapper/config files,  README and docs.\n"
+            "- For questions about implementation logic, prioritize source code modules.\n"
             "- Exclude tests, examples, generated files, and any '__init__.py'.\n"
             f"- Return ONLY a valid JSON array of file path strings from the provided list, with at most {MAX_RELEVANT_FILES} items. Output nothing but the JSON array."
             f"Repository file list:\n{file_tree}\n\n"
@@ -268,13 +268,15 @@ async def select_relevant_files(
     return [p for p in selected_files if p in set(file_paths)][:MAX_RELEVANT_FILES]
 
 
-async def collect_plugin_context(plugin: str, project_id: int, user_query: str) -> str:
+async def collect_plugin_context(
+    plugin: str, project: str, project_id: int, user_query: str
+) -> str:
     """
     Three-stage context retrieval of code base:
         1. Ask LLM which files are most relevant for the user's query.
         2. Fetch those files, then scan for imports to identify dependent modules.
         3. Fetch the dependencies and return the combined contents.
-    Returns a string containing the concatenated relevant code snippets,
+    Returns a string containing the concatenated relevant source code files,
         separated by file and with a header.
     """
 
@@ -288,12 +290,17 @@ def _log_stage(stage: str, files: list[str]):
             files,
         )
 
+    header = (
+        f"Relevant retrieved code of the '{plugin}' plugin "
+        f"(https://gitlab.dkrz.de/{project}/plugins4freva/{plugin}):\n\n"
+    )
+
     # ── Stage 0: fetch the repository tree with all files ────────────────────
     file_paths = fetch_repo_tree(project_id)
     if not file_paths:
         return "(repository is empty)"
 
-    # ── Stage 1: ask the LLM to select relevant files ─────────────────────────
+    # ── Stage 1: let LLM select relevant files ─────────────────────────
     base_files = await select_relevant_files(plugin, user_query, file_paths)
     _log_stage("Initial", base_files)
 
@@ -305,10 +312,11 @@ def _log_stage(stage: str, files: list[str]):
     dep_files = await select_relevant_files(plugin, init_code, tree_remaining, dep=True)
     _log_stage("Dependency", dep_files)
     if not dep_files:
-        return init_code
+        return header + init_code
 
     dep_code = fetch_plugin_code(project_id, dep_files, MAX_TOTAL_CODE_CHARS // 3)
-    return init_code + "\n\n# ── Dependency files ──\n\n" + dep_code
+    code_content = init_code + "\n\n### ── Dependency files ── ###\n\n" + dep_code
+    return header + code_content
 
 
 def validate_plugin_call(
@@ -407,25 +415,19 @@ async def plugin_code_search(user_query: str) -> str:
         str: Relevant code context fetched from source files of the plugin repository;
         or an error message if the plugin call is not authorized / code retrieval fails.
     """
-    plugin_name, project_name = await detect_plugin_project(user_query)
-    # return f"plugin_name: {plugin_name}, project_name: {project_name}"  # only for benchmarking the plugin detection stage
+    plugin, project = await detect_plugin_project(user_query)
+    # return f"plugin_name: {plugin}, project_name: {project}"  # only for benchmarking the plugin detection stage
 
     # Validate the plugin call
-    project_id = get_project_id(plugin_name, project_name)
-    result, message = validate_plugin_call(plugin_name, project_name, project_id)
+    project_id = get_project_id(plugin, project)
+    result, message = validate_plugin_call(plugin, project, project_id)
     if not result:
         return message
 
     # Fetch the plugin code and return it with a header
-    logger.info("Fetching source code for plugin '%s'", plugin_name)
+    logger.info("Fetching source code for plugin '%s'", plugin)
     try:
-        code_content = await collect_plugin_context(plugin_name, project_id, user_query)  # type: ignore
+        return await collect_plugin_context(plugin, project, project_id, user_query)  # type: ignore
     except Exception as e:
-        logger.warning("Failed to fetch plugin code for '%s': %s", plugin_name, e)
-        return f"Failed to retrieve source code for plugin '{plugin_name}': {e}"
-
-    header = (
-        f"Relevant retrieved code of the '{plugin_name}' plugin "
-        f"(https://gitlab.dkrz.de/{project_name}/plugins4freva/{plugin_name}):\n\n"
-    )
-    return header + code_content
+        logger.warning("Failed to fetch plugin code for '%s': %s", plugin, e)
+        return f"Failed to retrieve source code for plugin '{plugin}': {e}"