dvc.yaml

# This file defines the pipeline for running all strong solvers and
# then scoring them on the dev set. We have a different stage defined
# for each solver type since the commands differ significantly. All
# scoring is done in the same stage (in a dvc foreach loop) since the
# scoring command is the same for all solvers.

vars:
  # Used for specifying details for retrieverless solvers:
  - llm_defaults: &llm_defaults
      llm_args: ""
      suffix: ""

stages:
  solve_sqa:
    matrix:
      model:
        - claude-4.6
        - gemini-3.1-pro-preview
        - o3_high
      split:
        - dev
        - test
    # run the solver then rename the resulting file to have a nice name:
    cmd:
      INSPECT_EVAL_LOG_FILE_PATTERN=task_sqa_solver_sqa_${item.model}
      uv run --extra sqa
      inspect eval astabench/evals/sqa/task.py@sqa --display plain
      --log-dir ${item.split}_dvc_logs/solver_outputs/
      --solver astabench/solvers/sqa/sqa.py@sqa_solver
      -T split=${item.split}
      -T scorer_model=${scorer_model}
      -S completion_model=${item.model}
      --limit=${limit}
      --retry-on-error=10
      --log-shared
      --no-score;
      mv "$(ls -t ${item.split}_dvc_logs/solver_outputs/*${item.model}.eval 2>/dev/null | head -n1)" "${item.split}_dvc_logs/solver_outputs/task_sqa_solver_sqa_${item.model}.eval"
    params:
      - limit
      - scorer_model
      - sqa_solver_version
    outs:
      - ${item.split}_dvc_logs/solver_outputs/task_sqa_solver_sqa_${item.model}.eval

  solve_elicit:
    # run the solver then rename the resulting file to have a nice name:
    foreach:
      - dev
      - test
    do:
      cmd:
        INSPECT_EVAL_LOG_FILE_PATTERN=task_sqa_solver_elicit
        uv run --extra sqa
        inspect eval astabench/evals/sqa/task.py@sqa --display plain
        --log-dir ${item}_dvc_logs/solver_outputs/
        --solver astabench/solvers/sqa/elicit/memorized_solver.py@elicit_solver
        -T split=${item}
        -T scorer_model=${scorer_model}
        --limit=${limit}
        --log-shared
        --no-score;
        mv "$(ls -t ${item}_dvc_logs/solver_outputs/*task_sqa_solver_elicit.eval 2>/dev/null | head -n1)" "${item}_dvc_logs/solver_outputs/task_sqa_solver_elicit.eval"
      params:
        - scorer_model
        - limit
      outs:
        - ${item}_dvc_logs/solver_outputs/task_sqa_solver_elicit.eval

  solve_you:
    # run the solver then rename the resulting file to have a nice name:
    foreach:
      - dev
      - test
    do:
      cmd:
        INSPECT_EVAL_LOG_FILE_PATTERN=task_sqa_solver_you
        uv run --extra sqa
        inspect eval astabench/evals/sqa/task.py@sqa --display plain
        --log-dir ${item}_dvc_logs/solver_outputs/
        --solver astabench/solvers/sqa/formatted_youcom.py@formatted_solver -S api_type='research'
        -T split=${item}
        -T scorer_model=${scorer_model}
        --limit=${limit}
        --log-shared
        --no-score;
        mv "$(ls -t ${item}_dvc_logs/solver_outputs/*task_sqa_solver_you.eval 2>/dev/null | head -n1)" "${item}_dvc_logs/solver_outputs/task_sqa_solver_you.eval"
      params:
        - scorer_model
        - limit
      outs:
        - ${item}_dvc_logs/solver_outputs/task_sqa_solver_you.eval

  solve_perplexity_dr:
    # run the solver then rename the resulting file to have a nice name:
    foreach:
      - dev
      - test
    do:
      cmd:
        INSPECT_EVAL_LOG_FILE_PATTERN=task_sqa_solver_perplexity_dr
        uv run --extra sqa
        inspect eval astabench/evals/sqa/task.py@sqa --display plain
        --log-dir ${item}_dvc_logs/solver_outputs/
        -T with_search_tools=False
        --model 'perplexity/sonar-deep-research'
        --solver astabench/solvers/sqa/formatted_perplexity.py@formatted_solver
        -T split=${item}
        -T scorer_model=${scorer_model}
        --limit=${limit}
        --log-shared
        --no-score;
        mv "$(ls -t ${item}_dvc_logs/solver_outputs/*task_sqa_solver_perplexity_dr.eval 2>/dev/null | head -n1)" "${item}_dvc_logs/solver_outputs/task_sqa_solver_perplexity_dr.eval"
      params:
        - scorer_model
        - limit
      outs:
        - ${item}_dvc_logs/solver_outputs/task_sqa_solver_perplexity_dr.eval

  solve_storm:
    # run the solver then rename the resulting file to have a nice name:
    foreach:
      - dev
      - test
    do:
      cmd:
        INSPECT_EVAL_LOG_FILE_PATTERN=task_sqa_solver_storm
        uv run --extra storm --python 3.11
        inspect eval astabench/evals/sqa/task.py@sqa --display plain
        --log-dir ${item}_dvc_logs/solver_outputs/
        --solver astabench/solvers/sqa/storm_solver.py@storm_solver
        -T split=${item}
        -T scorer_model=${scorer_model}
        --limit=${limit}
        --log-shared
        --no-score;
        mv "$(ls -t ${item}_dvc_logs/solver_outputs/*task_sqa_solver_storm.eval 2>/dev/null | head -n1)" "${item}_dvc_logs/solver_outputs/task_sqa_solver_storm.eval"
      params:
        - scorer_model
        - limit
      outs:
        - ${item}_dvc_logs/solver_outputs/task_sqa_solver_storm.eval

  solve_scispace:
    matrix:
      split:
        - dev
        - test
    # run the solver then rename the resulting file to have a nice name:
    cmd:
      INSPECT_EVAL_LOG_FILE_PATTERN=task_sqa_solver_scispace
      uv run
      inspect eval astabench/evals/sqa/task.py@sqa --display plain
      --log-dir ${item.split}_dvc_logs/solver_outputs/
      --solver astabench/solvers/sqa/scispace/scispace.py@formatted_solver
      -T scorer_model=${scorer_model}
      -T split=${item.split}
      --limit=${limit}
      --retry-on-error=10
      --log-shared
      --no-score;
      mv "$(ls -t ${item.split}_dvc_logs/solver_outputs/*scispace.eval 2>/dev/null | head -n1)" "${item.split}_dvc_logs/solver_outputs/task_sqa_solver_scispace.eval"
    params:
      - limit
      - scorer_model
      - sqa_solver_version
    outs:
      - ${item.split}_dvc_logs/solver_outputs/task_sqa_solver_scispace.eval

  solve_openscholar:
    foreach:
      - dev
      - test
    # run the solver then rename the resulting file to have a nice name:
    do:
      cmd:
        INSPECT_EVAL_LOG_FILE_PATTERN=task_sqa_solver_openscholar
        uv run --extra sqa
        inspect eval astabench/evals/sqa/task.py@sqa --display plain
        --log-dir ${item}_dvc_logs/solver_outputs/
        --solver astabench/solvers/sqa/openscholar/memorized_solver.py
        -S path=astabench/solvers/sqa/openscholar/openscholar_cache_${item}.json
        -T scorer_model=${scorer_model}
        -T split=${item}
        --limit=${limit}
        --retry-on-error=10
        --log-shared
        --no-score;
        mv "$(ls -t ${item}_dvc_logs/solver_outputs/*openscholar.eval 2>/dev/null | head -n1)" "${item}_dvc_logs/solver_outputs/task_sqa_solver_openscholar.eval"
      params:
        - limit
        - scorer_model
      outs:
        - ${item}_dvc_logs/solver_outputs/task_sqa_solver_openscholar.eval

  # run retrieverless solvers:
  solve_llm:
    matrix:
      model:
        # fancy yaml syntax for handling structs:
        - <<: *llm_defaults
          llm_name: openai/o4-mini
          llm_args: "--reasoning-effort high --reasoning-tokens 8192 -M responses_store=false --reasoning-history none"
        - <<: *llm_defaults
          llm_name: anthropic/claude-sonnet-4-6
          llm_args: "--reasoning-tokens 8192"
          suffix: "-thinking"
        - <<: *llm_defaults
          llm_name: google/gemini-3.1-pro-preview
        - <<: *llm_defaults
          llm_name: anthropic/claude-sonnet-4-6
        - <<: *llm_defaults
          llm_name: openai/o3
          llm_args: "--reasoning-effort high --reasoning-tokens 8192 -M responses_store=false --reasoning-history none"
      split:
        - dev
        - test
    # run the solver then rename the resulting file to have a nice name:
    cmd:
      INSPECT_EVAL_LOG_FILE_PATTERN=task_sqa_solver_${item.model.llm_name}${item.model.suffix}
      uv run --extra sqa
      inspect eval astabench/evals/sqa/task.py@sqa --display plain
      --log-dir ${item.split}_dvc_logs/solver_outputs/
      --solver astabench/solvers/sqa/formatted_llm.py@formatted_solver
      --model ${item.model.llm_name} ${item.model.llm_args}
      -T split=${item.split}
      -T scorer_model=${scorer_model}
      -T excerpt_prompt=False
      --limit=${limit}
      --retry-on-error=10
      --log-shared
      --no-score;
      [[ "${item.model.llm_name}" == */* ]] && mkdir -p ${item.split}_dvc_logs/solver_outputs/task_sqa_solver_$(dirname "${item.model.llm_name}");
      mv "$(ls -t ${item.split}_dvc_logs/solver_outputs/*${item.model.llm_name}${item.model.suffix}.eval 2>/dev/null | head -n1)" "${item.split}_dvc_logs/solver_outputs/task_sqa_solver_${item.model.llm_name}${item.model.suffix}.eval"
    params:
      - limit
      - scorer_model
      - sqa_solver_version
    outs:
      - ${item.split}_dvc_logs/solver_outputs/task_sqa_solver_${item.model.llm_name}${item.model.suffix}.eval

  solve_futurehouse:
    matrix:
      model:
        - name: fhouse_crow
          solver_args: "-S agent=CROW"
        - name: fhouse_falcon
          solver_args: "-S agent=FALCON"
      split:
        - dev
        - test
    # run the solver then rename the resulting file to have a nice name:
    cmd:
      INSPECT_EVAL_LOG_FILE_PATTERN=task_sqa_solver_${item.model.name}
      uv run --extra futurehouse
      inspect eval astabench/evals/sqa/task.py@sqa --display plain
      --log-dir ${item.split}_dvc_logs/solver_outputs/
      --solver astabench/solvers/futurehouse/futurehouse_solver.py
      -T scorer_model=${scorer_model}
      -T split=${item.split}
      -S max_wait_time=900 ${item.model.solver_args}
      --limit=${limit}
      --retry-on-error=10
      --log-shared
      --no-score;
      mv "$(ls -t ${item.split}_dvc_logs/solver_outputs/*${item.model.name}.eval 2>/dev/null | head -n1)" "${item.split}_dvc_logs/solver_outputs/task_sqa_solver_${item.model.name}.eval"
    params:
      - limit
      - scorer_model
      - sqa_solver_version
    outs:
      - ${item.split}_dvc_logs/solver_outputs/task_sqa_solver_${item.model.name}.eval

  solve_memorized:
    matrix:
      model:
        - name: openai_deep_research
          solver_args: ""
      split:
        - dev
        - test
    # run the solver then rename the resulting file to have a nice name:
    cmd:
      INSPECT_EVAL_LOG_FILE_PATTERN=task_sqa_solver_${item.model.name}
      uv run --extra sqa
      inspect eval astabench/evals/sqa/task.py@sqa --display plain
      --log-dir ${item.split}_dvc_logs/solver_outputs/
      --solver astabench/solvers/sqa/general_memorized/memorized_solver.py@formatted_solver
      -T scorer_model=${scorer_model}
      -T split=${item.split}
      -S sys_name_or_path=${item.model.name} ${item.model.solver_args}
      --limit=${limit}
      --retry-on-error=10
      --log-shared
      --no-score;
      mv "$(ls -t ${item.split}_dvc_logs/solver_outputs/*${item.model.name}.eval 2>/dev/null | head -n1)" "${item.split}_dvc_logs/solver_outputs/task_sqa_solver_${item.model.name}.eval"
    params:
      - limit
      - scorer_model
      - sqa_solver_version
    outs:
      - ${item.split}_dvc_logs/solver_outputs/task_sqa_solver_${item.model.name}.eval

  score_all_solvers:
    matrix:
      model:
        # retrieverless solvers:
        - name: openai/o4-mini
          is_retrieverless: true
        - name: openai/o3
          is_retrieverless: true
        - name: anthropic/claude-sonnet-4-6-thinking
          is_retrieverless: true
        - name: google/gemini-3.1-pro-preview
          is_retrieverless: true
        - name: anthropic/claude-sonnet-4-6
          is_retrieverless: true
        # sqa solvers:
        - name: sqa_claude-4.6
          is_retrieverless: false
        - name: sqa_gemini-3.1
          is_retrieverless: false
        - name: sqa_o3_high
          is_retrieverless: false
        # strong baselines:
        - name: elicit
          is_retrieverless: false
        - name: storm
          is_retrieverless: false
        - name: scispace
          is_retrieverless: false
        - name: fhouse_crow
          is_retrieverless: true
        - name: fhouse_falcon
          is_retrieverless: true
        - name: openai_deep_research
          is_retrieverless: false
        - name: you
          is_retrieverless: false
        - name: perplexity_dr
          is_retrieverless: false
        - name: openscholar
          is_retrieverless: false
      split:
        - test
        - dev
    # copy the solver output, then score it
    cmd:
      echo "Scoring";[[ "${item.model.name}" == */* ]] && mkdir -p ${item.split}_dvc_logs/scored/task_sqa_solver_${item.model.name};
      cp ${item.split}_dvc_logs/solver_outputs/task_sqa_solver_${item.model.name}.eval ${item.split}_dvc_logs/scored/task_sqa_solver_${item.model.name}.eval;
      uv run
      inspect score
      --overwrite
      --display plain
      --scorer astabench/evals/sqa/task.py@score_all
      -S scorer_model=${scorer_model}
      -S is_retrieverless=${item.model.is_retrieverless}
      ${item.split}_dvc_logs/scored/task_sqa_solver_${item.model.name}.eval
    params:
      - sqa_scorer_version
      - scorer_model
    deps:
      - ${item.split}_dvc_logs/solver_outputs/task_sqa_solver_${item.model.name}.eval
    outs:
      - ${item.split}_dvc_logs/scored/task_sqa_solver_${item.model.name}.eval:
          persist: true

  log_any_remaining_errors_and_record_scores:
    matrix:
      model:
        - openai/o4-mini
        - openai/o3
        - anthropic/claude-sonnet-4-6-thinking
        - google/gemini-3.1-pro-preview
        - anthropic/claude-sonnet-4-6
        - sqa_claude-4.6
        - sqa_gemini-3.1
        - sqa_o3_high
        - elicit
        - storm
        - scispace
        - fhouse_crow
        - fhouse_falcon
        - openai_deep_research
        - you
        - perplexity_dr
        - openscholar
      split:
        - dev
        - test
    cmd: echo "Collecting errors";[[ "${item.model}" == */* ]] && mkdir -p ${item.split}_dvc_logs/errors/task_sqa_solver_$(dirname "${item.model}"); mkdir -p ${item.split}_dvc_logs/scores/task_sqa_solver_$(dirname "${item.model}");
      uv run scripts/log_errors.py
      ${item.split}_dvc_logs/scored/task_sqa_solver_${item.model}.eval
      ${item.split}_dvc_logs/errors/task_sqa_solver_${item.model}.md
      ${item.split}_dvc_logs/scores/task_sqa_solver_${item.model}.md
    deps:
      - ${item.split}_dvc_logs/scored/task_sqa_solver_${item.model}.eval
    outs:
      - ${item.split}_dvc_logs/errors/task_sqa_solver_${item.model}.md
      - ${item.split}_dvc_logs/scores/task_sqa_solver_${item.model}.md

  extract_model_responses:
    matrix:
      model:
        - openai/o4-mini
        - openai/o3
        - anthropic/claude-sonnet-4-6-thinking
        - google/gemini-3.1-pro-preview
        - anthropic/claude-sonnet-4-6
        - sqa_claude-4.6
        - sqa_gemini-3.1
        - sqa_o3_high
        - elicit
        - storm
        - scispace
        - fhouse_crow
        - fhouse_falcon
        - openai_deep_research
        - you
        - perplexity_dr
        - openscholar
      split:
        - dev
        - test
    cmd: echo "Extracting responses"; [[ "${item.model}" == */* ]] && mkdir -p ${item.split}_dvc_logs/model_responses/task_sqa_solver_$(dirname "${item.model}");
      uv run scripts/extract_model_responses.py
      ${item.split}_dvc_logs/solver_outputs/task_sqa_solver_${item.model}.eval
      ${item.split}_dvc_logs/model_responses/task_sqa_solver_${item.model}_responses.csv
    deps:
      - ${item.split}_dvc_logs/solver_outputs/task_sqa_solver_${item.model}.eval
    outs:
      - ${item.split}_dvc_logs/model_responses/task_sqa_solver_${item.model}_responses.csv

  create_nice_logs:
    matrix:
      model:
        - openai/o4-mini
        - openai/o3
        - anthropic/claude-sonnet-4-6-thinking
        - google/gemini-3.1-pro-preview
        - anthropic/claude-sonnet-4-6
        - sqa_claude-4.6
        - sqa_gemini-3.1
        - sqa_o3_high
        - elicit
        - storm
        - scispace
        - fhouse_crow
        - fhouse_falcon
        - openai_deep_research
        - you
        - perplexity_dr
        - openscholar
      split:
        - dev
        - test
    params:
      - scorer_model
    cmd: echo "Creating logs"; [[ "${item.model}" == */* ]] && mkdir -p ${item.split}_dvc_logs/debug_logs/task_sqa_solver_$(dirname "${item.model}");
      uv run scripts/create_debug_logs.py
      ${item.split}_dvc_logs/scored/
      task_sqa_solver_${item.model}.eval
      ${item.split}_dvc_logs/debug_logs/
    deps:
      - ${item.split}_dvc_logs/scored/task_sqa_solver_${item.model}.eval
    outs:
      - ${item.split}_dvc_logs/debug_logs/task_sqa_solver_${item.model}_rubric_eval.csv
      - ${item.split}_dvc_logs/debug_logs/task_sqa_solver_${item.model}_citation_eval.csv
      - ${item.split}_dvc_logs/debug_logs/task_sqa_solver_${item.model}_answer_precision_eval.csv