r2l-e-0003-toolrl-bfcl.json

{
  "schema_version": "1.0.0",
  "record_id": "r2l-e-0003-toolrl-bfcl",
  "record_status": "draft",
  "disclaimer": "DRAFT SOURCE EXTRACTION: values are transcribed from the cited source and remain ineligible for synthesis until two named authors independently verify them.",
  "provenance": {
    "evidence_class": "reported_by_source",
    "source": {
      "citation_key": "qian2025toolrlreward",
      "title": "ToolRL: Reward is All Tool Learning Needs",
      "persistent_identifier": {
        "type": "arxiv",
        "value": "2504.13958",
        "url": "https://arxiv.org/abs/2504.13958"
      },
      "publication_version": "source version cited by the manuscript",
      "publication_date": "2025",
      "accessed_at": "2026-07-23"
    },
    "extraction": {
      "extracted_at": "2026-07-23",
      "extracted_by": "Codex-assisted manuscript revision",
      "verification_status": "unverified",
      "verification_checks": []
    },
    "synthesis_eligibility": {
      "status": "pending",
      "reason": "Pending independent verification by two named manuscript authors."
    },
    "evidence_details": {
      "input_record_ids": [],
      "formula": null,
      "normalization": null,
      "uncertainty_procedure": null,
      "assumptions": [],
      "theorem_locator": null,
      "supporting_record_ids": [],
      "synthesis_rationale": null
    },
    "change_log": [
      {
        "changed_at": "2026-07-23",
        "changed_by": "Codex-assisted manuscript revision",
        "reason": "Created a source-reported draft for a manuscript quantitative claim."
      }
    ]
  },
  "system": {
    "name": "ToolRL Qwen2.5-3B-Instruct",
    "version_or_checkpoint": "source-reported paper configuration",
    "lineage": "rl_for_deliberation",
    "implementation_reference": null
  },
  "task": {
    "domain": "language model reasoning or tool use",
    "name": "BFCL V3",
    "version": "source-reported version",
    "split": "BFCL V3 full evaluation",
    "environment_or_dataset": "BFCL V3",
    "unit_of_analysis": "one evaluation item",
    "evaluation_protocol": "BFCL V3 overall-accuracy evaluation reported in the source's full-results table."
  },
  "configuration": {
    "configuration_id": "toolrl_qwen25_3b_grpo_cold_bfclv3",
    "training_summary": "Cold-start GRPO using the source's tool-use reward design.",
    "inference_summary": "Generate and execute typed tool calls under the BFCL V3 protocol.",
    "parameters": []
  },
  "r2l_interface": {
    "reasoning_object": {
      "type": "typed tool calls, arguments, and returned observations",
      "description": "typed tool calls, arguments, and returned observations",
      "origin": "learned",
      "explicit": true
    },
    "functional_coupling": {
      "operations": [
        "generated",
        "verified",
        "consumed"
      ],
      "affects": [
        "policy_optimization",
        "action_selection"
      ]
    },
    "learning_signals": [
      "tool selection reward",
      "format reward",
      "answer reward",
      "execution reward"
    ],
    "integration_points": [
      "reward",
      "tool_use"
    ],
    "grounding": {
      "target": "The task inputs, executable checks, or reference answers declared by the source.",
      "mechanism": "The source-specific reward or verifier connects the explicit trace to task feedback.",
      "diagnostic_ids": []
    }
  },
  "resources": [
    {
      "resource_id": "training_examples",
      "type": "demonstrations",
      "phase": "evaluation",
      "reporting_status": "not_reported",
      "quantity": null,
      "unit": null,
      "accounting_scope": "Training examples used by target and comparators",
      "notes": "The manuscript extraction does not assert this quantity; verify it from the source."
    }
  ],
  "controls": {
    "comparators": [
      {
        "comparator_id": "raw_model",
        "name": "Qwen2.5-3B-Instruct raw",
        "configuration_summary": "Unadapted source checkpoint.",
        "resource_matching": [
          {
            "resource_id": "training_examples",
            "status": "not_reported",
            "explanation": "Resource parity requires confirmation during human verification."
          }
        ]
      },
      {
        "comparator_id": "sft4k",
        "name": "Qwen2.5-3B-Instruct SFT4k",
        "configuration_summary": "Supervised fine-tuning on 4,000 examples.",
        "resource_matching": [
          {
            "resource_id": "training_examples",
            "status": "not_reported",
            "explanation": "Resource parity requires confirmation during human verification."
          }
        ]
      }
    ],
    "no_comparator_reason": null,
    "ablations": [],
    "leakage_and_contamination_checks": [
      "Use only source-reported contamination checks; none are inferred in this extraction."
    ],
    "privileged_information_checks": [
      "Verifier, reference-answer, and tool access must be checked during human verification."
    ]
  },
  "outcomes": [
    {
      "measurement_id": "bfcl_v3_grpo_cold",
      "metric": "BFCL V3 overall accuracy percent",
      "evaluation_layer": "outcome",
      "population_or_split": "BFCL V3 full evaluation",
      "estimate": {
        "kind": "numeric",
        "value": 52.98,
        "unit": "percent",
        "denominator": {
          "status": "not_reported",
          "value": null,
          "description": "The source-local extraction does not assert a denominator."
        },
        "aggregation": "Source-reported aggregate for the stated configuration."
      },
      "uncertainty": {
        "status": "not_reported",
        "description": "The cited result does not report an uncertainty interval for this value."
      },
      "comparator_id": "raw_model",
      "source_locator": "Qian et al., Table 1, Qwen2.5-3B-Instruct, Ours (GRPO Cold Start), Overall Acc.",
      "permitted_interpretation": "The source reports 52.98 percent overall accuracy for the stated ToolRL configuration."
    },
    {
      "measurement_id": "bfcl_v3_raw",
      "metric": "BFCL V3 overall accuracy percent",
      "evaluation_layer": "outcome",
      "population_or_split": "BFCL V3 full evaluation",
      "estimate": {
        "kind": "numeric",
        "value": 33.04,
        "unit": "percent",
        "denominator": {
          "status": "not_reported",
          "value": null,
          "description": "The source-local extraction does not assert a denominator."
        },
        "aggregation": "Source-reported aggregate for the stated configuration."
      },
      "uncertainty": {
        "status": "not_reported",
        "description": "The cited result does not report an uncertainty interval for this value."
      },
      "comparator_id": "raw_model",
      "source_locator": "Qian et al., Table 1, Qwen2.5-3B-Instruct (Raw), Overall Acc.",
      "permitted_interpretation": "The source reports 33.04 percent overall accuracy for the raw-model comparator."
    },
    {
      "measurement_id": "bfcl_v3_sft4k",
      "metric": "BFCL V3 overall accuracy percent",
      "evaluation_layer": "outcome",
      "population_or_split": "BFCL V3 full evaluation",
      "estimate": {
        "kind": "numeric",
        "value": 41.97,
        "unit": "percent",
        "denominator": {
          "status": "not_reported",
          "value": null,
          "description": "The source-local extraction does not assert a denominator."
        },
        "aggregation": "Source-reported aggregate for the stated configuration."
      },
      "uncertainty": {
        "status": "not_reported",
        "description": "The cited result does not report an uncertainty interval for this value."
      },
      "comparator_id": "sft4k",
      "source_locator": "Qian et al., Table 1, Qwen2.5-3B-Instruct (SFT4k), Overall Acc.",
      "permitted_interpretation": "The source reports 41.97 percent overall accuracy for the SFT4k comparator."
    }
  ],
  "diagnostics": [],
  "claim_scope": {
    "supported_interpretations": [
      "The target exceeds both stated comparators in the source's BFCL V3 full-results table."
    ],
    "unsupported_inferences": [
      "The differences are not a cross-benchmark aggregate or evidence of universal tool-use improvement."
    ]
  },
  "limitations": [
    {
      "limitation_id": "toolrl_denominator",
      "source": "extractor_identified",
      "category": "missing_uncertainty",
      "description": "The extraction retains no item denominator or uncertainty interval.",
      "affected_measurement_ids": [
        "bfcl_v3_grpo_cold",
        "bfcl_v3_raw",
        "bfcl_v3_sft4k"
      ]
    },
    {
      "limitation_id": "toolrl_local",
      "source": "extractor_identified",
      "category": "external_validity",
      "description": "The comparison is local to BFCL V3 and the stated Qwen2.5-3B configurations.",
      "affected_measurement_ids": [
        "bfcl_v3_grpo_cold",
        "bfcl_v3_raw",
        "bfcl_v3_sft4k"
      ]
    }
  ],
  "extensions": {
    "review_study_id": "R2L-S-0038",
    "evidence_record_id": "R2L-E-0003",
    "claim_direction": "positive_local"
  }
}