Skip to content

Commit 7dad759

Browse files
chubes4Chris Huber
andauthored
Fix Cook gate-feedback declaration and recovery budgets (#11947)
* fix(cook): classify gate feedback declarations Classify missing npm gate scripts before provider admission and preserve declared gate-feedback recovery budgets.\n\nAI assistance: OpenAI gpt-5.6-sol via OpenCode implemented and tested this change; Chris Huber reviewed and remains responsible for every line. * test(cli): initialize release readiness sources AI assistance: OpenAI gpt-5.6-sol via OpenCode inspected failed CI logs, identified stale CLI test fixtures, and ran the exact compile gate plus focused tests. Chris Huber reviewed and remains responsible for every line. --------- Co-authored-by: Chris Huber <chris@chubes.net>
1 parent 3d030b8 commit 7dad759

5 files changed

Lines changed: 239 additions & 9 deletions

File tree

crates/homeboy-agents/src/agent_task_cook_loop.rs

Lines changed: 47 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,9 @@ use crate::agent_task::{
66
AgentTaskRequest, AgentTaskSourceRef, AgentTaskWorkspaceMode, AGENT_TASK_REQUEST_SCHEMA,
77
};
88
use crate::agent_task_gate::{
9-
text_tail, AgentTaskGateDiagnosticProducer, AgentTaskGateDiagnosticRecord, AgentTaskGateReport,
10-
AgentTaskGateRevealPolicy, AgentTaskGateStatus, AgentTaskGateVisibility,
9+
text_tail, AgentTaskGateDiagnosticProducer, AgentTaskGateDiagnosticRecord,
10+
AgentTaskGateFailureClassification, AgentTaskGateReport, AgentTaskGateRevealPolicy,
11+
AgentTaskGateStatus, AgentTaskGateVisibility,
1112
};
1213
use crate::agent_task_promotion::{AgentTaskPromotionReport, AgentTaskPromotionStatus};
1314
use crate::agent_task_review_dossier::AiFilledReviewForm;
@@ -175,6 +176,8 @@ pub struct AgentTaskCookLoopGateFailure {
175176
#[serde(default, skip_serializing_if = "String::is_empty")]
176177
pub command: String,
177178
pub exit_code: i32,
179+
#[serde(default)]
180+
pub classification: AgentTaskGateFailureClassification,
178181
#[serde(default, skip_serializing_if = "String::is_empty")]
179182
pub stdout_tail: String,
180183
#[serde(default, skip_serializing_if = "String::is_empty")]
@@ -249,6 +252,9 @@ pub fn evaluate_cook_loop(options: AgentTaskCookLoopOptions) -> AgentTaskCookLoo
249252
apply_failure_progression_to_quality(&mut quality, &failure_progression);
250253
let should_retry = options.promotion_report.status == AgentTaskPromotionStatus::GateFailed
251254
&& !failed_gates.is_empty()
255+
&& failed_gates
256+
.iter()
257+
.all(|gate| gate.classification == AgentTaskGateFailureClassification::CandidateCode)
252258
&& !baseline_red
253259
&& retry_budget_remaining > 0;
254260
// Deterministic gates take precedence: a red gate must be fixed before the
@@ -608,6 +614,11 @@ fn gate_failure(
608614
reveal_policy: gate.reveal_policy,
609615
command,
610616
exit_code: gate.exit_code,
617+
classification: gate
618+
.failure_evidence
619+
.as_ref()
620+
.map(|evidence| evidence.classification)
621+
.unwrap_or_default(),
611622
stdout_tail,
612623
stderr_tail,
613624
summary,
@@ -641,6 +652,7 @@ fn agent_visible_gate_failure(
641652
reveal_policy: failure.reveal_policy,
642653
command: String::new(),
643654
exit_code: failure.exit_code,
655+
classification: failure.classification,
644656
stdout_tail: String::new(),
645657
stderr_tail: String::new(),
646658
summary: format!(
@@ -657,6 +669,7 @@ fn agent_visible_gate_failure(
657669
reveal_policy: failure.reveal_policy,
658670
command: String::new(),
659671
exit_code: failure.exit_code,
672+
classification: failure.classification,
660673
stdout_tail: String::new(),
661674
stderr_tail: String::new(),
662675
summary: "private deterministic gate failed; evidence redacted".to_string(),
@@ -670,6 +683,7 @@ fn agent_visible_gate_failure(
670683
reveal_policy: failure.reveal_policy,
671684
command: String::new(),
672685
exit_code: failure.exit_code,
686+
classification: failure.classification,
673687
stdout_tail: String::new(),
674688
stderr_tail: String::new(),
675689
summary: "private deterministic gate failed".to_string(),
@@ -981,6 +995,33 @@ mod tests {
981995
assert_eq!(request.policy.write, "artifacts_only");
982996
}
983997

998+
#[test]
999+
fn gate_declaration_failure_never_creates_code_remediation() {
1000+
let mut gate = failed_gate();
1001+
gate.failure_evidence
1002+
.as_mut()
1003+
.expect("failure evidence")
1004+
.classification = AgentTaskGateFailureClassification::GateDeclaration;
1005+
let report = evaluate_cook_loop(AgentTaskCookLoopOptions {
1006+
source_request: source_request(),
1007+
promotion_report: promotion_report(AgentTaskPromotionStatus::GateFailed, vec![gate]),
1008+
attempt: 1,
1009+
max_attempts: 3,
1010+
source_run_id: Some("run-declaration".to_string()),
1011+
current_diff: String::new(),
1012+
require_review_form: false,
1013+
review_form: None,
1014+
metadata: Value::Null,
1015+
});
1016+
1017+
assert_eq!(report.status, AgentTaskCookLoopStatus::RetriesExhausted);
1018+
assert!(report.follow_up_request.is_none());
1019+
assert_eq!(
1020+
report.failed_gates[0].classification,
1021+
AgentTaskGateFailureClassification::GateDeclaration
1022+
);
1023+
}
1024+
9841025
#[test]
9851026
fn red_gate_preserves_executor_provider_configuration() {
9861027
let mut source = source_request();
@@ -1381,6 +1422,7 @@ mod tests {
13811422
"producer output is opaque",
13821423
String::new(),
13831424
Some(AgentTaskGateFailureEvidence {
1425+
classification: AgentTaskGateFailureClassification::CandidateCode,
13841426
summary: "producer reported a failure".to_string(),
13851427
command: "opaque-gate".to_string(),
13861428
exit_code: 1,
@@ -1451,6 +1493,7 @@ mod tests {
14511493
reveal_policy: AgentTaskGateRevealPolicy::FullEvidence,
14521494
command: "opaque-gate".to_string(),
14531495
exit_code: 101,
1496+
classification: AgentTaskGateFailureClassification::CandidateCode,
14541497
stdout_tail: String::new(),
14551498
stderr_tail: String::new(),
14561499
summary: String::new(),
@@ -1898,6 +1941,7 @@ mod tests {
18981941
"running tests",
18991942
"boom",
19001943
Some(AgentTaskGateFailureEvidence {
1944+
classification: AgentTaskGateFailureClassification::CandidateCode,
19011945
summary: "opaque gate failed".to_string(),
19021946
command: "opaque-gate".to_string(),
19031947
exit_code: 101,
@@ -1942,6 +1986,7 @@ mod tests {
19421986
"secret fixture mismatch",
19431987
"private evaluator stack trace",
19441988
Some(AgentTaskGateFailureEvidence {
1989+
classification: AgentTaskGateFailureClassification::CandidateCode,
19451990
summary: "secret fixture mismatch on randomized private corpus".to_string(),
19461991
command: "./hidden-heldout-check --fixture secret".to_string(),
19471992
exit_code: 7,

crates/homeboy-agents/src/agent_task_gate.rs

Lines changed: 126 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -254,6 +254,75 @@ impl VerifyGateOptions {
254254
pub(crate) fn required_toolchains(&self) -> Vec<AgentTaskGateToolchainRequirement> {
255255
self.gate_toolchains.clone()
256256
}
257+
258+
/// Reject repository-owned npm gate declarations that no candidate patch can
259+
/// repair before a provider is admitted.
260+
pub(crate) fn preflight_declarations(&self, workspace: &Path) -> Result<()> {
261+
for command in self.verify.iter().chain(&self.private_verify) {
262+
let Some(script) = npm_run_script(command) else {
263+
continue;
264+
};
265+
let manifest_path = workspace.join("package.json");
266+
let manifest = fs::read_to_string(&manifest_path).map_err(|error| {
267+
Error::internal_io(error.to_string(), Some(manifest_path.display().to_string()))
268+
})?;
269+
let manifest: serde_json::Value = serde_json::from_str(&manifest).map_err(|error| {
270+
Error::validation_invalid_argument(
271+
"gate declaration",
272+
format!("invalid package manifest: {error}"),
273+
Some(manifest_path.display().to_string()),
274+
None,
275+
)
276+
})?;
277+
if manifest.pointer(&format!("/scripts/{script}")).is_some() {
278+
continue;
279+
}
280+
let package = manifest
281+
.get("name")
282+
.and_then(serde_json::Value::as_str)
283+
.unwrap_or("unnamed package");
284+
let remediation = format!(
285+
"Add \"{script}\": \"<command>\" to {}/scripts for package `{package}`, or change/remove the declared Cook gate `{command}`.",
286+
manifest_path.display(),
287+
);
288+
let mut error = Error::validation_invalid_argument(
289+
"gate declaration",
290+
format!(
291+
"declared npm gate `{command}` is invalid for package `{package}`: {} has no `scripts.{script}`. {remediation}",
292+
manifest_path.display(),
293+
),
294+
Some(manifest_path.display().to_string()),
295+
None,
296+
);
297+
error.details = json!({
298+
"failure_classification": "gate_declaration",
299+
"command": command,
300+
"package": package,
301+
"manifest": manifest_path,
302+
"missing_script": script,
303+
"remediation": remediation,
304+
});
305+
return Err(error);
306+
}
307+
Ok(())
308+
}
309+
310+
pub(crate) fn has_npm_run_declaration(&self) -> bool {
311+
self.verify
312+
.iter()
313+
.chain(&self.private_verify)
314+
.any(|command| npm_run_script(command).is_some())
315+
}
316+
}
317+
318+
fn npm_run_script(command: &str) -> Option<&str> {
319+
let tokens: Vec<_> = command.split_whitespace().collect();
320+
if tokens.len() == 3 && tokens[0] == "npm" && tokens[1] == "run" && !tokens[2].starts_with('-')
321+
{
322+
Some(tokens[2])
323+
} else {
324+
None
325+
}
257326
}
258327

259328
impl Default for VerifyGateOptions {
@@ -726,6 +795,8 @@ impl From<AgentTaskGateStatus> for HomeboyGateStatus {
726795

727796
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
728797
pub struct AgentTaskGateFailureEvidence {
798+
#[serde(default)]
799+
pub classification: AgentTaskGateFailureClassification,
729800
pub summary: String,
730801
pub command: String,
731802
pub exit_code: i32,
@@ -740,6 +811,14 @@ pub struct AgentTaskGateFailureEvidence {
740811
pub diagnostics: Vec<AgentTaskGateDiagnosticRecord>,
741812
}
742813

814+
#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)]
815+
#[serde(rename_all = "snake_case")]
816+
pub enum AgentTaskGateFailureClassification {
817+
#[default]
818+
CandidateCode,
819+
GateDeclaration,
820+
}
821+
743822
pub const AGENT_TASK_GATE_DIAGNOSTIC_RECORD_SCHEMA: &str = "homeboy/gate-diagnostic-record/v1";
744823

745824
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
@@ -2098,12 +2177,29 @@ fn gate_failure_evidence(
20982177
) -> AgentTaskGateFailureEvidence {
20992178
let stdout_tail = text_tail(stdout, 20);
21002179
let stderr_tail = text_tail(stderr, 20);
2101-
let summary = format!("deterministic gate failed with exit code {exit_code}: {command}");
2102-
let agent_feedback = format!(
2103-
"A deterministic verification gate failed after the candidate patch was applied. Fix the code so `{command}` passes, using the captured stdout/stderr tails as the primary failure evidence."
2104-
);
2180+
let missing_script = npm_run_script(command).filter(|script| {
2181+
stderr.contains(&format!("Missing script: \"{script}\""))
2182+
|| stderr.contains(&format!("Missing script: {script}"))
2183+
});
2184+
let classification = missing_script
2185+
.is_some()
2186+
.then_some(AgentTaskGateFailureClassification::GateDeclaration)
2187+
.unwrap_or(AgentTaskGateFailureClassification::CandidateCode);
2188+
let summary = match missing_script {
2189+
Some(script) => format!("declared npm gate is missing script `{script}`: {command}"),
2190+
None => format!("deterministic gate failed with exit code {exit_code}: {command}"),
2191+
};
2192+
let agent_feedback = match missing_script {
2193+
Some(script) => format!(
2194+
"The declared gate is invalid, not candidate-code feedback. Add `scripts.{script}` to the relevant package.json or change/remove `{command}` before rerunning Cook."
2195+
),
2196+
None => format!(
2197+
"A deterministic verification gate failed after the candidate patch was applied. Fix the code so `{command}` passes, using the captured stdout/stderr tails as the primary failure evidence."
2198+
),
2199+
};
21052200

21062201
AgentTaskGateFailureEvidence {
2202+
classification,
21072203
summary,
21082204
command: command.to_string(),
21092205
exit_code,
@@ -3755,6 +3851,32 @@ mod tests {
37553851
}
37563852
}
37573853

3854+
#[test]
3855+
fn npm_missing_script_is_a_non_retryable_gate_declaration_failure() {
3856+
let workspace = tempfile::tempdir().expect("workspace");
3857+
fs::write(
3858+
workspace.path().join("package.json"),
3859+
r#"{"name":"fixture-package","scripts":{"test":"true"}}"#,
3860+
)
3861+
.expect("manifest");
3862+
let gates = VerifyGateOptions {
3863+
verify: vec!["npm run typecheck".to_string()],
3864+
..Default::default()
3865+
};
3866+
3867+
let error = gates
3868+
.preflight_declarations(workspace.path())
3869+
.expect_err("missing script is a declaration failure");
3870+
3871+
assert_eq!(error.details["failure_classification"], "gate_declaration");
3872+
assert_eq!(error.details["package"], "fixture-package");
3873+
assert_eq!(error.details["missing_script"], "typecheck");
3874+
assert!(error.details["remediation"]
3875+
.as_str()
3876+
.expect("remediation")
3877+
.contains("scripts"));
3878+
}
3879+
37583880
/// A symlinked invocation temp alias must yield sandbox paths that are
37593881
/// simultaneously **short** and **non-symlink**.
37603882
///

crates/homeboy-agents/src/agent_task_service/cook.rs

Lines changed: 54 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2454,6 +2454,19 @@ fn review_budget_authority(
24542454
})
24552455
}
24562456

2457+
fn child_execution_budget(
2458+
scope: CookFollowUpBudgetScope,
2459+
budget_limit: &AgentTaskExecutionBudget,
2460+
) -> AgentTaskExecutionBudget {
2461+
match scope {
2462+
CookFollowUpBudgetScope::Cook => budget_limit.clone(),
2463+
CookFollowUpBudgetScope::FreshCookReview
2464+
| CookFollowUpBudgetScope::CandidateAdoptionReview => {
2465+
AgentTaskExecutionBudget::new(1, 0, 0)
2466+
}
2467+
}
2468+
}
2469+
24572470
/// Append and dispatch one remediation attempt from an authenticated promoted
24582471
/// candidate. Both ordinary Cook feedback and external candidate adoption use
24592472
/// this boundary so their budget, provenance, and baseline authority match.
@@ -2638,8 +2651,13 @@ where
26382651
vec![follow_up_request],
26392652
);
26402653
follow_up_plan.options = plan.options.clone();
2641-
follow_up_plan.options.execution_budget = AgentTaskExecutionBudget::new(1, 0, 0);
2642-
follow_up_plan.options.retry.max_attempts = 1;
2654+
// Gate-feedback is a child execution, not a fresh one-shot policy.
2655+
// Review-only continuations retain their separately bounded plan.
2656+
follow_up_plan.options.execution_budget =
2657+
child_execution_budget(budget_scope, &budget_limit);
2658+
if budget_scope != CookFollowUpBudgetScope::Cook {
2659+
follow_up_plan.options.retry.max_attempts = 1;
2660+
}
26432661
(next_attempt, next_run_id, follow_up_plan, None)
26442662
}
26452663
};
@@ -3335,6 +3353,40 @@ where
33353353
options.max_attempts,
33363354
&options.ai_tool,
33373355
);
3356+
if options.gates.has_npm_run_declaration() {
3357+
let gate_workspace = options.source_worktree_path.as_deref().ok_or_else(|| {
3358+
Error::validation_invalid_argument(
3359+
"workspace",
3360+
"Cook requires a workspace before gate declaration preflight",
3361+
Some(options.to_worktree.clone()),
3362+
None,
3363+
)
3364+
})?;
3365+
if let Err(error) = options
3366+
.gates
3367+
.preflight_declarations(std::path::Path::new(gate_workspace))
3368+
{
3369+
let error = with_pre_execution_phase(error, "gate_declaration_preflight");
3370+
record_pre_execution_failure(
3371+
&options.initial_plan,
3372+
&options.initial_run_id,
3373+
&error,
3374+
"gate_declaration_preflight",
3375+
)?;
3376+
return Ok(pre_execution_failure_report(
3377+
options.cook_id.clone(),
3378+
Vec::new(),
3379+
pre_execution_failure_details(
3380+
agent_task_lifecycle::exact_record(&options.initial_run_id)
3381+
.ok()
3382+
.as_ref(),
3383+
&error,
3384+
),
3385+
error,
3386+
Some(&options.initial_run_id),
3387+
));
3388+
}
3389+
}
33383390
let required_toolchains = options.gates.required_toolchains();
33393391
let preflight = if required_toolchains.is_empty()
33403392
&& options.gates.gate_package_artifacts.is_empty()

0 commit comments

Comments
 (0)