Skip to content

Commit 3064c7e

Browse files
chubes4chubes4
authored
Keep fixture-matrix discovery failures batch-scoped (#885) [AI: OpenAI GPT-5.6 Sol via OpenCode]
OpenAI GPT-5.6 Sol via OpenCode coordinated evidence review and reconciliation; Chris Huber approved this green control-plane repair.
2 parents 6ff5b3f + 0a4593c commit 3064c7e

3 files changed

Lines changed: 162 additions & 54 deletions

File tree

‎bench/static-site-fixture-matrix.bench.mjs‎

Lines changed: 99 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -415,60 +415,43 @@ export async function runFixtureMatrixBatch({ fixtures, batchIndex, matrix, outp
415415
staticSiteImporterSlug: options.staticSiteImporterSlug,
416416
dependencyOverrides,
417417
});
418-
const dependencyPlan = await discoverFixtureDependencyPlan({ fixtures, outputDirectory, staticSiteImporterPath, options, dependencyOverlays, batchSuffix });
419-
const resolvedDependencyPlan = await resolveHostDependencyPlan(dependencyPlan, path.join(outputDirectory, 'dependency-cache'));
420-
const batchRecipe = buildFixtureMatrixRecipe({
421-
matrix: batchMatrix,
422-
runId: batchMatrix.id,
423-
attemptId: batchSuffix,
424-
artifactsDirectory: outputDirectory,
425-
playgroundArtifactsDirectory: options.playgroundArtifactsDirectory || '/wordpress/wp-content/uploads/static-site-importer-fixture-matrix',
426-
wordpressVersion: options.wordpressVersion,
427-
staticSiteImporterPath,
428-
staticSiteImporterPlugin: options.staticSiteImporterPlugin,
429-
staticSiteImporterSlug: options.staticSiteImporterSlug,
430-
dependencyPlan: resolvedDependencyPlan,
431-
dependencyOverrides,
432-
dependencyOverlays,
433-
svgFontEvidence: true,
434-
...fixtureMatrixRecipeInput(normalizeFixtureMatrixRunConfig(Object.fromEntries(Object.keys(FIXTURE_MATRIX_RUN_FIELDS).map((key) => [key, options[key]])))),
435-
...visualParityRecipeInput(options),
436-
...liveWpParityRecipeInput(options),
437-
...runtimePresentationEvidenceRecipeInput(options),
438-
});
439418
const batchRecipeFile = path.join(outputDirectory, `wp-codebox-static-site-fixture-matrix-batch-${batchSuffix}.json`);
440419
const outputFile = path.join(outputDirectory, `wp-codebox-output-batch-${batchSuffix}.json`);
441420
const codeboxArtifactsDirectory = batchCodeboxArtifactsDirectory(outputDirectory, batchSuffix);
442421
const artifactRefs = batchArtifactRefs({ outputDirectory, batchSuffix, batchRecipeFile, outputFile, codeboxArtifactsDirectory });
443-
writeJsonArtifact(batchRecipeFile, batchRecipe);
444-
progress?.emit(recovery ? 'recovery' : 'batch', 'started', { fixture_id: fixtures[0]?.id || '', batch: batchNumber, recovery });
445-
progress?.emit('fixture', 'started', { fixture_id: fixtures[0]?.id || '', batch: batchNumber, recovery });
446422

447-
let batchRuntime = null;
423+
let dependencyPlan = null;
424+
let resolvedDependencyPlan = null;
425+
let batchRecipe = null;
448426
let batchError = null;
449427
let childCommandFailure = null;
450-
let childRecipeRunMs = 0;
451-
const childRecipeRunStartedAt = nowMs();
428+
452429
try {
453-
batchRuntime = await runWpCodeboxRecipe({
454-
recipeFile: batchRecipeFile,
455-
artifactsDir: codeboxArtifactsDirectory,
456-
outputFile,
457-
cwd: outputDirectory,
458-
wpCodeboxBin: options.wpCodeboxBin,
459-
inactivityTimeoutMs: batchInactivityTimeoutMs(options),
460-
onInactivity: ({ timeout_ms }) => {
461-
progress?.emit('batch', 'timeout', { fixture_id: fixtures[0]?.id || '', batch: batchNumber, recovery, timeout_ms });
462-
if (recovery) progress?.emit('fixture', 'timeout', { fixture_id: fixtures[0]?.id || '', batch: batchNumber, recovery, timeout_ms });
463-
},
430+
dependencyPlan = await discoverFixtureDependencyPlan({ fixtures, outputDirectory, staticSiteImporterPath, options, dependencyOverlays, batchSuffix });
431+
resolvedDependencyPlan = await resolveHostDependencyPlan(dependencyPlan, path.join(outputDirectory, 'dependency-cache'));
432+
batchRecipe = buildFixtureMatrixRecipe({
433+
matrix: batchMatrix,
434+
runId: batchMatrix.id,
435+
attemptId: batchSuffix,
436+
artifactsDirectory: outputDirectory,
437+
playgroundArtifactsDirectory: options.playgroundArtifactsDirectory || '/wordpress/wp-content/uploads/static-site-importer-fixture-matrix',
438+
wordpressVersion: options.wordpressVersion,
439+
staticSiteImporterPath,
440+
staticSiteImporterPlugin: options.staticSiteImporterPlugin,
441+
staticSiteImporterSlug: options.staticSiteImporterSlug,
442+
dependencyPlan: resolvedDependencyPlan,
443+
dependencyOverrides,
444+
dependencyOverlays,
445+
svgFontEvidence: true,
446+
...fixtureMatrixRecipeInput(normalizeFixtureMatrixRunConfig(Object.fromEntries(Object.keys(FIXTURE_MATRIX_RUN_FIELDS).map((key) => [key, options[key]])))),
447+
...visualParityRecipeInput(options),
448+
...liveWpParityRecipeInput(options),
449+
...runtimePresentationEvidenceRecipeInput(options),
464450
});
451+
writeJsonArtifact(batchRecipeFile, batchRecipe);
465452
} catch (error) {
453+
const failureStage = !dependencyPlan ? 'dependency_discovery' : !resolvedDependencyPlan ? 'dependency_resolution' : 'recipe_build';
466454
batchError = error;
467-
batchRuntime = {
468-
exitCode: error?.code ?? 1,
469-
outputFile,
470-
json: parseJsonText(error?.stdout),
471-
};
472455
childCommandFailure = buildWpCodeboxChildCommandFailure({
473456
error,
474457
fixtures,
@@ -480,9 +463,54 @@ export async function runFixtureMatrixBatch({ fixtures, batchIndex, matrix, outp
480463
artifactsDir: codeboxArtifactsDirectory,
481464
wpCodeboxBin: options.wpCodeboxBin,
482465
artifactRefs,
466+
failureStage,
483467
});
484-
} finally {
485-
childRecipeRunMs = elapsedMs(childRecipeRunStartedAt);
468+
}
469+
470+
progress?.emit(recovery ? 'recovery' : 'batch', batchError ? 'failed' : 'started', { fixture_id: fixtures[0]?.id || '', batch: batchNumber, recovery, ...(batchError ? { failure_stage: childCommandFailure?.failure_stage } : {}) });
471+
if (!batchError) {
472+
progress?.emit('fixture', 'started', { fixture_id: fixtures[0]?.id || '', batch: batchNumber, recovery });
473+
}
474+
475+
let batchRuntime = null;
476+
let childRecipeRunMs = 0;
477+
const childRecipeRunStartedAt = nowMs();
478+
if (!batchError) {
479+
try {
480+
batchRuntime = await runWpCodeboxRecipe({
481+
recipeFile: batchRecipeFile,
482+
artifactsDir: codeboxArtifactsDirectory,
483+
outputFile,
484+
cwd: outputDirectory,
485+
wpCodeboxBin: options.wpCodeboxBin,
486+
inactivityTimeoutMs: batchInactivityTimeoutMs(options),
487+
onInactivity: ({ timeout_ms }) => {
488+
progress?.emit('batch', 'timeout', { fixture_id: fixtures[0]?.id || '', batch: batchNumber, recovery, timeout_ms });
489+
if (recovery) progress?.emit('fixture', 'timeout', { fixture_id: fixtures[0]?.id || '', batch: batchNumber, recovery, timeout_ms });
490+
},
491+
});
492+
} catch (error) {
493+
batchError = error;
494+
batchRuntime = {
495+
exitCode: error?.code ?? 1,
496+
outputFile,
497+
json: parseJsonText(error?.stdout),
498+
};
499+
childCommandFailure = buildWpCodeboxChildCommandFailure({
500+
error,
501+
fixtures,
502+
batchNumber,
503+
batchSuffix,
504+
batchId: `batch-${String(batchNumber).padStart(3, '0')}`,
505+
batchRecipeFile,
506+
outputFile,
507+
artifactsDir: codeboxArtifactsDirectory,
508+
wpCodeboxBin: options.wpCodeboxBin,
509+
artifactRefs,
510+
});
511+
} finally {
512+
childRecipeRunMs = elapsedMs(childRecipeRunStartedAt);
513+
}
486514
}
487515

488516
const batchRun = fixtureMatrixBatchRunSummary({
@@ -494,6 +522,7 @@ export async function runFixtureMatrixBatch({ fixtures, batchIndex, matrix, outp
494522
codeboxArtifactsDirectory,
495523
batchRuntime,
496524
batchError,
525+
exitCode: childCommandFailure?.exit_status,
497526
performance: {
498527
child_recipe_run_ms: childRecipeRunMs,
499528
},
@@ -513,7 +542,7 @@ export async function runFixtureMatrixBatch({ fixtures, batchIndex, matrix, outp
513542
visualParity: fixtureMatrixGateConfig(normalizeFixtureMatrixRunConfig(Object.fromEntries(Object.keys(FIXTURE_MATRIX_RUN_FIELDS).map((key) => [key, options[key]])))).visualParity,
514543
liveWpParity: liveWpParityCollectorInput(options),
515544
dependencyOverrides,
516-
dependencyOverlays: batchRecipe.inputs.dependency_overlays || [],
545+
dependencyOverlays: batchRecipe?.inputs?.dependency_overlays || [],
517546
});
518547
const visualCompare = materializeVisualCompareArtifacts({
519548
result: batchResult,
@@ -547,6 +576,20 @@ export async function runFixtureMatrixBatch({ fixtures, batchIndex, matrix, outp
547576
};
548577
}
549578

579+
// Dependency discovery, host resolution, and recipe build failures are
580+
// planning failures, not sandbox corruption. Re-running each fixture
581+
// individually would fail identically — return the typed failure directly.
582+
if (childCommandFailure?.failure_stage && childCommandFailure.failure_stage !== 'recipe_run') {
583+
return {
584+
batchRun,
585+
batchResult: editorCanvas.result,
586+
visualParityArtifacts: visualCompare.artifacts,
587+
editorCanvasArtifacts: editorCanvas.artifacts,
588+
error: batchError,
589+
childCommandFailures: [childCommandFailure],
590+
};
591+
}
592+
550593
// A recipe-level failure leaves the sandbox's state untrustworthy. Re-run each
551594
// fixture in a fresh sandbox so one stalled step cannot classify its batch peers.
552595
progress?.emit('recovery', 'started', { fixture_id: fixtures[0]?.id || '', batch: batchNumber, recovery: true });
@@ -1411,7 +1454,7 @@ export function fixtureMatrixBatchRunSummary(input = {}) {
14111454
recipe_file: input.batchRecipeFile || '',
14121455
output_file: input.outputFile || '',
14131456
codebox_artifacts_directory: input.codeboxArtifactsDirectory || '',
1414-
exit_code: batchRuntime?.exitCode ?? 0,
1457+
exit_code: batchRuntime?.exitCode ?? input.exitCode ?? 0,
14151458
error: batchError ? batchError.message : '',
14161459
stderr_tail: batchError ? textTail(batchError.stderr) : '',
14171460
stdout_tail: batchError ? textTail(batchError.stdout) : '',
@@ -1447,17 +1490,18 @@ function runtimeSummary(runtime, runtimeError) {
14471490
};
14481491
}
14491492

1450-
function buildWpCodeboxChildCommandFailure({ error, fixtures, batchNumber, batchSuffix, batchId, batchRecipeFile, outputFile, artifactsDir, wpCodeboxBin: bin, artifactRefs }) {
1451-
const command = wpCodeboxRecipeRunCommand({ recipeFile: batchRecipeFile, artifactsDir, outputFile, wpCodeboxBin: bin });
1493+
function buildWpCodeboxChildCommandFailure({ error, fixtures, batchNumber, batchSuffix, batchId, batchRecipeFile, outputFile, artifactsDir, wpCodeboxBin: bin, artifactRefs, failureStage = 'recipe_run' }) {
1494+
const isRecipeRun = failureStage === 'recipe_run';
1495+
const command = isRecipeRun ? wpCodeboxRecipeRunCommand({ recipeFile: batchRecipeFile, artifactsDir, outputFile, wpCodeboxBin: bin }) : null;
14521496
return {
14531497
schema: 'homeboy/child-command-failure/v1',
14541498
kind: 'child_command_failed',
1455-
label: `WP Codebox recipe-run batch ${batchSuffix}`,
1499+
label: isRecipeRun ? `WP Codebox recipe-run batch ${batchSuffix}` : `Dependency ${failureStage} batch ${batchSuffix}`,
14561500
batch: batchNumber,
14571501
batch_id: batchId || `batch-${batchSuffix}`,
14581502
fixture_ids: normalizeFixtureIds(fixtures),
1459-
command: command.command,
1460-
command_argv: command.argv,
1503+
...(command ? { command: command.command } : {}),
1504+
...(command ? { command_argv: command.argv } : {}),
14611505
exit_status: exitStatus(error),
14621506
error_code: error?.code,
14631507
error_signal: error?.signal,
@@ -1466,9 +1510,10 @@ function buildWpCodeboxChildCommandFailure({ error, fixtures, batchNumber, batch
14661510
recipe_file: batchRecipeFile,
14671511
output_file: outputFile,
14681512
artifacts_directory: artifactsDir,
1469-
replay_command: wpCodeboxReplayCommand({ recipeFile: batchRecipeFile, artifactsDir, wpCodeboxBin: bin }),
1513+
...(isRecipeRun ? { replay_command: wpCodeboxReplayCommand({ recipeFile: batchRecipeFile, artifactsDir, wpCodeboxBin: bin }) } : {}),
14701514
artifact_refs: artifactRefs,
1471-
message: error?.message || 'WP Codebox recipe-run failed',
1515+
failure_stage: failureStage,
1516+
message: error?.message || (isRecipeRun ? 'WP Codebox recipe-run failed' : `Dependency ${failureStage} failed`),
14721517
};
14731518
}
14741519

‎lib/fixture-matrix/collectors/run-intake.mjs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -902,6 +902,7 @@ function childCommandFailureDiagnostic(failure) {
902902
artifacts_directory: failure.artifacts_directory || failure.artifactsDirectory,
903903
replay_command: failure.replay_command || failure.replayCommand,
904904
artifact_refs: failure.artifact_refs || failure.artifactRefs,
905+
failure_stage: failure.failure_stage,
905906
reason: diagnosticMessage(failure) || 'WP Codebox child command failed.',
906907
message: diagnosticMessage(failure) || 'WP Codebox child command failed.',
907908
});

‎tools/fixture-matrix.test.mjs‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5009,6 +5009,24 @@ async function runWpCodeboxRecipe(options = {}) {
50095009
fs.writeFileSync(capturedRecipes, JSON.stringify(captured));
50105010
}
50115011
if (recipe.includes('plan-artifact-dependencies')) {
5012+
// Parse batch number from the discovery artifacts path (e.g. .../discovery/001-fixture-01/).
5013+
const discoveryMatch = String(options.recipeFile).match(/discovery\\/(\\d{3})/);
5014+
const discoveryBatch = discoveryMatch ? Number(discoveryMatch[1]) : 0;
5015+
inFlight += 1;
5016+
peakInFlight = Math.max(peakInFlight, inFlight);
5017+
recordPeak();
5018+
const unit = Number(process.env.SSI_TEST_RECIPE_UNIT_MS || '15');
5019+
const delay = discoveryBatch * unit;
5020+
await new Promise((resolve) => setTimeout(resolve, Math.max(1, delay)));
5021+
inFlight -= 1;
5022+
const throwDiscoveryBatch = Number(process.env.SSI_TEST_RECIPE_DISCOVERY_THROW_BATCH || '0');
5023+
if (throwDiscoveryBatch && throwDiscoveryBatch === discoveryBatch) {
5024+
const error = new Error('discovery failed for batch ' + discoveryBatch);
5025+
error.code = 19;
5026+
error.stdout = '';
5027+
error.stderr = 'discovery boom';
5028+
throw error;
5029+
}
50125030
fs.mkdirSync(options.artifactsDir, { recursive: true });
50135031
fs.writeFileSync(require('node:path').join(options.artifactsDir, 'dependency-plan.json'), JSON.stringify({ schema: 'static-site-importer/runtime-dependency-plan/v1', artifact_sha256: 'a'.repeat(64), entries: [] }));
50145032
return { exitCode: 0, outputFile: options.outputFile, json: {} };
@@ -5076,6 +5094,7 @@ const CONCURRENCY_ENV_KEYS = [
50765094
'SSI_TEST_RECIPE_BATCH_COUNT',
50775095
'SSI_TEST_RECIPE_UNIT_MS',
50785096
'SSI_TEST_RECIPE_THROW_BATCH',
5097+
'SSI_TEST_RECIPE_DISCOVERY_THROW_BATCH',
50795098
'SSI_TEST_RECIPE_CAPTURE_FILE',
50805099
];
50815100

@@ -5266,6 +5285,49 @@ test('runFixtureMatrix isolates a throwing batch so sibling batches still comple
52665285
}
52675286
});
52685287

5288+
test('runFixtureMatrix isolates a dependency-discovery failure so sibling batches still complete', async () => {
5289+
const snapshot = snapshotConcurrencyEnv();
5290+
const workspace = setupConcurrencyWorkspace('ssi-discovery-isolation-', 4);
5291+
process.env.HOMEBOY_WP_CODEBOX_RECIPE_HELPER = workspace.helperPath;
5292+
process.env.SSI_TEST_RECIPE_BATCH_COUNT = '4';
5293+
process.env.SSI_TEST_RECIPE_UNIT_MS = '5';
5294+
process.env.SSI_TEST_RECIPE_DISCOVERY_THROW_BATCH = '2';
5295+
5296+
try {
5297+
const { summary, runtimeError } = await runFixtureMatrix({
5298+
id: 'discovery-isolation-matrix',
5299+
fixtureRoot: workspace.fixtureRoot,
5300+
outputDirectory: workspace.outputDirectory,
5301+
staticSiteImporterPath: workspace.staticSiteImporter,
5302+
run: true,
5303+
batchSize: 1,
5304+
concurrency: 4,
5305+
visualParity: false,
5306+
});
5307+
5308+
// The discovery failure surfaces as the runtime error + exit code, but the
5309+
// run still produced a full summary rather than rejecting.
5310+
assert.ok(runtimeError);
5311+
assert.match(runtimeError.message, /discovery failed/);
5312+
assert.equal(summary.runtime.exit_code, 19);
5313+
5314+
// Exactly one child-command failure with the correct stage.
5315+
const failures = summary.runtime.child_command_failures;
5316+
assert.equal(failures.length, 1);
5317+
assert.equal(failures[0].batch_id, 'batch-002');
5318+
assert.equal(failures[0].failure_stage, 'dependency_discovery');
5319+
assert.equal(failures[0].exit_status, 19);
5320+
5321+
// All four batches still ran; the three non-throwing siblings succeeded,
5322+
// proving one batch's discovery failure did not sink the others.
5323+
assert.equal(summary.runtime.batches.length, 4);
5324+
assert.equal(summary.result_summary.succeeded, 3);
5325+
assert.equal(summary.result_summary.failed, 1);
5326+
} finally {
5327+
restoreConcurrencyEnv(snapshot);
5328+
}
5329+
});
5330+
52695331
test('runFixtureMatrix recovers healthy fixtures from a poisoned batch sandbox', async () => {
52705332
const root = mkdtempSync(path.join(tmpdir(), 'ssi-fixture-recovery-'));
52715333
const staticSiteImporter = path.join(root, 'static-site-importer');

0 commit comments

Comments
 (0)