feat: wire UnityRetryService into the local provider's build retry (opt-in)

UnityBuildDiagnosticsService (classifier) + UnityRecoveryService (budget-
gated decision) + UnityRetryService (the retry-loop driver, including real
Library backup/nuke recovery actions) were all fully built with zero live
callers anywhere in the pipeline - no lifecycle point existed between a
Unity build's exit code and orchestrator's success/failure decision, so
every non-zero exit was treated identically regardless of whether it was a
known-transient condition (licensing race, stale ScriptAssemblies, etc.)
or a real failure.

OrchestratorSystem.Run gets one purely-additive optional trailing
parameter, onExitCode?: (code: number) => void, invoked with the real
exit code right before the promise settles either way. Every existing
caller (212 call sites across 28 files, confirmed via grep) that doesn't
pass it gets identical behavior - same thrown string on failure, same
resolved output on success.

LocalOrchestrator.runTaskInWorkflow wraps this in a retry loop entirely
encapsulated inside the provider when --enableBuildRetry is set (default
false - a meaningful behavior change like automatic Library-nuking
recovery shouldn't silently activate for existing users). The
ProviderInterface contract and standardBuildAutomation's call site are
completely untouched - no ripple into aws/k8s/gcp-cloud-run/azure-aci/
github-actions/gitlab-ci/remote-powershell/ansible/cli providers.

logText for classification reads the real Unity Editor log content
($LOG_FILE, exported by BuildAutomationWorkflow.BuildWorkflow for the
bare-local path and populated by remote-cli-log-stream from unity-editor's
own -logfile output) rather than just the wrapper's stdout capture, since
UnityBuildDiagnosticsService's crash/license/compile patterns need to
match against genuine Editor log content.

Scoped to build only (not test), local/local-system provider only, using
UnityRecoveryService's existing default budgets (no new budget-config
CLI surface added). Adds the previously-missing test coverage:
unity-recovery-service.test.ts and unity-retry-service.test.ts (neither
existed before this).

Author: frostebite

plugins/orchestrator/src/cli-plugin/build-parameters-adapter.ts

@@ -153,6 +153,7 @@ export function createBuildParametersFromCliOptions(options: Record<string, any>
   // ── reliability ───────────────────────────────────────────────────
   bp.gitIntegrityCheck = options.gitIntegrityCheck === true || options.gitIntegrityCheck === 'true';
   bp.gitAutoRecover = options.gitAutoRecover === true || options.gitAutoRecover === 'true';
+  bp.enableBuildRetry = options.enableBuildRetry === true || options.enableBuildRetry === 'true';
   bp.cleanReservedFilenames =
     options.cleanReservedFilenames === true || options.cleanReservedFilenames === 'true';
   bp.buildArchiveEnabled =

plugins/orchestrator/src/cli-plugin/orchestrator-options-plugin.ts

@@ -346,6 +346,16 @@ export function configureOrchestratorOptions(yargs: any): void {
     default: false,
   });
 
+  yargs.option('enableBuildRetry', {
+    description:
+      'Enable automatic classify/decide/retry recovery for failed Unity builds on the bare-host ' +
+      '`local`/`local-system` provider strategy (UnityRetryService, budget-gated). Default off: a ' +
+      'single failed attempt still throws exactly as before -- retry can back up or nuke the Library ' +
+      'folder as a recovery action, which is a meaningful behavior change existing users must opt into.',
+    type: 'boolean',
+    default: false,
+  });
+
   yargs.option('skipInContainerClone', {
     description:
       'Skip the in-container git clone and reuse a pre-hydrated workspace bind-mounted by the caller. ' +

plugins/orchestrator/src/model/build-parameters.ts

@@ -150,6 +150,12 @@ class BuildParameters {
   // ── reliability ─────────────────────────────────────────────────────
   gitIntegrityCheck!: boolean;
   gitAutoRecover!: boolean;
+  // Opt-in (default false) automatic classify -> decide -> retry loop for
+  // failed Unity runs on the bare-host `local`/`local-system` provider's
+  // build path only (see UnityRetryService). Off by default because retry
+  // can nuke/backup the Library folder as a recovery action -- a meaningful
+  // behavior change that must not silently activate for existing users.
+  enableBuildRetry!: boolean;
   cleanReservedFilenames!: boolean;
   buildArchiveEnabled!: boolean;
   buildArchivePath!: string;

plugins/orchestrator/src/model/orchestrator/providers/local/index.retry.test.ts

@@ -0,0 +1,204 @@
+import { describe, it, expect, beforeEach, afterEach, vi, type Mocked } from 'vitest';
+import fs from 'node:fs';
+import path from 'node:path';
+
+vi.mock('node:fs');
+vi.mock('../../services/core/orchestrator-system', () => ({
+  OrchestratorSystem: { Run: vi.fn() },
+}));
+vi.mock('../../services/reliability/unity-retry-service', () => ({
+  UnityRetryService: { executeWithRetry: vi.fn() },
+}));
+vi.mock('../../services/reliability/unity-recovery-service', () => ({
+  UnityRecoveryService: { createDefaultBudgets: vi.fn(() => ({ mocked: true })) },
+}));
+
+import LocalOrchestrator from './index';
+import { OrchestratorSystem } from '../../services/core/orchestrator-system';
+import { UnityRetryService } from '../../services/reliability/unity-retry-service';
+import { UnityRecoveryService } from '../../services/reliability/unity-recovery-service';
+import Orchestrator from '../../orchestrator';
+import BuildParameters from '../../../build-parameters';
+
+const mockFs = fs as Mocked<typeof fs>;
+const mockRun = OrchestratorSystem.Run as unknown as Mocked<typeof OrchestratorSystem.Run>;
+const mockExecuteWithRetry = UnityRetryService.executeWithRetry as unknown as Mocked<
+  typeof UnityRetryService.executeWithRetry
+>;
+
+/**
+ * Integration coverage for the --enableBuildRetry wiring point: the
+ * isBareLocalProvider path inside LocalOrchestrator.runTaskInWorkflow that
+ * decides between "single attempt, throw on failure" (today's behavior,
+ * default) and "delegate to UnityRetryService.executeWithRetry" (opt-in).
+ * UnityRetryService's own retry/recovery algorithm is covered exhaustively
+ * in unity-retry-service.test.ts -- this file only asserts the call pattern
+ * difference at the provider boundary.
+ */
+describe('LocalOrchestrator.runTaskInWorkflow -- enableBuildRetry wiring', () => {
+  const originalPlatform = process.platform;
+  const provider = new LocalOrchestrator();
+
+  const runTask = () =>
+    provider.runTaskInWorkflow('build-guid', 'unity-image', 'echo hi', '/mnt', '/mnt/', [], []);
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    Object.defineProperty(process, 'platform', { value: 'linux' });
+    mockFs.readFileSync.mockImplementation(() => {
+      throw new Error('ENOENT');
+    });
+  });
+
+  afterEach(() => {
+    Object.defineProperty(process, 'platform', { value: originalPlatform });
+  });
+
+  it('enableBuildRetry=false (default): calls OrchestratorSystem.Run directly once and never touches UnityRetryService', async () => {
+    Orchestrator.buildParameters = { enableBuildRetry: false, projectPath: '.' } as BuildParameters;
+    mockRun.mockResolvedValue('build output');
+
+    const result = await runTask();
+
+    expect(result).toBe('build output');
+    expect(mockRun).toHaveBeenCalledTimes(1);
+    expect(mockRun).toHaveBeenCalledWith('echo hi');
+    expect(mockExecuteWithRetry).not.toHaveBeenCalled();
+  });
+
+  it('enableBuildRetry left unset behaves identically to false (safe default for hosts not on this build parameters version)', async () => {
+    Orchestrator.buildParameters = { projectPath: '.' } as BuildParameters;
+    mockRun.mockResolvedValue('build output');
+
+    await runTask();
+
+    expect(mockRun).toHaveBeenCalledTimes(1);
+    expect(mockExecuteWithRetry).not.toHaveBeenCalled();
+  });
+
+  it('enableBuildRetry=false still throws exactly as before on failure -- zero behavior change for the common case', async () => {
+    Orchestrator.buildParameters = { enableBuildRetry: false, projectPath: '.' } as BuildParameters;
+    mockRun.mockRejectedValue('raw wrapper output');
+
+    await expect(runTask()).rejects.toBe('raw wrapper output');
+    expect(mockExecuteWithRetry).not.toHaveBeenCalled();
+  });
+
+  it('enableBuildRetry=true: delegates to UnityRetryService.executeWithRetry with UnityRecoveryService default budgets instead of calling OrchestratorSystem.Run directly', async () => {
+    Orchestrator.buildParameters = {
+      enableBuildRetry: true,
+      projectPath: 'MyProject',
+    } as BuildParameters;
+    mockExecuteWithRetry.mockResolvedValue({
+      succeeded: true,
+      attempts: 2,
+      lastDiagnostics: { failureCategory: 'SUCCESS' } as any,
+      actionsPerformed: ['retry-licensing'],
+    });
+
+    const result = await runTask();
+
+    expect(result).toBe('');
+    expect(mockRun).not.toHaveBeenCalled();
+    expect(mockExecuteWithRetry).toHaveBeenCalledTimes(1);
+    expect(UnityRecoveryService.createDefaultBudgets).toHaveBeenCalledTimes(1);
+
+    const [projectPathArgument, runUnityArgument, optionsArgument] =
+      mockExecuteWithRetry.mock.calls[0];
+    expect(projectPathArgument).toBe(path.join(process.cwd(), 'MyProject'));
+    expect(typeof runUnityArgument).toBe('function');
+    expect(optionsArgument).toEqual({ budgets: { mocked: true } });
+  });
+
+  it('enableBuildRetry=true and the retry loop ultimately fails: throws a descriptive Error (not the raw output string)', async () => {
+    Orchestrator.buildParameters = { enableBuildRetry: true, projectPath: '.' } as BuildParameters;
+    mockExecuteWithRetry.mockResolvedValue({
+      succeeded: false,
+      attempts: 6,
+      lastDiagnostics: {
+        failureCategory: 'CRASH',
+        failureSummary: { remediationHint: 'Clear ScriptAssemblies or nuke Library.' },
+      } as any,
+      actionsPerformed: ['nuke-library'],
+    });
+
+    await expect(runTask()).rejects.toThrow(
+      /Unity build failed after 6 attempt\(s\).*\[CRASH\].*nuke-library/s,
+    );
+  });
+
+  it('the runUnity callback passed to UnityRetryService runs the command with suppressError=true, captures the real exit code, and prefers the on-disk Editor log over wrapper stdout', async () => {
+    Orchestrator.buildParameters = { enableBuildRetry: true, projectPath: '.' } as BuildParameters;
+
+    let capturedRunUnity:
+      | (() => Promise<{
+          exitCode: number;
+          logText: string;
+          runtimeSeconds: number;
+        }>)
+      | undefined;
+
+    mockExecuteWithRetry.mockImplementation(async (_projectPath, runUnity) => {
+      capturedRunUnity = runUnity;
+
+      return {
+        succeeded: true,
+        attempts: 1,
+        lastDiagnostics: { failureCategory: 'SUCCESS' } as any,
+        actionsPerformed: [],
+      };
+    });
+    mockRun.mockImplementation(
+      async (
+        _command: string,
+        _suppressError?: boolean,
+        _suppressLogs?: boolean,
+        _outputCallback?: (output: string) => void,
+        onExitCode?: (code: number) => void,
+      ) => {
+        onExitCode?.(1);
+
+        return 'wrapper stdout only';
+      },
+    );
+    mockFs.readFileSync.mockReturnValue('genuine Unity Editor log content' as any);
+
+    await runTask();
+
+    expect(capturedRunUnity).toBeDefined();
+    const attemptResult = await capturedRunUnity!();
+
+    expect(mockRun).toHaveBeenCalledWith('echo hi', true, false, undefined, expect.any(Function));
+    expect(attemptResult.exitCode).toBe(1);
+    expect(attemptResult.logText).toBe('genuine Unity Editor log content');
+    expect(mockFs.readFileSync).toHaveBeenCalledWith(
+      path.join(process.cwd(), 'temp', 'job-log.txt'),
+      'utf8',
+    );
+  });
+
+  it('falls back to wrapper stdout as logText when the Editor log file cannot be read', async () => {
+    Orchestrator.buildParameters = { enableBuildRetry: true, projectPath: '.' } as BuildParameters;
+
+    let capturedRunUnity: (() => Promise<{ exitCode: number; logText: string }>) | undefined;
+    mockExecuteWithRetry.mockImplementation(async (_projectPath, runUnity) => {
+      capturedRunUnity = runUnity as any;
+
+      return {
+        succeeded: true,
+        attempts: 1,
+        lastDiagnostics: { failureCategory: 'SUCCESS' } as any,
+        actionsPerformed: [],
+      };
+    });
+    mockRun.mockResolvedValue('wrapper stdout only');
+    mockFs.readFileSync.mockImplementation(() => {
+      throw new Error('ENOENT: no such file');
+    });
+
+    await runTask();
+    const attemptResult = await capturedRunUnity!();
+
+    expect(attemptResult.logText).toBe('wrapper stdout only');
+  });
+});

plugins/orchestrator/src/model/orchestrator/providers/local/index.ts

@@ -1,3 +1,5 @@
+import fs from 'node:fs';
+import path from 'node:path';
 import BuildParameters from '../../../build-parameters';
 import { OrchestratorSystem } from '../../services/core/orchestrator-system';
 import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
@@ -7,6 +9,9 @@ import OrchestratorSecret from '../../options/orchestrator-secret';
 import { ProviderResource } from '../provider-resource';
 import { ProviderWorkflow } from '../provider-workflow';
 import { quote } from 'shell-quote';
+import Orchestrator from '../../orchestrator';
+import { UnityRetryService } from '../../services/reliability/unity-retry-service';
+import { UnityRecoveryService } from '../../services/reliability/unity-recovery-service';
 
 class LocalOrchestrator implements ProviderInterface {
   listResources(): Promise<ProviderResource[]> {
@@ -89,6 +94,7 @@ class LocalOrchestrator implements ProviderInterface {
     OrchestratorLogger.log(commands);
 
     // On Windows, many built-in hooks use POSIX shell syntax. Execute via bash if available.
+    let command = commands;
     if (process.platform === 'win32') {
       const inline = commands
         .replace(/\r/g, '')
@@ -97,12 +103,94 @@ class LocalOrchestrator implements ProviderInterface {
         .join(' ; ');
 
       // Use shell-quote to properly escape the command string, preventing command injection
-      const bashWrapped = `bash -lc ${quote([inline])}`;
+      command = `bash -lc ${quote([inline])}`;
+    }
 
-      return await OrchestratorSystem.Run(bashWrapped);
+    // Opt-in (--enableBuildRetry, default off): wrap the whole invocation in
+    // UnityRetryService's classify -> decide -> retry loop, entirely
+    // encapsulated inside this provider. Every provider strategy other than
+    // local/local-system goes through their own OrchestratorSystem.Run calls
+    // untouched, and the ProviderInterface contract is unchanged -- callers
+    // above (e.g. BuildAutomationWorkflow) keep awaiting a plain
+    // Promise<string>, same as before this feature existed.
+    if (Orchestrator.buildParameters?.enableBuildRetry) {
+      return await LocalOrchestrator.runWithRetry(command);
     }
 
-    return await OrchestratorSystem.Run(commands);
+    return await OrchestratorSystem.Run(command);
+  }
+
+  /**
+   * Runs `command` under UnityRetryService.executeWithRetry: on a non-zero
+   * exit, classify the failure (UnityBuildDiagnosticsService), decide on a
+   * budget-gated recovery action (UnityRecoveryService -- may back up/nuke
+   * the Library folder or clear subfolders), perform it, then retry the
+   * same command again, up to UnityRetryService's own attempt cap.
+   *
+   * `logText` is read from the real Unity Editor log rather than only the
+   * wrapper's captured stdout/stderr: dist/platforms/ubuntu/steps/runsteps.sh
+   * (via build.sh/activate.sh) invokes `unity-editor -logfile /dev/stdout`,
+   * and BuildAutomationWorkflow.BuildWorkflow pipes that whole pipeline
+   * through `node <builder> -m remote-cli-log-stream --logFile "$LOG_FILE"`,
+   * which appends every line to $LOG_FILE. For the bare-local provider,
+   * $LOG_FILE is exported as `$(pwd)/temp/job-log.txt` (see
+   * BuildAutomationWorkflow.BuildWorkflow), so that file accumulates the
+   * genuine Editor log content that UnityBuildDiagnosticsService's crash/
+   * license/compile patterns are meant to match against -- not just whatever
+   * subset `command`'s own stdout capture happens to contain.
+   */
+  private static async runWithRetry(command: string): Promise<string> {
+    const bp = Orchestrator.buildParameters;
+    const projectPath = path.isAbsolute(bp.projectPath || '')
+      ? bp.projectPath
+      : path.join(process.cwd(), bp.projectPath || '.');
+    const logFilePath = path.join(process.cwd(), 'temp', 'job-log.txt');
+
+    let lastOutput = '';
+
+    const result = await UnityRetryService.executeWithRetry(
+      projectPath,
+      async () => {
+        const startedAtMs = Date.now();
+        let exitCode = 0;
+
+        // suppressError=true: on a non-zero exit, OrchestratorSystem.Run
+        // resolves with the captured output instead of throwing, so the
+        // retry loop can inspect the result and decide whether to retry --
+        // exactly mirroring how the non-retry path above lets a throw
+        // propagate on the single, non-retried attempt.
+        const stdout = await OrchestratorSystem.Run(command, true, false, undefined, (code) => {
+          exitCode = code;
+        });
+        lastOutput = stdout;
+
+        const runtimeSeconds = (Date.now() - startedAtMs) / 1000;
+        const logText = LocalOrchestrator.readEditorLog(logFilePath) || stdout;
+
+        return { exitCode, logText, runtimeSeconds };
+      },
+      { budgets: UnityRecoveryService.createDefaultBudgets() },
+    );
+
+    if (!result.succeeded) {
+      const category = result.lastDiagnostics?.failureCategory ?? 'UNKNOWN';
+      const hint = result.lastDiagnostics?.failureSummary?.remediationHint ?? '';
+      throw new Error(
+        `Unity build failed after ${result.attempts} attempt(s) via UnityRetryService ` +
+          `[${category}]${hint ? `: ${hint}` : ''}. Actions performed: ` +
+          `${result.actionsPerformed.join(', ') || 'none'}.\n${lastOutput}`,
+      );
+    }
+
+    return lastOutput;
+  }
+
+  private static readEditorLog(logFilePath: string): string {
+    try {
+      return fs.readFileSync(logFilePath, 'utf8');
+    } catch {
+      return '';
+    }
   }
 }
 export default LocalOrchestrator;

plugins/orchestrator/src/model/orchestrator/services/core/orchestrator-system.ts

@@ -22,6 +22,13 @@ export class OrchestratorSystem {
     suppressLogs = false,
     // eslint-disable-next-line no-unused-vars
     outputCallback?: (output: string) => void,
+    // Invoked with the real (non-normalized) child process exit code right
+    // before the promise settles -- on both the success and failure paths.
+    // Purely additive: existing callers that don't pass this get identical
+    // behavior (same thrown string, same resolved output) to before this
+    // parameter was added.
+    // eslint-disable-next-line no-unused-vars
+    onExitCode?: (code: number) => void,
   ) {
     for (const element of command.split(`\n`)) {
       if (!suppressLogs) {
@@ -53,6 +60,7 @@ export class OrchestratorSystem {
         if (!suppressLogs) {
           RemoteClientLogger.log(`[${code}]`);
         }
+        onExitCode?.(code ?? -1);
         if (code !== 0 && !suppressError) {
           throwError(output);
         }