diff --git a/packages/core/src/agent/task-builder.ts b/packages/core/src/agent/task-builder.ts index 423a04a8bc..6487d2e88a 100644 --- a/packages/core/src/agent/task-builder.ts +++ b/packages/core/src/agent/task-builder.ts @@ -1,5 +1,6 @@ import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model'; import type { ModelRuntime } from '@/ai-model/models'; +import { findActionInActionSpaceOrThrow } from '@/common'; import type { AbstractInterface } from '@/device'; import type Service from '@/service'; import { setTimingFieldOnce } from '@/task-timing'; @@ -218,20 +219,15 @@ export class TaskBuilder { ): Promise { const planType = plan.type; const actionSpace = this.actionSpace; - const action = actionSpace.find((item) => item.name === planType); + const action = findActionInActionSpaceOrThrow(planType, actionSpace); const param = plan.param; - if (!action) { - throw new Error(`Action type '${planType}' not found`); - } - - const locateFields = action - ? findAllMidsceneLocatorField(action.paramSchema) - : []; + const locateFields = findAllMidsceneLocatorField(action.paramSchema); - const requiredLocateFields = action - ? findAllMidsceneLocatorField(action.paramSchema, true) - : []; + const requiredLocateFields = findAllMidsceneLocatorField( + action.paramSchema, + true, + ); locateFields.forEach((field) => { if (param[field]) { diff --git a/packages/core/src/ai-model/workflows/planning/locate-normalization.ts b/packages/core/src/ai-model/workflows/planning/locate-normalization.ts index c635a1360c..a8440fd696 100644 --- a/packages/core/src/ai-model/workflows/planning/locate-normalization.ts +++ b/packages/core/src/ai-model/workflows/planning/locate-normalization.ts @@ -26,7 +26,7 @@ export function normalizePlanningActionLocateFields( ): void { actions.forEach((action) => { const actionInActionSpace = actionSpace.find( - (item) => item.name === action.type, + (actionInSpace) => actionInSpace.name === action.type, ); if (!actionInActionSpace) { debug('skip locate normalization for action outside actionSpace', action); @@ -34,9 +34,9 @@ export function normalizePlanningActionLocateFields( } debug('actionInActionSpace matched', actionInActionSpace); - const locateFields = findAllMidsceneLocatorField( - actionInActionSpace.paramSchema, - ); + const locateFields = actionInActionSpace + ? findAllMidsceneLocatorField(actionInActionSpace.paramSchema) + : []; debug('locateFields', locateFields); diff --git a/packages/core/tests/unit-test/locate-normalization.test.ts b/packages/core/tests/unit-test/locate-normalization.test.ts index 65198573b9..ad1943d03e 100644 --- a/packages/core/tests/unit-test/locate-normalization.test.ts +++ b/packages/core/tests/unit-test/locate-normalization.test.ts @@ -24,7 +24,7 @@ const locateResultContext = { }; describe('normalizePlanningActionLocateFields', () => { - it('skips locate normalization when a planned action is not in the action space', () => { + it('leaves actions unchanged when no action schema is available for locate normalization', () => { const adaptPlanningParamToPixelBbox = vi.fn(); const actions: PlanningAction[] = [ { diff --git a/packages/core/tests/unit-test/task-builder.test.ts b/packages/core/tests/unit-test/task-builder.test.ts index c1d5d3237b..dbba3df22d 100644 --- a/packages/core/tests/unit-test/task-builder.test.ts +++ b/packages/core/tests/unit-test/task-builder.test.ts @@ -130,6 +130,29 @@ describe('TaskBuilder', () => { ]); }); + it('throws when building an executable task for an action outside actionSpace', async () => { + const mockInterface = new MockInterface([defineActionSleep()]); + const insightService = { + contextRetrieverFn: vi.fn(), + locate: vi.fn(), + } as unknown as Service; + const taskBuilder = new TaskBuilder({ + interfaceInstance: mockInterface, + service: insightService, + actionSpace: mockInterface.actionSpace(), + }); + + await expect( + taskBuilder.build( + [{ type: 'Tap', thought: 'tap missing action', param: {} }], + mockModelRuntime, + mockModelRuntime, + ), + ).rejects.toThrow( + /Action type 'Tap' is not in the current action space. Available actions: Sleep/, + ); + }); + it('supports fast-path action delays for system actions', async () => { vi.useFakeTimers();