Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 7 additions & 11 deletions packages/core/src/agent/task-builder.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';
import type { ModelRuntime } from '@/ai-model/models';
import { findActionInActionSpaceOrThrow } from '@/common';
import type { AbstractInterface } from '@/device';
import type Service from '@/service';
import { setTimingFieldOnce } from '@/task-timing';
Expand Down Expand Up @@ -218,20 +219,15 @@ export class TaskBuilder {
): Promise<void> {
const planType = plan.type;
const actionSpace = this.actionSpace;
const action = actionSpace.find((item) => item.name === planType);
const action = findActionInActionSpaceOrThrow(planType, actionSpace);
const param = plan.param;

if (!action) {
throw new Error(`Action type '${planType}' not found`);
}

const locateFields = action
? findAllMidsceneLocatorField(action.paramSchema)
: [];
const locateFields = findAllMidsceneLocatorField(action.paramSchema);

const requiredLocateFields = action
? findAllMidsceneLocatorField(action.paramSchema, true)
: [];
const requiredLocateFields = findAllMidsceneLocatorField(
action.paramSchema,
true,
);

locateFields.forEach((field) => {
if (param[field]) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,17 +26,17 @@ export function normalizePlanningActionLocateFields(
): void {
actions.forEach((action) => {
const actionInActionSpace = actionSpace.find(
(item) => item.name === action.type,
(actionInSpace) => actionInSpace.name === action.type,
);
if (!actionInActionSpace) {
debug('skip locate normalization for action outside actionSpace', action);
return;
}

debug('actionInActionSpace matched', actionInActionSpace);
const locateFields = findAllMidsceneLocatorField(
actionInActionSpace.paramSchema,
);
const locateFields = actionInActionSpace
? findAllMidsceneLocatorField(actionInActionSpace.paramSchema)
: [];
Comment on lines +37 to +39

debug('locateFields', locateFields);

Expand Down
2 changes: 1 addition & 1 deletion packages/core/tests/unit-test/locate-normalization.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ const locateResultContext = {
};

describe('normalizePlanningActionLocateFields', () => {
it('skips locate normalization when a planned action is not in the action space', () => {
it('leaves actions unchanged when no action schema is available for locate normalization', () => {
const adaptPlanningParamToPixelBbox = vi.fn();
const actions: PlanningAction[] = [
{
Expand Down
23 changes: 23 additions & 0 deletions packages/core/tests/unit-test/task-builder.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,29 @@ describe('TaskBuilder', () => {
]);
});

it('throws when building an executable task for an action outside actionSpace', async () => {
const mockInterface = new MockInterface([defineActionSleep()]);
const insightService = {
contextRetrieverFn: vi.fn(),
locate: vi.fn(),
} as unknown as Service;
const taskBuilder = new TaskBuilder({
interfaceInstance: mockInterface,
service: insightService,
actionSpace: mockInterface.actionSpace(),
});

await expect(
taskBuilder.build(
[{ type: 'Tap', thought: 'tap missing action', param: {} }],
mockModelRuntime,
mockModelRuntime,
),
).rejects.toThrow(
/Action type 'Tap' is not in the current action space. Available actions: Sleep/,
);
});

it('supports fast-path action delays for system actions', async () => {
vi.useFakeTimers();

Expand Down
Loading