Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 1 addition & 32 deletions src/tools/common/get_actor_output.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import type { InternalToolArgs, ToolEntry, ToolInputSchema } from '../../types.j
import { compileSchema } from '../../utils/ajv.js';
import { getValuesByDotKeys, parseCommaSeparatedList } from '../../utils/generic.js';
import { buildMCPResponse } from '../../utils/mcp.js';
import { cleanEmptyProperties } from '../../utils/schema_generation.js';
import { datasetItemsOutputSchema } from '../structured_output_schemas.js';

/**
Expand All @@ -28,38 +29,6 @@ const getActorOutputArgs = z.object({
.describe('Maximum number of items to return (default: 100).'),
});

/**
* Cleans empty properties (null, undefined, empty strings, empty arrays, empty objects) from an object
* @param obj - The object to clean
* @returns The cleaned object or undefined if the result is empty
*/
export function cleanEmptyProperties(obj: unknown): unknown {
if (obj === null || obj === undefined || obj === '') {
return undefined;
}

if (typeof obj !== 'object') {
return obj;
}

if (Array.isArray(obj)) {
const cleaned = obj
.map((item) => cleanEmptyProperties(item))
.filter((item) => item !== undefined);
return cleaned.length > 0 ? cleaned : undefined;
}

const cleaned: Record<string, unknown> = {};
for (const [key, value] of Object.entries(obj)) {
const cleanedValue = cleanEmptyProperties(value);
if (cleanedValue !== undefined) {
cleaned[key] = cleanedValue;
}
}

return Object.keys(cleaned).length > 0 ? cleaned : undefined;
}

/**
* This tool is used specifically for retrieving Actor output.
* It is a simplified version of the get-dataset-items tool.
Expand Down
38 changes: 8 additions & 30 deletions src/tools/core/actor_run_response.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { getWidgetConfig, WIDGET_URIS } from '../../resources/widgets.js';
import { logHttpError } from '../../utils/logging.js';
import { buildMCPResponse } from '../../utils/mcp.js';
import { formatRunStatusMessage, type ProgressTracker, TERMINAL_RUN_STATUSES } from '../../utils/progress.js';
import { cleanEmptyProperties } from '../../utils/schema_generation.js';

Comment thread
jirispilka marked this conversation as resolved.
/** Cap on `storages.keyValueStores.default.keys` array length. */
const KV_KEYS_LIMIT = 50;
Expand All @@ -33,10 +34,6 @@ const ITEM_COUNT_PROBE_LIMIT = 1;
*/
const ITEM_COUNT_PROBE_DELAYS_MS = [0, 1000, 2000, 2000] as const;

async function sleep(ms: number): Promise<void> {
await new Promise<void>((resolve) => { setTimeout(resolve, ms); });
}

/** Sentinel used by `raceAbort` to signal that the abort signal won the race. */
const ABORT = Symbol('ABORT');

Expand Down Expand Up @@ -138,11 +135,6 @@ export type FetchActorRunResult = {
// Helpers
// -----------------------------------------------------------------------------

/** Translate Apify slash-notation field paths to dot-notation. */
function slashToDot(field: string): string {
return field.replace(/\//g, '.');
}

/**
* Apify expands array indices in dataset fields (e.g. `entities.hashtags.0.text`,
* `entities.hashtags.1.text`, ... `entities.hashtags.14.text`), so deeply-nested or
Expand Down Expand Up @@ -175,20 +167,7 @@ export function collapseArrayIndices(fields: string[]): string[] {
* `call-actor` / `get-actor-run`, and `get-dataset` for the raw API passthrough).
*/
export function normalizeDatasetFields(fields: string[]): string[] {
return collapseArrayIndices(fields.map(slashToDot));
}

/**
* Drop undefined and null keys. Apify's SDK returns null for fields it doesn't have (e.g.
* an unnamed default dataset's `name`), and the response shape declares no nullable fields, so we
* filter both to keep the response clean and pass `getActorRunOutputSchema` validation.
*/
function omitNullish<T extends Record<string, unknown>>(obj: T): T {
const out: Record<string, unknown> = {};
for (const [k, v] of Object.entries(obj)) {
if (v !== undefined && v !== null) out[k] = v;
}
return out as T;
return collapseArrayIndices(fields.map((f) => f.replace(/\//g, '.')));
}

function toIsoString(value: Date | string | undefined | null): string | undefined {
Expand All @@ -199,27 +178,26 @@ function toIsoString(value: Date | string | undefined | null): string | undefine
function buildStats(run: ActorRun): RunResponse['stats'] | undefined {
const stats = run.stats as ActorRun['stats'] | undefined;
if (!stats) return undefined;
const out = omitNullish({
return cleanEmptyProperties({
runTimeSecs: stats.runTimeSecs,
computeUnits: stats.computeUnits,
memMaxBytes: stats.memMaxBytes,
});
return Object.keys(out).length > 0 ? out : undefined;
}) as RunResponse['stats'] | undefined;
}

function buildRunDataset(run: ActorRun, datasetMeta: Dataset | null, resolvedItemCount?: number): RunDataset | undefined {
if (!run.defaultDatasetId) return undefined;
if (!datasetMeta) {
return { id: run.defaultDatasetId };
}
return omitNullish({
return cleanEmptyProperties({
id: datasetMeta.id,
name: datasetMeta.name,
title: datasetMeta.title,
itemCount: resolvedItemCount ?? datasetMeta.itemCount,
cleanItemCount: datasetMeta.cleanItemCount,
fields: datasetMeta.fields ? normalizeDatasetFields(datasetMeta.fields) : undefined,
});
}) as RunDataset;
}

function buildRunKeyValueStore(run: ActorRun, listKeysResult: KeyValueClientListKeysResult | null): RunKeyValueStore | undefined {
Expand All @@ -236,7 +214,7 @@ function buildRunKeyValueStore(run: ActorRun, listKeysResult: KeyValueClientList
// we know the page count equals the total; when truncated, omit keyCount and let the agent
// detect "more keys exist" from `keys.length === KV_KEYS_LIMIT`.
const keyCount = listKeysResult.isTruncated ? undefined : keys.length;
return omitNullish({ id: run.defaultKeyValueStoreId, keys, keyCount });
return cleanEmptyProperties({ id: run.defaultKeyValueStoreId, keys, keyCount }) as RunKeyValueStore;
}

function errMessage(error: unknown): string {
Expand Down Expand Up @@ -268,7 +246,7 @@ async function resolveItemCountWithLagFallback(
let lastTotal = 0;
for (const delay of delays) {
if (delay > 0) {
const sleepResult = await raceAbort(sleep(delay), abortSignal);
const sleepResult = await raceAbort(new Promise<void>((resolve) => { setTimeout(resolve, delay); }), abortSignal);
if (sleepResult === ABORT) return lastTotal;
}
const result = await raceAbort(
Expand Down
31 changes: 4 additions & 27 deletions src/tools/structured_output_schemas.ts
Original file line number Diff line number Diff line change
Expand Up @@ -393,8 +393,6 @@ export const getActorRunOutputSchema = {
* (e.g. `{ url: { type: 'string' }, price: { type: 'number' } }`).
*/
export function buildEnrichedDirectActorOutputSchema(itemProperties: Record<string, unknown>) {
const { datasets } = getActorRunOutputSchema.properties.storages.properties;
const { default: defaultDataset } = datasets.properties;
const itemsSchema = {
type: 'object' as const,
description: 'JSON Schema for rows in the dataset at `storages.datasets.default.id` — describes row '
Expand All @@ -403,31 +401,10 @@ export function buildEnrichedDirectActorOutputSchema(itemProperties: Record<stri
+ 'dataset id and a `fields` projection drawn from this schema.',
properties: itemProperties,
};
return {
...getActorRunOutputSchema,
properties: {
...getActorRunOutputSchema.properties,
storages: {
...getActorRunOutputSchema.properties.storages,
properties: {
...getActorRunOutputSchema.properties.storages.properties,
datasets: {
...datasets,
properties: {
...datasets.properties,
default: {
...defaultDataset,
properties: {
...defaultDataset.properties,
itemsSchema,
},
},
},
},
},
},
},
};
const clone = structuredClone(getActorRunOutputSchema);
const datasetDefaultProps = clone.properties.storages.properties.datasets.properties.default.properties as Record<string, unknown>;
datasetDefaultProps.itemsSchema = itemsSchema;
return clone;
}

/**
Expand Down
47 changes: 42 additions & 5 deletions src/utils/schema_generation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,20 @@ export type SchemaGenerationOptions = {
clean?: boolean;
};

// Local counterpart to the dataset API's `clean=true` — empty arrays carry no schema info.
export function removeEmptyArrays(obj: unknown): unknown {
/**
* Local counterpart to the dataset API's `clean=true` — empty arrays carry no schema info.
* Strips only empty arrays; keeps null / '' / empty objects so schema inference still sees those fields.
* Stricter sibling: {@link cleanEmptyProperties} also strips nullish and empty strings.
*/
export function cleanEmptyArrays(obj: unknown): unknown {
if (Array.isArray(obj)) {
return obj.map(removeEmptyArrays);
return obj.map(cleanEmptyArrays);
}
if (typeof obj !== 'object' || obj === null) {
return obj;
}
return Object.entries(obj).reduce((acc, [key, value]) => {
const processed = removeEmptyArrays(value);
const processed = cleanEmptyArrays(value);
if (Array.isArray(processed) && processed.length === 0) {
return acc;
}
Expand All @@ -39,6 +43,39 @@ export function removeEmptyArrays(obj: unknown): unknown {
}, {} as Record<string, unknown>);
}

/**
* Cleans empty properties (null, undefined, empty strings, empty arrays, empty objects) from an object.
* Looser sibling: {@link cleanEmptyArrays} strips only empty arrays.
* @param obj - The object to clean
* @returns The cleaned object or undefined if the result is empty
*/
export function cleanEmptyProperties(obj: unknown): unknown {
if (obj === null || obj === undefined || obj === '') {
return undefined;
}

if (typeof obj !== 'object') {
return obj;
}

if (Array.isArray(obj)) {
const cleaned = obj
.map((item) => cleanEmptyProperties(item))
.filter((item) => item !== undefined);
return cleaned.length > 0 ? cleaned : undefined;
}

const cleaned: Record<string, unknown> = {};
for (const [key, value] of Object.entries(obj)) {
const cleanedValue = cleanEmptyProperties(value);
if (cleanedValue !== undefined) {
cleaned[key] = cleanedValue;
}
}

return Object.keys(cleaned).length > 0 ? cleaned : undefined;
}

const FORMAT_DETECTORS: [string, (s: string) => boolean][] = [
['date-time', (s) => /^\d{4}-\d{2}-\d{2}[Tt ]\d{2}:\d{2}:\d{2}(\.\d+)?([Zz]|[+-]\d{2}:\d{2})?$/.test(s)],
['date', (s) => /^\d{4}-\d{2}-\d{2}$/.test(s)],
Expand Down Expand Up @@ -149,7 +186,7 @@ export function generateSchemaFromItems(
const itemsToUse = datasetItems.slice(0, limit);
if (itemsToUse.length === 0) return null;

const processed = clean ? itemsToUse.map(removeEmptyArrays) : itemsToUse;
const processed = clean ? itemsToUse.map(cleanEmptyArrays) : itemsToUse;

const itemSchemas = processed.map(inferSchema);
const merged = itemSchemas.reduce(mergeSchemas);
Expand Down
18 changes: 9 additions & 9 deletions tests/unit/schema_generation.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { describe, expect, it } from 'vitest';
import {
generateSchemaFromItems,
type JsonSchemaProperty,
removeEmptyArrays,
cleanEmptyArrays,
} from '../../src/utils/schema_generation.js';

/** Extract item-level properties from a generated array schema. */
Expand Down Expand Up @@ -288,23 +288,23 @@ describe('generateSchemaFromItems — user-reported regression', () => {
});
});

describe('removeEmptyArrays', () => {
describe('cleanEmptyArrays', () => {
it('drops keys whose value is an empty array', () => {
expect(removeEmptyArrays({ kept: 1, dropped: [] })).toEqual({ kept: 1 });
expect(cleanEmptyArrays({ kept: 1, dropped: [] })).toEqual({ kept: 1 });
});

it('recurses into nested objects', () => {
expect(removeEmptyArrays({ a: { kept: 1, dropped: [] } })).toEqual({ a: { kept: 1 } });
expect(cleanEmptyArrays({ a: { kept: 1, dropped: [] } })).toEqual({ a: { kept: 1 } });
});

it('recurses into array elements', () => {
expect(removeEmptyArrays([{ x: [] }, { y: 1 }])).toEqual([{}, { y: 1 }]);
expect(cleanEmptyArrays([{ x: [] }, { y: 1 }])).toEqual([{}, { y: 1 }]);
});

it('preserves primitives, null, and non-empty arrays', () => {
expect(removeEmptyArrays(null)).toBeNull();
expect(removeEmptyArrays(42)).toBe(42);
expect(removeEmptyArrays('s')).toBe('s');
expect(removeEmptyArrays([1, 2, 3])).toEqual([1, 2, 3]);
expect(cleanEmptyArrays(null)).toBeNull();
expect(cleanEmptyArrays(42)).toBe(42);
expect(cleanEmptyArrays('s')).toBe('s');
expect(cleanEmptyArrays([1, 2, 3])).toEqual([1, 2, 3]);
});
});
Loading