|
5 | 5 | */ |
6 | 6 |
|
7 | 7 | import { ApifyClient } from '../apify_client.js'; |
8 | | -import { ACTOR_PRICING_MODEL } from '../const.js'; |
| 8 | +import { ACTOR_PRICING_MODEL, INPUT_FIELDS_FALLBACK_LIMIT, INPUT_SCHEMA_FETCH_CONCURRENCY } from '../const.js'; |
9 | 9 | import type { PaymentProvider } from '../payments/types.js'; |
10 | | -import type { ActorStoreList } from '../types.js'; |
| 10 | +import { actorDefinitionPrunedCache, inputFieldsCache } from '../state.js'; |
| 11 | +import type { ActorInputField, ActorInputSchema, ActorStoreList } from '../types.js'; |
| 12 | +import { runWithConcurrency } from './generic.js'; |
| 13 | +import { logHttpError } from './logging.js'; |
11 | 14 |
|
12 | 15 | /** |
13 | 16 | * Used in search Actors tool to search above the input supplied limit, |
@@ -85,3 +88,112 @@ export function filterRentalActors( |
85 | 88 | || userRentedActorIds.includes(actor.id), |
86 | 89 | ); |
87 | 90 | } |
| 91 | + |
| 92 | +/** |
| 93 | + * Extract minimal field info (name, type, required) from an ActorInputSchema. |
| 94 | + * |
| 95 | + * Heuristic: Apify Actor input schemas use `sectionCaption` to group fields into UI sections. |
| 96 | + * Fields before the first sectionCaption are the "main" inputs (e.g. search query, URL, limit). |
| 97 | + * Fields after are advanced config (proxy, custom code, deprecated, etc.). |
| 98 | + * |
| 99 | + * We return only the first section to keep search results compact. |
| 100 | + * Fallback: if no sectionCaption exists, cap at INPUT_FIELDS_FALLBACK_LIMIT fields. |
| 101 | + */ |
| 102 | +export function extractInputFields(input: ActorInputSchema): ActorInputField[] { |
| 103 | + const entries = Object.entries(input.properties ?? {}); |
| 104 | + const requiredSet = new Set(input.required ?? []); |
| 105 | + |
| 106 | + // Find the index of the first field with sectionCaption |
| 107 | + const firstSectionIdx = entries.findIndex( |
| 108 | + ([, prop]) => 'sectionCaption' in (prop as Record<string, unknown>), |
| 109 | + ); |
| 110 | + |
| 111 | + // If sectionCaption found, take only fields before it; otherwise cap at fallback limit |
| 112 | + const selectedEntries = firstSectionIdx > 0 |
| 113 | + ? entries.slice(0, firstSectionIdx) |
| 114 | + : entries.slice(0, INPUT_FIELDS_FALLBACK_LIMIT); |
| 115 | + |
| 116 | + return selectedEntries.map(([name, prop]) => ({ |
| 117 | + name, |
| 118 | + type: prop.type ?? 'unknown', |
| 119 | + required: requiredSet.has(name), |
| 120 | + })); |
| 121 | +} |
| 122 | + |
| 123 | +/** |
| 124 | + * Look up cached input fields for an actor by fullName. |
| 125 | + * Checks actorDefinitionPrunedCache for a cached definition with input schema. |
| 126 | + * Returns null on cache miss. |
| 127 | + */ |
| 128 | +function getCachedInputFields(fullName: string): ActorInputField[] | null { |
| 129 | + // Check dedicated input fields cache first |
| 130 | + const cached = inputFieldsCache.get(fullName); |
| 131 | + if (cached) return cached; |
| 132 | + |
| 133 | + // Fall back to full actor definition cache (populated by fetch-actor-details / call-actor) |
| 134 | + const cachedDef = actorDefinitionPrunedCache.get(fullName); |
| 135 | + if (cachedDef?.definition?.input) { |
| 136 | + const fields = extractInputFields(cachedDef.definition.input); |
| 137 | + inputFieldsCache.set(fullName, fields); |
| 138 | + return fields; |
| 139 | + } |
| 140 | + return null; |
| 141 | +} |
| 142 | + |
| 143 | +/** |
| 144 | + * Fetch input schema fields for a single actor from the API. |
| 145 | + * Returns null on failure. |
| 146 | + */ |
| 147 | +async function fetchInputFieldsForActor( |
| 148 | + fullName: string, |
| 149 | + apifyClient: ApifyClient, |
| 150 | +): Promise<ActorInputField[] | null> { |
| 151 | + try { |
| 152 | + const buildClient = await apifyClient.actor(fullName).defaultBuild(); |
| 153 | + const build = await buildClient.get(); |
| 154 | + if (build?.actorDefinition?.input) { |
| 155 | + const input = build.actorDefinition.input as ActorInputSchema; |
| 156 | + const fields = extractInputFields(input); |
| 157 | + inputFieldsCache.set(fullName, fields); |
| 158 | + return fields; |
| 159 | + } |
| 160 | + return null; |
| 161 | + } catch (error) { |
| 162 | + logHttpError(error, `Failed to fetch input schema for '${fullName}'`, { actorName: fullName }); |
| 163 | + return null; |
| 164 | + } |
| 165 | +} |
| 166 | + |
| 167 | +/** |
| 168 | + * Fetch input schemas for a list of actors with bounded concurrency. |
| 169 | + * Checks caches for hits, fetches misses from API. |
| 170 | + * Returns a Map<actorFullName, ActorInputField[]>. |
| 171 | + */ |
| 172 | +export async function fetchInputFieldsForActors( |
| 173 | + actors: ActorStoreList[], |
| 174 | + apifyClient: ApifyClient, |
| 175 | +): Promise<Map<string, ActorInputField[]>> { |
| 176 | + const result = new Map<string, ActorInputField[]>(); |
| 177 | + |
| 178 | + // Resolve cache hits first |
| 179 | + const toFetch: { fullName: string }[] = []; |
| 180 | + for (const actor of actors) { |
| 181 | + const fullName = `${actor.username}/${actor.name}`; |
| 182 | + const cached = getCachedInputFields(fullName); |
| 183 | + if (cached) { |
| 184 | + result.set(fullName, cached); |
| 185 | + } else { |
| 186 | + toFetch.push({ fullName }); |
| 187 | + } |
| 188 | + } |
| 189 | + |
| 190 | + // Fetch cache misses with bounded concurrency |
| 191 | + await runWithConcurrency(toFetch, INPUT_SCHEMA_FETCH_CONCURRENCY, async ({ fullName }) => { |
| 192 | + const fields = await fetchInputFieldsForActor(fullName, apifyClient); |
| 193 | + if (fields) { |
| 194 | + result.set(fullName, fields); |
| 195 | + } |
| 196 | + }); |
| 197 | + |
| 198 | + return result; |
| 199 | +} |
0 commit comments