diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.md index 1be517df4052..18b3b3a99d6d 100644 --- a/docs/upgrading/upgrading_v4.md +++ b/docs/upgrading/upgrading_v4.md @@ -1866,11 +1866,14 @@ Because the in-memory queue lives entirely within a single process and is never #### Out-of-band key-value files (e.g. a hand-placed `INPUT.json`) -`FileSystemStorageBackend` only fully tracks records it wrote itself (those have a `.__metadata__.json` sidecar). It still reads a value file placed in the store directory out-of-band — such as a hand-written or platform-provided `INPUT.json` — by probing the requested key plus the `.json` and `.txt` extensions. A few behaviors around these "bare" files changed in v4: +Keys are literal. `aaa` and `aaa.json` are two distinct keys, and `FileSystemStorageBackend` never infers a key from a file's extension. In v3 a hand-placed `aaa.json` in the store directory was readable as `aaa`; in v4 it is not read, not listed, and gets deleted by the purge of the default store on start like any other untracked file. -- **Extensionless bare files report `application/octet-stream`.** In v3 a bare value file with no extension was read as `text/plain`. In v4 the client is a plain byte transport and only infers a content type from a real extension, so an extensionless file now comes back as `application/octet-stream`. Give the file a `.json` or `.txt` extension if you need a more specific type. +The one exception is the run input. `FileSystemStorageBackend` only fully tracks records it wrote itself (those have a `.__metadata__.json` sidecar), but for the `INPUT` key (and the configured `inputKey`) it still reads a value file placed in the store directory out-of-band — such as a hand-written or platform-provided `INPUT.json` — by probing the requested key plus the `.json` extension. A few behaviors around these "bare" files changed in v4: + +- **Only `INPUT` and `INPUT.json` are probed.** v3 also fell back to `INPUT.txt` and `INPUT.bin`. In v4 those files are not read (`getValue('INPUT')` returns `undefined`), are not listed, and are no longer exempt from the purge of the default store on start, so they get deleted like any other untracked file. Rename a `.txt`/`.bin` input to `INPUT.json` (or drop the extension) before upgrading. +- **Extensionless bare files report `application/octet-stream`.** In v3 a bare value file with no extension was read as `text/plain`. In v4 the client is a plain byte transport and only infers a content type from a real extension, so an extensionless file now comes back as `application/octet-stream`. Give the file a `.json` extension if you need a more specific type. - **Malformed bare files are no longer silently swallowed.** In v3 a bare `INPUT.json` containing invalid JSON was treated as a missing record (`getValue` returned `undefined`). In v4 the raw bytes are returned verbatim and parsing happens in the `KeyValueStore` frontend, so a malformed value now surfaces a parse error at read time instead of looking absent. -- **Bare files are enumerated by `listKeys` under their actual on-disk name.** A bare `INPUT.json` (or `.txt`/`.bin`) shows up in `listKeys` as `INPUT.json` and reads back cleanly under that key via `getValue` / `recordExists` / `getPublicUrl`; the logical `INPUT` lookup keeps resolving the same file as well. An extensionless bare file is listed as `INPUT`. If both a tracked `INPUT` record and a bare `INPUT.json` exist, the tracked record wins and the bare variant is not listed. Everything `listKeys` needs is read from the filesystem index, so this no longer triggers the per-read O(n) directory scans the v3 fallback performed. +- **Bare files are enumerated by `listKeys` under their actual on-disk name.** A bare `INPUT.json` shows up in `listKeys` as `INPUT.json` and reads back cleanly under that key via `getValue` / `recordExists` / `getPublicUrl`; the logical `INPUT` lookup keeps resolving the same file as well. An extensionless bare file is listed as `INPUT`. If both a tracked `INPUT` record and a bare `INPUT.json` exist, the tracked record wins and the bare variant is not listed. Everything `listKeys` needs is read from the filesystem index, so this no longer triggers the per-read O(n) directory scans the v3 fallback performed. ## Only if you tuned autoscaling diff --git a/packages/fs-storage/src/file-system-storage.ts b/packages/fs-storage/src/file-system-storage.ts index 9baf57d1769e..274c16b992bd 100644 --- a/packages/fs-storage/src/file-system-storage.ts +++ b/packages/fs-storage/src/file-system-storage.ts @@ -66,8 +66,8 @@ export interface FileSystemStorageOptions { * * Like the conventional `INPUT`, this key may live in the default key-value store as a bare value * file with no metadata sidecar (e.g. the Apify CLI writes the effective input to `__CLI_INPUT.json` - * and points the run at that key). It is therefore readable out-of-band (``, `.json`, - * `.txt`, `.bin`) and preserved when the default store is purged, exactly like `INPUT`, + * and points the run at that key). It is therefore readable out-of-band (``, `.json`) and + * preserved when the default store is purged, exactly like `INPUT`, * which is always kept regardless of this setting. * * @default 'INPUT' diff --git a/packages/fs-storage/src/resource-clients/key-value-store.ts b/packages/fs-storage/src/resource-clients/key-value-store.ts index 5008d54c9362..148ddbd232e0 100644 --- a/packages/fs-storage/src/resource-clients/key-value-store.ts +++ b/packages/fs-storage/src/resource-clients/key-value-store.ts @@ -14,18 +14,16 @@ import { CachedIdClient } from './cached-id-client.js'; /** * Out-of-band ("bare") value-file fallbacks tried when a run-input lookup misses the tracked record, so a - * lookup for `INPUT` also matches a hand-placed `INPUT.json`/`.txt`/`.bin`. Passed to the native + * lookup for `INPUT` also matches a hand-placed `INPUT.json`. Passed to the native * `resolveValue`/`resolveExistingKey`, which do the probing and re-keying. * * Each entry declares the content type to report on a match — the native client does no MIME * inference. An empty `contentType` is its sentinel for "keep the synthesized - * `application/octet-stream`", used for the extensionless key and `.bin`. + * `application/octet-stream`", used for the extensionless key. */ const BARE_FILE_FALLBACKS: { extension: string; contentType: string }[] = [ { extension: '', contentType: '' }, { extension: '.json', contentType: 'application/json; charset=utf-8' }, - { extension: '.txt', contentType: 'text/plain; charset=utf-8' }, - { extension: '.bin', contentType: '' }, ]; /** The conventional run-input key, always treated as one alongside the configured `inputKey`. */ diff --git a/packages/fs-storage/test/fs-fallback.test.ts b/packages/fs-storage/test/fs-fallback.test.ts index 4406027f8949..956c3d45f39f 100644 --- a/packages/fs-storage/test/fs-fallback.test.ts +++ b/packages/fs-storage/test/fs-fallback.test.ts @@ -1,5 +1,5 @@ import { randomUUID } from 'node:crypto'; -import { mkdir, rm, writeFile } from 'node:fs/promises'; +import { mkdir, readdir, rm, writeFile } from 'node:fs/promises'; import { resolve } from 'node:path'; import { FileSystemStorageBackend } from '@crawlee/fs-storage'; @@ -206,19 +206,17 @@ describe('fallback to fs for reading', () => { }); // For each run-input bare file: the on-disk filename, the literal key that reads it directly, the -// content type the client reports (`.json`/`.txt` infer from the extension; the extensionless `INPUT` -// and `.bin` report the synthesized `application/octet-stream`), and a unique payload so a read can be -// proven to have returned *this* file and not a sibling. +// content type the client reports (`.json` infers from the extension; the extensionless `INPUT` +// reports the synthesized `application/octet-stream`), and a unique payload so a read can be proven to +// have returned *this* file and not a sibling. const BARE_VARIANTS = [ { file: 'INPUT', literalKey: 'INPUT', contentType: 'application/octet-stream' }, { file: 'INPUT.json', literalKey: 'INPUT.json', contentType: 'application/json; charset=utf-8' }, - { file: 'INPUT.txt', literalKey: 'INPUT.txt', contentType: 'text/plain; charset=utf-8' }, - { file: 'INPUT.bin', literalKey: 'INPUT.bin', contentType: 'application/octet-stream' }, ].map((variant) => ({ ...variant, payload: `payload of ${variant.file}` })); // Each run-input bare file must be reachable by exactly two keys — the logical `INPUT` (which probes -// the `['', '.json', '.txt', '.bin']` ladder, first match wins) and its own literal on-disk name — and -// NOT via a *different* extension's literal name (a bare `INPUT.txt` is not `INPUT.json`). Here each +// the `['', '.json']` ladder, first match wins) and its own literal on-disk name — and NOT via a +// *different* extension's literal name (a bare `INPUT` is not `INPUT.json`). Here each // variant lives in its own store so the logical-`INPUT` lookup resolves it unambiguously. describe('run-input bare-file reachability (one variant per store)', () => { const tmpLocation = resolve(import.meta.dirname, './tmp/fs-reachability-isolated'); @@ -271,7 +269,7 @@ describe('run-input bare-file reachability (one variant per store)', () => { }); }); -// The sharper cross-talk check: with *all four* variants in one store, each literal key must read back +// The sharper cross-talk check: with *both* variants in one store, each literal key must read back // its own bytes (never a sibling's), and the logical `INPUT` must resolve the first ladder match — the // extensionless `INPUT`. This is what fails if literal-name probing ever widens to other extensions. describe('run-input bare-file reachability (all variants in one store)', () => { @@ -314,3 +312,45 @@ describe('run-input bare-file reachability (all variants in one store)', () => { }); }); }); + +// v3 also probed `INPUT.txt` and `INPUT.bin`. The ladder is now `''`/`.json` only: those files are +// neither resolved by the logical `INPUT` nor by their literal name, are not listed, and are not exempt +// from the default-store purge. +describe('legacy INPUT.txt / INPUT.bin bare files', () => { + const tmpLocation = resolve(import.meta.dirname, './tmp/fs-legacy-input-variants'); + const legacyFiles = ['INPUT.txt', 'INPUT.bin']; + + afterEach(async () => { + await rm(tmpLocation, { force: true, recursive: true }); + }); + + test.each(legacyFiles)('a bare %s is not readable', async (file) => { + const storage = new FileSystemStorageBackend({ localDataDirectory: tmpLocation }); + const dir = resolve(storage.keyValueStoresDirectory, 'default'); + await mkdir(dir, { recursive: true }); + await writeFile(resolve(dir, file), `payload of ${file}`); + + const store = await storage.createKeyValueStoreBackend(); + + expect(await store.getValue('INPUT')).toBeUndefined(); + expect(await store.recordExists('INPUT')).toBe(false); + expect(await store.getValue(file)).toBeUndefined(); + expect(await store.recordExists(file)).toBe(false); + expect((await store.listKeys()).items).toEqual([]); + }); + + test('purge removes them from the default store', async () => { + const storage = new FileSystemStorageBackend({ localDataDirectory: tmpLocation }); + const dir = resolve(storage.keyValueStoresDirectory, 'default'); + await mkdir(dir, { recursive: true }); + await writeFile(resolve(dir, 'INPUT.json'), '{}'); + for (const file of legacyFiles) { + await writeFile(resolve(dir, file), `payload of ${file}`); + } + + await storage.purge(); + + const remaining = await readdir(dir); + expect(remaining.filter((file) => !file.startsWith('__metadata__'))).toEqual(['INPUT.json']); + }); +});