Skip to content

Commit 46fb6d3

Browse files
authored
Fix BigInt cache corruption in consolidated v3 (#413)
Fixes #410 `loadConsolidatedV3` converted int64/uint64 `fill_value` to `BigInt` in place on cache entries, so the next `store.get` failed re-serializing them through `JSON.stringify`. `_openV3` already runs `ensureCorrectScalar` on its own fresh copy, so the pre-conversion was dead code.
1 parent 7047960 commit 46fb6d3

File tree

2 files changed

+55
-12
lines changed

2 files changed

+55
-12
lines changed

packages/zarrita/__tests__/consolidated.test.ts

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,60 @@ describe("withConsolidatedMetadata (v3)", () => {
178178
withConsolidatedMetadata(new FileSystemStore(root), { format: "v3" });
179179
await expect(try_open).rejects.toThrowError(NotFoundError);
180180
});
181+
182+
// Regression for https://github.com/manzt/zarrita.js/issues/410: opening an
183+
// int64/uint64 array from a consolidated store used to poison the cache
184+
// with a BigInt fill_value, so the next `store.get` would throw
185+
// "Do not know how to serialize a BigInt" from `JSON.stringify`.
186+
it("opens int64/uint64 arrays without poisoning the cache", async () => {
187+
let baseMeta = {
188+
node_type: "array",
189+
zarr_format: 3,
190+
shape: [4],
191+
chunk_grid: {
192+
name: "regular",
193+
configuration: { chunk_shape: [4] },
194+
},
195+
chunk_key_encoding: {
196+
name: "default",
197+
configuration: { separator: "/" },
198+
},
199+
codecs: [{ name: "bytes", configuration: { endian: "little" } }],
200+
attributes: {},
201+
};
202+
let rootMeta = {
203+
zarr_format: 3,
204+
node_type: "group",
205+
attributes: {},
206+
consolidated_metadata: {
207+
kind: "inline",
208+
must_understand: false,
209+
metadata: {
210+
time: { ...baseMeta, data_type: "int64", fill_value: 0 },
211+
band: { ...baseMeta, data_type: "int16", fill_value: 0 },
212+
},
213+
},
214+
};
215+
let bytes = new TextEncoder().encode(JSON.stringify(rootMeta));
216+
let backing = {
217+
async get(key: string) {
218+
return key === "/zarr.json" ? bytes : undefined;
219+
},
220+
};
221+
let store = await withConsolidatedMetadata(backing, { format: "v3" });
222+
let grp = await open(store, { kind: "group" });
223+
let time = await open(grp.resolve("time"), { kind: "array" });
224+
let band = await open(grp.resolve("band"), { kind: "array" });
225+
expect(time.dtype).toBe("int64");
226+
expect(band.dtype).toBe("int16");
227+
// `ensureCorrectScalar` still runs on the freshly-decoded copy inside
228+
// `_openV3`, so int64 fill_values are converted to BigInt.
229+
expect(typeof time.fillValue).toBe("bigint");
230+
// Re-opening the same array must also work — proves the cache entry
231+
// stayed JSON-serializable after the first open.
232+
let timeAgain = await open(grp.resolve("time"), { kind: "array" });
233+
expect(timeAgain.dtype).toBe("int64");
234+
});
181235
});
182236

183237
describe("withConsolidatedMetadata (format array)", () => {

packages/zarrita/src/extension/consolidation.ts

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -9,17 +9,11 @@ import type {
99
ArrayMetadata,
1010
ArrayMetadataV2,
1111
Attributes,
12-
DataType,
1312
GroupMetadata,
1413
GroupMetadataV2,
1514
} from "../metadata.js";
1615
import { VERSION_COUNTER } from "../open.js";
17-
import {
18-
ensureCorrectScalar,
19-
jsonDecodeObject,
20-
jsonEncodeObject,
21-
rethrowUnless,
22-
} from "../util.js";
16+
import { jsonDecodeObject, jsonEncodeObject, rethrowUnless } from "../util.js";
2317
import { defineStoreExtension } from "./define.js";
2418

2519
type ConsolidatedMetadataV2 = {
@@ -161,11 +155,6 @@ async function loadConsolidatedV3(
161155
)) {
162156
let normalized = path.startsWith("/") ? path : `/${path}`;
163157
let key = `${normalized}/zarr.json` as AbsolutePath;
164-
if (meta.node_type === "array") {
165-
(meta as ArrayMetadata<DataType>).fill_value = ensureCorrectScalar(
166-
meta as ArrayMetadata<DataType>,
167-
);
168-
}
169158
knownMeta[key] = meta;
170159
}
171160
return knownMeta;

0 commit comments

Comments
 (0)