Skip to content

Commit cbd5d0d

Browse files
mpiannuccimpiannucci
committed
Fix store compat with wasm
1 parent 5f87b8d commit cbd5d0d

File tree

7 files changed

+39
-155
lines changed

7 files changed

+39
-155
lines changed

‎icechunk-js/icechunk.wasi-browser.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ const {
3636
type: 'module',
3737
})
3838

39+
3940
return worker
4041
},
4142
overwriteImports(importObject) {

‎icechunk-js/icechunk.wasi.cjs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ if (__nodeFs.existsSync(__wasmDebugFilePath)) {
3939
__wasmFilePath = __wasmDebugFilePath
4040
} else if (!__nodeFs.existsSync(__wasmFilePath)) {
4141
try {
42-
__wasmFilePath = __nodePath.resolve('@earthmover/icechunk-wasm32-wasi')
42+
__wasmFilePath = require.resolve('@earthmover/icechunk-wasm32-wasi/icechunk.wasm32-wasi.wasm')
4343
} catch {
4444
throw new Error('Cannot find icechunk.wasm32-wasi.wasm file, and @earthmover/icechunk-wasm32-wasi package is not installed.')
4545
}

‎icechunk-js/index.d.ts‎

Lines changed: 3 additions & 143 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,6 @@ export declare class Session {
5151
amend(message: string): Promise<string>
5252
flush(message: string): Promise<string>
5353
rebase(): Promise<void>
54-
/** Get all virtual chunk locations referenced by this session */
55-
allVirtualChunkLocations(): Promise<Array<string>>
5654
}
5755
export type JsSession = Session
5856

@@ -66,32 +64,19 @@ export declare class Storage {
6664
* especially useful for WASM builds where native Rust networking is unavailable.
6765
*/
6866
static newCustom(backend: { canWrite: () => Promise<boolean>; getObjectRange: (args: StorageGetObjectRangeArgs) => Promise<StorageGetObjectResponse>; putObject: (args: StoragePutObjectArgs) => Promise<StorageVersionedUpdateResult>; copyObject: (args: StorageCopyObjectArgs) => Promise<StorageVersionedUpdateResult>; listObjects: (prefix: string) => Promise<Array<StorageListInfo>>; deleteBatch: (args: StorageDeleteBatchArgs) => Promise<StorageDeleteObjectsResult>; getObjectLastModified: (path: string) => Promise<Date>; getObjectConditional: (args: StorageGetObjectConditionalArgs) => Promise<StorageGetModifiedResult> }): Storage
69-
static newLocalFilesystem(path: string): Promise<Storage>
70-
static newS3(bucket: string, prefix?: string | undefined | null, credentials?: S3Credentials | undefined | null, options?: S3Options | undefined | null): Storage
71-
static newR2(bucket?: string | undefined | null, prefix?: string | undefined | null, accountId?: string | undefined | null, credentials?: S3Credentials | undefined | null, options?: S3Options | undefined | null): Storage
72-
static newTigris(bucket: string, prefix?: string | undefined | null, credentials?: S3Credentials | undefined | null, options?: S3Options | undefined | null, useWeakConsistency?: boolean | undefined | null): Storage
73-
static newS3ObjectStore(bucket: string, prefix?: string | undefined | null, credentials?: S3Credentials | undefined | null, options?: S3Options | undefined | null): Promise<Storage>
74-
static newGcs(bucket: string, prefix?: string | undefined | null, credentials?: GcsCredentials | undefined | null, config?: Record<string, string> | undefined | null): Storage
75-
static newAzureBlob(account: string, container: string, prefix?: string | undefined | null, credentials?: AzureCredentials | undefined | null, config?: Record<string, string> | undefined | null): Promise<Storage>
76-
static newHttp(baseUrl: string, config?: Record<string, string> | undefined | null): Storage
77-
static newS3WithRefreshableCredentials(bucket: string, prefix: string | undefined | null, credentialsCallback: () => Promise<S3StaticCredentials>, initialCredentials?: S3StaticCredentials | undefined | null, options?: S3Options | undefined | null): Storage
78-
static newR2WithRefreshableCredentials(bucket: string | undefined | null, prefix: string | undefined | null, accountId: string | undefined | null, credentialsCallback: () => Promise<S3StaticCredentials>, initialCredentials?: S3StaticCredentials | undefined | null, options?: S3Options | undefined | null): Storage
79-
static newTigrisWithRefreshableCredentials(bucket: string, prefix: string | undefined | null, credentialsCallback: () => Promise<S3StaticCredentials>, initialCredentials?: S3StaticCredentials | undefined | null, options?: S3Options | undefined | null, useWeakConsistency?: boolean | undefined | null): Storage
80-
static newS3ObjectStoreWithRefreshableCredentials(bucket: string, prefix: string | undefined | null, credentialsCallback: () => Promise<S3StaticCredentials>, initialCredentials?: S3StaticCredentials | undefined | null, options?: S3Options | undefined | null): Promise<Storage>
81-
static newGcsWithRefreshableCredentials(bucket: string, prefix: string | undefined | null, credentialsCallback: () => Promise<GcsBearerCredential>, initialCredentials?: GcsBearerCredential | undefined | null, config?: Record<string, string> | undefined | null): Storage
8267
}
8368
export type JsStorage = Storage
8469

8570
export declare class Store {
86-
get(key: string): Promise<Buffer | null>
71+
get(key: string): Promise<Uint8Array | null>
8772
/**
8873
* Fetch a byte range from a key.
8974
*
9075
* Accepts zarrita's RangeQuery format:
9176
* { offset: number, length: number } - fetch length bytes starting at offset
9277
* { suffixLength: number } - fetch the last suffixLength bytes
9378
*/
94-
getRange(key: string, range: RangeQuery): Promise<Buffer | null>
79+
getRange(key: string, range: RangeQuery): Promise<Uint8Array | null>
9580
set(key: string, value: Buffer): Promise<void>
9681
exists(key: string): Promise<boolean>
9782
delete(key: string): Promise<void>
@@ -109,32 +94,9 @@ export declare class Store {
10994
deleteDir(prefix: string): Promise<void>
11095
getsize(key: string): Promise<number>
11196
getsizePrefix(prefix: string): Promise<number>
112-
/**
113-
* Set a single virtual reference to a chunk
114-
*
115-
* For checksum validation, provide either etag_checksum (string) or last_modified (JS Date object).
116-
* If both are provided, etag_checksum takes precedence.
117-
*/
118-
setVirtualRef(key: string, location: string, offset: number, length: number, etagChecksum: string | undefined | null, lastModified: Date | undefined | null, validateContainer: boolean): Promise<void>
119-
/**
120-
* Set multiple virtual references for the same array
121-
* Returns the indices of failed chunk references if any
122-
*/
123-
setVirtualRefs(arrayPath: string, chunks: Array<VirtualChunkSpec>, validateContainers: boolean): Promise<Array<Array<number>> | null>
12497
}
12598
export type JsStore = Store
12699

127-
/** Azure credentials */
128-
export type AzureCredentials =
129-
| { type: 'FromEnv' }
130-
| { type: 'Static', field0: AzureStaticCredentials }
131-
132-
/** Azure static credentials */
133-
export type AzureStaticCredentials =
134-
| { type: 'AccessKey', field0: string }
135-
| { type: 'SasToken', field0: string }
136-
| { type: 'BearerToken', field0: string }
137-
138100
/** Caching configuration */
139101
export interface CachingConfig {
140102
numSnapshotNodes?: number
@@ -155,12 +117,6 @@ export interface CompressionConfig {
155117
level?: number
156118
}
157119

158-
/** Credentials for virtual chunk access */
159-
export type Credentials =
160-
| { type: 'S3', field0: S3Credentials }
161-
| { type: 'Gcs', field0: GcsCredentials }
162-
| { type: 'Azure', field0: AzureCredentials }
163-
164120
export interface DiffOptions {
165121
fromBranch?: string
166122
fromTag?: string
@@ -189,25 +145,6 @@ export interface FeatureFlag {
189145
enabled: boolean
190146
}
191147

192-
/** GCS bearer credential with optional expiry */
193-
export interface GcsBearerCredential {
194-
bearer: string
195-
expiresAfter?: Date
196-
}
197-
198-
/** GCS credentials */
199-
export type GcsCredentials =
200-
| { type: 'Anonymous' }
201-
| { type: 'FromEnv' }
202-
| { type: 'Static', field0: GcsStaticCredentials }
203-
204-
/** GCS static credentials */
205-
export type GcsStaticCredentials =
206-
| { type: 'ServiceAccount', field0: string }
207-
| { type: 'ServiceAccountKey', field0: string }
208-
| { type: 'ApplicationCredentials', field0: string }
209-
| { type: 'BearerToken', field0: string }
210-
211148
export interface ManifestFileInfo {
212149
id: string
213150
sizeBytes: number
@@ -219,17 +156,6 @@ export interface MovedNode {
219156
to: string
220157
}
221158

222-
/** Object store configuration for virtual chunk containers */
223-
export type ObjectStoreConfig =
224-
| { type: 'InMemory' }
225-
| { type: 'LocalFileSystem', field0: string }
226-
| { type: 'Http', field0: Record<string, string> }
227-
| { type: 'S3Compatible', field0: S3Options }
228-
| { type: 'S3', field0: S3Options }
229-
| { type: 'Gcs', field0: Record<string, string> }
230-
| { type: 'Azure', field0: Record<string, string> }
231-
| { type: 'Tigris', field0: S3Options }
232-
233159
/**
234160
* Range query matching zarrita's RangeQuery type:
235161
* { offset: number, length: number } | { suffixLength: number }
@@ -246,28 +172,7 @@ export interface ReadonlySessionOptions {
246172
snapshotId?: string
247173
}
248174

249-
/**
250-
* Repository configuration
251-
*
252-
* The `manifest` field accepts a JSON object matching the serde serialization
253-
* of `ManifestConfig`. Example:
254-
* ```js
255-
* {
256-
* manifest: {
257-
* preload: {
258-
* max_total_refs: 1000,
259-
* preload_if: { true: null },
260-
* max_arrays_to_scan: 10
261-
* },
262-
* splitting: {
263-
* split_sizes: [
264-
* [{ path_matches: { regex: ".*" } }, [{ condition: "any", num_chunks: 100 }]]
265-
* ]
266-
* }
267-
* }
268-
* }
269-
* ```
270-
*/
175+
/** Repository configuration (WASM build — no virtual chunk support) */
271176
export interface RepositoryConfig {
272177
inlineChunkThresholdBytes?: number
273178
getPartialValuesConcurrency?: number
@@ -280,33 +185,6 @@ export interface RepositoryConfig {
280185
* The object is deserialized using serde, matching the Rust ManifestConfig structure.
281186
*/
282187
manifest?: any
283-
/** Virtual chunk containers configuration */
284-
virtualChunkContainers?: Record<string, VirtualChunkContainer>
285-
}
286-
287-
/** S3 credentials */
288-
export type S3Credentials =
289-
| { type: 'FromEnv' }
290-
| { type: 'Anonymous' }
291-
| { type: 'Static', field0: S3StaticCredentials }
292-
293-
/** S3 options */
294-
export interface S3Options {
295-
region?: string
296-
endpointUrl?: string
297-
allowHttp?: boolean
298-
anonymous?: boolean
299-
forcePathStyle?: boolean
300-
networkStreamTimeoutSeconds?: number
301-
requesterPays?: boolean
302-
}
303-
304-
/** S3 static credentials */
305-
export interface S3StaticCredentials {
306-
accessKeyId: string
307-
secretAccessKey: string
308-
sessionToken?: string
309-
expiresAfter?: Date
310188
}
311189

312190
export interface SnapshotInfo {
@@ -445,21 +323,3 @@ export interface VersionOptions {
445323
tag?: string
446324
snapshotId?: string
447325
}
448-
449-
/** Virtual chunk container configuration */
450-
export interface VirtualChunkContainer {
451-
name?: string
452-
urlPrefix: string
453-
store: JsObjectStoreConfig
454-
}
455-
456-
/** Specification for a virtual chunk reference */
457-
export interface VirtualChunkSpec {
458-
index: Array<number>
459-
location: string
460-
offset: number
461-
length: number
462-
etagChecksum?: string
463-
/** Last modified datetime (accepts JS Date object) */
464-
lastModified?: Date
465-
}

‎icechunk-js/index.js‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,6 @@
33
// @ts-nocheck
44
/* auto-generated by NAPI-RS */
55

6-
const { createRequire } = require('node:module')
7-
require = createRequire(__filename)
8-
96
const { readFileSync } = require('node:fs')
107
let nativeBinding = null
118
const loadErrors = []
@@ -108,7 +105,24 @@ function requireNative() {
108105
}
109106
} else if (process.platform === 'win32') {
110107
if (process.arch === 'x64') {
108+
if (process.config?.variables?.shlib_suffix === 'dll.a' || process.config?.variables?.node_target_type === 'shared_library') {
109+
try {
110+
return require('./icechunk.win32-x64-gnu.node')
111+
} catch (e) {
112+
loadErrors.push(e)
113+
}
111114
try {
115+
const binding = require('@earthmover/icechunk-win32-x64-gnu')
116+
const bindingPackageVersion = require('@earthmover/icechunk-win32-x64-gnu/package.json').version
117+
if (bindingPackageVersion !== '2.0.0-alpha.13' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
118+
throw new Error(`Native binding package version mismatch, expected 2.0.0-alpha.13 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
119+
}
120+
return binding
121+
} catch (e) {
122+
loadErrors.push(e)
123+
}
124+
} else {
125+
try {
112126
return require('./icechunk.win32-x64-msvc.node')
113127
} catch (e) {
114128
loadErrors.push(e)
@@ -123,6 +137,7 @@ function requireNative() {
123137
} catch (e) {
124138
loadErrors.push(e)
125139
}
140+
}
126141
} else if (process.arch === 'ia32') {
127142
try {
128143
return require('./icechunk.win32-ia32-msvc.node')
@@ -521,13 +536,17 @@ if (!nativeBinding || process.env.NAPI_RS_FORCE_WASI) {
521536
wasiBindingError = err
522537
}
523538
}
524-
if (!nativeBinding) {
539+
if (!nativeBinding || process.env.NAPI_RS_FORCE_WASI) {
525540
try {
526541
wasiBinding = require('@earthmover/icechunk-wasm32-wasi')
527542
nativeBinding = wasiBinding
528543
} catch (err) {
529544
if (process.env.NAPI_RS_FORCE_WASI) {
530-
wasiBindingError.cause = err
545+
if (!wasiBindingError) {
546+
wasiBindingError = err
547+
} else {
548+
wasiBindingError.cause = err
549+
}
531550
loadErrors.push(err)
532551
}
533552
}

‎icechunk-js/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@earthmover/icechunk",
3-
"version": "2.0.0-alpha.13",
3+
"version": "2.0.0-alpha.14",
44
"description": "JavaScript/TypeScript bindings for Icechunk",
55
"main": "index.js",
66
"repository": {

‎icechunk-js/src/store.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ use bytes::Bytes;
44
use futures::TryStreamExt;
55
use icechunk::format::ByteRange;
66
use icechunk::store::{Store, StoreErrorKind};
7-
use napi::bindgen_prelude::Buffer;
7+
use napi::bindgen_prelude::{Buffer, Uint8Array};
88
use napi_derive::napi;
99

1010
use crate::errors::IntoNapiResult;
@@ -88,10 +88,10 @@ fn normalize_key(key: &str) -> &str {
8888
#[napi]
8989
impl JsStore {
9090
#[napi]
91-
pub async fn get(&self, key: String) -> napi::Result<Option<Buffer>> {
91+
pub async fn get(&self, key: String) -> napi::Result<Option<Uint8Array>> {
9292
let key = normalize_key(&key);
9393
match self.0.get(key, &ByteRange::ALL).await {
94-
Ok(bytes) => Ok(Some(bytes.to_vec().into())),
94+
Ok(bytes) => Ok(Some(Uint8Array::from(bytes.to_vec()))),
9595
Err(e) if matches!(e.kind, StoreErrorKind::NotFound(_)) => Ok(None),
9696
Err(e) => Err(napi::Error::from_reason(e.to_string())),
9797
}
@@ -107,7 +107,7 @@ impl JsStore {
107107
&self,
108108
key: String,
109109
range: JsRangeQuery,
110-
) -> napi::Result<Option<Buffer>> {
110+
) -> napi::Result<Option<Uint8Array>> {
111111
let byte_range = if let Some(suffix) = range.suffix_length {
112112
ByteRange::Last(suffix as u64)
113113
} else {
@@ -119,7 +119,7 @@ impl JsStore {
119119
};
120120
let key = normalize_key(&key);
121121
match self.0.get(key, &byte_range).await {
122-
Ok(bytes) => Ok(Some(bytes.to_vec().into())),
122+
Ok(bytes) => Ok(Some(Uint8Array::from(bytes.to_vec()))),
123123
Err(e) if matches!(e.kind, StoreErrorKind::NotFound(_)) => Ok(None),
124124
Err(e) => Err(napi::Error::from_reason(e.to_string())),
125125
}

‎icechunk-js/wasi-worker-browser.mjs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
import { instantiateNapiModuleSync, MessageHandler, WASI } from '@napi-rs/wasm-runtime'
22

3+
const errorOutputs = []
4+
35
const handler = new MessageHandler({
46
onLoad({ wasmModule, wasmMemory }) {
57
const wasi = new WASI({
@@ -10,6 +12,7 @@ const handler = new MessageHandler({
1012
printErr: function() {
1113
// eslint-disable-next-line no-console
1214
console.error.apply(console, arguments)
15+
1316
},
1417
})
1518
return instantiateNapiModuleSync(wasmModule, {
@@ -25,6 +28,7 @@ const handler = new MessageHandler({
2528
},
2629
})
2730
},
31+
2832
})
2933

3034
globalThis.onmessage = function (e) {

0 commit comments

Comments
 (0)