diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..b9691ef --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "tests/arrow-wasm"] + path = tests/arrow-wasm + url = https://github.com/kylebarron/arrow-wasm diff --git a/package.json b/package.json index 44c5928..2b983eb 100644 --- a/package.json +++ b/package.json @@ -34,13 +34,19 @@ "devDependencies": { "@rollup/plugin-terser": "^0.4.3", "@rollup/plugin-typescript": "^11.1.2", + "@types/node": "^20.6.1", "@types/tape": "^4.13.2", "apache-arrow": "^13", + "arrow-wasm-arrow1": "./tests/arrow-wasm/pkg_arrow1/", + "arrow-wasm-arrow2": "./tests/arrow-wasm/pkg_arrow2/", "rollup": "^3.26.2", "rollup-plugin-dts": "^5.3.0", "rust-arrow-ffi": "./tests/rust-arrow-ffi/pkg/", "ts-node": "^10.9.1", "typescript": "^4.7.4", "vitest": "^0.33.0" + }, + "volta": { + "node": "20.6.1" } } diff --git a/tests/arrow-wasm b/tests/arrow-wasm new file mode 160000 index 0000000..970861a --- /dev/null +++ b/tests/arrow-wasm @@ -0,0 +1 @@ +Subproject commit 970861a5d817e1b8f202844134aa747347ec7dbe diff --git a/tests/ffi.test.ts b/tests/ffi.test.ts index a5818f4..40e3084 100644 --- a/tests/ffi.test.ts +++ b/tests/ffi.test.ts @@ -2,17 +2,18 @@ import { readFileSync } from "fs"; import { describe, it, expect } from "vitest"; import * as arrow from "apache-arrow"; import * as wasm from "rust-arrow-ffi"; +import * as arrowWasm from "arrow-wasm-arrow2"; import { arrowTableToFFI, arraysEqual, loadIPCTableFromDisk } from "./utils"; import { parseField, parseVector } from "../src"; import { Type } from "../src/types"; -wasm.setPanicHook(); +arrowWasm.setPanicHook(); -// @ts-expect-error -const WASM_MEMORY: WebAssembly.Memory = wasm.__wasm.memory; +const WASM_MEMORY: WebAssembly.Memory = arrowWasm.wasmMemory() const TEST_TABLE = loadIPCTableFromDisk("tests/table.arrow"); -const FFI_TABLE = arrowTableToFFI(TEST_TABLE); +const WASM_TABLE = arrowWasm.Table.fromIPCStream(arrow.tableToIPC(TEST_TABLE, "stream")) +const FFI_TABLE = WASM_TABLE.toFFI(); interface FixtureType { data: any; @@ -240,12 +241,12 @@ describe("large_binary", (t) => { // Then read the large table const tableBuffer = readFileSync("tests/large_table.arrow"); - let ffiTable = wasm.arrowIPCToFFI(tableBuffer); + let ffiTable = arrowWasm.Table.fromIPCStream(tableBuffer).intoFFI(); // This is hard-coded based on the order in pyarrow_generate_data.py let columnIndex = 0; - const fieldPtr = ffiTable.schemaAddr(columnIndex); + const fieldPtr = ffiTable.schemaAddr(); const field = parseField(WASM_MEMORY.buffer, fieldPtr); expect(field.name).toStrictEqual("large_binary"); diff --git a/tests/record-batch.test.ts b/tests/record-batch.test.ts index 40ae0ed..e165891 100644 --- a/tests/record-batch.test.ts +++ b/tests/record-batch.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from "vitest"; -import * as wasm from "rust-arrow-ffi"; +import * as arrowWasm from "arrow-wasm-arrow2"; import { arraysEqual, loadIPCTableFromDisk, @@ -7,11 +7,11 @@ import { } from "./utils"; import { parseRecordBatch } from "../src"; import { readFileSync } from "fs"; +import assert from "assert"; -wasm.setPanicHook(); +arrowWasm.setPanicHook(); -// @ts-expect-error -const WASM_MEMORY: WebAssembly.Memory = wasm.__wasm.memory; +const WASM_MEMORY: WebAssembly.Memory = arrowWasm.wasmMemory(); const TEST_TABLE = loadIPCTableFromDisk("tests/table.arrow"); const FFI_RECORD_BATCH = arrowTableToFFIRecordBatch(TEST_TABLE); @@ -20,8 +20,8 @@ describe("record batch", (t) => { function test(copy: boolean) { const newRecordBatch = parseRecordBatch( WASM_MEMORY.buffer, - FFI_RECORD_BATCH.array_addr(), - FFI_RECORD_BATCH.field_addr(), + FFI_RECORD_BATCH.arrayAddr(), + FFI_RECORD_BATCH.schemaAddr(), copy ); @@ -53,12 +53,16 @@ describe("record batch with large types", (t) => { function test(copy: boolean) { const tableBuffer = readFileSync("tests/large_table.arrow"); - let ffiRecordBatch = wasm.arrowIPCToFFIRecordBatch(tableBuffer); + const wasmTable = arrowWasm.Table.fromIPCFile(tableBuffer); + assert(wasmTable.numBatches === 1, "Should have one batch"); + const wasmRecordBatch = wasmTable.recordBatch(0); + assert(wasmRecordBatch, "wasm record batch should not be undefined"); + let ffiRecordBatch = wasmRecordBatch.intoFFI(); const newRecordBatch = parseRecordBatch( WASM_MEMORY.buffer, - ffiRecordBatch.array_addr(), - ffiRecordBatch.field_addr(), + ffiRecordBatch.arrayAddr(), + ffiRecordBatch.schemaAddr(), copy ); diff --git a/tests/utils.ts b/tests/utils.ts index 5037628..de0288b 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -1,6 +1,6 @@ import { readFileSync } from "fs"; import * as arrow from "apache-arrow"; -import * as wasm from "rust-arrow-ffi"; +import * as arrowWasm from "arrow-wasm-arrow2"; export function loadIPCTableFromDisk(path: string): arrow.Table { const buffer = readFileSync(path); @@ -8,15 +8,27 @@ export function loadIPCTableFromDisk(path: string): arrow.Table { } /** Put an Arrow Table in Wasm memory and expose it via FFI */ -export function arrowTableToFFI(table: arrow.Table): wasm.FFIArrowTable { - return wasm.arrowIPCToFFI(arrow.tableToIPC(table, "file")); +export function arrowTableToFFI(table: arrow.Table): arrowWasm.FFITable { + return arrowWasm.Table.fromIPCStream( + arrow.tableToIPC(table, "stream") + ).intoFFI(); } /** Put an Arrow Table in Wasm memory and expose it via an FFI RecordBatch */ export function arrowTableToFFIRecordBatch( table: arrow.Table -): wasm.FFIArrowRecordBatch { - return wasm.arrowIPCToFFIRecordBatch(arrow.tableToIPC(table, "file")); +): arrowWasm.FFIRecordBatch { + const wasmTable = arrowWasm.Table.fromIPCStream( + arrow.tableToIPC(table, "stream") + ); + if (wasmTable.numBatches !== 1) { + throw new Error(`expected only one batch, got ${wasmTable.numBatches}`); + } + const wasmRecordBatch = wasmTable.recordBatch(0); + if (!wasmRecordBatch) { + throw new Error("wasm record batch is undefined"); + } + return wasmRecordBatch.intoFFI(); } export function arraysEqual( diff --git a/yarn.lock b/yarn.lock index dfc3688..f323e43 100644 --- a/yarn.lock +++ b/yarn.lock @@ -464,6 +464,13 @@ __metadata: languageName: node linkType: hard +"@types/node@npm:^20.6.1": + version: 20.6.2 + resolution: "@types/node@npm:20.6.2" + checksum: 96fe5303872640a173f3fd43e289a451776ed5b8f0090094447c6790b43f23fb607eea8268af0829cef4d132e5afa0bfa4cd871aa7412e9042a414a698e9e971 + languageName: node + linkType: hard + "@types/pad-left@npm:2.1.1": version: 2.1.1 resolution: "@types/pad-left@npm:2.1.1" @@ -715,8 +722,11 @@ __metadata: dependencies: "@rollup/plugin-terser": ^0.4.3 "@rollup/plugin-typescript": ^11.1.2 + "@types/node": ^20.6.1 "@types/tape": ^4.13.2 apache-arrow: ^13 + arrow-wasm-arrow1: ./tests/arrow-wasm/pkg_arrow1/ + arrow-wasm-arrow2: ./tests/arrow-wasm/pkg_arrow2/ rollup: ^3.26.2 rollup-plugin-dts: ^5.3.0 rust-arrow-ffi: ./tests/rust-arrow-ffi/pkg/ @@ -728,6 +738,20 @@ __metadata: languageName: unknown linkType: soft +"arrow-wasm-arrow1@file:./tests/arrow-wasm/pkg_arrow1/::locator=arrow-js-ffi%40workspace%3A.": + version: 0.1.0 + resolution: "arrow-wasm-arrow1@file:./tests/arrow-wasm/pkg_arrow1/#./tests/arrow-wasm/pkg_arrow1/::hash=98dbb1&locator=arrow-js-ffi%40workspace%3A." + checksum: 012bbb0849532fb9aa42889c058850692c5ff9fd512aa9c50f6999f8ff341b2f0b826c6a70a47c799b9e3a39db8dddb2275455ec00dfc1657b678dba94ff8744 + languageName: node + linkType: hard + +"arrow-wasm-arrow2@file:./tests/arrow-wasm/pkg_arrow2/::locator=arrow-js-ffi%40workspace%3A.": + version: 0.1.0 + resolution: "arrow-wasm-arrow2@file:./tests/arrow-wasm/pkg_arrow2/#./tests/arrow-wasm/pkg_arrow2/::hash=94e2c0&locator=arrow-js-ffi%40workspace%3A." + checksum: ef606ea606fc68373f9d023d473b16c7612bb4af03a005dec2e27d6b0a6ed2dc22b9f993fce63b0e4a1120468dc8720b4582809520857aa6eeebb38e75a44252 + languageName: node + linkType: hard + "assertion-error@npm:^1.1.0": version: 1.1.0 resolution: "assertion-error@npm:1.1.0"