Skip to content

Commit 918f6f7

Browse files
texodustexodus
committed
Allow bytes and string for JSON/CSV input data with format option
Signed-off-by: Andrew Stein <steinlink@gmail.com>
1 parent 0aa19d5 commit 918f6f7

11 files changed

Lines changed: 317 additions & 49 deletions

File tree

‎rust/perspective-client/docs/client/table.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,10 @@ is stored and all calculation occurs.
3333
data.
3434
- `name` - The name of the table. This will be generated if it is not
3535
provided.
36+
- `format` - The explicit format of the input data, can be one of
37+
`"json"`, `"columns"`, `"csv"` or `"arrow"`. This overrides
38+
language-specific type dispatch behavior, which allows stringified and
39+
byte array alternative inputs.
3640

3741
<div class="javascript">
3842

@@ -51,6 +55,16 @@ import * as fs from "node:fs/promises";
5155
const table2 = await client.table(await fs.readFile("superstore.arrow"));
5256
```
5357

58+
Load a CSV from a `UInt8Array` (the default for this type is Arrow) using a
59+
format override:
60+
61+
```javascript
62+
const enc = new TextEncoder();
63+
const table = await client.table(enc.encode("x,y\n1,2\n3,4"), {
64+
format: "csv",
65+
});
66+
```
67+
5468
Create a table with an `index`:
5569

5670
```javascript

‎rust/perspective-client/src/rust/lib.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,9 @@ pub mod utils;
2828

2929
pub use crate::client::{Client, ClientHandler, Features, SystemInfo};
3030
pub use crate::session::{ProxySession, Session};
31-
pub use crate::table::{Schema, Table, TableInitOptions, UpdateOptions, ValidateExpressionsData};
31+
pub use crate::table::{
32+
Schema, Table, TableInitOptions, TableReadFormat, UpdateOptions, ValidateExpressionsData,
33+
};
3234
pub use crate::table_data::{TableData, UpdateData};
3335
pub use crate::view::{OnUpdateMode, OnUpdateOptions, View, ViewWindow};
3436

‎rust/perspective-client/src/rust/table.rs‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,34 @@ use crate::view::View;
3131

3232
pub type Schema = HashMap<String, ColumnType>;
3333

34+
#[derive(Clone, Copy, Debug, Serialize, Deserialize, TS)]
35+
pub enum TableReadFormat {
36+
#[serde(rename = "csv")]
37+
Csv,
38+
39+
#[serde(rename = "json")]
40+
JsonString,
41+
42+
#[serde(rename = "columns")]
43+
ColumnsString,
44+
45+
#[serde(rename = "arrow")]
46+
Arrow,
47+
}
48+
49+
impl TableReadFormat {
50+
pub fn parse(value: Option<String>) -> Result<Option<Self>, String> {
51+
Ok(match value.as_deref() {
52+
Some("csv") => Some(TableReadFormat::Csv),
53+
Some("json") => Some(TableReadFormat::JsonString),
54+
Some("columns") => Some(TableReadFormat::ColumnsString),
55+
Some("arrow") => Some(TableReadFormat::Arrow),
56+
None => None,
57+
Some(x) => return Err(format!("Unknown format \"{}\"", x)),
58+
})
59+
}
60+
}
61+
3462
/// Options which impact the behavior of [`Client::table`], as well as
3563
/// subsequent calls to [`Table::update`].
3664
#[derive(Clone, Debug, Default, Serialize, Deserialize, TS)]
@@ -39,6 +67,10 @@ pub struct TableInitOptions {
3967
#[ts(optional)]
4068
pub name: Option<String>,
4169

70+
#[serde(default)]
71+
#[ts(optional)]
72+
pub format: Option<TableReadFormat>,
73+
4274
/// This [`Table`] should use the column named by the `index` parameter as
4375
/// the `index`, which causes [`Table::update`] and [`Client::table`] input
4476
/// to either insert or update existing rows based on `index` column
@@ -101,6 +133,7 @@ impl From<TableInitOptions> for TableOptions {
101133
#[derive(Clone, Debug, Default, Deserialize, Serialize, TS)]
102134
pub struct UpdateOptions {
103135
pub port_id: Option<u32>,
136+
pub format: Option<TableReadFormat>,
104137
}
105138

106139
#[derive(Clone, Debug, Serialize, Deserialize)]

‎rust/perspective-js/src/rust/client.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,11 +76,11 @@ impl Client {
7676
value: &JsTableInitData,
7777
options: Option<JsTableInitOptions>,
7878
) -> ApiResult<Table> {
79-
let args = TableData::from_js_value(value)?;
8079
let options = options
8180
.into_serde_ext::<Option<TableInitOptions>>()?
8281
.unwrap_or_default();
8382

83+
let args = TableData::from_js_value(value, options.format)?;
8484
Ok(Table(self.client.table(args, options).await?))
8585
}
8686

‎rust/perspective-js/src/rust/table.rs‎

Lines changed: 60 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ use extend::ext;
1414
use js_sys::{Array, ArrayBuffer, Function, Object, Reflect, Uint8Array, JSON};
1515
use perspective_client::config::*;
1616
use perspective_client::proto::*;
17-
use perspective_client::{assert_table_api, TableData, UpdateData, UpdateOptions};
17+
use perspective_client::{assert_table_api, TableData, TableReadFormat, UpdateData, UpdateOptions};
1818
use wasm_bindgen::convert::TryFromJsValue;
1919
use wasm_bindgen::prelude::*;
2020
use wasm_bindgen_futures::spawn_local;
@@ -45,9 +45,9 @@ impl Vec<(String, ColumnType)> {
4545

4646
#[ext]
4747
pub(crate) impl TableData {
48-
fn from_js_value(value: &JsValue) -> ApiResult<TableData> {
48+
fn from_js_value(value: &JsValue, format: Option<TableReadFormat>) -> ApiResult<TableData> {
4949
let err_fn = || JsValue::from(format!("Failed to construct Table {:?}", value));
50-
if let Some(result) = UpdateData::from_js_value_partial(value)? {
50+
if let Some(result) = UpdateData::from_js_value_partial(value, format)? {
5151
Ok(result.into())
5252
} else if value.is_instance_of::<Object>() && Reflect::has(value, &"__get_model".into())? {
5353
let val = Reflect::get(value, &"__get_model".into())?
@@ -87,19 +87,53 @@ pub(crate) impl TableData {
8787

8888
#[ext]
8989
pub(crate) impl UpdateData {
90-
fn from_js_value_partial(value: &JsValue) -> ApiResult<Option<UpdateData>> {
90+
fn from_js_value_partial(
91+
value: &JsValue,
92+
format: Option<TableReadFormat>,
93+
) -> ApiResult<Option<UpdateData>> {
9194
let err_fn = || JsValue::from(format!("Failed to construct Table {:?}", value));
9295
if value.is_undefined() {
9396
Err(err_fn().into())
9497
} else if value.is_string() {
95-
Ok(Some(UpdateData::Csv(value.as_string().into_apierror()?)))
98+
match format {
99+
None | Some(TableReadFormat::Csv) => {
100+
Ok(Some(UpdateData::Csv(value.as_string().into_apierror()?)))
101+
},
102+
Some(TableReadFormat::JsonString) => Ok(Some(UpdateData::JsonRows(
103+
value.as_string().into_apierror()?,
104+
))),
105+
Some(TableReadFormat::ColumnsString) => Ok(Some(UpdateData::JsonColumns(
106+
value.as_string().into_apierror()?,
107+
))),
108+
Some(TableReadFormat::Arrow) => Ok(Some(UpdateData::Arrow(
109+
value.as_string().into_apierror()?.into_bytes().into(),
110+
))),
111+
}
96112
} else if value.is_instance_of::<ArrayBuffer>() {
97113
let uint8array = Uint8Array::new(value);
98114
let slice = uint8array.to_vec();
99-
Ok(Some(UpdateData::Arrow(slice.into())))
115+
match format {
116+
Some(TableReadFormat::Csv) => Ok(Some(UpdateData::Csv(String::from_utf8(slice)?))),
117+
Some(TableReadFormat::JsonString) => {
118+
Ok(Some(UpdateData::JsonRows(String::from_utf8(slice)?)))
119+
},
120+
Some(TableReadFormat::ColumnsString) => {
121+
Ok(Some(UpdateData::JsonColumns(String::from_utf8(slice)?)))
122+
},
123+
None | Some(TableReadFormat::Arrow) => Ok(Some(UpdateData::Arrow(slice.into()))),
124+
}
100125
} else if let Some(uint8array) = value.dyn_ref::<Uint8Array>() {
101126
let slice = uint8array.to_vec();
102-
Ok(Some(UpdateData::Arrow(slice.into())))
127+
match format {
128+
Some(TableReadFormat::Csv) => Ok(Some(UpdateData::Csv(String::from_utf8(slice)?))),
129+
Some(TableReadFormat::JsonString) => {
130+
Ok(Some(UpdateData::JsonRows(String::from_utf8(slice)?)))
131+
},
132+
Some(TableReadFormat::ColumnsString) => {
133+
Ok(Some(UpdateData::JsonColumns(String::from_utf8(slice)?)))
134+
},
135+
None | Some(TableReadFormat::Arrow) => Ok(Some(UpdateData::Arrow(slice.into()))),
136+
}
103137
} else if value.is_instance_of::<Array>() {
104138
let rows = JSON::stringify(value)?.as_string().into_apierror()?;
105139
Ok(Some(UpdateData::JsonRows(rows)))
@@ -108,8 +142,8 @@ pub(crate) impl UpdateData {
108142
}
109143
}
110144

111-
fn from_js_value(value: &JsValue) -> ApiResult<UpdateData> {
112-
match TableData::from_js_value(value)? {
145+
fn from_js_value(value: &JsValue, format: Option<TableReadFormat>) -> ApiResult<UpdateData> {
146+
match TableData::from_js_value(value, format)? {
113147
TableData::Schema(_) => Err(ApiError::new(
114148
"Method cannot be called with `Schema` argument",
115149
)),
@@ -237,16 +271,28 @@ impl Table {
237271

238272
#[doc = inherit_docs!("table/replace.md")]
239273
#[wasm_bindgen]
240-
pub async fn remove(&self, value: &JsValue) -> ApiResult<()> {
241-
let input = UpdateData::from_js_value(value)?;
274+
pub async fn remove(&self, value: &JsValue, options: Option<JsUpdateOptions>) -> ApiResult<()> {
275+
let options = options
276+
.into_serde_ext::<Option<UpdateOptions>>()?
277+
.unwrap_or_default();
278+
279+
let input = UpdateData::from_js_value(value, options.format)?;
242280
self.0.remove(input).await?;
243281
Ok(())
244282
}
245283

246284
#[doc = inherit_docs!("table/replace.md")]
247285
#[wasm_bindgen]
248-
pub async fn replace(&self, input: &JsValue) -> ApiResult<()> {
249-
let input = UpdateData::from_js_value(input)?;
286+
pub async fn replace(
287+
&self,
288+
input: &JsValue,
289+
options: Option<JsUpdateOptions>,
290+
) -> ApiResult<()> {
291+
let options = options
292+
.into_serde_ext::<Option<UpdateOptions>>()?
293+
.unwrap_or_default();
294+
295+
let input = UpdateData::from_js_value(input, options.format)?;
250296
self.0.replace(input).await?;
251297
Ok(())
252298
}
@@ -258,11 +304,11 @@ impl Table {
258304
input: &JsTableInitData,
259305
options: Option<JsUpdateOptions>,
260306
) -> ApiResult<()> {
261-
let input = UpdateData::from_js_value(input)?;
262307
let options = options
263308
.into_serde_ext::<Option<UpdateOptions>>()?
264309
.unwrap_or_default();
265310

311+
let input = UpdateData::from_js_value(input, options.format)?;
266312
self.0.update(input, options).await?;
267313
Ok(())
268314
}

‎rust/perspective-js/src/rust/utils/errors.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
1212

1313
use std::fmt::Display;
14+
use std::string::FromUtf8Error;
1415

1516
use perspective_client::ClientError;
1617
use wasm_bindgen::prelude::*;
@@ -84,7 +85,8 @@ define_api_error!(
8485
futures::channel::oneshot::Canceled,
8586
base64::DecodeError,
8687
chrono::ParseError,
87-
prost::DecodeError
88+
prost::DecodeError,
89+
FromUtf8Error
8890
);
8991

9092
#[wasm_bindgen(inline_js = r#"

‎rust/perspective-js/test/js/constructors/csv.spec.ts‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
2+
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
3+
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
4+
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
5+
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
6+
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
7+
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
8+
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
9+
// ┃ This file is part of the Perspective library, distributed under the terms ┃
10+
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
11+
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
12+
13+
import { test, expect } from "@finos/perspective-test";
14+
import perspective from "../perspective_client";
15+
16+
const CSV = "x,y,z\n1,2,3\n4,5,6";
17+
18+
test.describe("CSV Constructors", function () {
19+
test("Handles String format", async function () {
20+
const table = await perspective.table(CSV);
21+
const v = await table.view();
22+
const json = await v.to_json();
23+
expect(json).toEqual([
24+
{ x: 1, y: 2, z: 3 },
25+
{ x: 4, y: 5, z: 6 },
26+
]);
27+
});
28+
29+
test("Handles String format with explicit format option", async function () {
30+
const table = await perspective.table(CSV, {
31+
format: "csv",
32+
});
33+
const v = await table.view();
34+
const json = await v.to_json();
35+
expect(json).toEqual([
36+
{ x: 1, y: 2, z: 3 },
37+
{ x: 4, y: 5, z: 6 },
38+
]);
39+
});
40+
41+
test("Handles ArrayBuffer format", async function () {
42+
var enc = new TextEncoder();
43+
const table = await perspective.table(enc.encode(CSV).buffer, {
44+
format: "csv",
45+
});
46+
const v = await table.view();
47+
const json = await v.to_json();
48+
expect(json).toEqual([
49+
{ x: 1, y: 2, z: 3 },
50+
{ x: 4, y: 5, z: 6 },
51+
]);
52+
});
53+
54+
test("Handles UInt8Arry format", async function () {
55+
var enc = new TextEncoder();
56+
const table = await perspective.table(enc.encode(CSV), {
57+
format: "csv",
58+
});
59+
const v = await table.view();
60+
const json = await v.to_json();
61+
expect(json).toEqual([
62+
{ x: 1, y: 2, z: 3 },
63+
{ x: 4, y: 5, z: 6 },
64+
]);
65+
});
66+
});

‎rust/perspective-js/test/js/constructors/json.spec.ts‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
import { test, expect } from "@finos/perspective-test";
1414
import perspective from "../perspective_client";
1515

16+
const TEST_JSON = { x: [1, 4], y: [2, 5], z: [3, 6] };
17+
1618
test.describe("JSON", function () {
1719
test.describe("Integer columns", function () {
1820
test("Integer columns can be updated with all JSON numeric types and cousins", async function () {
@@ -41,4 +43,48 @@ test.describe("JSON", function () {
4143
]);
4244
});
4345
});
46+
47+
test("Handles String format", async function () {
48+
const table = await perspective.table(JSON.stringify(TEST_JSON), {
49+
format: "columns",
50+
});
51+
const v = await table.view();
52+
const json = await v.to_json();
53+
expect(json).toEqual([
54+
{ x: 1, y: 2, z: 3 },
55+
{ x: 4, y: 5, z: 6 },
56+
]);
57+
});
58+
59+
test("Handles UInt8Array format", async function () {
60+
const enc = new TextEncoder();
61+
const table = await perspective.table(
62+
enc.encode(JSON.stringify(TEST_JSON)),
63+
{
64+
format: "columns",
65+
}
66+
);
67+
const v = await table.view();
68+
const json = await v.to_json();
69+
expect(json).toEqual([
70+
{ x: 1, y: 2, z: 3 },
71+
{ x: 4, y: 5, z: 6 },
72+
]);
73+
});
74+
75+
test("Handles ArrayBuffer format", async function () {
76+
const enc = new TextEncoder();
77+
const table = await perspective.table(
78+
enc.encode(JSON.stringify(TEST_JSON)).buffer,
79+
{
80+
format: "columns",
81+
}
82+
);
83+
const v = await table.view();
84+
const json = await v.to_json();
85+
expect(json).toEqual([
86+
{ x: 1, y: 2, z: 3 },
87+
{ x: 4, y: 5, z: 6 },
88+
]);
89+
});
4490
});

0 commit comments

Comments
 (0)