Skip to content

Commit b17c28d

Browse files
NickCrewscpcloud
authored andcommitted
feat(duckdb): add to_json() to Table and duckdb backend
Fixes #10413
1 parent c42945a commit b17c28d

File tree

4 files changed

+120
-1
lines changed

4 files changed

+120
-1
lines changed

ibis/backends/__init__.py

+26
Original file line numberDiff line numberDiff line change
@@ -586,6 +586,32 @@ def to_delta(
586586
with expr.to_pyarrow_batches(params=params) as batch_reader:
587587
write_deltalake(path, batch_reader, **kwargs)
588588

589+
@util.experimental
590+
def to_json(
591+
self,
592+
expr: ir.Table,
593+
path: str | Path,
594+
**kwargs: Any,
595+
) -> None:
596+
"""Write the results of `expr` to a json file of [{column -> value}, ...] objects.
597+
598+
This method is eager and will execute the associated expression
599+
immediately.
600+
601+
Parameters
602+
----------
603+
expr
604+
The ibis expression to execute and persist to Delta Lake table.
605+
path
606+
The data source. A string or Path to the Delta Lake table.
607+
kwargs
608+
Additional, backend-specifc keyword arguments.
609+
"""
610+
backend = expr._find_backend(use_default=True)
611+
raise NotImplementedError(
612+
f"{backend.__class__.__name__} does not support writing to JSON."
613+
)
614+
589615

590616
class CanListCatalog(abc.ABC):
591617
@abc.abstractmethod

ibis/backends/duckdb/__init__.py

+40-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import warnings
99
from operator import itemgetter
1010
from pathlib import Path
11-
from typing import TYPE_CHECKING, Any
11+
from typing import TYPE_CHECKING, Any, Literal
1212

1313
import duckdb
1414
import pyarrow as pa
@@ -1566,6 +1566,45 @@ def to_geo(
15661566
with self._safe_raw_sql(copy_cmd):
15671567
pass
15681568

1569+
@util.experimental
1570+
def to_json(
1571+
self,
1572+
expr: ir.Table,
1573+
path: str | Path,
1574+
*,
1575+
compression: Literal["auto", "none", "gzip", "zstd"] = "auto",
1576+
dateformat: str | None = None,
1577+
timestampformat: str | None = None,
1578+
) -> None:
1579+
"""Write the results of `expr` to a json file of [{column -> value}, ...] objects.
1580+
1581+
This method is eager and will execute the associated expression
1582+
immediately.
1583+
See https://duckdb.org/docs/sql/statements/copy.html#json-options
1584+
for more info.
1585+
1586+
Parameters
1587+
----------
1588+
expr
1589+
The ibis expression to execute and persist to Delta Lake table.
1590+
path
1591+
URLs such as S3 buckets are supported.
1592+
compression
1593+
Compression codec to use. One of "auto", "none", "gzip", "zstd".
1594+
dateformat
1595+
Date format string.
1596+
timestampformat
1597+
Timestamp format string.
1598+
"""
1599+
opts = f", COMPRESSION '{compression.upper()}'"
1600+
if dateformat:
1601+
opts += f", DATEFORMAT '{dateformat}'"
1602+
if timestampformat:
1603+
opts += f", TIMESTAMPFORMAT '{timestampformat}'"
1604+
self.raw_sql(
1605+
f"COPY ({self.compile(expr)}) TO '{path!s}' (FORMAT JSON, ARRAY true{opts});"
1606+
)
1607+
15691608
def _get_schema_using_query(self, query: str) -> sch.Schema:
15701609
with self._safe_raw_sql(f"DESCRIBE {query}") as cur:
15711610
rows = cur.fetch_arrow_table()

ibis/backends/tests/test_export.py

+32
Original file line numberDiff line numberDiff line change
@@ -347,6 +347,38 @@ def test_table_to_csv(tmp_path, backend, awards_players):
347347
backend.assert_frame_equal(awards_players.to_pandas(), df)
348348

349349

350+
@pytest.mark.notimpl(
351+
[
352+
"athena",
353+
"bigquery",
354+
"clickhouse",
355+
"databricks",
356+
"datafusion",
357+
"druid",
358+
"exasol",
359+
"flink",
360+
"impala",
361+
"mssql",
362+
"mysql",
363+
"oracle",
364+
"polars",
365+
"postgres",
366+
"pyspark",
367+
"risingwave",
368+
"snowflake",
369+
"sqlite",
370+
"trino",
371+
],
372+
reason="haven't gotten to them yet. Might be easy!",
373+
raises=NotImplementedError,
374+
)
375+
def test_to_json(backend, tmp_path, awards_players):
376+
out_path = tmp_path / "out.json"
377+
awards_players.to_json(out_path)
378+
df = pd.read_json(out_path, orient="records")
379+
backend.assert_frame_equal(awards_players.to_pandas(), df)
380+
381+
350382
@pytest.mark.notimpl(
351383
["duckdb"],
352384
reason="cannot inline WriteOptions objects",

ibis/expr/types/core.py

+22
Original file line numberDiff line numberDiff line change
@@ -771,6 +771,28 @@ def to_delta(
771771
"""
772772
self._find_backend(use_default=True).to_delta(self, path, **kwargs)
773773

774+
@experimental
775+
def to_json(
776+
self,
777+
path: str | Path,
778+
**kwargs: Any,
779+
) -> None:
780+
"""Write the results of `expr` to a json file of [{column -> value}, ...] objects.
781+
782+
This method is eager and will execute the associated expression
783+
immediately.
784+
785+
Parameters
786+
----------
787+
expr
788+
The ibis expression to execute and persist to Delta Lake table.
789+
path
790+
The data source. A string or Path to the Delta Lake table.
791+
kwargs
792+
Additional, backend-specifc keyword arguments.
793+
"""
794+
self._find_backend(use_default=True).to_json(self, path, **kwargs)
795+
774796
@experimental
775797
def to_torch(
776798
self,

0 commit comments

Comments
 (0)