Skip to content

Commit 9607b4a

Browse files
a-hirotaclaude
andcommitted
Add utc_timestamps parameter to cudf.io.parquet.ParquetWriter
Behavioral change: allow chunked writes to emit timestamp columns with isAdjustedToUTC=false, as DataFrame.to_parquet does. The default (True) keeps the current output unchanged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 557faa7 commit 9607b4a

2 files changed

Lines changed: 30 additions & 0 deletions

File tree

‎python/cudf/cudf/io/parquet.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1858,6 +1858,10 @@ class ParquetWriter:
18581858
store_schema : bool, default False
18591859
If ``True``, enable computing and writing arrow schema to Parquet
18601860
file footer's key-value metadata section for faithful round-tripping.
1861+
utc_timestamps : bool, default True
1862+
If ``True``, timestamp columns are written with
1863+
``isAdjustedToUTC=true``. If ``False``, they are written with
1864+
``isAdjustedToUTC=false``, matching ``DataFrame.to_parquet``.
18611865
18621866
See Also
18631867
--------
@@ -1877,6 +1881,7 @@ def __init__(
18771881
max_dictionary_size: int = 1048576,
18781882
use_dictionary: bool = True,
18791883
store_schema: bool = False,
1884+
utc_timestamps: bool = True,
18801885
):
18811886
filepaths_or_buffers = (
18821887
list(filepath_or_buffer)
@@ -1895,6 +1900,7 @@ def __init__(
18951900
self.max_dictionary_size = max_dictionary_size
18961901
self.use_dictionary = use_dictionary
18971902
self.write_arrow_schema = store_schema
1903+
self.utc_timestamps = utc_timestamps
18981904

18991905
def write_table(self, table, partitions_info=None) -> None:
19001906
"""Writes a single table to the file"""
@@ -2001,6 +2007,7 @@ def _initialize_chunked_state(
20012007
.max_page_size_rows(self.max_page_size_rows)
20022008
.max_dictionary_size(self.max_dictionary_size)
20032009
.write_arrow_schema(self.write_arrow_schema)
2010+
.utc_timestamps(self.utc_timestamps)
20042011
.build()
20052012
)
20062013
options.set_dictionary_policy(dict_policy)

‎python/cudf/cudf/tests/input_output/test_parquet.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import decimal
66
import glob
77
import hashlib
8+
import json
89
import math
910
import os
1011
import pathlib
@@ -1706,6 +1707,28 @@ def test_parquet_writer_int96_timestamps(tmp_path, pdf, gdf):
17061707
assert_eq(expect, got, check_categorical=False, check_dtype=False)
17071708

17081709

1710+
@pytest.mark.parametrize(
1711+
"kwargs, expected", [({}, True), ({"utc_timestamps": False}, False)]
1712+
)
1713+
def test_parquet_writer_chunked_utc_timestamps(tmp_path, kwargs, expected):
1714+
gdf = cudf.DataFrame(
1715+
{"ts": cudf.Series([0, 1_000_000, None], dtype="datetime64[us]")}
1716+
)
1717+
fname = tmp_path / "gdf.parquet"
1718+
1719+
with ParquetWriter(fname, **kwargs) as writer:
1720+
writer.write_table(gdf)
1721+
writer.write_table(gdf)
1722+
1723+
logical_type = json.loads(
1724+
pq.ParquetFile(fname).schema.column(0).logical_type.to_json()
1725+
)
1726+
assert logical_type["isAdjustedToUTC"] is expected
1727+
assert_eq(
1728+
cudf.read_parquet(fname), cudf.concat([gdf, gdf], ignore_index=True)
1729+
)
1730+
1731+
17091732
def test_multifile_parquet_folder(tmp_path):
17101733
test_pdf1 = pd.DataFrame(
17111734
{

0 commit comments

Comments
 (0)