Skip to content

Commit 64c5194

Browse files
cpcloudcpcloud
committed
refactor(pyspark): only allow timezoned timestamps with pyspark < 3.4
1 parent 80a4bbf commit 64c5194

File tree

5 files changed

+103
-16
lines changed

5 files changed

+103
-16
lines changed

‎ibis/backends/pyspark/datatypes.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ def from_ibis(cls, dtype):
126126
else:
127127
if not SUPPORTS_TIMESTAMP_NTZ:
128128
raise com.UnsupportedBackendType(
129-
"PySpark<3.5 doesn't properly support timestamps without a timezone"
129+
"PySpark<3.4 doesn't properly support timestamps without a timezone"
130130
)
131131
return pt.TimestampNTZType()
132132
else:

‎ibis/backends/pyspark/tests/test_ddl.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -181,9 +181,9 @@ def test_create_table_reserved_identifier(con, alltypes, keyword_t):
181181

182182

183183
@pytest.mark.xfail_version(
184-
pyspark=["pyspark<3.5"],
184+
pyspark=["pyspark<3.4"],
185185
raises=ValueError,
186-
reason="PySparkAnalysisException is not available in PySpark <3.5",
186+
reason="PySparkAnalysisException is not available in PySpark <3.4",
187187
)
188188
def test_create_database_exists(con):
189189
con.create_database(dbname := util.gen_name("dbname"))
@@ -197,9 +197,9 @@ def test_create_database_exists(con):
197197

198198

199199
@pytest.mark.xfail_version(
200-
pyspark=["pyspark<3.5"],
200+
pyspark=["pyspark<3.4"],
201201
raises=ValueError,
202-
reason="PySparkAnalysisException is not available in PySpark <3.5",
202+
reason="PySparkAnalysisException is not available in PySpark <3.4",
203203
)
204204
def test_drop_database_exists(con):
205205
con.create_database(dbname := util.gen_name("dbname"))

‎ibis/backends/tests/test_generic.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1130,7 +1130,7 @@ def test_typeof(con):
11301130
@pytest.mark.notimpl(["polars"], reason="incorrect answer")
11311131
@pytest.mark.notyet(["impala"], reason="can't find table in subquery")
11321132
@pytest.mark.notimpl(["datafusion", "druid"])
1133-
@pytest.mark.xfail_version(pyspark=["pyspark<3.5"])
1133+
@pytest.mark.xfail_version(pyspark=["pyspark<3.4"])
11341134
@pytest.mark.notyet(["exasol"], raises=ExaQueryError, reason="not supported by exasol")
11351135
@pytest.mark.notyet(
11361136
["risingwave"],

‎ibis/backends/tests/test_struct.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -253,8 +253,8 @@ def test_keyword_fields(con, nullable):
253253
reason="doesn't seem to support IN-style subqueries on structs",
254254
)
255255
@pytest.mark.xfail_version(
256-
pyspark=["pyspark<3.5"],
257-
reason="requires pyspark 3.5",
256+
pyspark=["pyspark<3.4"],
257+
reason="requires pyspark 3.4",
258258
raises=PySparkAnalysisException,
259259
)
260260
@pytest.mark.notimpl(

‎ibis/backends/tests/test_temporal.py‎

Lines changed: 95 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -293,9 +293,42 @@ def test_timestamp_extract_week_of_year(backend, alltypes, df):
293293
@pytest.mark.parametrize(
294294
("ibis_unit", "pandas_unit"),
295295
[
296-
param("Y", "Y", id="year"),
297-
param("Q", "Q", id="quarter"),
298-
param("M", "M", id="month"),
296+
param(
297+
"Y",
298+
"Y",
299+
id="year",
300+
marks=[
301+
pytest.mark.xfail_version(
302+
pyspark=["pyspark<3.4"],
303+
reason="no support for timezoneless timestamps",
304+
raises=UserWarning,
305+
),
306+
],
307+
),
308+
param(
309+
"Q",
310+
"Q",
311+
id="quarter",
312+
marks=[
313+
pytest.mark.xfail_version(
314+
pyspark=["pyspark<3.4"],
315+
reason="no support for timezoneless timestamps",
316+
raises=UserWarning,
317+
),
318+
],
319+
),
320+
param(
321+
"M",
322+
"M",
323+
id="month",
324+
marks=[
325+
pytest.mark.xfail_version(
326+
pyspark=["pyspark<3.4"],
327+
reason="no support for timezoneless timestamps",
328+
raises=UserWarning,
329+
),
330+
],
331+
),
299332
param(
300333
"W",
301334
"W",
@@ -306,9 +339,24 @@ def test_timestamp_extract_week_of_year(backend, alltypes, df):
306339
raises=AssertionError,
307340
reason="implemented, but doesn't match other backends",
308341
),
342+
pytest.mark.xfail_version(
343+
pyspark=["pyspark<3.4"],
344+
reason="no support for timezoneless timestamps",
345+
raises=UserWarning,
346+
),
347+
],
348+
),
349+
param(
350+
"D",
351+
"D",
352+
marks=[
353+
pytest.mark.xfail_version(
354+
pyspark=["pyspark<3.4"],
355+
reason="no support for timezoneless timestamps",
356+
raises=UserWarning,
357+
),
309358
],
310359
),
311-
param("D", "D"),
312360
param(
313361
"h",
314362
"h",
@@ -432,6 +480,11 @@ def test_timestamp_truncate(backend, alltypes, df, ibis_unit, pandas_unit):
432480
),
433481
],
434482
)
483+
@pytest.mark.xfail_version(
484+
pyspark=["pyspark<3.4"],
485+
reason="no support for timezoneless timestamps",
486+
raises=UserWarning,
487+
)
435488
@pytest.mark.notimpl(["druid"], raises=com.OperationNotDefinedError)
436489
def test_date_truncate(backend, alltypes, df, unit):
437490
expr = alltypes.timestamp_col.date().truncate(unit).name("tmp")
@@ -501,10 +554,31 @@ def test_date_truncate(backend, alltypes, df, unit):
501554
reason="Bind error: Invalid unit: week",
502555
),
503556
sqlite_without_ymd_intervals,
557+
pytest.mark.xfail_version(
558+
pyspark=["pyspark<3.4"], raises=AssertionError
559+
),
560+
],
561+
),
562+
param(
563+
"D",
564+
pd.offsets.DateOffset,
565+
marks=[
566+
sqlite_without_ymd_intervals,
567+
pytest.mark.xfail_version(
568+
pyspark=["pyspark<3.4"], raises=AssertionError
569+
),
570+
],
571+
),
572+
param(
573+
"h",
574+
pd.Timedelta,
575+
marks=[
576+
sqlite_without_hms_intervals,
577+
pytest.mark.xfail_version(
578+
pyspark=["pyspark<3.4"], raises=AssertionError
579+
),
504580
],
505581
),
506-
param("D", pd.offsets.DateOffset, marks=sqlite_without_ymd_intervals),
507-
param("h", pd.Timedelta, marks=sqlite_without_hms_intervals),
508582
param("m", pd.Timedelta, marks=sqlite_without_hms_intervals),
509583
param("s", pd.Timedelta, marks=sqlite_without_hms_intervals),
510584
param(
@@ -770,6 +844,10 @@ def convert_to_offset(x):
770844
raises=PySparkConnectGrpcException,
771845
reason="arrow conversion breaks",
772846
),
847+
pytest.mark.xfail_version(
848+
pyspark=["pyspark<3.4"],
849+
reason="no support for timezoneless timestamps",
850+
),
773851
pytest.mark.notyet(
774852
["databricks"],
775853
raises=AssertionError,
@@ -830,6 +908,10 @@ def convert_to_offset(x):
830908
raises=PySparkConnectGrpcException,
831909
reason="arrow conversion breaks",
832910
),
911+
pytest.mark.xfail_version(
912+
pyspark=["pyspark<3.4"],
913+
reason="no support for timezoneless timestamps",
914+
),
833915
pytest.mark.notyet(
834916
["databricks"],
835917
raises=AssertionError,
@@ -876,6 +958,9 @@ def test_temporal_binop(backend, con, alltypes, df, expr_fn, expected_fn):
876958
reason="TableException: DAY_INTERVAL_TYPES precision is not supported: 5",
877959
),
878960
sqlite_without_ymd_intervals,
961+
pytest.mark.xfail_version(
962+
pyspark=["pyspark<3.4"], raises=AssertionError
963+
),
879964
],
880965
),
881966
param("5W", plus, id="weeks-plus", marks=sqlite_without_ymd_intervals),
@@ -940,7 +1025,8 @@ def test_timestamp_comparison_filter(backend, con, alltypes, df, func_name):
9401025
comparison_fn(alltypes.timestamp_col.cast("timestamp('UTC')"), ts)
9411026
)
9421027

943-
col = df.timestamp_col.dt.tz_localize("UTC")
1028+
if getattr((col := df.timestamp_col).dtype, "tz", None) is None:
1029+
col = df.timestamp_col.dt.tz_localize("UTC")
9441030
expected = df[comparison_fn(col, ts)]
9451031
result = con.execute(expr)
9461032

@@ -973,7 +1059,8 @@ def test_timestamp_comparison_filter_numpy(backend, con, alltypes, df, func_name
9731059

9741060
ts = pd.Timestamp(ts.item(), tz="UTC")
9751061

976-
col = df.timestamp_col.dt.tz_localize("UTC")
1062+
if getattr((col := df.timestamp_col).dtype, "tz", None) is None:
1063+
col = df.timestamp_col.dt.tz_localize("UTC")
9771064
expected = df[comparison_fn(col, ts)]
9781065
result = con.execute(expr)
9791066

0 commit comments

Comments
 (0)