Skip to content

Commit 8c79a0c

Browse files
committed
use read_delta_path_with_cdf
1 parent 22ce95f commit 8c79a0c

1 file changed

Lines changed: 4 additions & 20 deletions

File tree

integration_tests/src/main/python/delta_lake_write_test.py

Lines changed: 4 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -487,23 +487,15 @@ def test_delta_write_round_trip_cdf_write_opt(spark_tmp_path, enable_deletion_ve
487487
gen_df(spark, gen_list).coalesce(1).write.format("delta"), enable_deletion_vectors)
488488
.option("delta.enableChangeDataFeed", "true")
489489
.save(path),
490-
lambda spark, path: spark.read.format("delta")
491-
.option("readChangeFeed", "true")
492-
.option("startingVersion", 0)
493-
.load(path)
494-
.drop("_commit_timestamp"),
490+
read_delta_path_with_cdf,
495491
data_path,
496492
conf=confs)
497493
assert_gpu_and_cpu_writes_are_equal_collect(
498494
lambda spark, path: get_writer_with_deletion_vector_property_set(
499495
gen_df(spark, gen_list).coalesce(1).write.format("delta"), enable_deletion_vectors)
500496
.mode("overwrite")
501497
.save(path),
502-
lambda spark, path: spark.read.format("delta")
503-
.option("readChangeFeed", "true")
504-
.option("startingVersion", 0)
505-
.load(path)
506-
.drop("_commit_timestamp"),
498+
read_delta_path_with_cdf,
507499
data_path,
508500
conf=confs)
509501

@@ -528,22 +520,14 @@ def setup_tables(spark):
528520
.mode("append")
529521
.option("delta.enableChangeDataFeed", "true")
530522
.save(path),
531-
lambda spark, path: spark.read.format("delta")
532-
.option("readChangeFeed", "true")
533-
.option("startingVersion", 0)
534-
.load(path)
535-
.drop("_commit_timestamp"),
523+
read_delta_path_with_cdf,
536524
data_path,
537525
conf=confs)
538526
assert_gpu_and_cpu_writes_are_equal_collect(
539527
lambda spark, path: gen_df(spark, gen_list).coalesce(1).write.format("delta")
540528
.mode("overwrite")
541529
.save(path),
542-
lambda spark, path: spark.read.format("delta")
543-
.option("readChangeFeed", "true")
544-
.option("startingVersion", 0)
545-
.load(path)
546-
.drop("_commit_timestamp"),
530+
read_delta_path_with_cdf,
547531
data_path,
548532
conf=confs)
549533
with_cpu_session(lambda spark: assert_gpu_and_cpu_delta_logs_equivalent(spark, data_path))

0 commit comments

Comments
 (0)