diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.00000-12-3ac0d3a9-e19f-4bef-a39a-30030476b8aa-0-00001.parquet.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.00000-12-3ac0d3a9-e19f-4bef-a39a-30030476b8aa-0-00001.parquet.crc new file mode 100644 index 000000000..806c6f2ce Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.00000-12-3ac0d3a9-e19f-4bef-a39a-30030476b8aa-0-00001.parquet.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.00000-9-8b7ad7ff-1bf1-4522-9b6b-da181d84a8d6-0-00001.parquet.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.00000-9-8b7ad7ff-1bf1-4522-9b6b-da181d84a8d6-0-00001.parquet.crc new file mode 100644 index 000000000..5eb594a06 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.00000-9-8b7ad7ff-1bf1-4522-9b6b-da181d84a8d6-0-00001.parquet.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.data-5987f8ac-013a-4827-8559-f7e150264139.parquet.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.data-5987f8ac-013a-4827-8559-f7e150264139.parquet.crc deleted file mode 100644 index 0f3255e1f..000000000 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.data-5987f8ac-013a-4827-8559-f7e150264139.parquet.crc and /dev/null differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.data-9e95f806-d9a9-4263-b277-e39cadb79dd6.parquet.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.data-9e95f806-d9a9-4263-b277-e39cadb79dd6.parquet.crc deleted file mode 100644 index efb28c093..000000000 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.data-9e95f806-d9a9-4263-b277-e39cadb79dd6.parquet.crc and /dev/null differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.delete-242a4468-1e89-489f-aa1b-eafd83a379db.parquet.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.delete-242a4468-1e89-489f-aa1b-eafd83a379db.parquet.crc new file mode 100644 index 000000000..e5807f19f Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.delete-242a4468-1e89-489f-aa1b-eafd83a379db.parquet.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.delete-2ca427ee-335e-412b-85d9-cb2ffd9ecfde.parquet.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.delete-2ca427ee-335e-412b-85d9-cb2ffd9ecfde.parquet.crc new file mode 100644 index 000000000..1f74cb4bb Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.delete-2ca427ee-335e-412b-85d9-cb2ffd9ecfde.parquet.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.delete-6b31fafe-0aa5-4197-b4e8-052dbc2afa98.parquet.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.delete-6b31fafe-0aa5-4197-b4e8-052dbc2afa98.parquet.crc new file mode 100644 index 000000000..73f539085 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.delete-6b31fafe-0aa5-4197-b4e8-052dbc2afa98.parquet.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.delete-93d19556-6cbf-4720-a9a3-3cd5004ad532.parquet.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.delete-93d19556-6cbf-4720-a9a3-3cd5004ad532.parquet.crc new file mode 100644 index 000000000..f3aa3cf07 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.delete-93d19556-6cbf-4720-a9a3-3cd5004ad532.parquet.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.delete-f58783ca-d36b-4a53-ac49-873d47e858e7.parquet.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.delete-f58783ca-d36b-4a53-ac49-873d47e858e7.parquet.crc deleted file mode 100644 index 4bee8dc8c..000000000 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.delete-f58783ca-d36b-4a53-ac49-873d47e858e7.parquet.crc and /dev/null differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.delete-f8a8f040-7d1d-4f80-a7c5-42ae37572eec.parquet.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.delete-f8a8f040-7d1d-4f80-a7c5-42ae37572eec.parquet.crc deleted file mode 100644 index 4d00d991a..000000000 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/.delete-f8a8f040-7d1d-4f80-a7c5-42ae37572eec.parquet.crc and /dev/null differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/00000-12-3ac0d3a9-e19f-4bef-a39a-30030476b8aa-0-00001.parquet b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/00000-12-3ac0d3a9-e19f-4bef-a39a-30030476b8aa-0-00001.parquet new file mode 100644 index 000000000..dc6fa2201 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/00000-12-3ac0d3a9-e19f-4bef-a39a-30030476b8aa-0-00001.parquet differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/00000-9-8b7ad7ff-1bf1-4522-9b6b-da181d84a8d6-0-00001.parquet b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/00000-9-8b7ad7ff-1bf1-4522-9b6b-da181d84a8d6-0-00001.parquet new file mode 100644 index 000000000..195c77d1c Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/00000-9-8b7ad7ff-1bf1-4522-9b6b-da181d84a8d6-0-00001.parquet differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/data-5987f8ac-013a-4827-8559-f7e150264139.parquet b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/data-5987f8ac-013a-4827-8559-f7e150264139.parquet deleted file mode 100644 index 811b61f9d..000000000 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/data-5987f8ac-013a-4827-8559-f7e150264139.parquet and /dev/null differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/data-9e95f806-d9a9-4263-b277-e39cadb79dd6.parquet b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/data-9e95f806-d9a9-4263-b277-e39cadb79dd6.parquet deleted file mode 100644 index 9aea36156..000000000 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/data-9e95f806-d9a9-4263-b277-e39cadb79dd6.parquet and /dev/null differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/delete-242a4468-1e89-489f-aa1b-eafd83a379db.parquet b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/delete-242a4468-1e89-489f-aa1b-eafd83a379db.parquet new file mode 100644 index 000000000..f1cb75bae Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/delete-242a4468-1e89-489f-aa1b-eafd83a379db.parquet differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/delete-2ca427ee-335e-412b-85d9-cb2ffd9ecfde.parquet b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/delete-2ca427ee-335e-412b-85d9-cb2ffd9ecfde.parquet new file mode 100644 index 000000000..2088f7796 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/delete-2ca427ee-335e-412b-85d9-cb2ffd9ecfde.parquet differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/delete-6b31fafe-0aa5-4197-b4e8-052dbc2afa98.parquet b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/delete-6b31fafe-0aa5-4197-b4e8-052dbc2afa98.parquet new file mode 100644 index 000000000..6c0bf3dc4 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/delete-6b31fafe-0aa5-4197-b4e8-052dbc2afa98.parquet differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/delete-93d19556-6cbf-4720-a9a3-3cd5004ad532.parquet b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/delete-93d19556-6cbf-4720-a9a3-3cd5004ad532.parquet new file mode 100644 index 000000000..580283adb Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/delete-93d19556-6cbf-4720-a9a3-3cd5004ad532.parquet differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/delete-f58783ca-d36b-4a53-ac49-873d47e858e7.parquet b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/delete-f58783ca-d36b-4a53-ac49-873d47e858e7.parquet deleted file mode 100644 index fd33564ae..000000000 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/delete-f58783ca-d36b-4a53-ac49-873d47e858e7.parquet and /dev/null differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/delete-f8a8f040-7d1d-4f80-a7c5-42ae37572eec.parquet b/data/persistent/equality_deletes/warehouse/mydb/mytable/data/delete-f8a8f040-7d1d-4f80-a7c5-42ae37572eec.parquet deleted file mode 100644 index b0ed6674e..000000000 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/data/delete-f8a8f040-7d1d-4f80-a7c5-42ae37572eec.parquet and /dev/null differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.34f7dec7-90c5-4cd5-b158-5782b73fc010-m0.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.34f7dec7-90c5-4cd5-b158-5782b73fc010-m0.avro.crc new file mode 100644 index 000000000..910e76517 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.34f7dec7-90c5-4cd5-b158-5782b73fc010-m0.avro.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.44f48ecd-dd48-42c0-8639-76cc51702031-m0.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.44f48ecd-dd48-42c0-8639-76cc51702031-m0.avro.crc deleted file mode 100644 index ac7856995..000000000 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.44f48ecd-dd48-42c0-8639-76cc51702031-m0.avro.crc and /dev/null differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.61648895-78fc-44d6-bf55-298a7614c4f8-m0.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.61648895-78fc-44d6-bf55-298a7614c4f8-m0.avro.crc new file mode 100644 index 000000000..bd8a0b9a6 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.61648895-78fc-44d6-bf55-298a7614c4f8-m0.avro.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.6afce3c7-c302-42aa-91ab-bcbe4f01cb97-m0.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.6afce3c7-c302-42aa-91ab-bcbe4f01cb97-m0.avro.crc deleted file mode 100644 index e5a59af06..000000000 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.6afce3c7-c302-42aa-91ab-bcbe4f01cb97-m0.avro.crc and /dev/null differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.8057d23a-ed01-40cb-bfd6-44b145234c6d-m0.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.8057d23a-ed01-40cb-bfd6-44b145234c6d-m0.avro.crc new file mode 100644 index 000000000..4e9578155 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.8057d23a-ed01-40cb-bfd6-44b145234c6d-m0.avro.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.91bf4420-2bae-484f-b724-8184d56d3029-m0.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.91bf4420-2bae-484f-b724-8184d56d3029-m0.avro.crc new file mode 100644 index 000000000..f80318e8d Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.91bf4420-2bae-484f-b724-8184d56d3029-m0.avro.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.bcc5469e-83b4-4a41-be7e-af79ed029353-m0.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.bcc5469e-83b4-4a41-be7e-af79ed029353-m0.avro.crc new file mode 100644 index 000000000..1485e1f0e Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.bcc5469e-83b4-4a41-be7e-af79ed029353-m0.avro.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.c4028cec-4266-45e9-bf74-77cbf1b55328-m0.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.c4028cec-4266-45e9-bf74-77cbf1b55328-m0.avro.crc new file mode 100644 index 000000000..a05f477a4 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.c4028cec-4266-45e9-bf74-77cbf1b55328-m0.avro.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.e08e7f38-46b9-40b7-9f1d-bd0fe9e77386-m0.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.e08e7f38-46b9-40b7-9f1d-bd0fe9e77386-m0.avro.crc deleted file mode 100644 index ddad3cf56..000000000 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.e08e7f38-46b9-40b7-9f1d-bd0fe9e77386-m0.avro.crc and /dev/null differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.ee85b137-904a-4591-8054-6b5d4993c9bd-m0.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.ee85b137-904a-4591-8054-6b5d4993c9bd-m0.avro.crc deleted file mode 100644 index 241eaf1f8..000000000 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.ee85b137-904a-4591-8054-6b5d4993c9bd-m0.avro.crc and /dev/null differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-1046545856685507949-1-ee85b137-904a-4591-8054-6b5d4993c9bd.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-1046545856685507949-1-ee85b137-904a-4591-8054-6b5d4993c9bd.avro.crc deleted file mode 100644 index fb7454fbe..000000000 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-1046545856685507949-1-ee85b137-904a-4591-8054-6b5d4993c9bd.avro.crc and /dev/null differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-1584331123492059582-2-91bf4420-2bae-484f-b724-8184d56d3029.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-1584331123492059582-2-91bf4420-2bae-484f-b724-8184d56d3029.avro.crc new file mode 100644 index 000000000..ee7b83472 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-1584331123492059582-2-91bf4420-2bae-484f-b724-8184d56d3029.avro.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-1584331123492059582-3-91bf4420-2bae-484f-b724-8184d56d3029.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-1584331123492059582-3-91bf4420-2bae-484f-b724-8184d56d3029.avro.crc new file mode 100644 index 000000000..b5f23c464 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-1584331123492059582-3-91bf4420-2bae-484f-b724-8184d56d3029.avro.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-1916084761853986166-1-61648895-78fc-44d6-bf55-298a7614c4f8.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-1916084761853986166-1-61648895-78fc-44d6-bf55-298a7614c4f8.avro.crc new file mode 100644 index 000000000..f46569920 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-1916084761853986166-1-61648895-78fc-44d6-bf55-298a7614c4f8.avro.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-2894602912036338632-1-e08e7f38-46b9-40b7-9f1d-bd0fe9e77386.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-2894602912036338632-1-e08e7f38-46b9-40b7-9f1d-bd0fe9e77386.avro.crc deleted file mode 100644 index a60bbc033..000000000 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-2894602912036338632-1-e08e7f38-46b9-40b7-9f1d-bd0fe9e77386.avro.crc and /dev/null differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-3340507003387467420-1-8057d23a-ed01-40cb-bfd6-44b145234c6d.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-3340507003387467420-1-8057d23a-ed01-40cb-bfd6-44b145234c6d.avro.crc new file mode 100644 index 000000000..7df2875c9 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-3340507003387467420-1-8057d23a-ed01-40cb-bfd6-44b145234c6d.avro.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-3512576891615857142-1-6afce3c7-c302-42aa-91ab-bcbe4f01cb97.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-3512576891615857142-1-6afce3c7-c302-42aa-91ab-bcbe4f01cb97.avro.crc deleted file mode 100644 index 005916f94..000000000 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-3512576891615857142-1-6afce3c7-c302-42aa-91ab-bcbe4f01cb97.avro.crc and /dev/null differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-7342794868382145167-3-34f7dec7-90c5-4cd5-b158-5782b73fc010.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-7342794868382145167-3-34f7dec7-90c5-4cd5-b158-5782b73fc010.avro.crc new file mode 100644 index 000000000..2ff49a223 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-7342794868382145167-3-34f7dec7-90c5-4cd5-b158-5782b73fc010.avro.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-842401149381792626-2-c4028cec-4266-45e9-bf74-77cbf1b55328.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-842401149381792626-2-c4028cec-4266-45e9-bf74-77cbf1b55328.avro.crc new file mode 100644 index 000000000..6246b46ba Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-842401149381792626-2-c4028cec-4266-45e9-bf74-77cbf1b55328.avro.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-853766660775201079-1-bcc5469e-83b4-4a41-be7e-af79ed029353.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-853766660775201079-1-bcc5469e-83b4-4a41-be7e-af79ed029353.avro.crc new file mode 100644 index 000000000..06b8623fa Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-853766660775201079-1-bcc5469e-83b4-4a41-be7e-af79ed029353.avro.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-8985322175058482040-1-44f48ecd-dd48-42c0-8639-76cc51702031.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-8985322175058482040-1-44f48ecd-dd48-42c0-8639-76cc51702031.avro.crc deleted file mode 100644 index fe98570dd..000000000 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.snap-8985322175058482040-1-44f48ecd-dd48-42c0-8639-76cc51702031.avro.crc and /dev/null differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.v1.metadata.json.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.v1.metadata.json.crc index e7c64c8e3..5e42549da 100644 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.v1.metadata.json.crc and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.v1.metadata.json.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.v2.metadata.json.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.v2.metadata.json.crc index b4fef03c0..e00da7b33 100644 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.v2.metadata.json.crc and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.v2.metadata.json.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.v3.metadata.json.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.v3.metadata.json.crc index 7013a2514..459767589 100644 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.v3.metadata.json.crc and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.v3.metadata.json.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.v4.metadata.json.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.v4.metadata.json.crc index 35922527b..2b7ae3c99 100644 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.v4.metadata.json.crc and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.v4.metadata.json.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.v5.metadata.json.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.v5.metadata.json.crc index c6dc82315..65ccc8734 100644 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.v5.metadata.json.crc and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.v5.metadata.json.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.v6.metadata.json.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.v6.metadata.json.crc new file mode 100644 index 000000000..2b5026524 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.v6.metadata.json.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.v7.metadata.json.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.v7.metadata.json.crc new file mode 100644 index 000000000..24fd9e208 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.v7.metadata.json.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.version-hint.text.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.version-hint.text.crc index 398edb952..dba98e89d 100644 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.version-hint.text.crc and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/.version-hint.text.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/34f7dec7-90c5-4cd5-b158-5782b73fc010-m0.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/34f7dec7-90c5-4cd5-b158-5782b73fc010-m0.avro new file mode 100644 index 000000000..58a349de2 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/34f7dec7-90c5-4cd5-b158-5782b73fc010-m0.avro differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/44f48ecd-dd48-42c0-8639-76cc51702031-m0.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/44f48ecd-dd48-42c0-8639-76cc51702031-m0.avro deleted file mode 100644 index 192c8a286..000000000 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/44f48ecd-dd48-42c0-8639-76cc51702031-m0.avro and /dev/null differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/61648895-78fc-44d6-bf55-298a7614c4f8-m0.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/61648895-78fc-44d6-bf55-298a7614c4f8-m0.avro new file mode 100644 index 000000000..a198a220d Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/61648895-78fc-44d6-bf55-298a7614c4f8-m0.avro differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/6afce3c7-c302-42aa-91ab-bcbe4f01cb97-m0.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/6afce3c7-c302-42aa-91ab-bcbe4f01cb97-m0.avro deleted file mode 100644 index b60eb4078..000000000 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/6afce3c7-c302-42aa-91ab-bcbe4f01cb97-m0.avro and /dev/null differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/8057d23a-ed01-40cb-bfd6-44b145234c6d-m0.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/8057d23a-ed01-40cb-bfd6-44b145234c6d-m0.avro new file mode 100644 index 000000000..1599936c3 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/8057d23a-ed01-40cb-bfd6-44b145234c6d-m0.avro differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/91bf4420-2bae-484f-b724-8184d56d3029-m0.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/91bf4420-2bae-484f-b724-8184d56d3029-m0.avro new file mode 100644 index 000000000..b208946ec Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/91bf4420-2bae-484f-b724-8184d56d3029-m0.avro differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/bcc5469e-83b4-4a41-be7e-af79ed029353-m0.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/bcc5469e-83b4-4a41-be7e-af79ed029353-m0.avro new file mode 100644 index 000000000..374f47205 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/bcc5469e-83b4-4a41-be7e-af79ed029353-m0.avro differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/c4028cec-4266-45e9-bf74-77cbf1b55328-m0.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/c4028cec-4266-45e9-bf74-77cbf1b55328-m0.avro new file mode 100644 index 000000000..029e572f2 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/c4028cec-4266-45e9-bf74-77cbf1b55328-m0.avro differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/e08e7f38-46b9-40b7-9f1d-bd0fe9e77386-m0.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/e08e7f38-46b9-40b7-9f1d-bd0fe9e77386-m0.avro deleted file mode 100644 index 1b131af26..000000000 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/e08e7f38-46b9-40b7-9f1d-bd0fe9e77386-m0.avro and /dev/null differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/ee85b137-904a-4591-8054-6b5d4993c9bd-m0.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/ee85b137-904a-4591-8054-6b5d4993c9bd-m0.avro deleted file mode 100644 index 0fd593f32..000000000 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/ee85b137-904a-4591-8054-6b5d4993c9bd-m0.avro and /dev/null differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-1046545856685507949-1-ee85b137-904a-4591-8054-6b5d4993c9bd.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-1046545856685507949-1-ee85b137-904a-4591-8054-6b5d4993c9bd.avro deleted file mode 100644 index 1a66ac4f5..000000000 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-1046545856685507949-1-ee85b137-904a-4591-8054-6b5d4993c9bd.avro and /dev/null differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-1584331123492059582-2-91bf4420-2bae-484f-b724-8184d56d3029.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-1584331123492059582-2-91bf4420-2bae-484f-b724-8184d56d3029.avro new file mode 100644 index 000000000..401818d88 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-1584331123492059582-2-91bf4420-2bae-484f-b724-8184d56d3029.avro differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-1584331123492059582-3-91bf4420-2bae-484f-b724-8184d56d3029.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-1584331123492059582-3-91bf4420-2bae-484f-b724-8184d56d3029.avro new file mode 100644 index 000000000..c345a81ba Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-1584331123492059582-3-91bf4420-2bae-484f-b724-8184d56d3029.avro differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-1916084761853986166-1-61648895-78fc-44d6-bf55-298a7614c4f8.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-1916084761853986166-1-61648895-78fc-44d6-bf55-298a7614c4f8.avro new file mode 100644 index 000000000..140cebdc9 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-1916084761853986166-1-61648895-78fc-44d6-bf55-298a7614c4f8.avro differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-2894602912036338632-1-e08e7f38-46b9-40b7-9f1d-bd0fe9e77386.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-2894602912036338632-1-e08e7f38-46b9-40b7-9f1d-bd0fe9e77386.avro deleted file mode 100644 index a77c1cfd2..000000000 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-2894602912036338632-1-e08e7f38-46b9-40b7-9f1d-bd0fe9e77386.avro and /dev/null differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-3340507003387467420-1-8057d23a-ed01-40cb-bfd6-44b145234c6d.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-3340507003387467420-1-8057d23a-ed01-40cb-bfd6-44b145234c6d.avro new file mode 100644 index 000000000..8bef512e9 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-3340507003387467420-1-8057d23a-ed01-40cb-bfd6-44b145234c6d.avro differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-3512576891615857142-1-6afce3c7-c302-42aa-91ab-bcbe4f01cb97.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-3512576891615857142-1-6afce3c7-c302-42aa-91ab-bcbe4f01cb97.avro deleted file mode 100644 index 113e7b215..000000000 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-3512576891615857142-1-6afce3c7-c302-42aa-91ab-bcbe4f01cb97.avro and /dev/null differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-7342794868382145167-3-34f7dec7-90c5-4cd5-b158-5782b73fc010.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-7342794868382145167-3-34f7dec7-90c5-4cd5-b158-5782b73fc010.avro new file mode 100644 index 000000000..c80ac708f Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-7342794868382145167-3-34f7dec7-90c5-4cd5-b158-5782b73fc010.avro differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-842401149381792626-2-c4028cec-4266-45e9-bf74-77cbf1b55328.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-842401149381792626-2-c4028cec-4266-45e9-bf74-77cbf1b55328.avro new file mode 100644 index 000000000..d60fa3e20 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-842401149381792626-2-c4028cec-4266-45e9-bf74-77cbf1b55328.avro differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-853766660775201079-1-bcc5469e-83b4-4a41-be7e-af79ed029353.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-853766660775201079-1-bcc5469e-83b4-4a41-be7e-af79ed029353.avro new file mode 100644 index 000000000..12ff1c3bd Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-853766660775201079-1-bcc5469e-83b4-4a41-be7e-af79ed029353.avro differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-8985322175058482040-1-44f48ecd-dd48-42c0-8639-76cc51702031.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-8985322175058482040-1-44f48ecd-dd48-42c0-8639-76cc51702031.avro deleted file mode 100644 index 7a0f61f1e..000000000 Binary files a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-8985322175058482040-1-44f48ecd-dd48-42c0-8639-76cc51702031.avro and /dev/null differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v1.metadata.json b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v1.metadata.json index 86679dde6..eb5e84000 100644 --- a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v1.metadata.json +++ b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v1.metadata.json @@ -1,10 +1,10 @@ { "format-version" : 2, - "table-uuid" : "7c269e29-15d2-48a6-bc83-4919d38e3041", + "table-uuid" : "96247900-66da-4f86-9cbe-c81dbcf8420f", "location" : "data/persistent/equality_deletes/warehouse/mydb/mytable", "last-sequence-number" : 0, - "last-updated-ms" : 1746122109432, - "last-column-id" : 2, + "last-updated-ms" : 1758879194794, + "last-column-id" : 3, "current-schema-id" : 0, "schemas" : [ { "type" : "struct", @@ -12,13 +12,18 @@ "fields" : [ { "id" : 1, "name" : "id", - "required" : true, + "required" : false, "type" : "int" }, { "id" : 2, "name" : "name", "required" : false, "type" : "string" + }, { + "id" : 3, + "name" : "bir", + "required" : false, + "type" : "date" } ] } ], "default-spec-id" : 0, @@ -32,11 +37,15 @@ "order-id" : 0, "fields" : [ ] } ], - "properties" : { }, + "properties" : { + "owner" : "zhangjun", + "write.parquet.compression-codec" : "zstd" + }, "current-snapshot-id" : -1, "refs" : { }, "snapshots" : [ ], "statistics" : [ ], + "partition-statistics" : [ ], "snapshot-log" : [ ], "metadata-log" : [ ] } \ No newline at end of file diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v2.metadata.json b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v2.metadata.json index 4670b89c8..d1539989d 100644 --- a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v2.metadata.json +++ b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v2.metadata.json @@ -1,10 +1,10 @@ { "format-version" : 2, - "table-uuid" : "7c269e29-15d2-48a6-bc83-4919d38e3041", + "table-uuid" : "96247900-66da-4f86-9cbe-c81dbcf8420f", "location" : "data/persistent/equality_deletes/warehouse/mydb/mytable", "last-sequence-number" : 1, - "last-updated-ms" : 1746122109975, - "last-column-id" : 2, + "last-updated-ms" : 1758879443926, + "last-column-id" : 3, "current-schema-id" : 0, "schemas" : [ { "type" : "struct", @@ -12,13 +12,18 @@ "fields" : [ { "id" : 1, "name" : "id", - "required" : true, + "required" : false, "type" : "int" }, { "id" : 2, "name" : "name", "required" : false, "type" : "string" + }, { + "id" : 3, + "name" : "bir", + "required" : false, + "type" : "date" } ] } ], "default-spec-id" : 0, @@ -32,41 +37,50 @@ "order-id" : 0, "fields" : [ ] } ], - "properties" : { }, - "current-snapshot-id" : 1046545856685507949, + "properties" : { + "owner" : "zhangjun", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 853766660775201079, "refs" : { "main" : { - "snapshot-id" : 1046545856685507949, + "snapshot-id" : 853766660775201079, "type" : "branch" } }, "snapshots" : [ { "sequence-number" : 1, - "snapshot-id" : 1046545856685507949, - "timestamp-ms" : 1746122109975, + "snapshot-id" : 853766660775201079, + "timestamp-ms" : 1758879443926, "summary" : { "operation" : "append", + "spark.app.id" : "local-1758878991225", "added-data-files" : "1", - "added-records" : "3", - "added-files-size" : "685", + "added-records" : "4", + "added-files-size" : "935", "changed-partition-count" : "1", - "total-records" : "3", - "total-files-size" : "685", + "total-records" : "4", + "total-files-size" : "935", "total-data-files" : "1", "total-delete-files" : "0", "total-position-deletes" : "0", - "total-equality-deletes" : "0" + "total-equality-deletes" : "0", + "engine-version" : "3.5.1", + "app-id" : "local-1758878991225", + "engine-name" : "spark", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" }, - "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-1046545856685507949-1-ee85b137-904a-4591-8054-6b5d4993c9bd.avro", + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-853766660775201079-1-bcc5469e-83b4-4a41-be7e-af79ed029353.avro", "schema-id" : 0 } ], "statistics" : [ ], + "partition-statistics" : [ ], "snapshot-log" : [ { - "timestamp-ms" : 1746122109975, - "snapshot-id" : 1046545856685507949 + "timestamp-ms" : 1758879443926, + "snapshot-id" : 853766660775201079 } ], "metadata-log" : [ { - "timestamp-ms" : 1746122109432, + "timestamp-ms" : 1758879194794, "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v1.metadata.json" } ] } \ No newline at end of file diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v3.metadata.json b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v3.metadata.json index 5abd9c789..7ed0eed5b 100644 --- a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v3.metadata.json +++ b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v3.metadata.json @@ -1,10 +1,10 @@ { "format-version" : 2, - "table-uuid" : "7c269e29-15d2-48a6-bc83-4919d38e3041", + "table-uuid" : "96247900-66da-4f86-9cbe-c81dbcf8420f", "location" : "data/persistent/equality_deletes/warehouse/mydb/mytable", "last-sequence-number" : 2, - "last-updated-ms" : 1746122110110, - "last-column-id" : 2, + "last-updated-ms" : 1758879495787, + "last-column-id" : 3, "current-schema-id" : 0, "schemas" : [ { "type" : "struct", @@ -12,13 +12,18 @@ "fields" : [ { "id" : 1, "name" : "id", - "required" : true, + "required" : false, "type" : "int" }, { "id" : 2, "name" : "name", "required" : false, "type" : "string" + }, { + "id" : 3, + "name" : "bir", + "required" : false, + "type" : "date" } ] } ], "default-spec-id" : 0, @@ -32,68 +37,78 @@ "order-id" : 0, "fields" : [ ] } ], - "properties" : { }, - "current-snapshot-id" : 3512576891615857142, + "properties" : { + "owner" : "zhangjun", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 7342794868382145167, "refs" : { "main" : { - "snapshot-id" : 3512576891615857142, + "snapshot-id" : 7342794868382145167, "type" : "branch" } }, "snapshots" : [ { "sequence-number" : 1, - "snapshot-id" : 1046545856685507949, - "timestamp-ms" : 1746122109975, + "snapshot-id" : 853766660775201079, + "timestamp-ms" : 1758879443926, "summary" : { "operation" : "append", + "spark.app.id" : "local-1758878991225", "added-data-files" : "1", - "added-records" : "3", - "added-files-size" : "685", + "added-records" : "4", + "added-files-size" : "935", "changed-partition-count" : "1", - "total-records" : "3", - "total-files-size" : "685", + "total-records" : "4", + "total-files-size" : "935", "total-data-files" : "1", "total-delete-files" : "0", "total-position-deletes" : "0", - "total-equality-deletes" : "0" + "total-equality-deletes" : "0", + "engine-version" : "3.5.1", + "app-id" : "local-1758878991225", + "engine-name" : "spark", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" }, - "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-1046545856685507949-1-ee85b137-904a-4591-8054-6b5d4993c9bd.avro", + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-853766660775201079-1-bcc5469e-83b4-4a41-be7e-af79ed029353.avro", "schema-id" : 0 }, { "sequence-number" : 2, - "snapshot-id" : 3512576891615857142, - "parent-snapshot-id" : 1046545856685507949, - "timestamp-ms" : 1746122110110, + "snapshot-id" : 7342794868382145167, + "parent-snapshot-id" : 853766660775201079, + "timestamp-ms" : 1758879495787, "summary" : { - "operation" : "overwrite", + "operation" : "delete", "added-equality-delete-files" : "1", "added-delete-files" : "1", - "added-files-size" : "451", + "added-files-size" : "466", "added-equality-deletes" : "1", "changed-partition-count" : "1", - "total-records" : "3", - "total-files-size" : "1136", + "total-records" : "4", + "total-files-size" : "1401", "total-data-files" : "1", "total-delete-files" : "1", "total-position-deletes" : "0", - "total-equality-deletes" : "1" + "total-equality-deletes" : "1", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" }, - "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-3512576891615857142-1-6afce3c7-c302-42aa-91ab-bcbe4f01cb97.avro", + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-7342794868382145167-1-34f7dec7-90c5-4cd5-b158-5782b73fc010.avro", "schema-id" : 0 } ], "statistics" : [ ], + "partition-statistics" : [ ], "snapshot-log" : [ { - "timestamp-ms" : 1746122109975, - "snapshot-id" : 1046545856685507949 + "timestamp-ms" : 1758879443926, + "snapshot-id" : 853766660775201079 }, { - "timestamp-ms" : 1746122110110, - "snapshot-id" : 3512576891615857142 + "timestamp-ms" : 1758879495787, + "snapshot-id" : 7342794868382145167 } ], "metadata-log" : [ { - "timestamp-ms" : 1746122109432, + "timestamp-ms" : 1758879194794, "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v1.metadata.json" }, { - "timestamp-ms" : 1746122109975, + "timestamp-ms" : 1758879443926, "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v2.metadata.json" } ] } \ No newline at end of file diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v4.metadata.json b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v4.metadata.json index 604f8e4ce..cb35e0c61 100644 --- a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v4.metadata.json +++ b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v4.metadata.json @@ -1,10 +1,10 @@ { "format-version" : 2, - "table-uuid" : "7c269e29-15d2-48a6-bc83-4919d38e3041", + "table-uuid" : "96247900-66da-4f86-9cbe-c81dbcf8420f", "location" : "data/persistent/equality_deletes/warehouse/mydb/mytable", "last-sequence-number" : 3, - "last-updated-ms" : 1746122110222, - "last-column-id" : 2, + "last-updated-ms" : 1758879496119, + "last-column-id" : 3, "current-schema-id" : 0, "schemas" : [ { "type" : "struct", @@ -12,13 +12,18 @@ "fields" : [ { "id" : 1, "name" : "id", - "required" : true, + "required" : false, "type" : "int" }, { "id" : 2, "name" : "name", "required" : false, "type" : "string" + }, { + "id" : 3, + "name" : "bir", + "required" : false, + "type" : "date" } ] } ], "default-spec-id" : 0, @@ -32,94 +37,106 @@ "order-id" : 0, "fields" : [ ] } ], - "properties" : { }, - "current-snapshot-id" : 8985322175058482040, + "properties" : { + "owner" : "zhangjun", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 1584331123492059582, "refs" : { "main" : { - "snapshot-id" : 8985322175058482040, + "snapshot-id" : 1584331123492059582, "type" : "branch" } }, "snapshots" : [ { "sequence-number" : 1, - "snapshot-id" : 1046545856685507949, - "timestamp-ms" : 1746122109975, + "snapshot-id" : 853766660775201079, + "timestamp-ms" : 1758879443926, "summary" : { "operation" : "append", + "spark.app.id" : "local-1758878991225", "added-data-files" : "1", - "added-records" : "3", - "added-files-size" : "685", + "added-records" : "4", + "added-files-size" : "935", "changed-partition-count" : "1", - "total-records" : "3", - "total-files-size" : "685", + "total-records" : "4", + "total-files-size" : "935", "total-data-files" : "1", "total-delete-files" : "0", "total-position-deletes" : "0", - "total-equality-deletes" : "0" + "total-equality-deletes" : "0", + "engine-version" : "3.5.1", + "app-id" : "local-1758878991225", + "engine-name" : "spark", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" }, - "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-1046545856685507949-1-ee85b137-904a-4591-8054-6b5d4993c9bd.avro", + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-853766660775201079-1-bcc5469e-83b4-4a41-be7e-af79ed029353.avro", "schema-id" : 0 }, { "sequence-number" : 2, - "snapshot-id" : 3512576891615857142, - "parent-snapshot-id" : 1046545856685507949, - "timestamp-ms" : 1746122110110, + "snapshot-id" : 7342794868382145167, + "parent-snapshot-id" : 853766660775201079, + "timestamp-ms" : 1758879495787, "summary" : { - "operation" : "overwrite", + "operation" : "delete", "added-equality-delete-files" : "1", "added-delete-files" : "1", - "added-files-size" : "451", + "added-files-size" : "466", "added-equality-deletes" : "1", "changed-partition-count" : "1", - "total-records" : "3", - "total-files-size" : "1136", + "total-records" : "4", + "total-files-size" : "1401", "total-data-files" : "1", "total-delete-files" : "1", "total-position-deletes" : "0", - "total-equality-deletes" : "1" + "total-equality-deletes" : "1", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" }, - "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-3512576891615857142-1-6afce3c7-c302-42aa-91ab-bcbe4f01cb97.avro", + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-7342794868382145167-1-34f7dec7-90c5-4cd5-b158-5782b73fc010.avro", "schema-id" : 0 }, { "sequence-number" : 3, - "snapshot-id" : 8985322175058482040, - "parent-snapshot-id" : 3512576891615857142, - "timestamp-ms" : 1746122110222, + "snapshot-id" : 1584331123492059582, + "parent-snapshot-id" : 7342794868382145167, + "timestamp-ms" : 1758879496119, "summary" : { - "operation" : "append", - "added-data-files" : "1", - "added-records" : "3", - "added-files-size" : "734", + "operation" : "delete", + "added-equality-delete-files" : "1", + "added-delete-files" : "1", + "added-files-size" : "463", + "added-equality-deletes" : "1", "changed-partition-count" : "1", - "total-records" : "6", - "total-files-size" : "1870", - "total-data-files" : "2", - "total-delete-files" : "1", + "total-records" : "4", + "total-files-size" : "1864", + "total-data-files" : "1", + "total-delete-files" : "2", "total-position-deletes" : "0", - "total-equality-deletes" : "1" + "total-equality-deletes" : "2", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" }, - "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-8985322175058482040-1-44f48ecd-dd48-42c0-8639-76cc51702031.avro", + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-1584331123492059582-2-91bf4420-2bae-484f-b724-8184d56d3029.avro", "schema-id" : 0 } ], "statistics" : [ ], + "partition-statistics" : [ ], "snapshot-log" : [ { - "timestamp-ms" : 1746122109975, - "snapshot-id" : 1046545856685507949 + "timestamp-ms" : 1758879443926, + "snapshot-id" : 853766660775201079 }, { - "timestamp-ms" : 1746122110110, - "snapshot-id" : 3512576891615857142 + "timestamp-ms" : 1758879495787, + "snapshot-id" : 7342794868382145167 }, { - "timestamp-ms" : 1746122110222, - "snapshot-id" : 8985322175058482040 + "timestamp-ms" : 1758879496119, + "snapshot-id" : 1584331123492059582 } ], "metadata-log" : [ { - "timestamp-ms" : 1746122109432, + "timestamp-ms" : 1758879194794, "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v1.metadata.json" }, { - "timestamp-ms" : 1746122109975, + "timestamp-ms" : 1758879443926, "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v2.metadata.json" }, { - "timestamp-ms" : 1746122110110, + "timestamp-ms" : 1758879495787, "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v3.metadata.json" } ] } \ No newline at end of file diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v5.metadata.json b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v5.metadata.json index 29aa4e220..91315aca4 100644 --- a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v5.metadata.json +++ b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v5.metadata.json @@ -1,10 +1,10 @@ { "format-version" : 2, - "table-uuid" : "7c269e29-15d2-48a6-bc83-4919d38e3041", + "table-uuid" : "96247900-66da-4f86-9cbe-c81dbcf8420f", "location" : "data/persistent/equality_deletes/warehouse/mydb/mytable", "last-sequence-number" : 4, - "last-updated-ms" : 1746122110333, - "last-column-id" : 2, + "last-updated-ms" : 1758879496480, + "last-column-id" : 3, "current-schema-id" : 0, "schemas" : [ { "type" : "struct", @@ -12,13 +12,18 @@ "fields" : [ { "id" : 1, "name" : "id", - "required" : true, + "required" : false, "type" : "int" }, { "id" : 2, "name" : "name", "required" : false, "type" : "string" + }, { + "id" : 3, + "name" : "bir", + "required" : false, + "type" : "date" } ] } ], "default-spec-id" : 0, @@ -32,121 +37,140 @@ "order-id" : 0, "fields" : [ ] } ], - "properties" : { }, - "current-snapshot-id" : 2894602912036338632, + "properties" : { + "owner" : "zhangjun", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 842401149381792626, "refs" : { "main" : { - "snapshot-id" : 2894602912036338632, + "snapshot-id" : 842401149381792626, "type" : "branch" } }, "snapshots" : [ { "sequence-number" : 1, - "snapshot-id" : 1046545856685507949, - "timestamp-ms" : 1746122109975, + "snapshot-id" : 853766660775201079, + "timestamp-ms" : 1758879443926, "summary" : { "operation" : "append", + "spark.app.id" : "local-1758878991225", "added-data-files" : "1", - "added-records" : "3", - "added-files-size" : "685", + "added-records" : "4", + "added-files-size" : "935", "changed-partition-count" : "1", - "total-records" : "3", - "total-files-size" : "685", + "total-records" : "4", + "total-files-size" : "935", "total-data-files" : "1", "total-delete-files" : "0", "total-position-deletes" : "0", - "total-equality-deletes" : "0" + "total-equality-deletes" : "0", + "engine-version" : "3.5.1", + "app-id" : "local-1758878991225", + "engine-name" : "spark", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" }, - "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-1046545856685507949-1-ee85b137-904a-4591-8054-6b5d4993c9bd.avro", + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-853766660775201079-1-bcc5469e-83b4-4a41-be7e-af79ed029353.avro", "schema-id" : 0 }, { "sequence-number" : 2, - "snapshot-id" : 3512576891615857142, - "parent-snapshot-id" : 1046545856685507949, - "timestamp-ms" : 1746122110110, + "snapshot-id" : 7342794868382145167, + "parent-snapshot-id" : 853766660775201079, + "timestamp-ms" : 1758879495787, "summary" : { - "operation" : "overwrite", + "operation" : "delete", "added-equality-delete-files" : "1", "added-delete-files" : "1", - "added-files-size" : "451", + "added-files-size" : "466", "added-equality-deletes" : "1", "changed-partition-count" : "1", - "total-records" : "3", - "total-files-size" : "1136", + "total-records" : "4", + "total-files-size" : "1401", "total-data-files" : "1", "total-delete-files" : "1", "total-position-deletes" : "0", - "total-equality-deletes" : "1" + "total-equality-deletes" : "1", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" }, - "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-3512576891615857142-1-6afce3c7-c302-42aa-91ab-bcbe4f01cb97.avro", + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-7342794868382145167-1-34f7dec7-90c5-4cd5-b158-5782b73fc010.avro", "schema-id" : 0 }, { "sequence-number" : 3, - "snapshot-id" : 8985322175058482040, - "parent-snapshot-id" : 3512576891615857142, - "timestamp-ms" : 1746122110222, + "snapshot-id" : 1584331123492059582, + "parent-snapshot-id" : 7342794868382145167, + "timestamp-ms" : 1758879496119, "summary" : { - "operation" : "append", - "added-data-files" : "1", - "added-records" : "3", - "added-files-size" : "734", + "operation" : "delete", + "added-equality-delete-files" : "1", + "added-delete-files" : "1", + "added-files-size" : "463", + "added-equality-deletes" : "1", "changed-partition-count" : "1", - "total-records" : "6", - "total-files-size" : "1870", - "total-data-files" : "2", - "total-delete-files" : "1", + "total-records" : "4", + "total-files-size" : "1864", + "total-data-files" : "1", + "total-delete-files" : "2", "total-position-deletes" : "0", - "total-equality-deletes" : "1" + "total-equality-deletes" : "2", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" }, - "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-8985322175058482040-1-44f48ecd-dd48-42c0-8639-76cc51702031.avro", + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-1584331123492059582-2-91bf4420-2bae-484f-b724-8184d56d3029.avro", "schema-id" : 0 }, { "sequence-number" : 4, - "snapshot-id" : 2894602912036338632, - "parent-snapshot-id" : 8985322175058482040, - "timestamp-ms" : 1746122110333, + "snapshot-id" : 842401149381792626, + "parent-snapshot-id" : 1584331123492059582, + "timestamp-ms" : 1758879496480, "summary" : { - "operation" : "overwrite", + "operation" : "delete", "added-equality-delete-files" : "1", "added-delete-files" : "1", - "added-files-size" : "674", + "added-files-size" : "706", "added-equality-deletes" : "1", "changed-partition-count" : "1", - "total-records" : "6", - "total-files-size" : "2544", - "total-data-files" : "2", - "total-delete-files" : "2", + "total-records" : "4", + "total-files-size" : "2570", + "total-data-files" : "1", + "total-delete-files" : "3", "total-position-deletes" : "0", - "total-equality-deletes" : "2" + "total-equality-deletes" : "3", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" }, - "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-2894602912036338632-1-e08e7f38-46b9-40b7-9f1d-bd0fe9e77386.avro", + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-842401149381792626-2-c4028cec-4266-45e9-bf74-77cbf1b55328.avro", "schema-id" : 0 } ], "statistics" : [ ], + "partition-statistics" : [ ], "snapshot-log" : [ { - "timestamp-ms" : 1746122109975, - "snapshot-id" : 1046545856685507949 + "timestamp-ms" : 1758879443926, + "snapshot-id" : 853766660775201079 + }, { + "timestamp-ms" : 1758879495787, + "snapshot-id" : 7342794868382145167 + }, { + "timestamp-ms" : 1758879496119, + "snapshot-id" : 1584331123492059582 }, { - "timestamp-ms" : 1746122110110, - "snapshot-id" : 3512576891615857142 + "timestamp-ms" : 1758879496330, + "snapshot-id" : 7342794868382145167 }, { - "timestamp-ms" : 1746122110222, - "snapshot-id" : 8985322175058482040 + "timestamp-ms" : 1758879496404, + "snapshot-id" : 1584331123492059582 }, { - "timestamp-ms" : 1746122110333, - "snapshot-id" : 2894602912036338632 + "timestamp-ms" : 1758879496480, + "snapshot-id" : 842401149381792626 } ], "metadata-log" : [ { - "timestamp-ms" : 1746122109432, + "timestamp-ms" : 1758879194794, "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v1.metadata.json" }, { - "timestamp-ms" : 1746122109975, + "timestamp-ms" : 1758879443926, "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v2.metadata.json" }, { - "timestamp-ms" : 1746122110110, + "timestamp-ms" : 1758879495787, "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v3.metadata.json" }, { - "timestamp-ms" : 1746122110222, + "timestamp-ms" : 1758879496119, "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v4.metadata.json" } ] } \ No newline at end of file diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v6.metadata.json b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v6.metadata.json new file mode 100644 index 000000000..a9ce6fc8b --- /dev/null +++ b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v6.metadata.json @@ -0,0 +1,207 @@ +{ + "format-version" : 2, + "table-uuid" : "96247900-66da-4f86-9cbe-c81dbcf8420f", + "location" : "data/persistent/equality_deletes/warehouse/mydb/mytable", + "last-sequence-number" : 5, + "last-updated-ms" : 1758879647963, + "last-column-id" : 3, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "id", + "required" : false, + "type" : "int" + }, { + "id" : 2, + "name" : "name", + "required" : false, + "type" : "string" + }, { + "id" : 3, + "name" : "bir", + "required" : false, + "type" : "date" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "zhangjun", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 3340507003387467420, + "refs" : { + "main" : { + "snapshot-id" : 3340507003387467420, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 853766660775201079, + "timestamp-ms" : 1758879443926, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1758878991225", + "added-data-files" : "1", + "added-records" : "4", + "added-files-size" : "935", + "changed-partition-count" : "1", + "total-records" : "4", + "total-files-size" : "935", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0", + "engine-version" : "3.5.1", + "app-id" : "local-1758878991225", + "engine-name" : "spark", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" + }, + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-853766660775201079-1-bcc5469e-83b4-4a41-be7e-af79ed029353.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 7342794868382145167, + "parent-snapshot-id" : 853766660775201079, + "timestamp-ms" : 1758879495787, + "summary" : { + "operation" : "delete", + "added-equality-delete-files" : "1", + "added-delete-files" : "1", + "added-files-size" : "466", + "added-equality-deletes" : "1", + "changed-partition-count" : "1", + "total-records" : "4", + "total-files-size" : "1401", + "total-data-files" : "1", + "total-delete-files" : "1", + "total-position-deletes" : "0", + "total-equality-deletes" : "1", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" + }, + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-7342794868382145167-1-34f7dec7-90c5-4cd5-b158-5782b73fc010.avro", + "schema-id" : 0 + }, { + "sequence-number" : 3, + "snapshot-id" : 1584331123492059582, + "parent-snapshot-id" : 7342794868382145167, + "timestamp-ms" : 1758879496119, + "summary" : { + "operation" : "delete", + "added-equality-delete-files" : "1", + "added-delete-files" : "1", + "added-files-size" : "463", + "added-equality-deletes" : "1", + "changed-partition-count" : "1", + "total-records" : "4", + "total-files-size" : "1864", + "total-data-files" : "1", + "total-delete-files" : "2", + "total-position-deletes" : "0", + "total-equality-deletes" : "2", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" + }, + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-1584331123492059582-2-91bf4420-2bae-484f-b724-8184d56d3029.avro", + "schema-id" : 0 + }, { + "sequence-number" : 4, + "snapshot-id" : 842401149381792626, + "parent-snapshot-id" : 1584331123492059582, + "timestamp-ms" : 1758879496480, + "summary" : { + "operation" : "delete", + "added-equality-delete-files" : "1", + "added-delete-files" : "1", + "added-files-size" : "706", + "added-equality-deletes" : "1", + "changed-partition-count" : "1", + "total-records" : "4", + "total-files-size" : "2570", + "total-data-files" : "1", + "total-delete-files" : "3", + "total-position-deletes" : "0", + "total-equality-deletes" : "3", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" + }, + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-842401149381792626-2-c4028cec-4266-45e9-bf74-77cbf1b55328.avro", + "schema-id" : 0 + }, { + "sequence-number" : 5, + "snapshot-id" : 3340507003387467420, + "parent-snapshot-id" : 842401149381792626, + "timestamp-ms" : 1758879647963, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1758878991225", + "added-data-files" : "1", + "added-records" : "2", + "added-files-size" : "909", + "changed-partition-count" : "1", + "total-records" : "6", + "total-files-size" : "3479", + "total-data-files" : "2", + "total-delete-files" : "3", + "total-position-deletes" : "0", + "total-equality-deletes" : "3", + "engine-version" : "3.5.1", + "app-id" : "local-1758878991225", + "engine-name" : "spark", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" + }, + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-3340507003387467420-1-8057d23a-ed01-40cb-bfd6-44b145234c6d.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "partition-statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1758879443926, + "snapshot-id" : 853766660775201079 + }, { + "timestamp-ms" : 1758879495787, + "snapshot-id" : 7342794868382145167 + }, { + "timestamp-ms" : 1758879496119, + "snapshot-id" : 1584331123492059582 + }, { + "timestamp-ms" : 1758879496330, + "snapshot-id" : 7342794868382145167 + }, { + "timestamp-ms" : 1758879496404, + "snapshot-id" : 1584331123492059582 + }, { + "timestamp-ms" : 1758879496480, + "snapshot-id" : 842401149381792626 + }, { + "timestamp-ms" : 1758879647963, + "snapshot-id" : 3340507003387467420 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1758879194794, + "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1758879443926, + "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1758879495787, + "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1758879496119, + "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1758879496480, + "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v5.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v7.metadata.json b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v7.metadata.json new file mode 100644 index 000000000..a5554516c --- /dev/null +++ b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v7.metadata.json @@ -0,0 +1,235 @@ +{ + "format-version" : 2, + "table-uuid" : "96247900-66da-4f86-9cbe-c81dbcf8420f", + "location" : "data/persistent/equality_deletes/warehouse/mydb/mytable", + "last-sequence-number" : 6, + "last-updated-ms" : 1758879681766, + "last-column-id" : 3, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "id", + "required" : false, + "type" : "int" + }, { + "id" : 2, + "name" : "name", + "required" : false, + "type" : "string" + }, { + "id" : 3, + "name" : "bir", + "required" : false, + "type" : "date" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "zhangjun", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 1916084761853986166, + "refs" : { + "main" : { + "snapshot-id" : 1916084761853986166, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 853766660775201079, + "timestamp-ms" : 1758879443926, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1758878991225", + "added-data-files" : "1", + "added-records" : "4", + "added-files-size" : "935", + "changed-partition-count" : "1", + "total-records" : "4", + "total-files-size" : "935", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0", + "engine-version" : "3.5.1", + "app-id" : "local-1758878991225", + "engine-name" : "spark", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" + }, + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-853766660775201079-1-bcc5469e-83b4-4a41-be7e-af79ed029353.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 7342794868382145167, + "parent-snapshot-id" : 853766660775201079, + "timestamp-ms" : 1758879495787, + "summary" : { + "operation" : "delete", + "added-equality-delete-files" : "1", + "added-delete-files" : "1", + "added-files-size" : "466", + "added-equality-deletes" : "1", + "changed-partition-count" : "1", + "total-records" : "4", + "total-files-size" : "1401", + "total-data-files" : "1", + "total-delete-files" : "1", + "total-position-deletes" : "0", + "total-equality-deletes" : "1", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" + }, + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-7342794868382145167-1-34f7dec7-90c5-4cd5-b158-5782b73fc010.avro", + "schema-id" : 0 + }, { + "sequence-number" : 3, + "snapshot-id" : 1584331123492059582, + "parent-snapshot-id" : 7342794868382145167, + "timestamp-ms" : 1758879496119, + "summary" : { + "operation" : "delete", + "added-equality-delete-files" : "1", + "added-delete-files" : "1", + "added-files-size" : "463", + "added-equality-deletes" : "1", + "changed-partition-count" : "1", + "total-records" : "4", + "total-files-size" : "1864", + "total-data-files" : "1", + "total-delete-files" : "2", + "total-position-deletes" : "0", + "total-equality-deletes" : "2", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" + }, + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-1584331123492059582-2-91bf4420-2bae-484f-b724-8184d56d3029.avro", + "schema-id" : 0 + }, { + "sequence-number" : 4, + "snapshot-id" : 842401149381792626, + "parent-snapshot-id" : 1584331123492059582, + "timestamp-ms" : 1758879496480, + "summary" : { + "operation" : "delete", + "added-equality-delete-files" : "1", + "added-delete-files" : "1", + "added-files-size" : "706", + "added-equality-deletes" : "1", + "changed-partition-count" : "1", + "total-records" : "4", + "total-files-size" : "2570", + "total-data-files" : "1", + "total-delete-files" : "3", + "total-position-deletes" : "0", + "total-equality-deletes" : "3", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" + }, + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-842401149381792626-2-c4028cec-4266-45e9-bf74-77cbf1b55328.avro", + "schema-id" : 0 + }, { + "sequence-number" : 5, + "snapshot-id" : 3340507003387467420, + "parent-snapshot-id" : 842401149381792626, + "timestamp-ms" : 1758879647963, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1758878991225", + "added-data-files" : "1", + "added-records" : "2", + "added-files-size" : "909", + "changed-partition-count" : "1", + "total-records" : "6", + "total-files-size" : "3479", + "total-data-files" : "2", + "total-delete-files" : "3", + "total-position-deletes" : "0", + "total-equality-deletes" : "3", + "engine-version" : "3.5.1", + "app-id" : "local-1758878991225", + "engine-name" : "spark", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" + }, + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-3340507003387467420-1-8057d23a-ed01-40cb-bfd6-44b145234c6d.avro", + "schema-id" : 0 + }, { + "sequence-number" : 6, + "snapshot-id" : 1916084761853986166, + "parent-snapshot-id" : 3340507003387467420, + "timestamp-ms" : 1758879681766, + "summary" : { + "operation" : "delete", + "added-equality-delete-files" : "1", + "added-delete-files" : "1", + "added-files-size" : "466", + "added-equality-deletes" : "1", + "changed-partition-count" : "1", + "total-records" : "6", + "total-files-size" : "3945", + "total-data-files" : "2", + "total-delete-files" : "4", + "total-position-deletes" : "0", + "total-equality-deletes" : "4", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" + }, + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/snap-1916084761853986166-1-61648895-78fc-44d6-bf55-298a7614c4f8.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "partition-statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1758879443926, + "snapshot-id" : 853766660775201079 + }, { + "timestamp-ms" : 1758879495787, + "snapshot-id" : 7342794868382145167 + }, { + "timestamp-ms" : 1758879496119, + "snapshot-id" : 1584331123492059582 + }, { + "timestamp-ms" : 1758879496330, + "snapshot-id" : 7342794868382145167 + }, { + "timestamp-ms" : 1758879496404, + "snapshot-id" : 1584331123492059582 + }, { + "timestamp-ms" : 1758879496480, + "snapshot-id" : 842401149381792626 + }, { + "timestamp-ms" : 1758879647963, + "snapshot-id" : 3340507003387467420 + }, { + "timestamp-ms" : 1758879681766, + "snapshot-id" : 1916084761853986166 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1758879194794, + "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1758879443926, + "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1758879495787, + "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1758879496119, + "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1758879496480, + "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v5.metadata.json" + }, { + "timestamp-ms" : 1758879647963, + "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/v6.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/version-hint.text b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/version-hint.text index 7813681f5..c7930257d 100644 --- a/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/version-hint.text +++ b/data/persistent/equality_deletes/warehouse/mydb/mytable/metadata/version-hint.text @@ -1 +1 @@ -5 \ No newline at end of file +7 \ No newline at end of file diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=a/.00001-1760585152415-373476a7-2e0e-4faa-a399-c1e9226724b9-00001.parquet.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=a/.00001-1760585152415-373476a7-2e0e-4faa-a399-c1e9226724b9-00001.parquet.crc new file mode 100644 index 000000000..127cc153d Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=a/.00001-1760585152415-373476a7-2e0e-4faa-a399-c1e9226724b9-00001.parquet.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=a/00001-1760585152415-373476a7-2e0e-4faa-a399-c1e9226724b9-00001.parquet b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=a/00001-1760585152415-373476a7-2e0e-4faa-a399-c1e9226724b9-00001.parquet new file mode 100644 index 000000000..744d5a986 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=a/00001-1760585152415-373476a7-2e0e-4faa-a399-c1e9226724b9-00001.parquet differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=b/.00001-1760585152415-373476a7-2e0e-4faa-a399-c1e9226724b9-00002.parquet.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=b/.00001-1760585152415-373476a7-2e0e-4faa-a399-c1e9226724b9-00002.parquet.crc new file mode 100644 index 000000000..56ab2cade Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=b/.00001-1760585152415-373476a7-2e0e-4faa-a399-c1e9226724b9-00002.parquet.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=b/.delete-07681bdb-b7a0-4bbc-a7ea-db5e51918b3d.parquet.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=b/.delete-07681bdb-b7a0-4bbc-a7ea-db5e51918b3d.parquet.crc new file mode 100644 index 000000000..a7ceb508d Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=b/.delete-07681bdb-b7a0-4bbc-a7ea-db5e51918b3d.parquet.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=b/00001-1760585152415-373476a7-2e0e-4faa-a399-c1e9226724b9-00002.parquet b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=b/00001-1760585152415-373476a7-2e0e-4faa-a399-c1e9226724b9-00002.parquet new file mode 100644 index 000000000..8bfcff49e Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=b/00001-1760585152415-373476a7-2e0e-4faa-a399-c1e9226724b9-00002.parquet differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=b/delete-07681bdb-b7a0-4bbc-a7ea-db5e51918b3d.parquet b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=b/delete-07681bdb-b7a0-4bbc-a7ea-db5e51918b3d.parquet new file mode 100644 index 000000000..5506880c0 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=b/delete-07681bdb-b7a0-4bbc-a7ea-db5e51918b3d.parquet differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=c/.00001-1760585152415-373476a7-2e0e-4faa-a399-c1e9226724b9-00003.parquet.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=c/.00001-1760585152415-373476a7-2e0e-4faa-a399-c1e9226724b9-00003.parquet.crc new file mode 100644 index 000000000..11bd5fb34 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=c/.00001-1760585152415-373476a7-2e0e-4faa-a399-c1e9226724b9-00003.parquet.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=c/.delete-1929bcb5-210d-4b3d-9989-a10234740545.parquet.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=c/.delete-1929bcb5-210d-4b3d-9989-a10234740545.parquet.crc new file mode 100644 index 000000000..d9c87b272 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=c/.delete-1929bcb5-210d-4b3d-9989-a10234740545.parquet.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=c/00001-1760585152415-373476a7-2e0e-4faa-a399-c1e9226724b9-00003.parquet b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=c/00001-1760585152415-373476a7-2e0e-4faa-a399-c1e9226724b9-00003.parquet new file mode 100644 index 000000000..8e62dfd87 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=c/00001-1760585152415-373476a7-2e0e-4faa-a399-c1e9226724b9-00003.parquet differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=c/delete-1929bcb5-210d-4b3d-9989-a10234740545.parquet b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=c/delete-1929bcb5-210d-4b3d-9989-a10234740545.parquet new file mode 100644 index 000000000..1c66db669 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=c/delete-1929bcb5-210d-4b3d-9989-a10234740545.parquet differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=d/.00001-1760585152415-373476a7-2e0e-4faa-a399-c1e9226724b9-00004.parquet.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=d/.00001-1760585152415-373476a7-2e0e-4faa-a399-c1e9226724b9-00004.parquet.crc new file mode 100644 index 000000000..f04ecbf36 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=d/.00001-1760585152415-373476a7-2e0e-4faa-a399-c1e9226724b9-00004.parquet.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=d/00001-1760585152415-373476a7-2e0e-4faa-a399-c1e9226724b9-00004.parquet b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=d/00001-1760585152415-373476a7-2e0e-4faa-a399-c1e9226724b9-00004.parquet new file mode 100644 index 000000000..1693c898a Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=d/00001-1760585152415-373476a7-2e0e-4faa-a399-c1e9226724b9-00004.parquet differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=e/.00001-1760585154163-9544d795-f42e-4c94-863f-422e5a11b63e-00001.parquet.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=e/.00001-1760585154163-9544d795-f42e-4c94-863f-422e5a11b63e-00001.parquet.crc new file mode 100644 index 000000000..cda8ebf73 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=e/.00001-1760585154163-9544d795-f42e-4c94-863f-422e5a11b63e-00001.parquet.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=e/00001-1760585154163-9544d795-f42e-4c94-863f-422e5a11b63e-00001.parquet b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=e/00001-1760585154163-9544d795-f42e-4c94-863f-422e5a11b63e-00001.parquet new file mode 100644 index 000000000..42e60017f Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=e/00001-1760585154163-9544d795-f42e-4c94-863f-422e5a11b63e-00001.parquet differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=f/.00001-1760585154163-9544d795-f42e-4c94-863f-422e5a11b63e-00002.parquet.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=f/.00001-1760585154163-9544d795-f42e-4c94-863f-422e5a11b63e-00002.parquet.crc new file mode 100644 index 000000000..d565a923b Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=f/.00001-1760585154163-9544d795-f42e-4c94-863f-422e5a11b63e-00002.parquet.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=f/.delete-ad683036-fbdb-4976-b0d3-2d1fc4525299.parquet.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=f/.delete-ad683036-fbdb-4976-b0d3-2d1fc4525299.parquet.crc new file mode 100644 index 000000000..78b6fb0ce Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=f/.delete-ad683036-fbdb-4976-b0d3-2d1fc4525299.parquet.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=f/00001-1760585154163-9544d795-f42e-4c94-863f-422e5a11b63e-00002.parquet b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=f/00001-1760585154163-9544d795-f42e-4c94-863f-422e5a11b63e-00002.parquet new file mode 100644 index 000000000..960d3dd0e Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=f/00001-1760585154163-9544d795-f42e-4c94-863f-422e5a11b63e-00002.parquet differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=f/delete-ad683036-fbdb-4976-b0d3-2d1fc4525299.parquet b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=f/delete-ad683036-fbdb-4976-b0d3-2d1fc4525299.parquet new file mode 100644 index 000000000..92364d0ab Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/data/name=f/delete-ad683036-fbdb-4976-b0d3-2d1fc4525299.parquet differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.2b5d4ce3-154e-4498-a648-89be96ebd74a-m0.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.2b5d4ce3-154e-4498-a648-89be96ebd74a-m0.avro.crc new file mode 100644 index 000000000..6b8c21f50 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.2b5d4ce3-154e-4498-a648-89be96ebd74a-m0.avro.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.36f9b154-c2d2-411f-9f29-9d23d46d688b-m0.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.36f9b154-c2d2-411f-9f29-9d23d46d688b-m0.avro.crc new file mode 100644 index 000000000..549998692 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.36f9b154-c2d2-411f-9f29-9d23d46d688b-m0.avro.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.b096df95-e029-4163-a7b4-762554ad0da9-m0.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.b096df95-e029-4163-a7b4-762554ad0da9-m0.avro.crc new file mode 100644 index 000000000..628b5c286 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.b096df95-e029-4163-a7b4-762554ad0da9-m0.avro.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.ec04e9ce-e03b-4fbe-a707-3645c1938319-m0.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.ec04e9ce-e03b-4fbe-a707-3645c1938319-m0.avro.crc new file mode 100644 index 000000000..b00e2f7c6 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.ec04e9ce-e03b-4fbe-a707-3645c1938319-m0.avro.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.ff275eec-b23b-435f-a6a5-1ab7ce1c8749-m0.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.ff275eec-b23b-435f-a6a5-1ab7ce1c8749-m0.avro.crc new file mode 100644 index 000000000..cfc3128d4 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.ff275eec-b23b-435f-a6a5-1ab7ce1c8749-m0.avro.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.snap-3138177759849079112-1-ff275eec-b23b-435f-a6a5-1ab7ce1c8749.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.snap-3138177759849079112-1-ff275eec-b23b-435f-a6a5-1ab7ce1c8749.avro.crc new file mode 100644 index 000000000..7b5740a95 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.snap-3138177759849079112-1-ff275eec-b23b-435f-a6a5-1ab7ce1c8749.avro.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.snap-4416477559988274704-1-b096df95-e029-4163-a7b4-762554ad0da9.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.snap-4416477559988274704-1-b096df95-e029-4163-a7b4-762554ad0da9.avro.crc new file mode 100644 index 000000000..33b2df44c Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.snap-4416477559988274704-1-b096df95-e029-4163-a7b4-762554ad0da9.avro.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.snap-6022113614852838397-1-36f9b154-c2d2-411f-9f29-9d23d46d688b.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.snap-6022113614852838397-1-36f9b154-c2d2-411f-9f29-9d23d46d688b.avro.crc new file mode 100644 index 000000000..1a820cbc9 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.snap-6022113614852838397-1-36f9b154-c2d2-411f-9f29-9d23d46d688b.avro.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.snap-6447922032991303611-1-2b5d4ce3-154e-4498-a648-89be96ebd74a.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.snap-6447922032991303611-1-2b5d4ce3-154e-4498-a648-89be96ebd74a.avro.crc new file mode 100644 index 000000000..daaf48a64 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.snap-6447922032991303611-1-2b5d4ce3-154e-4498-a648-89be96ebd74a.avro.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.snap-883882341917253211-1-ec04e9ce-e03b-4fbe-a707-3645c1938319.avro.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.snap-883882341917253211-1-ec04e9ce-e03b-4fbe-a707-3645c1938319.avro.crc new file mode 100644 index 000000000..3d3d23e9d Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.snap-883882341917253211-1-ec04e9ce-e03b-4fbe-a707-3645c1938319.avro.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.v1.metadata.json.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.v1.metadata.json.crc new file mode 100644 index 000000000..40de4b23e Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.v1.metadata.json.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.v2.metadata.json.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.v2.metadata.json.crc new file mode 100644 index 000000000..f7dbc010d Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.v2.metadata.json.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.v3.metadata.json.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.v3.metadata.json.crc new file mode 100644 index 000000000..581b8d2e6 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.v3.metadata.json.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.v4.metadata.json.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.v4.metadata.json.crc new file mode 100644 index 000000000..dd4d3543d Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.v4.metadata.json.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.v5.metadata.json.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.v5.metadata.json.crc new file mode 100644 index 000000000..73ea83a2a Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.v5.metadata.json.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.v6.metadata.json.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.v6.metadata.json.crc new file mode 100644 index 000000000..12d3a08fc Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.v6.metadata.json.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.version-hint.text.crc b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.version-hint.text.crc new file mode 100644 index 000000000..d02405139 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/.version-hint.text.crc differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/2b5d4ce3-154e-4498-a648-89be96ebd74a-m0.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/2b5d4ce3-154e-4498-a648-89be96ebd74a-m0.avro new file mode 100644 index 000000000..ae9b36fa9 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/2b5d4ce3-154e-4498-a648-89be96ebd74a-m0.avro differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/36f9b154-c2d2-411f-9f29-9d23d46d688b-m0.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/36f9b154-c2d2-411f-9f29-9d23d46d688b-m0.avro new file mode 100644 index 000000000..430a5c18d Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/36f9b154-c2d2-411f-9f29-9d23d46d688b-m0.avro differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/b096df95-e029-4163-a7b4-762554ad0da9-m0.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/b096df95-e029-4163-a7b4-762554ad0da9-m0.avro new file mode 100644 index 000000000..c6a2a91f0 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/b096df95-e029-4163-a7b4-762554ad0da9-m0.avro differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/ec04e9ce-e03b-4fbe-a707-3645c1938319-m0.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/ec04e9ce-e03b-4fbe-a707-3645c1938319-m0.avro new file mode 100644 index 000000000..972b9409e Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/ec04e9ce-e03b-4fbe-a707-3645c1938319-m0.avro differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/ff275eec-b23b-435f-a6a5-1ab7ce1c8749-m0.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/ff275eec-b23b-435f-a6a5-1ab7ce1c8749-m0.avro new file mode 100644 index 000000000..9d91eaebc Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/ff275eec-b23b-435f-a6a5-1ab7ce1c8749-m0.avro differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-3138177759849079112-1-ff275eec-b23b-435f-a6a5-1ab7ce1c8749.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-3138177759849079112-1-ff275eec-b23b-435f-a6a5-1ab7ce1c8749.avro new file mode 100644 index 000000000..ebfc5b33b Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-3138177759849079112-1-ff275eec-b23b-435f-a6a5-1ab7ce1c8749.avro differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-4416477559988274704-1-b096df95-e029-4163-a7b4-762554ad0da9.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-4416477559988274704-1-b096df95-e029-4163-a7b4-762554ad0da9.avro new file mode 100644 index 000000000..fafeddf4d Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-4416477559988274704-1-b096df95-e029-4163-a7b4-762554ad0da9.avro differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-6022113614852838397-1-36f9b154-c2d2-411f-9f29-9d23d46d688b.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-6022113614852838397-1-36f9b154-c2d2-411f-9f29-9d23d46d688b.avro new file mode 100644 index 000000000..00e869763 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-6022113614852838397-1-36f9b154-c2d2-411f-9f29-9d23d46d688b.avro differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-6447922032991303611-1-2b5d4ce3-154e-4498-a648-89be96ebd74a.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-6447922032991303611-1-2b5d4ce3-154e-4498-a648-89be96ebd74a.avro new file mode 100644 index 000000000..a983288ba Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-6447922032991303611-1-2b5d4ce3-154e-4498-a648-89be96ebd74a.avro differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-883882341917253211-1-ec04e9ce-e03b-4fbe-a707-3645c1938319.avro b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-883882341917253211-1-ec04e9ce-e03b-4fbe-a707-3645c1938319.avro new file mode 100644 index 000000000..6ab9ab061 Binary files /dev/null and b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-883882341917253211-1-ec04e9ce-e03b-4fbe-a707-3645c1938319.avro differ diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v1.metadata.json b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v1.metadata.json new file mode 100644 index 000000000..884005928 --- /dev/null +++ b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v1.metadata.json @@ -0,0 +1,55 @@ +{ + "format-version" : 2, + "table-uuid" : "31ad6764-1255-4aa3-bff4-9a27a25d69a7", + "location" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned", + "last-sequence-number" : 0, + "last-updated-ms" : 1760585152178, + "last-column-id" : 3, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "id", + "required" : true, + "type" : "int" + }, { + "id" : 2, + "name" : "name", + "required" : false, + "type" : "string" + }, { + "id" : 3, + "name" : "bir", + "required" : false, + "type" : "date" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ { + "name" : "name", + "transform" : "identity", + "source-id" : 2, + "field-id" : 1000 + } ] + } ], + "last-partition-id" : 1000, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : -1, + "refs" : { }, + "snapshots" : [ ], + "statistics" : [ ], + "partition-statistics" : [ ], + "snapshot-log" : [ ], + "metadata-log" : [ ] +} \ No newline at end of file diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v2.metadata.json b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v2.metadata.json new file mode 100644 index 000000000..9b75da960 --- /dev/null +++ b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v2.metadata.json @@ -0,0 +1,86 @@ +{ + "format-version" : 2, + "table-uuid" : "31ad6764-1255-4aa3-bff4-9a27a25d69a7", + "location" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned", + "last-sequence-number" : 1, + "last-updated-ms" : 1760585153811, + "last-column-id" : 3, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "id", + "required" : true, + "type" : "int" + }, { + "id" : 2, + "name" : "name", + "required" : false, + "type" : "string" + }, { + "id" : 3, + "name" : "bir", + "required" : false, + "type" : "date" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ { + "name" : "name", + "transform" : "identity", + "source-id" : 2, + "field-id" : 1000 + } ] + } ], + "last-partition-id" : 1000, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 6447922032991303611, + "refs" : { + "main" : { + "snapshot-id" : 6447922032991303611, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 6447922032991303611, + "timestamp-ms" : 1760585153811, + "summary" : { + "operation" : "append", + "added-data-files" : "4", + "added-records" : "4", + "added-files-size" : "3568", + "changed-partition-count" : "4", + "total-records" : "4", + "total-files-size" : "3568", + "total-data-files" : "4", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" + }, + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-6447922032991303611-1-2b5d4ce3-154e-4498-a648-89be96ebd74a.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "partition-statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1760585153811, + "snapshot-id" : 6447922032991303611 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1760585152178, + "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v1.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v3.metadata.json b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v3.metadata.json new file mode 100644 index 000000000..e782d0772 --- /dev/null +++ b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v3.metadata.json @@ -0,0 +1,114 @@ +{ + "format-version" : 2, + "table-uuid" : "31ad6764-1255-4aa3-bff4-9a27a25d69a7", + "location" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned", + "last-sequence-number" : 2, + "last-updated-ms" : 1760585153998, + "last-column-id" : 3, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "id", + "required" : true, + "type" : "int" + }, { + "id" : 2, + "name" : "name", + "required" : false, + "type" : "string" + }, { + "id" : 3, + "name" : "bir", + "required" : false, + "type" : "date" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ { + "name" : "name", + "transform" : "identity", + "source-id" : 2, + "field-id" : 1000 + } ] + } ], + "last-partition-id" : 1000, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 4416477559988274704, + "refs" : { + "main" : { + "snapshot-id" : 4416477559988274704, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 6447922032991303611, + "timestamp-ms" : 1760585153811, + "summary" : { + "operation" : "append", + "added-data-files" : "4", + "added-records" : "4", + "added-files-size" : "3568", + "changed-partition-count" : "4", + "total-records" : "4", + "total-files-size" : "3568", + "total-data-files" : "4", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" + }, + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-6447922032991303611-1-2b5d4ce3-154e-4498-a648-89be96ebd74a.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 4416477559988274704, + "parent-snapshot-id" : 6447922032991303611, + "timestamp-ms" : 1760585153998, + "summary" : { + "operation" : "delete", + "added-equality-delete-files" : "1", + "added-delete-files" : "1", + "added-files-size" : "466", + "added-equality-deletes" : "1", + "changed-partition-count" : "1", + "total-records" : "4", + "total-files-size" : "4034", + "total-data-files" : "4", + "total-delete-files" : "1", + "total-position-deletes" : "0", + "total-equality-deletes" : "1", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" + }, + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-4416477559988274704-1-b096df95-e029-4163-a7b4-762554ad0da9.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "partition-statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1760585153811, + "snapshot-id" : 6447922032991303611 + }, { + "timestamp-ms" : 1760585153998, + "snapshot-id" : 4416477559988274704 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1760585152178, + "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1760585153811, + "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v2.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v4.metadata.json b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v4.metadata.json new file mode 100644 index 000000000..d87faca90 --- /dev/null +++ b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v4.metadata.json @@ -0,0 +1,142 @@ +{ + "format-version" : 2, + "table-uuid" : "31ad6764-1255-4aa3-bff4-9a27a25d69a7", + "location" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned", + "last-sequence-number" : 3, + "last-updated-ms" : 1760585154134, + "last-column-id" : 3, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "id", + "required" : true, + "type" : "int" + }, { + "id" : 2, + "name" : "name", + "required" : false, + "type" : "string" + }, { + "id" : 3, + "name" : "bir", + "required" : false, + "type" : "date" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ { + "name" : "name", + "transform" : "identity", + "source-id" : 2, + "field-id" : 1000 + } ] + } ], + "last-partition-id" : 1000, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 6022113614852838397, + "refs" : { + "main" : { + "snapshot-id" : 6022113614852838397, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 6447922032991303611, + "timestamp-ms" : 1760585153811, + "summary" : { + "operation" : "append", + "added-data-files" : "4", + "added-records" : "4", + "added-files-size" : "3568", + "changed-partition-count" : "4", + "total-records" : "4", + "total-files-size" : "3568", + "total-data-files" : "4", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" + }, + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-6447922032991303611-1-2b5d4ce3-154e-4498-a648-89be96ebd74a.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 4416477559988274704, + "parent-snapshot-id" : 6447922032991303611, + "timestamp-ms" : 1760585153998, + "summary" : { + "operation" : "delete", + "added-equality-delete-files" : "1", + "added-delete-files" : "1", + "added-files-size" : "466", + "added-equality-deletes" : "1", + "changed-partition-count" : "1", + "total-records" : "4", + "total-files-size" : "4034", + "total-data-files" : "4", + "total-delete-files" : "1", + "total-position-deletes" : "0", + "total-equality-deletes" : "1", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" + }, + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-4416477559988274704-1-b096df95-e029-4163-a7b4-762554ad0da9.avro", + "schema-id" : 0 + }, { + "sequence-number" : 3, + "snapshot-id" : 6022113614852838397, + "parent-snapshot-id" : 4416477559988274704, + "timestamp-ms" : 1760585154134, + "summary" : { + "operation" : "delete", + "added-equality-delete-files" : "1", + "added-delete-files" : "1", + "added-files-size" : "695", + "added-equality-deletes" : "1", + "changed-partition-count" : "1", + "total-records" : "4", + "total-files-size" : "4729", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "0", + "total-equality-deletes" : "2", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" + }, + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-6022113614852838397-1-36f9b154-c2d2-411f-9f29-9d23d46d688b.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "partition-statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1760585153811, + "snapshot-id" : 6447922032991303611 + }, { + "timestamp-ms" : 1760585153998, + "snapshot-id" : 4416477559988274704 + }, { + "timestamp-ms" : 1760585154134, + "snapshot-id" : 6022113614852838397 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1760585152178, + "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1760585153811, + "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1760585153998, + "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v3.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v5.metadata.json b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v5.metadata.json new file mode 100644 index 000000000..a8374a5b1 --- /dev/null +++ b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v5.metadata.json @@ -0,0 +1,169 @@ +{ + "format-version" : 2, + "table-uuid" : "31ad6764-1255-4aa3-bff4-9a27a25d69a7", + "location" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned", + "last-sequence-number" : 4, + "last-updated-ms" : 1760585154287, + "last-column-id" : 3, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "id", + "required" : true, + "type" : "int" + }, { + "id" : 2, + "name" : "name", + "required" : false, + "type" : "string" + }, { + "id" : 3, + "name" : "bir", + "required" : false, + "type" : "date" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ { + "name" : "name", + "transform" : "identity", + "source-id" : 2, + "field-id" : 1000 + } ] + } ], + "last-partition-id" : 1000, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 883882341917253211, + "refs" : { + "main" : { + "snapshot-id" : 883882341917253211, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 6447922032991303611, + "timestamp-ms" : 1760585153811, + "summary" : { + "operation" : "append", + "added-data-files" : "4", + "added-records" : "4", + "added-files-size" : "3568", + "changed-partition-count" : "4", + "total-records" : "4", + "total-files-size" : "3568", + "total-data-files" : "4", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" + }, + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-6447922032991303611-1-2b5d4ce3-154e-4498-a648-89be96ebd74a.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 4416477559988274704, + "parent-snapshot-id" : 6447922032991303611, + "timestamp-ms" : 1760585153998, + "summary" : { + "operation" : "delete", + "added-equality-delete-files" : "1", + "added-delete-files" : "1", + "added-files-size" : "466", + "added-equality-deletes" : "1", + "changed-partition-count" : "1", + "total-records" : "4", + "total-files-size" : "4034", + "total-data-files" : "4", + "total-delete-files" : "1", + "total-position-deletes" : "0", + "total-equality-deletes" : "1", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" + }, + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-4416477559988274704-1-b096df95-e029-4163-a7b4-762554ad0da9.avro", + "schema-id" : 0 + }, { + "sequence-number" : 3, + "snapshot-id" : 6022113614852838397, + "parent-snapshot-id" : 4416477559988274704, + "timestamp-ms" : 1760585154134, + "summary" : { + "operation" : "delete", + "added-equality-delete-files" : "1", + "added-delete-files" : "1", + "added-files-size" : "695", + "added-equality-deletes" : "1", + "changed-partition-count" : "1", + "total-records" : "4", + "total-files-size" : "4729", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "0", + "total-equality-deletes" : "2", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" + }, + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-6022113614852838397-1-36f9b154-c2d2-411f-9f29-9d23d46d688b.avro", + "schema-id" : 0 + }, { + "sequence-number" : 4, + "snapshot-id" : 883882341917253211, + "parent-snapshot-id" : 6022113614852838397, + "timestamp-ms" : 1760585154287, + "summary" : { + "operation" : "append", + "added-data-files" : "2", + "added-records" : "2", + "added-files-size" : "1783", + "changed-partition-count" : "2", + "total-records" : "6", + "total-files-size" : "6512", + "total-data-files" : "6", + "total-delete-files" : "2", + "total-position-deletes" : "0", + "total-equality-deletes" : "2", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" + }, + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-883882341917253211-1-ec04e9ce-e03b-4fbe-a707-3645c1938319.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "partition-statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1760585153811, + "snapshot-id" : 6447922032991303611 + }, { + "timestamp-ms" : 1760585153998, + "snapshot-id" : 4416477559988274704 + }, { + "timestamp-ms" : 1760585154134, + "snapshot-id" : 6022113614852838397 + }, { + "timestamp-ms" : 1760585154287, + "snapshot-id" : 883882341917253211 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1760585152178, + "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1760585153811, + "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1760585153998, + "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1760585154134, + "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v4.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v6.metadata.json b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v6.metadata.json new file mode 100644 index 000000000..8d755c8ed --- /dev/null +++ b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v6.metadata.json @@ -0,0 +1,197 @@ +{ + "format-version" : 2, + "table-uuid" : "31ad6764-1255-4aa3-bff4-9a27a25d69a7", + "location" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned", + "last-sequence-number" : 5, + "last-updated-ms" : 1760585154430, + "last-column-id" : 3, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "id", + "required" : true, + "type" : "int" + }, { + "id" : 2, + "name" : "name", + "required" : false, + "type" : "string" + }, { + "id" : 3, + "name" : "bir", + "required" : false, + "type" : "date" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ { + "name" : "name", + "transform" : "identity", + "source-id" : 2, + "field-id" : 1000 + } ] + } ], + "last-partition-id" : 1000, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 3138177759849079112, + "refs" : { + "main" : { + "snapshot-id" : 3138177759849079112, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 6447922032991303611, + "timestamp-ms" : 1760585153811, + "summary" : { + "operation" : "append", + "added-data-files" : "4", + "added-records" : "4", + "added-files-size" : "3568", + "changed-partition-count" : "4", + "total-records" : "4", + "total-files-size" : "3568", + "total-data-files" : "4", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" + }, + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-6447922032991303611-1-2b5d4ce3-154e-4498-a648-89be96ebd74a.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 4416477559988274704, + "parent-snapshot-id" : 6447922032991303611, + "timestamp-ms" : 1760585153998, + "summary" : { + "operation" : "delete", + "added-equality-delete-files" : "1", + "added-delete-files" : "1", + "added-files-size" : "466", + "added-equality-deletes" : "1", + "changed-partition-count" : "1", + "total-records" : "4", + "total-files-size" : "4034", + "total-data-files" : "4", + "total-delete-files" : "1", + "total-position-deletes" : "0", + "total-equality-deletes" : "1", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" + }, + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-4416477559988274704-1-b096df95-e029-4163-a7b4-762554ad0da9.avro", + "schema-id" : 0 + }, { + "sequence-number" : 3, + "snapshot-id" : 6022113614852838397, + "parent-snapshot-id" : 4416477559988274704, + "timestamp-ms" : 1760585154134, + "summary" : { + "operation" : "delete", + "added-equality-delete-files" : "1", + "added-delete-files" : "1", + "added-files-size" : "695", + "added-equality-deletes" : "1", + "changed-partition-count" : "1", + "total-records" : "4", + "total-files-size" : "4729", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "0", + "total-equality-deletes" : "2", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" + }, + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-6022113614852838397-1-36f9b154-c2d2-411f-9f29-9d23d46d688b.avro", + "schema-id" : 0 + }, { + "sequence-number" : 4, + "snapshot-id" : 883882341917253211, + "parent-snapshot-id" : 6022113614852838397, + "timestamp-ms" : 1760585154287, + "summary" : { + "operation" : "append", + "added-data-files" : "2", + "added-records" : "2", + "added-files-size" : "1783", + "changed-partition-count" : "2", + "total-records" : "6", + "total-files-size" : "6512", + "total-data-files" : "6", + "total-delete-files" : "2", + "total-position-deletes" : "0", + "total-equality-deletes" : "2", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" + }, + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-883882341917253211-1-ec04e9ce-e03b-4fbe-a707-3645c1938319.avro", + "schema-id" : 0 + }, { + "sequence-number" : 5, + "snapshot-id" : 3138177759849079112, + "parent-snapshot-id" : 883882341917253211, + "timestamp-ms" : 1760585154430, + "summary" : { + "operation" : "delete", + "added-equality-delete-files" : "1", + "added-delete-files" : "1", + "added-files-size" : "466", + "added-equality-deletes" : "1", + "changed-partition-count" : "1", + "total-records" : "6", + "total-files-size" : "6978", + "total-data-files" : "6", + "total-delete-files" : "3", + "total-position-deletes" : "0", + "total-equality-deletes" : "3", + "iceberg-version" : "Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)" + }, + "manifest-list" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/snap-3138177759849079112-1-ff275eec-b23b-435f-a6a5-1ab7ce1c8749.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "partition-statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1760585153811, + "snapshot-id" : 6447922032991303611 + }, { + "timestamp-ms" : 1760585153998, + "snapshot-id" : 4416477559988274704 + }, { + "timestamp-ms" : 1760585154134, + "snapshot-id" : 6022113614852838397 + }, { + "timestamp-ms" : 1760585154287, + "snapshot-id" : 883882341917253211 + }, { + "timestamp-ms" : 1760585154430, + "snapshot-id" : 3138177759849079112 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1760585152178, + "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1760585153811, + "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1760585153998, + "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1760585154134, + "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1760585154287, + "metadata-file" : "data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/v5.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/version-hint.text b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/version-hint.text new file mode 100644 index 000000000..62f945751 --- /dev/null +++ b/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned/metadata/version-hint.text @@ -0,0 +1 @@ +6 \ No newline at end of file diff --git a/src/deletes/equality_delete.cpp b/src/deletes/equality_delete.cpp index 825800098..5b567e08c 100644 --- a/src/deletes/equality_delete.cpp +++ b/src/deletes/equality_delete.cpp @@ -70,6 +70,19 @@ void IcebergMultiFileList::ScanEqualityDeleteFile(const IcebergManifestEntry &en auto global_id = column_index.GetPrimaryIndex(); global_id_to_result_id[global_id] = i; } + //! For the column(s) that are needed but aren't referenced, add them to the map + for (auto field_id : entry.equality_ids) { + auto global_column_id = id_to_global_column[field_id]; + ColumnIndex equality_index(global_column_id); + //! Check if the column needed by the equality delete is present + if (std::find(column_indexes.begin(), column_indexes.end(), equality_index) != column_indexes.end()) { + continue; + } + auto new_result_id = column_indexes.size() + equality_id_to_result_id.size(); + //! Create or get the result id mapping for this equality id + auto result_id = equality_id_to_result_id.emplace(field_id, new_result_id).first->second; + global_id_to_result_id[global_column_id] = result_id; + } //! Take only the relevant columns from the result InitializeFromOtherChunk(result, result_p, column_ids); @@ -85,16 +98,14 @@ void IcebergMultiFileList::ScanEqualityDeleteFile(const IcebergManifestEntry &en auto &vec = result.data[col_idx]; auto it = global_id_to_result_id.find(global_column_id); - if (it == global_id_to_result_id.end()) { - throw NotImplementedException("Equality deletes need the relevant columns to be selected"); - } - global_column_id = it->second; + D_ASSERT(it != global_id_to_result_id.end()); + auto result_column_id = it->second; for (idx_t i = 0; i < count; i++) { auto &row = rows[i]; auto constant = vec.GetValue(i); unique_ptr equality_filter; - auto bound_ref = make_uniq(col.type, global_column_id); + auto bound_ref = make_uniq(col.type, result_column_id); if (!constant.IsNull()) { //! Create a COMPARE_NOT_EQUAL expression equality_filter = diff --git a/src/iceberg_functions/iceberg_multi_file_list.cpp b/src/iceberg_functions/iceberg_multi_file_list.cpp index 8d3e7ce79..8e7232a5e 100644 --- a/src/iceberg_functions/iceberg_multi_file_list.cpp +++ b/src/iceberg_functions/iceberg_multi_file_list.cpp @@ -528,6 +528,38 @@ bool IcebergMultiFileList::ManifestMatchesFilter(const IcebergManifest &manifest return true; } +vector> +IcebergMultiFileList::GetEqualityDeletesForFile(const IcebergManifestEntry &data_file) const { + vector> result; + + //! Look through all the equality delete files with a *higher* sequence number + auto &metadata = GetMetadata(); + auto it = equality_delete_data.upper_bound(data_file.sequence_number); + for (; it != equality_delete_data.end(); it++) { + auto &files = it->second->files; + for (auto &file : files) { + auto &partition_spec = metadata.partition_specs.at(file.partition_spec_id); + if (partition_spec.IsPartitioned()) { + if (file.partition_spec_id != data_file.partition_spec_id) { + //! Not unpartitioned and the data does not share the same partition spec as the delete, skip the + //! delete file. + continue; + } + D_ASSERT(file.partition_values.size() == data_file.partition_values.size()); + for (idx_t i = 0; i < file.partition_values.size(); i++) { + if (file.partition_values[i] != data_file.partition_values[i]) { + //! Same partition spec id, but the partitioning information doesn't match, delete file doesn't + //! apply. + continue; + } + } + } + result.insert(result.end(), file.rows.begin(), file.rows.end()); + } + } + return result; +} + void IcebergMultiFileList::InitializeFiles(lock_guard &guard) { if (initialized) { return; diff --git a/src/iceberg_functions/iceberg_multi_file_reader.cpp b/src/iceberg_functions/iceberg_multi_file_reader.cpp index 12493ffdd..3b6df9e85 100644 --- a/src/iceberg_functions/iceberg_multi_file_reader.cpp +++ b/src/iceberg_functions/iceberg_multi_file_reader.cpp @@ -250,6 +250,55 @@ static void ApplyPartitionConstants(const IcebergMultiFileList &multi_file_list, } } +ReaderInitializeType IcebergMultiFileReader::InitializeReader(MultiFileReaderData &reader_data, + const MultiFileBindData &bind_data, + const vector &global_columns, + const vector &global_column_ids, + optional_ptr table_filters, + ClientContext &context, MultiFileGlobalState &gstate) { + + FinalizeBind(reader_data, bind_data.file_options, bind_data.reader_bind, global_columns, global_column_ids, context, + gstate.multi_file_reader_state.get()); + + //! Create a mapping from field_id -> column index + unordered_map id_to_global_column; + for (column_t i = 0; i < global_columns.size(); i++) { + auto &col = global_columns[i]; + D_ASSERT(!col.identifier.IsNull()); + id_to_global_column[col.identifier.GetValue()] = i; + } + + //! Get the data file that we're preparing to scan + const auto &multi_file_list = gstate.file_list.Cast(); + auto &reader = *reader_data.reader; + auto file_id = reader.file_list_idx.GetIndex(); + auto &data_file = multi_file_list.data_files[file_id]; + + //! Collect all the equality delete ids needed + unordered_set equality_delete_ids; + auto delete_rows = multi_file_list.GetEqualityDeletesForFile(data_file); + for (auto &row : delete_rows) { + auto &filters = row.get().filters; + for (auto &filter : filters) { + equality_delete_ids.insert(filter.first); + } + } + + //! Add the columns needed by the equality deletes if not present + auto new_global_column_ids = global_column_ids; + auto &equality_to_result_id = multi_file_list.equality_id_to_result_id; + new_global_column_ids.resize(global_column_ids.size() + equality_to_result_id.size()); + + for (auto it : equality_to_result_id) { + auto global_column_id = id_to_global_column[it.first]; + ColumnIndex equality_index(global_column_id); + new_global_column_ids[it.second] = equality_index; + } + + return CreateMapping(context, reader_data, global_columns, new_global_column_ids, table_filters, gstate.file_list, + bind_data.reader_bind, bind_data.virtual_columns); +} + void IcebergMultiFileReader::FinalizeBind(MultiFileReaderData &reader_data, const MultiFileOptions &file_options, const MultiFileReaderBindData &options, const vector &global_columns, @@ -292,33 +341,7 @@ void IcebergMultiFileReader::ApplyEqualityDeletes(ClientContext &context, DataCh const IcebergMultiFileList &multi_file_list, const IcebergManifestEntry &data_file, const vector &local_columns) { - vector> delete_rows; - - auto &metadata = multi_file_list.GetMetadata(); - auto delete_data_it = multi_file_list.equality_delete_data.upper_bound(data_file.sequence_number); - //! Look through all the equality delete files with a *higher* sequence number - for (; delete_data_it != multi_file_list.equality_delete_data.end(); delete_data_it++) { - auto &files = delete_data_it->second->files; - for (auto &file : files) { - auto &partition_spec = metadata.partition_specs.at(file.partition_spec_id); - if (partition_spec.IsPartitioned()) { - if (file.partition_spec_id != data_file.partition_spec_id) { - //! Not unpartitioned and the data does not share the same partition spec as the delete, skip the - //! delete file. - continue; - } - D_ASSERT(file.partition_values.size() == data_file.partition_values.size()); - for (idx_t i = 0; i < file.partition_values.size(); i++) { - if (file.partition_values[i] != data_file.partition_values[i]) { - //! Same partition spec id, but the partitioning information doesn't match, delete file doesn't - //! apply. - continue; - } - } - } - delete_rows.insert(delete_rows.end(), file.rows.begin(), file.rows.end()); - } - } + auto delete_rows = multi_file_list.GetEqualityDeletesForFile(data_file); if (delete_rows.empty()) { return; @@ -359,9 +382,9 @@ void IcebergMultiFileReader::ApplyEqualityDeletes(ClientContext &context, DataCh } else { equalities.push_back(make_uniq(Value::BOOLEAN(true))); } - } else { - equalities.push_back(expression->Copy()); + continue; } + equalities.push_back(expression->Copy()); } unique_ptr filter; @@ -399,17 +422,34 @@ void IcebergMultiFileReader::FinalizeChunk(ClientContext &context, const MultiFi DataChunk &input_chunk, DataChunk &output_chunk, ExpressionExecutor &executor, optional_ptr global_state) { - // Base class finalization first + D_ASSERT(global_state); + // Get the metadata for this file + const auto &multi_file_list = global_state->file_list->Cast(); + + //! Add the extra equality delete fields to output chunk. + idx_t diff = executor.expressions.size() - output_chunk.ColumnCount(); + (void)diff; + D_ASSERT(diff == multi_file_list.equality_id_to_result_id.size()); + if (diff > 0) { + int32_t start = input_chunk.ColumnCount() - diff; + for (int32_t i = 0; i < diff; i++) { + output_chunk.data.emplace_back(input_chunk.data[start + i]); + } + } + + //! Base class finalization first MultiFileReader::FinalizeChunk(context, bind_data, reader, reader_data, input_chunk, output_chunk, executor, global_state); - D_ASSERT(global_state); - // Get the metadata for this file - const auto &multi_file_list = dynamic_cast(*global_state->file_list); + auto &local_columns = reader.columns; auto file_id = reader.file_list_idx.GetIndex(); auto &data_file = multi_file_list.data_files[file_id]; - auto &local_columns = reader.columns; ApplyEqualityDeletes(context, output_chunk, multi_file_list, data_file, local_columns); + + //! Remove the extra columns we added to perform the equality delete filtering + for (idx_t i = 0; i < diff; i++) { + output_chunk.data.pop_back(); + } } bool IcebergMultiFileReader::ParseOption(const string &key, const Value &val, MultiFileOptions &options, diff --git a/src/include/iceberg_multi_file_list.hpp b/src/include/iceberg_multi_file_list.hpp index b56380a9c..2c5a371f0 100644 --- a/src/include/iceberg_multi_file_list.hpp +++ b/src/include/iceberg_multi_file_list.hpp @@ -53,6 +53,8 @@ struct IcebergMultiFileList : public MultiFileList { unique_ptr GetPositionalDeletesForFile(const string &file_path) const; void ProcessDeletes(const vector &global_columns, const vector &column_indexes) const; + vector> + GetEqualityDeletesForFile(const IcebergManifestEntry &data_file) const; public: //! MultiFileList API @@ -117,6 +119,8 @@ struct IcebergMultiFileList : public MultiFileList { //! All equality deletes with sequence numbers higher than that of the data_file apply to that data_file mutable map> equality_delete_data; mutable mutex delete_lock; + //! The columns needed by the equality deletes that aren't referenced by the scan + mutable unordered_map equality_id_to_result_id; bool initialized = false; const IcebergOptions &options; diff --git a/src/include/iceberg_multi_file_reader.hpp b/src/include/iceberg_multi_file_reader.hpp index 00377e192..c7652c0c9 100644 --- a/src/include/iceberg_multi_file_reader.hpp +++ b/src/include/iceberg_multi_file_reader.hpp @@ -50,6 +50,11 @@ struct IcebergMultiFileReader : public MultiFileReader { const MultiFileReaderBindData &bind_data, const MultiFileList &file_list, const vector &global_columns, const vector &global_column_ids) override; + ReaderInitializeType InitializeReader(MultiFileReaderData &reader_data, const MultiFileBindData &bind_data, + const vector &global_columns, + const vector &global_column_ids, + optional_ptr table_filters, ClientContext &context, + MultiFileGlobalState &gstate) override; void FinalizeBind(MultiFileReaderData &reader_data, const MultiFileOptions &file_options, const MultiFileReaderBindData &options, const vector &global_columns, const vector &global_column_ids, ClientContext &context, diff --git a/test/sql/local/equality_deletes.test b/test/sql/local/equality_deletes.test index 45d0f31ea..c670a9bc1 100644 --- a/test/sql/local/equality_deletes.test +++ b/test/sql/local/equality_deletes.test @@ -10,41 +10,149 @@ require iceberg require httpfs +# for unpartitioned table: +# create table by spark sql: create table mytable_partitioned(id int,name string,bir date ); + # First snapshot: -# add (id=1, name=a), (id=2, name=b), (id=3, name=b) -# Result: [(id=1, name=a), (id=2, name=b), (id=3, name=b)] +# insert into mytable values(1,'a',date '2025-01-01'),(2,'b',date '2025-01-02'),(3,'c',date '2025-01-03'),(4,'d',date '2025-01-04'); +# Result: [(id=1, name=a, bir=2025-01-01), (id=2, name=b, bir=2025-01-02), (id=3, name=c, bir=2025-01-03), (id=4, name=d, bir=2025-01-04)] # Second snapshot: # delete where name=b -# Result: [(id=1, name=a)] +# Result: [(id=1, name=a, bir=2025-01-01), (id=3, name=c, bir=2025-01-03), (id=4, name=d, bir=2025-01-04)] # Third snapshot: -# add (id=1, name=a), (id=2, name=b), (id=3, name=b) -# Result: [(id=1, name=a), (id=2, name=b), (id=1, name=b), (id=1, name=a)] +# delete where id = 1 +# Result: [ (id=3, name=c, bir=2025-01-03), (id=4, name=d, bir=2025-01-04)] # Fourth snapshot: -# delete where id=1 AND name=a -# Result: [(id=2, name=b), (id=1, name=b)] +# delete where id = 3 and name = c +# Result: [(id=4, name=d, bir=2025-01-04)] -query II +# Fifth snapshot: +# insert into mytable values(5,'e',date '2025-01-05'),(6,'f',date '2025-01-06'); +# Result:[(id=4, name=d, bir=2025-01-04),(5,'e',date '2025-01-05'),(6,'f',date '2025-01-06')] + +# Sixth snapshot: +# delete where name = f +# Result:[(id=4, name=d, bir=2025-01-04),(5,'e',date '2025-01-05')] + + +# all +query III SELECT * FROM ICEBERG_SCAN('__WORKING_DIRECTORY__/data/persistent/equality_deletes/warehouse/mydb/mytable'); ---- -2 b -1 b +5 e 2025-01-05 +4 d 2025-01-04 + +# only equality delete fields query II -SELECT name, id FROM ICEBERG_SCAN('data/persistent/equality_deletes/warehouse/mydb/mytable'); +SELECT id,name FROM ICEBERG_SCAN('__WORKING_DIRECTORY__/data/persistent/equality_deletes/warehouse/mydb/mytable'); ---- -b 2 -b 1 +5 e +4 d -query III -SELECT filename[17:32], name, id FROM ICEBERG_SCAN('data/persistent/equality_deletes/warehouse/mydb/mytable'); + +# no equality delete fields +query I +SELECT bir FROM ICEBERG_SCAN('__WORKING_DIRECTORY__/data/persistent/equality_deletes/warehouse/mydb/mytable'); ---- -equality_deletes b 2 -equality_deletes b 1 +2025-01-05 +2025-01-04 + +# partial equality delete field +query I +SELECT name FROM ICEBERG_SCAN('__WORKING_DIRECTORY__/data/persistent/equality_deletes/warehouse/mydb/mytable'); +---- +e +d + +# partial equality delete field and other +query II +SELECT name,bir FROM ICEBERG_SCAN('__WORKING_DIRECTORY__/data/persistent/equality_deletes/warehouse/mydb/mytable'); +---- +e 2025-01-05 +d 2025-01-04 + +# contain metadata +query II +SELECT filename[17:32], name FROM ICEBERG_SCAN('__WORKING_DIRECTORY__/data/persistent/equality_deletes/warehouse/mydb/mytable'); +---- +equality_deletes e +equality_deletes d + +query I +SELECT count(*) FROM ICEBERG_SCAN('__WORKING_DIRECTORY__/data/persistent/equality_deletes/warehouse/mydb/mytable'); +---- +2 + + +# for partitioned table: +# create table : mytable_partitioned(id int,name string,bir date ) partitioned by (name); + +# First snapshot: +# insert into mytable_partitioned values(1,'a',date '2025-01-01'),(2,'b',date '2025-01-02'),(3,'c',date '2025-01-03'),(4,'d',date '2025-01-04'); +# Result: [(id=1, name=a, bir=2025-01-01), (id=2, name=b, bir=2025-01-02), (id=3, name=c, bir=2025-01-03), (id=4, name=d, bir=2025-01-04)] -statement error -SELECT filename[17:32], name FROM ICEBERG_SCAN('data/persistent/equality_deletes/warehouse/mydb/mytable'); +# Second snapshot: +# delete where name=b +# Result: [(id=1, name=a, bir=2025-01-01), (id=3, name=c, bir=2025-01-03), (id=4, name=d, bir=2025-01-04)] + +# Third snapshot: +# delete where id = 3 and name = c +# Result: [(id=1, name=a, bir=2025-01-01), (id=4, name=d, bir=2025-01-04)] + +# Fourth snapshot: +# insert into mytable_partitioned values(5,'e',date '2025-01-05'),(6,'f',date '2025-01-06'); +# Result:[(id=1, name=a, bir=2025-01-01), (id=4, name=d, bir=2025-01-04), (5,'e',date '2025-01-05'),(6,'f',date '2025-01-06')] + +# Fifth snapshot: +# delete where name = f +# Result:[(id=1, name=a, bir=2025-01-01), (id=4, name=d, bir=2025-01-04), (5,'e',date '2025-01-05')] + +# only equality delete fields +query I +SELECT name FROM ICEBERG_SCAN('__WORKING_DIRECTORY__/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned'); +---- +e +a +d + +# no equality delete field +query I +SELECT id FROM ICEBERG_SCAN('__WORKING_DIRECTORY__/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned'); +---- +5 +1 +4 + +# partial equality delete field and other +query II +SELECT name,bir FROM ICEBERG_SCAN('__WORKING_DIRECTORY__/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned'); +---- +e 2025-01-05 +a 2025-01-01 +d 2025-01-04 + + +# contain metadata +query II +SELECT filename[17:32], name FROM ICEBERG_SCAN('__WORKING_DIRECTORY__/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned'); +---- +equality_deletes e +equality_deletes a +equality_deletes d + +query I +SELECT count(*) FROM ICEBERG_SCAN('__WORKING_DIRECTORY__/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned'); +---- +3 + +# all +query III +SELECT * FROM ICEBERG_SCAN('__WORKING_DIRECTORY__/data/persistent/equality_deletes/warehouse/mydb/mytable_partitioned'); ---- -Not implemented Error: Equality deletes need the relevant columns to be selected +5 e 2025-01-05 +1 a 2025-01-01 +4 d 2025-01-04 \ No newline at end of file