From 32e2f84ab797a98010c774a8ee3dbe61b6c17782 Mon Sep 17 00:00:00 2001 From: Corwin Joy Date: Thu, 8 May 2025 20:04:32 -0700 Subject: [PATCH 1/8] Add a time_travel test with an uncommited delta. Upgrade load_with_datetime to ignore any uncommited deltas in any subdirectory of delta_log. Signed-off-by: Corwin Joy --- .../core/src/kernel/snapshot/log_segment.rs | 3 ++ crates/core/src/logstore/mod.rs | 15 +++++++- crates/core/src/table/mod.rs | 15 +++++++- crates/core/tests/time_travel.rs | 35 ++++++++---------- .../_delta_log/.tmp/00000000000000000004.json | 4 ++ .../_delta_log/00000000000000000000.json | 9 +++++ .../_delta_log/00000000000000000001.json | 27 ++++++++++++++ .../_delta_log/00000000000000000002.json | 29 +++++++++++++++ .../_delta_log/00000000000000000003.json | 5 +++ ...4768-a43c-3eda0d2a499d-c000.snappy.parquet | Bin 0 -> 262 bytes ...41fe-a8f0-e65b746382dd-c000.snappy.parquet | Bin 0 -> 262 bytes ...4ca1-9074-a278c24c4449-c000.snappy.parquet | Bin 0 -> 262 bytes ...4790-b38a-6ee7e24456b1-c000.snappy.parquet | Bin 0 -> 262 bytes ...40ba-adae-ce66d1fcaef6-c000.snappy.parquet | Bin 0 -> 429 bytes ...47c3-88a6-abcfb049d3b4-c000.snappy.parquet | Bin 0 -> 429 bytes ...4df2-9c85-f34969ad3aa9-c000.snappy.parquet | Bin 0 -> 429 bytes ...4e2b-a42b-9731b2e490ae-c000.snappy.parquet | Bin 0 -> 429 bytes ...4c2c-a923-f6f89930a5c1-c000.snappy.parquet | Bin 0 -> 429 bytes ...4f13-8d07-599a21197296-c000.snappy.parquet | Bin 0 -> 429 bytes ...4562-98f6-5e6cfa3ae45d-c000.snappy.parquet | Bin 0 -> 429 bytes ...420c-861f-5a649e3d9674-c000.snappy.parquet | Bin 0 -> 429 bytes ...4581-b3cc-84502b0c314f-c000.snappy.parquet | Bin 0 -> 429 bytes ...4bd9-a080-73e02491c643-c000.snappy.parquet | Bin 0 -> 429 bytes ...4dda-8498-7bfb2940713b-c000.snappy.parquet | Bin 0 -> 429 bytes ...482d-b9a1-7e717b67f294-c000.snappy.parquet | Bin 0 -> 429 bytes ...41b6-81ef-5223cf40f025-c000.snappy.parquet | Bin 0 -> 429 bytes ...4b00-9e83-e31021a93cf9-c000.snappy.parquet | Bin 0 -> 429 bytes ...40af-98f5-2fccfa1b123f-c000.snappy.parquet | Bin 0 -> 429 bytes ...45b1-8d34-a0018cf73b70-c000.snappy.parquet | Bin 0 -> 429 bytes ...49c1-a888-81565a40161d-c000.snappy.parquet | Bin 0 -> 429 bytes ...4148-8475-e21d2a2935f8-c000.snappy.parquet | Bin 0 -> 429 bytes ...492b-9403-53e33b3778ac-c000.snappy.parquet | Bin 0 -> 429 bytes ...4643-816f-cbd30a3f8c1b-c000.snappy.parquet | Bin 0 -> 429 bytes ...48ee-93f6-0acf11199a0d-c000.snappy.parquet | Bin 0 -> 429 bytes ...4f4c-be15-135e15b398f4-c000.snappy.parquet | Bin 0 -> 429 bytes ...4fde-9acd-623e740be992-c000.snappy.parquet | Bin 0 -> 429 bytes ...4f34-8cd4-6688aad8585d-c000.snappy.parquet | Bin 0 -> 429 bytes ...48ce-9909-78da7294ffbd-c000.snappy.parquet | Bin 0 -> 429 bytes ...4a90-a8b4-578c9e9a218d-c000.snappy.parquet | Bin 0 -> 429 bytes ...43fb-b07f-975d2226b800-c000.snappy.parquet | Bin 0 -> 429 bytes ...4193-aa43-993cdf937fd3-c000.snappy.parquet | Bin 0 -> 429 bytes ...4871-9613-f5ad1940b689-c000.snappy.parquet | Bin 0 -> 429 bytes ...48fb-a03d-e356fcd1564a-c000.snappy.parquet | Bin 0 -> 429 bytes ...4c02-befa-90f056c2d77a-c000.snappy.parquet | Bin 0 -> 429 bytes ...461d-a3d3-8dc112766ff5-c000.snappy.parquet | Bin 0 -> 429 bytes 45 files changed, 119 insertions(+), 23 deletions(-) create mode 100644 crates/test/tests/data/simple_table_uncommited/_delta_log/.tmp/00000000000000000004.json create mode 100644 crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000000.json create mode 100644 crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000001.json create mode 100644 crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000002.json create mode 100644 crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000003.json create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00000-2befed33-c358-4768-a43c-3eda0d2a499d-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00000-a72b1fb3-f2df-41fe-a8f0-e65b746382dd-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00000-a922ea3b-ffc2-4ca1-9074-a278c24c4449-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00000-c1777d7d-89d9-4790-b38a-6ee7e24456b1-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00000-f17fcbf5-e0dc-40ba-adae-ce66d1fcaef6-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00001-7891c33d-cedc-47c3-88a6-abcfb049d3b4-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00001-bb70d2ba-c196-4df2-9c85-f34969ad3aa9-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00001-c506e79a-0bf8-4e2b-a42b-9731b2e490ae-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00003-508ae4aa-801c-4c2c-a923-f6f89930a5c1-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00003-53f42606-6cda-4f13-8d07-599a21197296-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00004-315835fe-fb44-4562-98f6-5e6cfa3ae45d-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00004-80938522-09c0-420c-861f-5a649e3d9674-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00004-95c9bc2c-ac85-4581-b3cc-84502b0c314f-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00005-94a0861b-6455-4bd9-a080-73e02491c643-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00006-46f2ff20-eb5d-4dda-8498-7bfb2940713b-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00006-63ce9deb-bc0f-482d-b9a1-7e717b67f294-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00007-3a0e4727-de0d-41b6-81ef-5223cf40f025-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00007-94f725e2-3963-4b00-9e83-e31021a93cf9-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00011-42f838f9-a911-40af-98f5-2fccfa1b123f-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00045-332fe409-7705-45b1-8d34-a0018cf73b70-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00049-d3095817-de74-49c1-a888-81565a40161d-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00058-b462c4cb-0c48-4148-8475-e21d2a2935f8-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00068-90650739-6a8e-492b-9403-53e33b3778ac-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00069-c78b4dd8-f955-4643-816f-cbd30a3f8c1b-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00077-2fcb1c7c-5390-48ee-93f6-0acf11199a0d-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00107-3f6c2aa0-fc28-4f4c-be15-135e15b398f4-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00112-07fd790a-11dc-4fde-9acd-623e740be992-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00116-bc66759e-6381-4f34-8cd4-6688aad8585d-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00121-d8bc3e53-d2f2-48ce-9909-78da7294ffbd-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00128-b31c3b81-24da-4a90-a8b4-578c9e9a218d-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00140-e9b1971d-d708-43fb-b07f-975d2226b800-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00143-03ceb88e-5283-4193-aa43-993cdf937fd3-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00150-ec6643fc-4963-4871-9613-f5ad1940b689-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00154-4630673a-5227-48fb-a03d-e356fcd1564a-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00164-bf40481c-4afd-4c02-befa-90f056c2d77a-c000.snappy.parquet create mode 100644 crates/test/tests/data/simple_table_uncommited/part-00190-8ac0ae67-fb1d-461d-a3d3-8dc112766ff5-c000.snappy.parquet diff --git a/crates/core/src/kernel/snapshot/log_segment.rs b/crates/core/src/kernel/snapshot/log_segment.rs index 32a7a0e27e..f333d4ee97 100644 --- a/crates/core/src/kernel/snapshot/log_segment.rs +++ b/crates/core/src/kernel/snapshot/log_segment.rs @@ -480,6 +480,7 @@ async fn list_log_files_with_checkpoint( let version_prefix = format!("{:020}", cp.version); let start_from = log_root.child(version_prefix.as_str()); + // QUESTION: DOES THIS NEED TO BE FILTERED TO ELIMINATE temp subdirs? let files = fs_client .list_with_offset(Some(log_root), &start_from) .try_collect::>() @@ -542,6 +543,8 @@ pub(super) async fn list_log_files( let mut commit_files = Vec::with_capacity(25); let mut checkpoint_files = Vec::with_capacity(10); + + // QUESTION: Does this need to be filtered to exclude subdirs? for meta in fs_client .list_with_offset(Some(log_root), &start_from) .try_collect::>() diff --git a/crates/core/src/logstore/mod.rs b/crates/core/src/logstore/mod.rs index 21d84ff0d7..e05f9eec88 100644 --- a/crates/core/src/logstore/mod.rs +++ b/crates/core/src/logstore/mod.rs @@ -534,14 +534,25 @@ pub async fn get_latest_version( // list files to find max version let version = async { let mut max_version: i64 = version_start; - let prefix = Some(log_store.log_path()); + let prefix = log_store.log_path(); let offset_path = commit_uri_from_version(max_version); let object_store = log_store.object_store(None); - let mut files = object_store.list_with_offset(prefix, &offset_path); + let mut files = object_store.list_with_offset(Some(prefix), &offset_path); let mut empty_stream = true; while let Some(obj_meta) = files.next().await { let obj_meta = obj_meta?; + let location_path: Path = obj_meta.location.clone(); + let parts: Vec<_> = location_path.prefix_match(prefix).unwrap().collect(); + if parts.len() > 1 { + // Per the spec, ignore any files in subdirectories. + // Spark may create these as uncommited transactions which we don't want + // + // https://github.com/delta-io/delta/blob/master/PROTOCOL.md#delta-log-entries + // "Delta files are stored as JSON in a directory at the *root* of the table + // named _delta_log, and ... make up the log of all changes that have occurred to a table." + continue; + } if let Some(log_version) = extract_version_from_filename(obj_meta.location.as_ref()) { max_version = max(max_version, log_version); // also cache timestamp for version, for faster time-travel diff --git a/crates/core/src/table/mod.rs b/crates/core/src/table/mod.rs index 3ce8ba57d8..ba45706baa 100644 --- a/crates/core/src/table/mod.rs +++ b/crates/core/src/table/mod.rs @@ -564,13 +564,24 @@ impl DeltaTable { ) -> Result<(), DeltaTableError> { let mut min_version: i64 = -1; let log_store = self.log_store(); - let prefix = Some(log_store.log_path()); + let prefix = log_store.log_path(); let offset_path = commit_uri_from_version(min_version); let object_store = log_store.object_store(None); - let mut files = object_store.list_with_offset(prefix, &offset_path); + let mut files = object_store.list_with_offset(Some(prefix), &offset_path); while let Some(obj_meta) = files.next().await { let obj_meta = obj_meta?; + let location_path: Path = obj_meta.location.clone(); + let parts: Vec<_> = location_path.prefix_match(prefix).unwrap().collect(); + if parts.len() > 1 { + // Per the spec, ignore any files in subdirectories. + // Spark may create these as uncommited transactions which we don't want + // + // https://github.com/delta-io/delta/blob/master/PROTOCOL.md#delta-log-entries + // "Delta files are stored as JSON in a directory at the *root* of the table + // named _delta_log, and ... make up the log of all changes that have occurred to a table." + continue; + } if let Some(log_version) = extract_version_from_filename(obj_meta.location.as_ref()) { if min_version == -1 { min_version = log_version diff --git a/crates/core/tests/time_travel.rs b/crates/core/tests/time_travel.rs index 3277a41961..2bff87ded5 100644 --- a/crates/core/tests/time_travel.rs +++ b/crates/core/tests/time_travel.rs @@ -5,14 +5,17 @@ use std::time::SystemTime; #[tokio::test] async fn time_travel_by_ds() { + // test time travel on a table with an uncommited delta in a .tmp subfolder + // git does not preserve mtime, so we need to manually set it in the test - let log_dir = "../test/tests/data/simple_table/_delta_log"; + let log_dir = "../test/tests/data/simple_table_uncommited/_delta_log"; let log_mtime_pair = vec![ ("00000000000000000000.json", "2020-05-01T22:47:31-07:00"), ("00000000000000000001.json", "2020-05-02T22:47:31-07:00"), ("00000000000000000002.json", "2020-05-03T22:47:31-07:00"), ("00000000000000000003.json", "2020-05-04T22:47:31-07:00"), - ("00000000000000000004.json", "2020-05-05T22:47:31-07:00"), + // Final file is uncommitted by Spark and is in a .tmp subdir + (".tmp/00000000000000000004.json", "2020-05-05T22:47:31-07:00"), ]; for (fname, ds) in log_mtime_pair { let ts: SystemTime = ds_to_ts(ds).into(); @@ -23,7 +26,7 @@ async fn time_travel_by_ds() { } let mut table = deltalake_core::open_table_with_ds( - "../test/tests/data/simple_table", + "../test/tests/data/simple_table_uncommited", "2020-05-01T00:47:31-07:00", ) .await @@ -32,7 +35,7 @@ async fn time_travel_by_ds() { assert_eq!(table.version(), 0); table = deltalake_core::open_table_with_ds( - "../test/tests/data/simple_table", + "../test/tests/data/simple_table_uncommited", "2020-05-02T22:47:31-07:00", ) .await @@ -40,7 +43,7 @@ async fn time_travel_by_ds() { assert_eq!(table.version(), 1); table = deltalake_core::open_table_with_ds( - "../test/tests/data/simple_table", + "../test/tests/data/simple_table_uncommited", "2020-05-02T23:47:31-07:00", ) .await @@ -48,7 +51,7 @@ async fn time_travel_by_ds() { assert_eq!(table.version(), 1); table = deltalake_core::open_table_with_ds( - "../test/tests/data/simple_table", + "../test/tests/data/simple_table_uncommited", "2020-05-03T22:47:31-07:00", ) .await @@ -56,15 +59,17 @@ async fn time_travel_by_ds() { assert_eq!(table.version(), 2); table = deltalake_core::open_table_with_ds( - "../test/tests/data/simple_table", + "../test/tests/data/simple_table_uncommited", "2020-05-04T22:47:31-07:00", ) .await .unwrap(); assert_eq!(table.version(), 3); - + + // Versions beyond 3 should be ignored. + // This is an uncommited delta in the .tmp subdirectory table = deltalake_core::open_table_with_ds( - "../test/tests/data/simple_table", + "../test/tests/data/simple_table_uncommited", "2020-05-05T21:47:31-07:00", ) .await @@ -72,20 +77,12 @@ async fn time_travel_by_ds() { assert_eq!(table.version(), 3); table = deltalake_core::open_table_with_ds( - "../test/tests/data/simple_table", - "2020-05-05T22:47:31-07:00", - ) - .await - .unwrap(); - assert_eq!(table.version(), 4); - - table = deltalake_core::open_table_with_ds( - "../test/tests/data/simple_table", + "../test/tests/data/simple_table_uncommited", "2020-05-25T22:47:31-07:00", ) .await .unwrap(); - assert_eq!(table.version(), 4); + assert_eq!(table.version(), 3); } fn ds_to_ts(ds: &str) -> DateTime { diff --git a/crates/test/tests/data/simple_table_uncommited/_delta_log/.tmp/00000000000000000004.json b/crates/test/tests/data/simple_table_uncommited/_delta_log/.tmp/00000000000000000004.json new file mode 100644 index 0000000000..8911adfbb2 --- /dev/null +++ b/crates/test/tests/data/simple_table_uncommited/_delta_log/.tmp/00000000000000000004.json @@ -0,0 +1,4 @@ +{"commitInfo":{"timestamp":1587968626537,"operation":"DELETE","operationParameters":{"predicate":"[\"((`id` % CAST(2 AS BIGINT)) = CAST(0 AS BIGINT))\"]"},"readVersion":3,"isBlindAppend":false}} +{"remove":{"path":"part-00001-bb70d2ba-c196-4df2-9c85-f34969ad3aa9-c000.snappy.parquet","deletionTimestamp":1587968626536,"dataChange":true}} +{"remove":{"path":"part-00000-f17fcbf5-e0dc-40ba-adae-ce66d1fcaef6-c000.snappy.parquet","deletionTimestamp":1587968626536,"dataChange":true}} +{"add":{"path":"part-00000-2befed33-c358-4768-a43c-3eda0d2a499d-c000.snappy.parquet","partitionValues":{},"size":262,"modificationTime":1587968626000,"dataChange":true}} diff --git a/crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000000.json b/crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000000.json new file mode 100644 index 0000000000..47dd9bd9db --- /dev/null +++ b/crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000000.json @@ -0,0 +1,9 @@ +{"commitInfo":{"timestamp":1587968586154,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isBlindAppend":true}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"5fba94ed-9794-4965-ba6e-6ee3c0d22af9","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1587968585495}} +{"add":{"path":"part-00000-a72b1fb3-f2df-41fe-a8f0-e65b746382dd-c000.snappy.parquet","partitionValues":{},"size":262,"modificationTime":1587968586000,"dataChange":true}} +{"add":{"path":"part-00001-c506e79a-0bf8-4e2b-a42b-9731b2e490ae-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968586000,"dataChange":true}} +{"add":{"path":"part-00003-508ae4aa-801c-4c2c-a923-f6f89930a5c1-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968586000,"dataChange":true}} +{"add":{"path":"part-00004-80938522-09c0-420c-861f-5a649e3d9674-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968586000,"dataChange":true}} +{"add":{"path":"part-00006-63ce9deb-bc0f-482d-b9a1-7e717b67f294-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968586000,"dataChange":true}} +{"add":{"path":"part-00007-94f725e2-3963-4b00-9e83-e31021a93cf9-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968586000,"dataChange":true}} diff --git a/crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000001.json b/crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000001.json new file mode 100644 index 0000000000..0af2ff5ac4 --- /dev/null +++ b/crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000001.json @@ -0,0 +1,27 @@ +{"commitInfo":{"timestamp":1587968596254,"operation":"MERGE","operationParameters":{"predicate":"(oldData.`id` = newData.`id`)"},"readVersion":0,"isBlindAppend":false}} +{"remove":{"path":"part-00006-63ce9deb-bc0f-482d-b9a1-7e717b67f294-c000.snappy.parquet","deletionTimestamp":1587968596250,"dataChange":true}} +{"remove":{"path":"part-00001-c506e79a-0bf8-4e2b-a42b-9731b2e490ae-c000.snappy.parquet","deletionTimestamp":1587968596253,"dataChange":true}} +{"remove":{"path":"part-00007-94f725e2-3963-4b00-9e83-e31021a93cf9-c000.snappy.parquet","deletionTimestamp":1587968596253,"dataChange":true}} +{"remove":{"path":"part-00003-508ae4aa-801c-4c2c-a923-f6f89930a5c1-c000.snappy.parquet","deletionTimestamp":1587968596253,"dataChange":true}} +{"remove":{"path":"part-00004-80938522-09c0-420c-861f-5a649e3d9674-c000.snappy.parquet","deletionTimestamp":1587968596253,"dataChange":true}} +{"add":{"path":"part-00000-a922ea3b-ffc2-4ca1-9074-a278c24c4449-c000.snappy.parquet","partitionValues":{},"size":262,"modificationTime":1587968595000,"dataChange":true}} +{"add":{"path":"part-00004-95c9bc2c-ac85-4581-b3cc-84502b0c314f-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} +{"add":{"path":"part-00005-94a0861b-6455-4bd9-a080-73e02491c643-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} +{"add":{"path":"part-00011-42f838f9-a911-40af-98f5-2fccfa1b123f-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} +{"add":{"path":"part-00045-332fe409-7705-45b1-8d34-a0018cf73b70-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} +{"add":{"path":"part-00049-d3095817-de74-49c1-a888-81565a40161d-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} +{"add":{"path":"part-00058-b462c4cb-0c48-4148-8475-e21d2a2935f8-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} +{"add":{"path":"part-00068-90650739-6a8e-492b-9403-53e33b3778ac-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} +{"add":{"path":"part-00069-c78b4dd8-f955-4643-816f-cbd30a3f8c1b-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} +{"add":{"path":"part-00077-2fcb1c7c-5390-48ee-93f6-0acf11199a0d-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} +{"add":{"path":"part-00107-3f6c2aa0-fc28-4f4c-be15-135e15b398f4-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} +{"add":{"path":"part-00112-07fd790a-11dc-4fde-9acd-623e740be992-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} +{"add":{"path":"part-00116-bc66759e-6381-4f34-8cd4-6688aad8585d-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} +{"add":{"path":"part-00121-d8bc3e53-d2f2-48ce-9909-78da7294ffbd-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} +{"add":{"path":"part-00128-b31c3b81-24da-4a90-a8b4-578c9e9a218d-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} +{"add":{"path":"part-00140-e9b1971d-d708-43fb-b07f-975d2226b800-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} +{"add":{"path":"part-00143-03ceb88e-5283-4193-aa43-993cdf937fd3-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} +{"add":{"path":"part-00150-ec6643fc-4963-4871-9613-f5ad1940b689-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} +{"add":{"path":"part-00154-4630673a-5227-48fb-a03d-e356fcd1564a-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} +{"add":{"path":"part-00164-bf40481c-4afd-4c02-befa-90f056c2d77a-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} +{"add":{"path":"part-00190-8ac0ae67-fb1d-461d-a3d3-8dc112766ff5-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} diff --git a/crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000002.json b/crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000002.json new file mode 100644 index 0000000000..8eeebb28d1 --- /dev/null +++ b/crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000002.json @@ -0,0 +1,29 @@ +{"commitInfo":{"timestamp":1587968604143,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":1,"isBlindAppend":false}} +{"add":{"path":"part-00000-c1777d7d-89d9-4790-b38a-6ee7e24456b1-c000.snappy.parquet","partitionValues":{},"size":262,"modificationTime":1587968602000,"dataChange":true}} +{"add":{"path":"part-00001-7891c33d-cedc-47c3-88a6-abcfb049d3b4-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968602000,"dataChange":true}} +{"add":{"path":"part-00003-53f42606-6cda-4f13-8d07-599a21197296-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968602000,"dataChange":true}} +{"add":{"path":"part-00004-315835fe-fb44-4562-98f6-5e6cfa3ae45d-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968602000,"dataChange":true}} +{"add":{"path":"part-00006-46f2ff20-eb5d-4dda-8498-7bfb2940713b-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968602000,"dataChange":true}} +{"add":{"path":"part-00007-3a0e4727-de0d-41b6-81ef-5223cf40f025-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968602000,"dataChange":true}} +{"remove":{"path":"part-00004-95c9bc2c-ac85-4581-b3cc-84502b0c314f-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} +{"remove":{"path":"part-00107-3f6c2aa0-fc28-4f4c-be15-135e15b398f4-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} +{"remove":{"path":"part-00011-42f838f9-a911-40af-98f5-2fccfa1b123f-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} +{"remove":{"path":"part-00190-8ac0ae67-fb1d-461d-a3d3-8dc112766ff5-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} +{"remove":{"path":"part-00121-d8bc3e53-d2f2-48ce-9909-78da7294ffbd-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} +{"remove":{"path":"part-00005-94a0861b-6455-4bd9-a080-73e02491c643-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} +{"remove":{"path":"part-00077-2fcb1c7c-5390-48ee-93f6-0acf11199a0d-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} +{"remove":{"path":"part-00112-07fd790a-11dc-4fde-9acd-623e740be992-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} +{"remove":{"path":"part-00128-b31c3b81-24da-4a90-a8b4-578c9e9a218d-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} +{"remove":{"path":"part-00000-a922ea3b-ffc2-4ca1-9074-a278c24c4449-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} +{"remove":{"path":"part-00068-90650739-6a8e-492b-9403-53e33b3778ac-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} +{"remove":{"path":"part-00164-bf40481c-4afd-4c02-befa-90f056c2d77a-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} +{"remove":{"path":"part-00045-332fe409-7705-45b1-8d34-a0018cf73b70-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} +{"remove":{"path":"part-00058-b462c4cb-0c48-4148-8475-e21d2a2935f8-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} +{"remove":{"path":"part-00140-e9b1971d-d708-43fb-b07f-975d2226b800-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} +{"remove":{"path":"part-00116-bc66759e-6381-4f34-8cd4-6688aad8585d-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} +{"remove":{"path":"part-00069-c78b4dd8-f955-4643-816f-cbd30a3f8c1b-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} +{"remove":{"path":"part-00154-4630673a-5227-48fb-a03d-e356fcd1564a-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} +{"remove":{"path":"part-00150-ec6643fc-4963-4871-9613-f5ad1940b689-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} +{"remove":{"path":"part-00049-d3095817-de74-49c1-a888-81565a40161d-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} +{"remove":{"path":"part-00000-a72b1fb3-f2df-41fe-a8f0-e65b746382dd-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} +{"remove":{"path":"part-00143-03ceb88e-5283-4193-aa43-993cdf937fd3-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} diff --git a/crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000003.json b/crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000003.json new file mode 100644 index 0000000000..7c411a8f24 --- /dev/null +++ b/crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000003.json @@ -0,0 +1,5 @@ +{"commitInfo":{"timestamp":1587968614187,"operation":"UPDATE","operationParameters":{"predicate":"((id#697L % cast(2 as bigint)) = cast(0 as bigint))"},"readVersion":2,"isBlindAppend":false}} +{"remove":{"path":"part-00003-53f42606-6cda-4f13-8d07-599a21197296-c000.snappy.parquet","deletionTimestamp":1587968614096,"dataChange":true}} +{"remove":{"path":"part-00006-46f2ff20-eb5d-4dda-8498-7bfb2940713b-c000.snappy.parquet","deletionTimestamp":1587968614096,"dataChange":true}} +{"add":{"path":"part-00000-f17fcbf5-e0dc-40ba-adae-ce66d1fcaef6-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968614000,"dataChange":true}} +{"add":{"path":"part-00001-bb70d2ba-c196-4df2-9c85-f34969ad3aa9-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968614000,"dataChange":true}} diff --git a/crates/test/tests/data/simple_table_uncommited/part-00000-2befed33-c358-4768-a43c-3eda0d2a499d-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00000-2befed33-c358-4768-a43c-3eda0d2a499d-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..37061709636c6f5050fdadc50e72d8549ec85e03 GIT binary patch literal 262 zcmXw#ze~h06vsnxytUFohY}Dxs3EbKyL59Cr{e|}IgX@xJ&Wm&OHxHj|8XayoWmR5 z`+R@E+db}C9Hqt6%8k;#A6@+hos1(CU)@AW)M|8%(p8!z>%pFg9A)6-PfFa`kl;B_ z;E5eRNe5nP>1FaVWB)Y*uCViVsy!}n(}K|sSFbbfW%r9(9nKe-8TwP$`pFonOrXIA z1^y4j)x0c+1trgyy1UNiA7EWO^f@Dp5|-bpsWm#6TcMk6BSd+xI5eD>rKX&60S~km VY9lsOfzoW9W*I{0JIpS8;0O1OPTT+h literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00000-a72b1fb3-f2df-41fe-a8f0-e65b746382dd-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00000-a72b1fb3-f2df-41fe-a8f0-e65b746382dd-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..37061709636c6f5050fdadc50e72d8549ec85e03 GIT binary patch literal 262 zcmXw#ze~h06vsnxytUFohY}Dxs3EbKyL59Cr{e|}IgX@xJ&Wm&OHxHj|8XayoWmR5 z`+R@E+db}C9Hqt6%8k;#A6@+hos1(CU)@AW)M|8%(p8!z>%pFg9A)6-PfFa`kl;B_ z;E5eRNe5nP>1FaVWB)Y*uCViVsy!}n(}K|sSFbbfW%r9(9nKe-8TwP$`pFonOrXIA z1^y4j)x0c+1trgyy1UNiA7EWO^f@Dp5|-bpsWm#6TcMk6BSd+xI5eD>rKX&60S~km VY9lsOfzoW9W*I{0JIpS8;0O1OPTT+h literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00000-a922ea3b-ffc2-4ca1-9074-a278c24c4449-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00000-a922ea3b-ffc2-4ca1-9074-a278c24c4449-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..37061709636c6f5050fdadc50e72d8549ec85e03 GIT binary patch literal 262 zcmXw#ze~h06vsnxytUFohY}Dxs3EbKyL59Cr{e|}IgX@xJ&Wm&OHxHj|8XayoWmR5 z`+R@E+db}C9Hqt6%8k;#A6@+hos1(CU)@AW)M|8%(p8!z>%pFg9A)6-PfFa`kl;B_ z;E5eRNe5nP>1FaVWB)Y*uCViVsy!}n(}K|sSFbbfW%r9(9nKe-8TwP$`pFonOrXIA z1^y4j)x0c+1trgyy1UNiA7EWO^f@Dp5|-bpsWm#6TcMk6BSd+xI5eD>rKX&60S~km VY9lsOfzoW9W*I{0JIpS8;0O1OPTT+h literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00000-c1777d7d-89d9-4790-b38a-6ee7e24456b1-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00000-c1777d7d-89d9-4790-b38a-6ee7e24456b1-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..37061709636c6f5050fdadc50e72d8549ec85e03 GIT binary patch literal 262 zcmXw#ze~h06vsnxytUFohY}Dxs3EbKyL59Cr{e|}IgX@xJ&Wm&OHxHj|8XayoWmR5 z`+R@E+db}C9Hqt6%8k;#A6@+hos1(CU)@AW)M|8%(p8!z>%pFg9A)6-PfFa`kl;B_ z;E5eRNe5nP>1FaVWB)Y*uCViVsy!}n(}K|sSFbbfW%r9(9nKe-8TwP$`pFonOrXIA z1^y4j)x0c+1trgyy1UNiA7EWO^f@Dp5|-bpsWm#6TcMk6BSd+xI5eD>rKX&60S~km VY9lsOfzoW9W*I{0JIpS8;0O1OPTT+h literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00000-f17fcbf5-e0dc-40ba-adae-ce66d1fcaef6-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00000-f17fcbf5-e0dc-40ba-adae-ce66d1fcaef6-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..466ecb955860fdef9846accbb4a7f79bc1debe46 GIT binary patch literal 429 zcmZ`$&r8EF82uW@=s~=tG>}6HI5w!Eb*@{77jNQeh-uGU}?CLhefGPHTj11^vo4WQ3{GJ1Hp#KMJrcN%9xsR;91S8^KAJu{eZ0M3Li;;tzPs{nTdiLeP@__y^C)5N6vmRh8 zM?k1MD60h*OJSV(PY7PGG&gBht!j8xzVlo)LW)MX_iNUCEESuwx~bZE!+e%zO3Ru} zAJ(i8`L>lwL+{3PSu9Ldv|5Wqo34>5pY0d2>3Xwy+<<$tvpdTx?@d*8Ru*2!!+?ig yKWQ^9Ju!)8I!WVrc%DQmjiTXD22l{jY81@lWE_u!L?tpD5K7JK?ks?r`KWKLq*>$u literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00001-7891c33d-cedc-47c3-88a6-abcfb049d3b4-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00001-7891c33d-cedc-47c3-88a6-abcfb049d3b4-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..2964979ef6db7dca39bfa9bbe4d6dae04308352b GIT binary patch literal 429 zcmZ`$&r8EF7)|3GJ&3oI2689?#|AaD&UNeX;!Qja5icT=F6k_$?baWPjQtxP{A>ND z8@p%@;eGG>-uGU}?D{Swh%n;$1Ow<2o4NKe`aK8ap#C2$Or2h0xR0#8gkb95oYYHM zy*}6T7oAIj5$T*^7iS7Nm@>+GHX|K6vQd|AS%Mwxd|J+z)w8b-mIw5YIb{$S!Fq(Y za*QZ*2W7Poaw)-C{Dcto$_PlaYIP&3@?GS*kxDkweOUA6W2yO+*G<*V8}9Qo(?->N z`ncwW%(tyf0lgbDWwC&&XpNDHfiB3j&-V-YbiLU;ZIFArvpdf#?@d>ARu*0;!a#&x yKWQ_gJUNL~I!WVrc#%XpjiTXD1yK;idKAp#WE_u!L?6THq>_zvA2+=DSZY4ybyKzThWk9tv{5ym zK5cj*^Ia=bK<~#)SuCI`T4Q8lpbIkX^TR?u-E6ndTjbvD?Jn}lTkERM%EAjp7>Lm8 yCv9ewCnvE=CutlHFOx{8Q8XN?APS;bkAiuejN@^T=tPAB${4)v!9r*TAN38y^jY=* literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00001-c506e79a-0bf8-4e2b-a42b-9731b2e490ae-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00001-c506e79a-0bf8-4e2b-a42b-9731b2e490ae-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..4cb7aae46305259afd5baa77685636cf4b1261e0 GIT binary patch literal 429 zcmZ`$&r8EF7)|3CJ&3oI2689?#|AaD&UI_?;!Qja5icT=F6k_$?baWPjQtxP{A>Lt z8@p%@;pKhb``$}3zqtxdjQ{DXzIvnveyiFJ?=O#R!_W+m&l zmuB&*3rR2{y>sm1Tp(<>eC%du!Eg1%lWnj_Vv;7i2gCB3<4tTdWf&p}i&0O0;zw3w`)c=EpuCpr)`-yds5KR5s(|ReZ zx0ibUs&h#&BAs*W;!Gh2Q$|_OW~4($HtNzHOR$5TPs{nTdiM3v@`(O1rwjrk7?03S zjuB<Lm8 yCv9ewCnvE=CutlHFOx{8Q8XN?APS;bkAiuejN@^T=tPAB${5`40Umb-AN398aaJ<` literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00003-53f42606-6cda-4f13-8d07-599a21197296-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00003-53f42606-6cda-4f13-8d07-599a21197296-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..a1caecfbace73080e53a4875572a74b87c91aca7 GIT binary patch literal 429 zcmZ`$&r8EF7)|3CJ&3oI2689?#|AaD&UNeX;!Qja5icT=F6k_$?baU?8T&Up_}BVP zH+In+!u#I$z3;t{+0A`O5Mjjg2?o$5HgoMR`aMVFp#C2$Or2d}xKFHugkb95p4LlQ zy}i`)SDj0O5$T*`7iS7Nm@>+GHX|K6vQd}rSb`nwd|J+z)w8dUmPhoDIb{$S!Fq&t za*QZ*2W7Poaw)-C{Dcto$_PlaYIP&3@?GS*kxDkwecbToW2yO+*G<*V8}9Qo(?->N z`n2JN%y+F!0lgnHWwC&&XpNDHfiB3j&kqavbhF()Z;^Yqx4XzIZ>_63D+@0aVIV@U ypR}1#o}9!gouqL*yi6jUM$vGnf+&b$JqqS=GLFYVq7xMkC}Z%t2MeJYeAG7|$W~wg literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00004-315835fe-fb44-4562-98f6-5e6cfa3ae45d-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00004-315835fe-fb44-4562-98f6-5e6cfa3ae45d-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..1cd670d1938d11793908ad9575052e04c257dcbe GIT binary patch literal 429 zcmZ`$&r8EF7)|4tJ&3oI2689?#|AaD&UNeX;!Qja5icT=F6k_$?baWPjQtxP{A>ND z8@p%@;eGG>-uGU}?D{Swh%n;$1Ow<2o4NKe`aK8ap#C2$Or2h0xR0#8gkb95oYYHM zy*}6T7oAIj5$T*^7iS7Nm@>+GHX|K6vQd|AS%Mwxd|J+z)w8b-mIw5YIb{$S!Fq(Y za*QZ*2W7Poaw)-C{Dcto$_PlaYIP&3@?GS*kxDkweOUA6W2yO+*G<*V8}9Qo(?->N z`ncwW%(tyf0lgbDWwC&&XpNDHfiB3j&-V-YbiLU;ZIFArvpdf#?@d>ARu*0;!a#&x yKWQ_gJUNL~I!WVrc#%XpjiTXD1yK;idKAp#WE_u!L?}6HI5w!Eb*`?%i#PE!M7)Sdx}>w1wp)KFGWKtH@UJ!1 zja@XSm-oH*y%#dSz6&v6iaj4A1Nzveu8rXL9FPOUKUkwWx#+}4*4}~<@o$dnm8f2y z>&1)81xA2-r^rQ_1P2jH=)k79M$T>l&AL?$ugZ6xt42uC2=`&bnva!YGgdcMyJ(ot(oAVtv)SW@ z6(a9OnKblHrpsb!ZbhrLNVJ(6oATLyA)9Ts+ovtKx4XEryz<^uRcB@4g**&+=na!L w)6x^uSfqRoq|zuFk7W=AQLHAxB2F&j%OFvS3`c}g^SXNrU~WF@8}S)dJOBUy literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00004-95c9bc2c-ac85-4581-b3cc-84502b0c314f-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00004-95c9bc2c-ac85-4581-b3cc-84502b0c314f-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..5770a94ab6136340e11bdadaaeeba5dcf36b772f GIT binary patch literal 429 zcmZ`$&r8EF7)|3CJ%|@cX&{FZaBNUR>s+@EFW$t{5b+`+>5|T3+HU=!$k@N(!N1mT zy0MGq5Z?E`?|tuu%&zZ3f(Rp?PcVQkv6*Y1qTh2s4(k8G!qmwnhWp6cO9-a^&2hby z)$4OTf6=)l7?I8yc5$YVgDIn|XEV~FBO7(;mL=H1&Zp&kSv~veV0l3Qm{SIU5v)gO zE60d3cTiRfA(s-I#ZL%PuZ(~+t5!FnD&Iw}8>wU?-G?=AK9-tKdEHd)yx~4iGi_AO zr;lr1$b8$%6wtddQx*%Tiq;sJ80dmb`+UEUPuH8x(+0V>JG=9|^4@e+XJz4qA`C?6 z^^-O;%9E2=rIR#{hZjkt(Lt z8@p%@;pKhb``$}3zqtxdjQ{DXzIvnveyiFJ?=O#R!_W+m&l zmuB&*3rR2{y>sm1Tp(<>eC%du!Eg1%lWnj_Vv;7i2gCB3<4t}6HI5w!Eb*@{77jNQeh*fmWoYT-Bj(oVLnSUrDe^g zPa9T!>8U^z>8OP%wQHcx(gi`am2Mb_kKI$7c7*=us literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00006-63ce9deb-bc0f-482d-b9a1-7e717b67f294-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00006-63ce9deb-bc0f-482d-b9a1-7e717b67f294-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..ec63649165cdab46a6642fe3e32f8be13979b618 GIT binary patch literal 429 zcmZ`$&r8EF82y@!(SvwPX&{FZaBNUR>s+@EFW$t{5b+`+>5|T3+HU=!$k@N(!N1l_ zH+In+!u#I$z3;t{`So3h0aNVx7#YyRHg)X|{GKCnVDJZPrcN)BxjWWDf)Vj=PU@AY zUZ3m5i^>H?fV*eNMVSN#5lU#^rnpPSHuT7j#mGU_gD8k%H4YYWGKnWaq7oSn38m(B_ZGn1eAGAm!B#~8 literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00007-3a0e4727-de0d-41b6-81ef-5223cf40f025-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00007-3a0e4727-de0d-41b6-81ef-5223cf40f025-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..533801e17f8f556a8e034d9bd7f7411d33ed8dfe GIT binary patch literal 429 zcmZ`$&r8EF7)|3C6~tRg138p{V}lx6=el)x@g|;zh!+t_mvk1>cIyvC#{LZt{TdWf&p}i&0PB!{hkAIQ2!4WrcN(0+(*`4LNN7jPU@ws zUZ3mvi_Rs%h;+`di!+5BOc`Z8n~@G3*{DmmEWr+TJ}u|V>e*KZ%LDqyoH7WEU_C-x zIYyMZgR)u(xs>26enN!xbw4flDPX`^aB zeO&WG=G#`LfZmOnvRFV>w8qH9Ko?}%=lg|xy54M_Hpso**`4Q=_ok~lD+@0aVIV@U ypR}1#o}9!gouqL*yhtLQM$vGnf+&b$JqqS=GLFYVq7xMkC}Z%tdkdi%eAG8Wz*c(z literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00007-94f725e2-3963-4b00-9e83-e31021a93cf9-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00007-94f725e2-3963-4b00-9e83-e31021a93cf9-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..ea2985c2bc2c2730d8d4442a6eb4e120b8e51c10 GIT binary patch literal 429 zcmZ`$&r8EF7)|3CJ&3oI2689?#|AaD&UNeX;!Qja5icT=F6k_$?baU?8T&Up_}BVP zH+In+!u#I$z3;t{+0A`O5Mjjg2?o$5HgoL``aMVFp#C2$Or2d}xKFHugkb95p4LlQ zy}i`)SDj0O5$T*`7iS7Nm@>+GHX|K6vQd}rSb`nwd|J+z)w8dUmPhoDIb{$S!Fq&t za*QZ*2W7Poaw)-C{Dcto$_PlaYIP&3@?GS*kxDkwecbToW2yO+*G<*V8}9Qo(?->N z`n2JN%y+F!0lgnHWwC&&XpNDHfiB3j&kqavbhF()Z;^Yqx4XzIZ>_63D+@0aVIV@U ypR}1#o}9!gouqL*yi6jUM$vGnf+&b$JqqS=GLFYVq7xMkC}Z%t2MeJYeAG7xcvemT literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00011-42f838f9-a911-40af-98f5-2fccfa1b123f-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00011-42f838f9-a911-40af-98f5-2fccfa1b123f-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..1cd670d1938d11793908ad9575052e04c257dcbe GIT binary patch literal 429 zcmZ`$&r8EF7)|4tJ&3oI2689?#|AaD&UNeX;!Qja5icT=F6k_$?baWPjQtxP{A>ND z8@p%@;eGG>-uGU}?D{Swh%n;$1Ow<2o4NKe`aK8ap#C2$Or2h0xR0#8gkb95oYYHM zy*}6T7oAIj5$T*^7iS7Nm@>+GHX|K6vQd|AS%Mwxd|J+z)w8b-mIw5YIb{$S!Fq(Y za*QZ*2W7Poaw)-C{Dcto$_PlaYIP&3@?GS*kxDkweOUA6W2yO+*G<*V8}9Qo(?->N z`ncwW%(tyf0lgbDWwC&&XpNDHfiB3j&-V-YbiLU;ZIFArvpdf#?@d>ARu*0;!a#&x yKWQ_gJUNL~I!WVrc#%XpjiTXD1yK;idKAp#WE_u!L?+GHX|K6vQd}rSb`nwd|J+z)w8dUmPhoDIb{$S!Fq&t za*QZ*2W7Poaw)-C{Dcto$_PlaYIP&3@?GS*kxDkwecbToW2yO+*G<*V8}9Qo(?->N z`n2JN%y+F!0lgnHWwC&&XpNDHfiB3j&kqavbhF()Z;^Yqx4XzIZ>_63D+@0aVIV@U ypR}1#o}9!gouqL*yi6jUM$vGnf+&b$JqqS=GLFYVq7xMkC}Z%t2MeJYeAG7|$W~wg literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00049-d3095817-de74-49c1-a888-81565a40161d-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00049-d3095817-de74-49c1-a888-81565a40161d-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..533801e17f8f556a8e034d9bd7f7411d33ed8dfe GIT binary patch literal 429 zcmZ`$&r8EF7)|3C6~tRg138p{V}lx6=el)x@g|;zh!+t_mvk1>cIyvC#{LZt{TdWf&p}i&0PB!{hkAIQ2!4WrcN(0+(*`4LNN7jPU@ws zUZ3mvi_Rs%h;+`di!+5BOc`Z8n~@G3*{DmmEWr+TJ}u|V>e*KZ%LDqyoH7WEU_C-x zIYyMZgR)u(xs>26enN!xbw4flDPX`^aB zeO&WG=G#`LfZmOnvRFV>w8qH9Ko?}%=lg|xy54M_Hpso**`4Q=_ok~lD+@0aVIV@U ypR}1#o}9!gouqL*yhtLQM$vGnf+&b$JqqS=GLFYVq7xMkC}Z%tdkdi%eAG8Wz*c(z literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00058-b462c4cb-0c48-4148-8475-e21d2a2935f8-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00058-b462c4cb-0c48-4148-8475-e21d2a2935f8-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..17a7a308f42d4f4eb92ddbf67b9c813a00c6921c GIT binary patch literal 429 zcmZ`$&r8EF7)|3CJ&52TrGXquz_CFMt#jQvym%8&L&S@Sq)R%BX}k4@B4huC2me~X z>BcUaLwMi&zW2QsGP}MD2_lSmKEVLG#AdF2ihj=lIjH{!3sWbT815r$FCm!vH^=o- zRND z8@p%@;eGG>-uGU}?D{Swh%n;$1Ow<2o4NKe`aK8ap#C2$Or2h0xR0#8gkb95oYYHM zy*}6T7oAIj5$T*^7iS7Nm@>+GHX|K6vQd|AS%Mwxd|J+z)w8b-mIw5YIb{$S!Fq(Y za*QZ*2W7Poaw)-C{Dcto$_PlaYIP&3@?GS*kxDkweOUA6W2yO+*G<*V8}9Qo(?->N z`ncwW%(tyf0lgbDWwC&&XpNDHfiB3j&-V-YbiLU;ZIFArvpdf#?@d>ARu*0;!a#&x yKWQ_gJUNL~I!WVrc#%XpjiTXD1yK;idKAp#WE_u!L?TdWf&p}i&0O0;zw3w`)c=EpuCpr)`-yds5KR5s(|ReZ zx0ibUs&h#&BAs*W;!Gh2Q$|_OW~4($HtNzHOR$5TPs{nTdiM3v@`(O1rwjrk7?03S zjuB<Lm8 yCv9ewCnvE=CutlHFOx{8Q8XN?APS;bkAiuejN@^T=tPAB${5`40Umb-AN398aaJ<` literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00077-2fcb1c7c-5390-48ee-93f6-0acf11199a0d-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00077-2fcb1c7c-5390-48ee-93f6-0acf11199a0d-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..df7f737162902c4b54c83142f6af578074176961 GIT binary patch literal 429 zcmZ`$&r8EF7)|3CMZ{Z5138p{V}lx6=el)x@g|;zh!+t_mvk1>cIyvC#{LZt{TdWf&p}i&0PB!{hkAIQ2!4WrcN(0+(*`4LNN7jPU@ws zUZ3mvi_Rs%h;+`di!+5BOc`Z8n~@G3*{DmmEWr+TJ}u|V>e*KZ%LDqyoH7WEU_C-x zIYyMZgR)u(xs>26enN!xbw4flDPX`^aB zeO&WG=G#`LfZmOnvRFV>w8qH9Ko?}%=lg|xy54M_Hpso**`4Q=_ok~lD+@0aVIV@U ypR}1#o}9!gouqL*yhtLQM$vGnf+&b$JqqS=GLFYVq7xMkC}Z%tdkdi%eAG8iXjX&( literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00107-3f6c2aa0-fc28-4f4c-be15-135e15b398f4-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00107-3f6c2aa0-fc28-4f4c-be15-135e15b398f4-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..ec63649165cdab46a6642fe3e32f8be13979b618 GIT binary patch literal 429 zcmZ`$&r8EF82y@!(SvwPX&{FZaBNUR>s+@EFW$t{5b+`+>5|T3+HU=!$k@N(!N1l_ zH+In+!u#I$z3;t{`So3h0aNVx7#YyRHg)X|{GKCnVDJZPrcN)BxjWWDf)Vj=PU@AY zUZ3m5i^>H?fV*eNMVSN#5lU#^rnpPSHuT7j#mGU_gD8k%H4YYWGKnWaq7oSn38m(B_ZGn1eAGAm!B#~8 literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00112-07fd790a-11dc-4fde-9acd-623e740be992-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00112-07fd790a-11dc-4fde-9acd-623e740be992-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..b5e5edad5e55405cda49241ab0bfa6bc71237653 GIT binary patch literal 429 zcmZ`$&r8EF7)|3CJq&Lt4dhS)jty#Po$J=&#hZ8f2XG%Om>7oH7WEU_C-R zIYyMZ!?Icmxsu>4enN!xZK4flDPX`^aB zd))9s=DSv=fZmUpvRFb@w8qH9Ko?}%=ZA%Sw%Kl}6HI5w!Eb*@{77jNQeh*fmWoYT-Bj(oVLnSUrDe^g zPa9T!>8U^z>8OP%wQHcx(gi`am2Mb_kKI$7c7*=us literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00121-d8bc3e53-d2f2-48ce-9909-78da7294ffbd-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00121-d8bc3e53-d2f2-48ce-9909-78da7294ffbd-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..960d2b6e47cb3b2079d3b15800927e18e342929a GIT binary patch literal 429 zcmZ`$&r8EF7)|3C1;txR138p{V}lx6=el)x@g|;zh!+t_mvk1>cIyvC#{LZt{TdWf&p}i&0PB!{hkAIQ2!4WrcN(0+(*`4LNN7jPU@ws zUZ3mvi_Rs%h;+`di!+5BOc`Z8n~@G3*{DmmEWr+TJ}u|V>e*KZ%LDqyoH7WEU_C-x zIYyMZgR)u(xs>26enN!xbw4flDPX`^aB zeO&WG=G#`LfZmOnvRFV>w8qH9Ko?}%=lg|xy54M_Hpso**`4Q=_ok~lD+@0aVIV@U ypR}1#o}9!gouqL*yhtLQM$vGnf+&b$JqqS=GLFYVq7xMkC}Z%tdkdi%eAG8u5LS%< literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00128-b31c3b81-24da-4a90-a8b4-578c9e9a218d-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00128-b31c3b81-24da-4a90-a8b4-578c9e9a218d-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..3b91e5be30c42fa560fdfce67f347a692072b1fb GIT binary patch literal 429 zcmZ`$&r8EF82uW@=s~=tG>}6HI5w!Eb*`?%i#PE!M7)Sdx}>w1wp)KFGWKtH@UJ!1 zja@XSm-oH*y%#dSz6&v6iaj4A1Nzveu8rXL9FPOUKUkwWx#+}4*4}~<@o$dnm8f2y z>&1)81xA2-r^rQ_1P2jH=)k79M$T>l&AL?$ugZ6xt42uC2=`&bnva!YGgdcMyJ(ot(oAVtv)SW@ z6(a9OnKblHrpsb!ZbhrLNVJ(6oATLyA)9Ts+ovtKx4XEryz<^uRcB@4g**&+=na!L w)6x^uSfqRoq|zuFk7W=AQLHAxB2F&j%OFvS3`c}g^SXNrU~WF@8}S)dJOBUy literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00140-e9b1971d-d708-43fb-b07f-975d2226b800-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00140-e9b1971d-d708-43fb-b07f-975d2226b800-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..ea2985c2bc2c2730d8d4442a6eb4e120b8e51c10 GIT binary patch literal 429 zcmZ`$&r8EF7)|3CJ&3oI2689?#|AaD&UNeX;!Qja5icT=F6k_$?baU?8T&Up_}BVP zH+In+!u#I$z3;t{+0A`O5Mjjg2?o$5HgoL``aMVFp#C2$Or2d}xKFHugkb95p4LlQ zy}i`)SDj0O5$T*`7iS7Nm@>+GHX|K6vQd}rSb`nwd|J+z)w8dUmPhoDIb{$S!Fq&t za*QZ*2W7Poaw)-C{Dcto$_PlaYIP&3@?GS*kxDkwecbToW2yO+*G<*V8}9Qo(?->N z`n2JN%y+F!0lgnHWwC&&XpNDHfiB3j&kqavbhF()Z;^Yqx4XzIZ>_63D+@0aVIV@U ypR}1#o}9!gouqL*yi6jUM$vGnf+&b$JqqS=GLFYVq7xMkC}Z%t2MeJYeAG7xcvemT literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00143-03ceb88e-5283-4193-aa43-993cdf937fd3-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00143-03ceb88e-5283-4193-aa43-993cdf937fd3-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..25fe1a24f16f5e8a38ed3c836e4bb61f42ae8138 GIT binary patch literal 429 zcmZ`$&r8EF7)|3CJ*c;o2689?#|AaD&UNeX;!Qja5icT=F6k_$?baWPjQtxP{A>ND z8@p%@;eGG>-uGU}?D{Swh%n;$1Ow<2o4NK0`aK8ap#C2$Or2h0xR0#8gkb959M?-( zy*}6T7oAIj5$T*^7iS7Nm@>+GHX|K6vQd|AS%Mwxd|J+z)w8b-mIw5YIb{$S!Fq(Y za*QZ*2W7Poaw)-C{Dcto$_PlaYIP&3@?GS*kxDkweOUA6W2yO+*G<*V8}9Qo(?->N z`ncwW%(tyf0lgbDWwC&&XpNDHfiB3j&-V-YbiLU;ZIFArvpdf#?@d>ARu*0;!a#&x yKWQ_gJUNL~I!WVrc#%XpjiTXD1yK;idKAp#WE_u!L?BcUaLwMi&zW2QsGP}MD2_lSmKEVLG#AdF2ihj=lIjH{!3sWbT815r$FCm!vH^=o- zRs+@EFW$t{5b+`+>5|T3+HU=!$k@N(!N1l_ zH+In+!u#I$z3;t{`So3h0aNVx7#YyTHg)YD{GKCnVDJZPrh1pi+$YvSf)Vj=PV1GZ zUZ3m5i^>H?fIDZ%MVSN#5lU#^rnp1KHgw63#mGU_gD8k%H4YYWGKnWaq7oSn38m(B_ZGn1eAG962v(>7 literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00164-bf40481c-4afd-4c02-befa-90f056c2d77a-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00164-bf40481c-4afd-4c02-befa-90f056c2d77a-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..c3cf2f57ca0058ef3d77ac1fa8c1e569a05480d2 GIT binary patch literal 429 zcmZ`$&r8EF7)|3CJ&2xC8pxpp92?ZoI@hhki#PE!M7)Sdx}>w1wp)KFGWKtH@UQio zZtS8tg!jGgd*6E@v+KK%Ai{{}6AYkBZ06dh==U6ugZh84Fm-Z?;XbnV5`w9Jb6hWF z_4-`TUvw@BMx=9wU7RW8V9F@#*^G4P$VOedWeIk$^JzI>R?ogVSRT+n=9EET1nUvn z$}ytM9hB8V$fX2l@e@MSDEoIgGT*i`1@vyrl*Iz7qBTY)2D%{AKHo3o)AeTav_bCe&h9*~yfV2p6+}T4>rpU|lW{x_5}l}UKpBJA-CGFF;G@0)h_F_% literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00190-8ac0ae67-fb1d-461d-a3d3-8dc112766ff5-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00190-8ac0ae67-fb1d-461d-a3d3-8dc112766ff5-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..be5bf31a8b174d20f341bff6689da4770272c5c5 GIT binary patch literal 429 zcmZ`$&r8EF7)|3CJ&1ToX&{FZaBNUR>s+@EFW$t{5b+`+>5|T3+HU=!$k@N(!N1mT zy0MGq5Z?E`?|tuu%&zZ3f(Rp?PcVQkv6*Y1qTh2s4(k8G!qmwnhWp6cO9-a^&2hby z)$4OTf6=)l7?I8yc5$YVgDIn|XEV~FBO7(;mL=H1&Zp&kSv~veV0l3Qm{SIU5v)gO zE60d3cTiRfA(s-I#ZL%PuZ(~+t5!FnD&Iw}8>wU?-G?=AK9-tKdEHd)yx~4iGi_AO zr;lr1$b8$%6wtddQx*%Tiq;sJ80dmb`+UEUPuH8x(+0V>JG=9|^4@e+XJz4qA`C?6 z^^-O;%9E2=rIR#{hZjkt( Date: Fri, 9 May 2025 17:18:20 -0700 Subject: [PATCH 2/8] Get rid of new test directory. Slight reformatting and test clarification. Signed-off-by: Corwin Joy --- crates/core/src/logstore/mod.rs | 2 +- crates/core/src/table/mod.rs | 4 +-- crates/core/tests/time_travel.rs | 34 +++++++++++------- .../_delta_log/.tmp/00000000000000000004.json | 4 --- .../_delta_log/00000000000000000000.json | 9 ----- .../_delta_log/00000000000000000001.json | 27 -------------- .../_delta_log/00000000000000000002.json | 29 --------------- .../_delta_log/00000000000000000003.json | 5 --- ...4768-a43c-3eda0d2a499d-c000.snappy.parquet | Bin 262 -> 0 bytes ...41fe-a8f0-e65b746382dd-c000.snappy.parquet | Bin 262 -> 0 bytes ...4ca1-9074-a278c24c4449-c000.snappy.parquet | Bin 262 -> 0 bytes ...4790-b38a-6ee7e24456b1-c000.snappy.parquet | Bin 262 -> 0 bytes ...40ba-adae-ce66d1fcaef6-c000.snappy.parquet | Bin 429 -> 0 bytes ...47c3-88a6-abcfb049d3b4-c000.snappy.parquet | Bin 429 -> 0 bytes ...4df2-9c85-f34969ad3aa9-c000.snappy.parquet | Bin 429 -> 0 bytes ...4e2b-a42b-9731b2e490ae-c000.snappy.parquet | Bin 429 -> 0 bytes ...4c2c-a923-f6f89930a5c1-c000.snappy.parquet | Bin 429 -> 0 bytes ...4f13-8d07-599a21197296-c000.snappy.parquet | Bin 429 -> 0 bytes ...4562-98f6-5e6cfa3ae45d-c000.snappy.parquet | Bin 429 -> 0 bytes ...420c-861f-5a649e3d9674-c000.snappy.parquet | Bin 429 -> 0 bytes ...4581-b3cc-84502b0c314f-c000.snappy.parquet | Bin 429 -> 0 bytes ...4bd9-a080-73e02491c643-c000.snappy.parquet | Bin 429 -> 0 bytes ...4dda-8498-7bfb2940713b-c000.snappy.parquet | Bin 429 -> 0 bytes ...482d-b9a1-7e717b67f294-c000.snappy.parquet | Bin 429 -> 0 bytes ...41b6-81ef-5223cf40f025-c000.snappy.parquet | Bin 429 -> 0 bytes ...4b00-9e83-e31021a93cf9-c000.snappy.parquet | Bin 429 -> 0 bytes ...40af-98f5-2fccfa1b123f-c000.snappy.parquet | Bin 429 -> 0 bytes ...45b1-8d34-a0018cf73b70-c000.snappy.parquet | Bin 429 -> 0 bytes ...49c1-a888-81565a40161d-c000.snappy.parquet | Bin 429 -> 0 bytes ...4148-8475-e21d2a2935f8-c000.snappy.parquet | Bin 429 -> 0 bytes ...492b-9403-53e33b3778ac-c000.snappy.parquet | Bin 429 -> 0 bytes ...4643-816f-cbd30a3f8c1b-c000.snappy.parquet | Bin 429 -> 0 bytes ...48ee-93f6-0acf11199a0d-c000.snappy.parquet | Bin 429 -> 0 bytes ...4f4c-be15-135e15b398f4-c000.snappy.parquet | Bin 429 -> 0 bytes ...4fde-9acd-623e740be992-c000.snappy.parquet | Bin 429 -> 0 bytes ...4f34-8cd4-6688aad8585d-c000.snappy.parquet | Bin 429 -> 0 bytes ...48ce-9909-78da7294ffbd-c000.snappy.parquet | Bin 429 -> 0 bytes ...4a90-a8b4-578c9e9a218d-c000.snappy.parquet | Bin 429 -> 0 bytes ...43fb-b07f-975d2226b800-c000.snappy.parquet | Bin 429 -> 0 bytes ...4193-aa43-993cdf937fd3-c000.snappy.parquet | Bin 429 -> 0 bytes ...4871-9613-f5ad1940b689-c000.snappy.parquet | Bin 429 -> 0 bytes ...48fb-a03d-e356fcd1564a-c000.snappy.parquet | Bin 429 -> 0 bytes ...4c02-befa-90f056c2d77a-c000.snappy.parquet | Bin 429 -> 0 bytes ...461d-a3d3-8dc112766ff5-c000.snappy.parquet | Bin 429 -> 0 bytes 44 files changed, 24 insertions(+), 90 deletions(-) delete mode 100644 crates/test/tests/data/simple_table_uncommited/_delta_log/.tmp/00000000000000000004.json delete mode 100644 crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000000.json delete mode 100644 crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000001.json delete mode 100644 crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000002.json delete mode 100644 crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000003.json delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00000-2befed33-c358-4768-a43c-3eda0d2a499d-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00000-a72b1fb3-f2df-41fe-a8f0-e65b746382dd-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00000-a922ea3b-ffc2-4ca1-9074-a278c24c4449-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00000-c1777d7d-89d9-4790-b38a-6ee7e24456b1-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00000-f17fcbf5-e0dc-40ba-adae-ce66d1fcaef6-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00001-7891c33d-cedc-47c3-88a6-abcfb049d3b4-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00001-bb70d2ba-c196-4df2-9c85-f34969ad3aa9-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00001-c506e79a-0bf8-4e2b-a42b-9731b2e490ae-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00003-508ae4aa-801c-4c2c-a923-f6f89930a5c1-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00003-53f42606-6cda-4f13-8d07-599a21197296-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00004-315835fe-fb44-4562-98f6-5e6cfa3ae45d-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00004-80938522-09c0-420c-861f-5a649e3d9674-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00004-95c9bc2c-ac85-4581-b3cc-84502b0c314f-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00005-94a0861b-6455-4bd9-a080-73e02491c643-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00006-46f2ff20-eb5d-4dda-8498-7bfb2940713b-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00006-63ce9deb-bc0f-482d-b9a1-7e717b67f294-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00007-3a0e4727-de0d-41b6-81ef-5223cf40f025-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00007-94f725e2-3963-4b00-9e83-e31021a93cf9-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00011-42f838f9-a911-40af-98f5-2fccfa1b123f-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00045-332fe409-7705-45b1-8d34-a0018cf73b70-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00049-d3095817-de74-49c1-a888-81565a40161d-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00058-b462c4cb-0c48-4148-8475-e21d2a2935f8-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00068-90650739-6a8e-492b-9403-53e33b3778ac-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00069-c78b4dd8-f955-4643-816f-cbd30a3f8c1b-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00077-2fcb1c7c-5390-48ee-93f6-0acf11199a0d-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00107-3f6c2aa0-fc28-4f4c-be15-135e15b398f4-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00112-07fd790a-11dc-4fde-9acd-623e740be992-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00116-bc66759e-6381-4f34-8cd4-6688aad8585d-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00121-d8bc3e53-d2f2-48ce-9909-78da7294ffbd-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00128-b31c3b81-24da-4a90-a8b4-578c9e9a218d-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00140-e9b1971d-d708-43fb-b07f-975d2226b800-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00143-03ceb88e-5283-4193-aa43-993cdf937fd3-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00150-ec6643fc-4963-4871-9613-f5ad1940b689-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00154-4630673a-5227-48fb-a03d-e356fcd1564a-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00164-bf40481c-4afd-4c02-befa-90f056c2d77a-c000.snappy.parquet delete mode 100644 crates/test/tests/data/simple_table_uncommited/part-00190-8ac0ae67-fb1d-461d-a3d3-8dc112766ff5-c000.snappy.parquet diff --git a/crates/core/src/logstore/mod.rs b/crates/core/src/logstore/mod.rs index e05f9eec88..6c3cb8045b 100644 --- a/crates/core/src/logstore/mod.rs +++ b/crates/core/src/logstore/mod.rs @@ -549,7 +549,7 @@ pub async fn get_latest_version( // Spark may create these as uncommited transactions which we don't want // // https://github.com/delta-io/delta/blob/master/PROTOCOL.md#delta-log-entries - // "Delta files are stored as JSON in a directory at the *root* of the table + // "Delta files are stored as JSON in a directory at the *root* of the table // named _delta_log, and ... make up the log of all changes that have occurred to a table." continue; } diff --git a/crates/core/src/table/mod.rs b/crates/core/src/table/mod.rs index ba45706baa..54a186928d 100644 --- a/crates/core/src/table/mod.rs +++ b/crates/core/src/table/mod.rs @@ -578,9 +578,9 @@ impl DeltaTable { // Spark may create these as uncommited transactions which we don't want // // https://github.com/delta-io/delta/blob/master/PROTOCOL.md#delta-log-entries - // "Delta files are stored as JSON in a directory at the *root* of the table + // "Delta files are stored as JSON in a directory at the *root* of the table // named _delta_log, and ... make up the log of all changes that have occurred to a table." - continue; + continue; } if let Some(log_version) = extract_version_from_filename(obj_meta.location.as_ref()) { if min_version == -1 { diff --git a/crates/core/tests/time_travel.rs b/crates/core/tests/time_travel.rs index 2bff87ded5..1faaaf5f58 100644 --- a/crates/core/tests/time_travel.rs +++ b/crates/core/tests/time_travel.rs @@ -8,14 +8,15 @@ async fn time_travel_by_ds() { // test time travel on a table with an uncommited delta in a .tmp subfolder // git does not preserve mtime, so we need to manually set it in the test - let log_dir = "../test/tests/data/simple_table_uncommited/_delta_log"; + let log_dir = "../test/tests/data/simple_table/_delta_log"; let log_mtime_pair = vec![ ("00000000000000000000.json", "2020-05-01T22:47:31-07:00"), ("00000000000000000001.json", "2020-05-02T22:47:31-07:00"), ("00000000000000000002.json", "2020-05-03T22:47:31-07:00"), ("00000000000000000003.json", "2020-05-04T22:47:31-07:00"), + ("00000000000000000004.json", "2020-05-05T22:47:31-07:00"), // Final file is uncommitted by Spark and is in a .tmp subdir - (".tmp/00000000000000000004.json", "2020-05-05T22:47:31-07:00"), + (".tmp/00000000000000000005.json", "2020-05-06T22:47:31-07:00"), ]; for (fname, ds) in log_mtime_pair { let ts: SystemTime = ds_to_ts(ds).into(); @@ -26,7 +27,7 @@ async fn time_travel_by_ds() { } let mut table = deltalake_core::open_table_with_ds( - "../test/tests/data/simple_table_uncommited", + "../test/tests/data/simple_table", "2020-05-01T00:47:31-07:00", ) .await @@ -35,7 +36,7 @@ async fn time_travel_by_ds() { assert_eq!(table.version(), 0); table = deltalake_core::open_table_with_ds( - "../test/tests/data/simple_table_uncommited", + "../test/tests/data/simple_table", "2020-05-02T22:47:31-07:00", ) .await @@ -43,7 +44,7 @@ async fn time_travel_by_ds() { assert_eq!(table.version(), 1); table = deltalake_core::open_table_with_ds( - "../test/tests/data/simple_table_uncommited", + "../test/tests/data/simple_table", "2020-05-02T23:47:31-07:00", ) .await @@ -51,7 +52,7 @@ async fn time_travel_by_ds() { assert_eq!(table.version(), 1); table = deltalake_core::open_table_with_ds( - "../test/tests/data/simple_table_uncommited", + "../test/tests/data/simple_table", "2020-05-03T22:47:31-07:00", ) .await @@ -59,17 +60,15 @@ async fn time_travel_by_ds() { assert_eq!(table.version(), 2); table = deltalake_core::open_table_with_ds( - "../test/tests/data/simple_table_uncommited", + "../test/tests/data/simple_table", "2020-05-04T22:47:31-07:00", ) .await .unwrap(); assert_eq!(table.version(), 3); - - // Versions beyond 3 should be ignored. - // This is an uncommited delta in the .tmp subdirectory + table = deltalake_core::open_table_with_ds( - "../test/tests/data/simple_table_uncommited", + "../test/tests/data/simple_table", "2020-05-05T21:47:31-07:00", ) .await @@ -77,12 +76,21 @@ async fn time_travel_by_ds() { assert_eq!(table.version(), 3); table = deltalake_core::open_table_with_ds( - "../test/tests/data/simple_table_uncommited", + "../test/tests/data/simple_table", + "2020-05-05T22:47:31-07:00", + ) + .await + .unwrap(); + assert_eq!(table.version(), 4); + + // Final append in .tmp subdir is uncommited and should be ignored + table = deltalake_core::open_table_with_ds( + "../test/tests/data/simple_table", "2020-05-25T22:47:31-07:00", ) .await .unwrap(); - assert_eq!(table.version(), 3); + assert_eq!(table.version(), 4); } fn ds_to_ts(ds: &str) -> DateTime { diff --git a/crates/test/tests/data/simple_table_uncommited/_delta_log/.tmp/00000000000000000004.json b/crates/test/tests/data/simple_table_uncommited/_delta_log/.tmp/00000000000000000004.json deleted file mode 100644 index 8911adfbb2..0000000000 --- a/crates/test/tests/data/simple_table_uncommited/_delta_log/.tmp/00000000000000000004.json +++ /dev/null @@ -1,4 +0,0 @@ -{"commitInfo":{"timestamp":1587968626537,"operation":"DELETE","operationParameters":{"predicate":"[\"((`id` % CAST(2 AS BIGINT)) = CAST(0 AS BIGINT))\"]"},"readVersion":3,"isBlindAppend":false}} -{"remove":{"path":"part-00001-bb70d2ba-c196-4df2-9c85-f34969ad3aa9-c000.snappy.parquet","deletionTimestamp":1587968626536,"dataChange":true}} -{"remove":{"path":"part-00000-f17fcbf5-e0dc-40ba-adae-ce66d1fcaef6-c000.snappy.parquet","deletionTimestamp":1587968626536,"dataChange":true}} -{"add":{"path":"part-00000-2befed33-c358-4768-a43c-3eda0d2a499d-c000.snappy.parquet","partitionValues":{},"size":262,"modificationTime":1587968626000,"dataChange":true}} diff --git a/crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000000.json b/crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000000.json deleted file mode 100644 index 47dd9bd9db..0000000000 --- a/crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000000.json +++ /dev/null @@ -1,9 +0,0 @@ -{"commitInfo":{"timestamp":1587968586154,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isBlindAppend":true}} -{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} -{"metaData":{"id":"5fba94ed-9794-4965-ba6e-6ee3c0d22af9","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1587968585495}} -{"add":{"path":"part-00000-a72b1fb3-f2df-41fe-a8f0-e65b746382dd-c000.snappy.parquet","partitionValues":{},"size":262,"modificationTime":1587968586000,"dataChange":true}} -{"add":{"path":"part-00001-c506e79a-0bf8-4e2b-a42b-9731b2e490ae-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968586000,"dataChange":true}} -{"add":{"path":"part-00003-508ae4aa-801c-4c2c-a923-f6f89930a5c1-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968586000,"dataChange":true}} -{"add":{"path":"part-00004-80938522-09c0-420c-861f-5a649e3d9674-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968586000,"dataChange":true}} -{"add":{"path":"part-00006-63ce9deb-bc0f-482d-b9a1-7e717b67f294-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968586000,"dataChange":true}} -{"add":{"path":"part-00007-94f725e2-3963-4b00-9e83-e31021a93cf9-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968586000,"dataChange":true}} diff --git a/crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000001.json b/crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000001.json deleted file mode 100644 index 0af2ff5ac4..0000000000 --- a/crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000001.json +++ /dev/null @@ -1,27 +0,0 @@ -{"commitInfo":{"timestamp":1587968596254,"operation":"MERGE","operationParameters":{"predicate":"(oldData.`id` = newData.`id`)"},"readVersion":0,"isBlindAppend":false}} -{"remove":{"path":"part-00006-63ce9deb-bc0f-482d-b9a1-7e717b67f294-c000.snappy.parquet","deletionTimestamp":1587968596250,"dataChange":true}} -{"remove":{"path":"part-00001-c506e79a-0bf8-4e2b-a42b-9731b2e490ae-c000.snappy.parquet","deletionTimestamp":1587968596253,"dataChange":true}} -{"remove":{"path":"part-00007-94f725e2-3963-4b00-9e83-e31021a93cf9-c000.snappy.parquet","deletionTimestamp":1587968596253,"dataChange":true}} -{"remove":{"path":"part-00003-508ae4aa-801c-4c2c-a923-f6f89930a5c1-c000.snappy.parquet","deletionTimestamp":1587968596253,"dataChange":true}} -{"remove":{"path":"part-00004-80938522-09c0-420c-861f-5a649e3d9674-c000.snappy.parquet","deletionTimestamp":1587968596253,"dataChange":true}} -{"add":{"path":"part-00000-a922ea3b-ffc2-4ca1-9074-a278c24c4449-c000.snappy.parquet","partitionValues":{},"size":262,"modificationTime":1587968595000,"dataChange":true}} -{"add":{"path":"part-00004-95c9bc2c-ac85-4581-b3cc-84502b0c314f-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} -{"add":{"path":"part-00005-94a0861b-6455-4bd9-a080-73e02491c643-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} -{"add":{"path":"part-00011-42f838f9-a911-40af-98f5-2fccfa1b123f-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} -{"add":{"path":"part-00045-332fe409-7705-45b1-8d34-a0018cf73b70-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} -{"add":{"path":"part-00049-d3095817-de74-49c1-a888-81565a40161d-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} -{"add":{"path":"part-00058-b462c4cb-0c48-4148-8475-e21d2a2935f8-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} -{"add":{"path":"part-00068-90650739-6a8e-492b-9403-53e33b3778ac-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} -{"add":{"path":"part-00069-c78b4dd8-f955-4643-816f-cbd30a3f8c1b-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} -{"add":{"path":"part-00077-2fcb1c7c-5390-48ee-93f6-0acf11199a0d-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} -{"add":{"path":"part-00107-3f6c2aa0-fc28-4f4c-be15-135e15b398f4-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} -{"add":{"path":"part-00112-07fd790a-11dc-4fde-9acd-623e740be992-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} -{"add":{"path":"part-00116-bc66759e-6381-4f34-8cd4-6688aad8585d-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} -{"add":{"path":"part-00121-d8bc3e53-d2f2-48ce-9909-78da7294ffbd-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} -{"add":{"path":"part-00128-b31c3b81-24da-4a90-a8b4-578c9e9a218d-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} -{"add":{"path":"part-00140-e9b1971d-d708-43fb-b07f-975d2226b800-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} -{"add":{"path":"part-00143-03ceb88e-5283-4193-aa43-993cdf937fd3-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} -{"add":{"path":"part-00150-ec6643fc-4963-4871-9613-f5ad1940b689-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} -{"add":{"path":"part-00154-4630673a-5227-48fb-a03d-e356fcd1564a-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} -{"add":{"path":"part-00164-bf40481c-4afd-4c02-befa-90f056c2d77a-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} -{"add":{"path":"part-00190-8ac0ae67-fb1d-461d-a3d3-8dc112766ff5-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968596000,"dataChange":true}} diff --git a/crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000002.json b/crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000002.json deleted file mode 100644 index 8eeebb28d1..0000000000 --- a/crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000002.json +++ /dev/null @@ -1,29 +0,0 @@ -{"commitInfo":{"timestamp":1587968604143,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":1,"isBlindAppend":false}} -{"add":{"path":"part-00000-c1777d7d-89d9-4790-b38a-6ee7e24456b1-c000.snappy.parquet","partitionValues":{},"size":262,"modificationTime":1587968602000,"dataChange":true}} -{"add":{"path":"part-00001-7891c33d-cedc-47c3-88a6-abcfb049d3b4-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968602000,"dataChange":true}} -{"add":{"path":"part-00003-53f42606-6cda-4f13-8d07-599a21197296-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968602000,"dataChange":true}} -{"add":{"path":"part-00004-315835fe-fb44-4562-98f6-5e6cfa3ae45d-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968602000,"dataChange":true}} -{"add":{"path":"part-00006-46f2ff20-eb5d-4dda-8498-7bfb2940713b-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968602000,"dataChange":true}} -{"add":{"path":"part-00007-3a0e4727-de0d-41b6-81ef-5223cf40f025-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968602000,"dataChange":true}} -{"remove":{"path":"part-00004-95c9bc2c-ac85-4581-b3cc-84502b0c314f-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} -{"remove":{"path":"part-00107-3f6c2aa0-fc28-4f4c-be15-135e15b398f4-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} -{"remove":{"path":"part-00011-42f838f9-a911-40af-98f5-2fccfa1b123f-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} -{"remove":{"path":"part-00190-8ac0ae67-fb1d-461d-a3d3-8dc112766ff5-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} -{"remove":{"path":"part-00121-d8bc3e53-d2f2-48ce-9909-78da7294ffbd-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} -{"remove":{"path":"part-00005-94a0861b-6455-4bd9-a080-73e02491c643-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} -{"remove":{"path":"part-00077-2fcb1c7c-5390-48ee-93f6-0acf11199a0d-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} -{"remove":{"path":"part-00112-07fd790a-11dc-4fde-9acd-623e740be992-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} -{"remove":{"path":"part-00128-b31c3b81-24da-4a90-a8b4-578c9e9a218d-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} -{"remove":{"path":"part-00000-a922ea3b-ffc2-4ca1-9074-a278c24c4449-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} -{"remove":{"path":"part-00068-90650739-6a8e-492b-9403-53e33b3778ac-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} -{"remove":{"path":"part-00164-bf40481c-4afd-4c02-befa-90f056c2d77a-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} -{"remove":{"path":"part-00045-332fe409-7705-45b1-8d34-a0018cf73b70-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} -{"remove":{"path":"part-00058-b462c4cb-0c48-4148-8475-e21d2a2935f8-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} -{"remove":{"path":"part-00140-e9b1971d-d708-43fb-b07f-975d2226b800-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} -{"remove":{"path":"part-00116-bc66759e-6381-4f34-8cd4-6688aad8585d-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} -{"remove":{"path":"part-00069-c78b4dd8-f955-4643-816f-cbd30a3f8c1b-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} -{"remove":{"path":"part-00154-4630673a-5227-48fb-a03d-e356fcd1564a-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} -{"remove":{"path":"part-00150-ec6643fc-4963-4871-9613-f5ad1940b689-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} -{"remove":{"path":"part-00049-d3095817-de74-49c1-a888-81565a40161d-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} -{"remove":{"path":"part-00000-a72b1fb3-f2df-41fe-a8f0-e65b746382dd-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} -{"remove":{"path":"part-00143-03ceb88e-5283-4193-aa43-993cdf937fd3-c000.snappy.parquet","deletionTimestamp":1587968604143,"dataChange":true}} diff --git a/crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000003.json b/crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000003.json deleted file mode 100644 index 7c411a8f24..0000000000 --- a/crates/test/tests/data/simple_table_uncommited/_delta_log/00000000000000000003.json +++ /dev/null @@ -1,5 +0,0 @@ -{"commitInfo":{"timestamp":1587968614187,"operation":"UPDATE","operationParameters":{"predicate":"((id#697L % cast(2 as bigint)) = cast(0 as bigint))"},"readVersion":2,"isBlindAppend":false}} -{"remove":{"path":"part-00003-53f42606-6cda-4f13-8d07-599a21197296-c000.snappy.parquet","deletionTimestamp":1587968614096,"dataChange":true}} -{"remove":{"path":"part-00006-46f2ff20-eb5d-4dda-8498-7bfb2940713b-c000.snappy.parquet","deletionTimestamp":1587968614096,"dataChange":true}} -{"add":{"path":"part-00000-f17fcbf5-e0dc-40ba-adae-ce66d1fcaef6-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968614000,"dataChange":true}} -{"add":{"path":"part-00001-bb70d2ba-c196-4df2-9c85-f34969ad3aa9-c000.snappy.parquet","partitionValues":{},"size":429,"modificationTime":1587968614000,"dataChange":true}} diff --git a/crates/test/tests/data/simple_table_uncommited/part-00000-2befed33-c358-4768-a43c-3eda0d2a499d-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00000-2befed33-c358-4768-a43c-3eda0d2a499d-c000.snappy.parquet deleted file mode 100644 index 37061709636c6f5050fdadc50e72d8549ec85e03..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 262 zcmXw#ze~h06vsnxytUFohY}Dxs3EbKyL59Cr{e|}IgX@xJ&Wm&OHxHj|8XayoWmR5 z`+R@E+db}C9Hqt6%8k;#A6@+hos1(CU)@AW)M|8%(p8!z>%pFg9A)6-PfFa`kl;B_ z;E5eRNe5nP>1FaVWB)Y*uCViVsy!}n(}K|sSFbbfW%r9(9nKe-8TwP$`pFonOrXIA z1^y4j)x0c+1trgyy1UNiA7EWO^f@Dp5|-bpsWm#6TcMk6BSd+xI5eD>rKX&60S~km VY9lsOfzoW9W*I{0JIpS8;0O1OPTT+h diff --git a/crates/test/tests/data/simple_table_uncommited/part-00000-a72b1fb3-f2df-41fe-a8f0-e65b746382dd-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00000-a72b1fb3-f2df-41fe-a8f0-e65b746382dd-c000.snappy.parquet deleted file mode 100644 index 37061709636c6f5050fdadc50e72d8549ec85e03..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 262 zcmXw#ze~h06vsnxytUFohY}Dxs3EbKyL59Cr{e|}IgX@xJ&Wm&OHxHj|8XayoWmR5 z`+R@E+db}C9Hqt6%8k;#A6@+hos1(CU)@AW)M|8%(p8!z>%pFg9A)6-PfFa`kl;B_ z;E5eRNe5nP>1FaVWB)Y*uCViVsy!}n(}K|sSFbbfW%r9(9nKe-8TwP$`pFonOrXIA z1^y4j)x0c+1trgyy1UNiA7EWO^f@Dp5|-bpsWm#6TcMk6BSd+xI5eD>rKX&60S~km VY9lsOfzoW9W*I{0JIpS8;0O1OPTT+h diff --git a/crates/test/tests/data/simple_table_uncommited/part-00000-a922ea3b-ffc2-4ca1-9074-a278c24c4449-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00000-a922ea3b-ffc2-4ca1-9074-a278c24c4449-c000.snappy.parquet deleted file mode 100644 index 37061709636c6f5050fdadc50e72d8549ec85e03..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 262 zcmXw#ze~h06vsnxytUFohY}Dxs3EbKyL59Cr{e|}IgX@xJ&Wm&OHxHj|8XayoWmR5 z`+R@E+db}C9Hqt6%8k;#A6@+hos1(CU)@AW)M|8%(p8!z>%pFg9A)6-PfFa`kl;B_ z;E5eRNe5nP>1FaVWB)Y*uCViVsy!}n(}K|sSFbbfW%r9(9nKe-8TwP$`pFonOrXIA z1^y4j)x0c+1trgyy1UNiA7EWO^f@Dp5|-bpsWm#6TcMk6BSd+xI5eD>rKX&60S~km VY9lsOfzoW9W*I{0JIpS8;0O1OPTT+h diff --git a/crates/test/tests/data/simple_table_uncommited/part-00000-c1777d7d-89d9-4790-b38a-6ee7e24456b1-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00000-c1777d7d-89d9-4790-b38a-6ee7e24456b1-c000.snappy.parquet deleted file mode 100644 index 37061709636c6f5050fdadc50e72d8549ec85e03..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 262 zcmXw#ze~h06vsnxytUFohY}Dxs3EbKyL59Cr{e|}IgX@xJ&Wm&OHxHj|8XayoWmR5 z`+R@E+db}C9Hqt6%8k;#A6@+hos1(CU)@AW)M|8%(p8!z>%pFg9A)6-PfFa`kl;B_ z;E5eRNe5nP>1FaVWB)Y*uCViVsy!}n(}K|sSFbbfW%r9(9nKe-8TwP$`pFonOrXIA z1^y4j)x0c+1trgyy1UNiA7EWO^f@Dp5|-bpsWm#6TcMk6BSd+xI5eD>rKX&60S~km VY9lsOfzoW9W*I{0JIpS8;0O1OPTT+h diff --git a/crates/test/tests/data/simple_table_uncommited/part-00000-f17fcbf5-e0dc-40ba-adae-ce66d1fcaef6-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00000-f17fcbf5-e0dc-40ba-adae-ce66d1fcaef6-c000.snappy.parquet deleted file mode 100644 index 466ecb955860fdef9846accbb4a7f79bc1debe46..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 429 zcmZ`$&r8EF82uW@=s~=tG>}6HI5w!Eb*@{77jNQeh-uGU}?CLhefGPHTj11^vo4WQ3{GJ1Hp#KMJrcN%9xsR;91S8^KAJu{eZ0M3Li;;tzPs{nTdiLeP@__y^C)5N6vmRh8 zM?k1MD60h*OJSV(PY7PGG&gBht!j8xzVlo)LW)MX_iNUCEESuwx~bZE!+e%zO3Ru} zAJ(i8`L>lwL+{3PSu9Ldv|5Wqo34>5pY0d2>3Xwy+<<$tvpdTx?@d*8Ru*2!!+?ig yKWQ^9Ju!)8I!WVrc%DQmjiTXD22l{jY81@lWE_u!L?tpD5K7JK?ks?r`KWKLq*>$u diff --git a/crates/test/tests/data/simple_table_uncommited/part-00001-7891c33d-cedc-47c3-88a6-abcfb049d3b4-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00001-7891c33d-cedc-47c3-88a6-abcfb049d3b4-c000.snappy.parquet deleted file mode 100644 index 2964979ef6db7dca39bfa9bbe4d6dae04308352b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 429 zcmZ`$&r8EF7)|3GJ&3oI2689?#|AaD&UNeX;!Qja5icT=F6k_$?baWPjQtxP{A>ND z8@p%@;eGG>-uGU}?D{Swh%n;$1Ow<2o4NKe`aK8ap#C2$Or2h0xR0#8gkb95oYYHM zy*}6T7oAIj5$T*^7iS7Nm@>+GHX|K6vQd|AS%Mwxd|J+z)w8b-mIw5YIb{$S!Fq(Y za*QZ*2W7Poaw)-C{Dcto$_PlaYIP&3@?GS*kxDkweOUA6W2yO+*G<*V8}9Qo(?->N z`ncwW%(tyf0lgbDWwC&&XpNDHfiB3j&-V-YbiLU;ZIFArvpdf#?@d>ARu*0;!a#&x yKWQ_gJUNL~I!WVrc#%XpjiTXD1yK;idKAp#WE_u!L?6THq>_zvA2+=DSZY4ybyKzThWk9tv{5ym zK5cj*^Ia=bK<~#)SuCI`T4Q8lpbIkX^TR?u-E6ndTjbvD?Jn}lTkERM%EAjp7>Lm8 yCv9ewCnvE=CutlHFOx{8Q8XN?APS;bkAiuejN@^T=tPAB${4)v!9r*TAN38y^jY=* diff --git a/crates/test/tests/data/simple_table_uncommited/part-00001-c506e79a-0bf8-4e2b-a42b-9731b2e490ae-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00001-c506e79a-0bf8-4e2b-a42b-9731b2e490ae-c000.snappy.parquet deleted file mode 100644 index 4cb7aae46305259afd5baa77685636cf4b1261e0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 429 zcmZ`$&r8EF7)|3CJ&3oI2689?#|AaD&UI_?;!Qja5icT=F6k_$?baWPjQtxP{A>Lt z8@p%@;pKhb``$}3zqtxdjQ{DXzIvnveyiFJ?=O#R!_W+m&l zmuB&*3rR2{y>sm1Tp(<>eC%du!Eg1%lWnj_Vv;7i2gCB3<4tTdWf&p}i&0O0;zw3w`)c=EpuCpr)`-yds5KR5s(|ReZ zx0ibUs&h#&BAs*W;!Gh2Q$|_OW~4($HtNzHOR$5TPs{nTdiM3v@`(O1rwjrk7?03S zjuB<Lm8 yCv9ewCnvE=CutlHFOx{8Q8XN?APS;bkAiuejN@^T=tPAB${5`40Umb-AN398aaJ<` diff --git a/crates/test/tests/data/simple_table_uncommited/part-00003-53f42606-6cda-4f13-8d07-599a21197296-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00003-53f42606-6cda-4f13-8d07-599a21197296-c000.snappy.parquet deleted file mode 100644 index a1caecfbace73080e53a4875572a74b87c91aca7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 429 zcmZ`$&r8EF7)|3CJ&3oI2689?#|AaD&UNeX;!Qja5icT=F6k_$?baU?8T&Up_}BVP zH+In+!u#I$z3;t{+0A`O5Mjjg2?o$5HgoMR`aMVFp#C2$Or2d}xKFHugkb95p4LlQ zy}i`)SDj0O5$T*`7iS7Nm@>+GHX|K6vQd}rSb`nwd|J+z)w8dUmPhoDIb{$S!Fq&t za*QZ*2W7Poaw)-C{Dcto$_PlaYIP&3@?GS*kxDkwecbToW2yO+*G<*V8}9Qo(?->N z`n2JN%y+F!0lgnHWwC&&XpNDHfiB3j&kqavbhF()Z;^Yqx4XzIZ>_63D+@0aVIV@U ypR}1#o}9!gouqL*yi6jUM$vGnf+&b$JqqS=GLFYVq7xMkC}Z%t2MeJYeAG7|$W~wg diff --git a/crates/test/tests/data/simple_table_uncommited/part-00004-315835fe-fb44-4562-98f6-5e6cfa3ae45d-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00004-315835fe-fb44-4562-98f6-5e6cfa3ae45d-c000.snappy.parquet deleted file mode 100644 index 1cd670d1938d11793908ad9575052e04c257dcbe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 429 zcmZ`$&r8EF7)|4tJ&3oI2689?#|AaD&UNeX;!Qja5icT=F6k_$?baWPjQtxP{A>ND z8@p%@;eGG>-uGU}?D{Swh%n;$1Ow<2o4NKe`aK8ap#C2$Or2h0xR0#8gkb95oYYHM zy*}6T7oAIj5$T*^7iS7Nm@>+GHX|K6vQd|AS%Mwxd|J+z)w8b-mIw5YIb{$S!Fq(Y za*QZ*2W7Poaw)-C{Dcto$_PlaYIP&3@?GS*kxDkweOUA6W2yO+*G<*V8}9Qo(?->N z`ncwW%(tyf0lgbDWwC&&XpNDHfiB3j&-V-YbiLU;ZIFArvpdf#?@d>ARu*0;!a#&x yKWQ_gJUNL~I!WVrc#%XpjiTXD1yK;idKAp#WE_u!L?}6HI5w!Eb*`?%i#PE!M7)Sdx}>w1wp)KFGWKtH@UJ!1 zja@XSm-oH*y%#dSz6&v6iaj4A1Nzveu8rXL9FPOUKUkwWx#+}4*4}~<@o$dnm8f2y z>&1)81xA2-r^rQ_1P2jH=)k79M$T>l&AL?$ugZ6xt42uC2=`&bnva!YGgdcMyJ(ot(oAVtv)SW@ z6(a9OnKblHrpsb!ZbhrLNVJ(6oATLyA)9Ts+ovtKx4XEryz<^uRcB@4g**&+=na!L w)6x^uSfqRoq|zuFk7W=AQLHAxB2F&j%OFvS3`c}g^SXNrU~WF@8}S)dJOBUy diff --git a/crates/test/tests/data/simple_table_uncommited/part-00004-95c9bc2c-ac85-4581-b3cc-84502b0c314f-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00004-95c9bc2c-ac85-4581-b3cc-84502b0c314f-c000.snappy.parquet deleted file mode 100644 index 5770a94ab6136340e11bdadaaeeba5dcf36b772f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 429 zcmZ`$&r8EF7)|3CJ%|@cX&{FZaBNUR>s+@EFW$t{5b+`+>5|T3+HU=!$k@N(!N1mT zy0MGq5Z?E`?|tuu%&zZ3f(Rp?PcVQkv6*Y1qTh2s4(k8G!qmwnhWp6cO9-a^&2hby z)$4OTf6=)l7?I8yc5$YVgDIn|XEV~FBO7(;mL=H1&Zp&kSv~veV0l3Qm{SIU5v)gO zE60d3cTiRfA(s-I#ZL%PuZ(~+t5!FnD&Iw}8>wU?-G?=AK9-tKdEHd)yx~4iGi_AO zr;lr1$b8$%6wtddQx*%Tiq;sJ80dmb`+UEUPuH8x(+0V>JG=9|^4@e+XJz4qA`C?6 z^^-O;%9E2=rIR#{hZjkt(Lt z8@p%@;pKhb``$}3zqtxdjQ{DXzIvnveyiFJ?=O#R!_W+m&l zmuB&*3rR2{y>sm1Tp(<>eC%du!Eg1%lWnj_Vv;7i2gCB3<4t}6HI5w!Eb*@{77jNQeh*fmWoYT-Bj(oVLnSUrDe^g zPa9T!>8U^z>8OP%wQHcx(gi`am2Mb_kKI$7c7*=us diff --git a/crates/test/tests/data/simple_table_uncommited/part-00006-63ce9deb-bc0f-482d-b9a1-7e717b67f294-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00006-63ce9deb-bc0f-482d-b9a1-7e717b67f294-c000.snappy.parquet deleted file mode 100644 index ec63649165cdab46a6642fe3e32f8be13979b618..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 429 zcmZ`$&r8EF82y@!(SvwPX&{FZaBNUR>s+@EFW$t{5b+`+>5|T3+HU=!$k@N(!N1l_ zH+In+!u#I$z3;t{`So3h0aNVx7#YyRHg)X|{GKCnVDJZPrcN)BxjWWDf)Vj=PU@AY zUZ3m5i^>H?fV*eNMVSN#5lU#^rnpPSHuT7j#mGU_gD8k%H4YYWGKnWaq7oSn38m(B_ZGn1eAGAm!B#~8 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00007-3a0e4727-de0d-41b6-81ef-5223cf40f025-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00007-3a0e4727-de0d-41b6-81ef-5223cf40f025-c000.snappy.parquet deleted file mode 100644 index 533801e17f8f556a8e034d9bd7f7411d33ed8dfe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 429 zcmZ`$&r8EF7)|3C6~tRg138p{V}lx6=el)x@g|;zh!+t_mvk1>cIyvC#{LZt{TdWf&p}i&0PB!{hkAIQ2!4WrcN(0+(*`4LNN7jPU@ws zUZ3mvi_Rs%h;+`di!+5BOc`Z8n~@G3*{DmmEWr+TJ}u|V>e*KZ%LDqyoH7WEU_C-x zIYyMZgR)u(xs>26enN!xbw4flDPX`^aB zeO&WG=G#`LfZmOnvRFV>w8qH9Ko?}%=lg|xy54M_Hpso**`4Q=_ok~lD+@0aVIV@U ypR}1#o}9!gouqL*yhtLQM$vGnf+&b$JqqS=GLFYVq7xMkC}Z%tdkdi%eAG8Wz*c(z diff --git a/crates/test/tests/data/simple_table_uncommited/part-00007-94f725e2-3963-4b00-9e83-e31021a93cf9-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00007-94f725e2-3963-4b00-9e83-e31021a93cf9-c000.snappy.parquet deleted file mode 100644 index ea2985c2bc2c2730d8d4442a6eb4e120b8e51c10..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 429 zcmZ`$&r8EF7)|3CJ&3oI2689?#|AaD&UNeX;!Qja5icT=F6k_$?baU?8T&Up_}BVP zH+In+!u#I$z3;t{+0A`O5Mjjg2?o$5HgoL``aMVFp#C2$Or2d}xKFHugkb95p4LlQ zy}i`)SDj0O5$T*`7iS7Nm@>+GHX|K6vQd}rSb`nwd|J+z)w8dUmPhoDIb{$S!Fq&t za*QZ*2W7Poaw)-C{Dcto$_PlaYIP&3@?GS*kxDkwecbToW2yO+*G<*V8}9Qo(?->N z`n2JN%y+F!0lgnHWwC&&XpNDHfiB3j&kqavbhF()Z;^Yqx4XzIZ>_63D+@0aVIV@U ypR}1#o}9!gouqL*yi6jUM$vGnf+&b$JqqS=GLFYVq7xMkC}Z%t2MeJYeAG7xcvemT diff --git a/crates/test/tests/data/simple_table_uncommited/part-00011-42f838f9-a911-40af-98f5-2fccfa1b123f-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00011-42f838f9-a911-40af-98f5-2fccfa1b123f-c000.snappy.parquet deleted file mode 100644 index 1cd670d1938d11793908ad9575052e04c257dcbe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 429 zcmZ`$&r8EF7)|4tJ&3oI2689?#|AaD&UNeX;!Qja5icT=F6k_$?baWPjQtxP{A>ND z8@p%@;eGG>-uGU}?D{Swh%n;$1Ow<2o4NKe`aK8ap#C2$Or2h0xR0#8gkb95oYYHM zy*}6T7oAIj5$T*^7iS7Nm@>+GHX|K6vQd|AS%Mwxd|J+z)w8b-mIw5YIb{$S!Fq(Y za*QZ*2W7Poaw)-C{Dcto$_PlaYIP&3@?GS*kxDkweOUA6W2yO+*G<*V8}9Qo(?->N z`ncwW%(tyf0lgbDWwC&&XpNDHfiB3j&-V-YbiLU;ZIFArvpdf#?@d>ARu*0;!a#&x yKWQ_gJUNL~I!WVrc#%XpjiTXD1yK;idKAp#WE_u!L?+GHX|K6vQd}rSb`nwd|J+z)w8dUmPhoDIb{$S!Fq&t za*QZ*2W7Poaw)-C{Dcto$_PlaYIP&3@?GS*kxDkwecbToW2yO+*G<*V8}9Qo(?->N z`n2JN%y+F!0lgnHWwC&&XpNDHfiB3j&kqavbhF()Z;^Yqx4XzIZ>_63D+@0aVIV@U ypR}1#o}9!gouqL*yi6jUM$vGnf+&b$JqqS=GLFYVq7xMkC}Z%t2MeJYeAG7|$W~wg diff --git a/crates/test/tests/data/simple_table_uncommited/part-00049-d3095817-de74-49c1-a888-81565a40161d-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00049-d3095817-de74-49c1-a888-81565a40161d-c000.snappy.parquet deleted file mode 100644 index 533801e17f8f556a8e034d9bd7f7411d33ed8dfe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 429 zcmZ`$&r8EF7)|3C6~tRg138p{V}lx6=el)x@g|;zh!+t_mvk1>cIyvC#{LZt{TdWf&p}i&0PB!{hkAIQ2!4WrcN(0+(*`4LNN7jPU@ws zUZ3mvi_Rs%h;+`di!+5BOc`Z8n~@G3*{DmmEWr+TJ}u|V>e*KZ%LDqyoH7WEU_C-x zIYyMZgR)u(xs>26enN!xbw4flDPX`^aB zeO&WG=G#`LfZmOnvRFV>w8qH9Ko?}%=lg|xy54M_Hpso**`4Q=_ok~lD+@0aVIV@U ypR}1#o}9!gouqL*yhtLQM$vGnf+&b$JqqS=GLFYVq7xMkC}Z%tdkdi%eAG8Wz*c(z diff --git a/crates/test/tests/data/simple_table_uncommited/part-00058-b462c4cb-0c48-4148-8475-e21d2a2935f8-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00058-b462c4cb-0c48-4148-8475-e21d2a2935f8-c000.snappy.parquet deleted file mode 100644 index 17a7a308f42d4f4eb92ddbf67b9c813a00c6921c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 429 zcmZ`$&r8EF7)|3CJ&52TrGXquz_CFMt#jQvym%8&L&S@Sq)R%BX}k4@B4huC2me~X z>BcUaLwMi&zW2QsGP}MD2_lSmKEVLG#AdF2ihj=lIjH{!3sWbT815r$FCm!vH^=o- zRND z8@p%@;eGG>-uGU}?D{Swh%n;$1Ow<2o4NKe`aK8ap#C2$Or2h0xR0#8gkb95oYYHM zy*}6T7oAIj5$T*^7iS7Nm@>+GHX|K6vQd|AS%Mwxd|J+z)w8b-mIw5YIb{$S!Fq(Y za*QZ*2W7Poaw)-C{Dcto$_PlaYIP&3@?GS*kxDkweOUA6W2yO+*G<*V8}9Qo(?->N z`ncwW%(tyf0lgbDWwC&&XpNDHfiB3j&-V-YbiLU;ZIFArvpdf#?@d>ARu*0;!a#&x yKWQ_gJUNL~I!WVrc#%XpjiTXD1yK;idKAp#WE_u!L?TdWf&p}i&0O0;zw3w`)c=EpuCpr)`-yds5KR5s(|ReZ zx0ibUs&h#&BAs*W;!Gh2Q$|_OW~4($HtNzHOR$5TPs{nTdiM3v@`(O1rwjrk7?03S zjuB<Lm8 yCv9ewCnvE=CutlHFOx{8Q8XN?APS;bkAiuejN@^T=tPAB${5`40Umb-AN398aaJ<` diff --git a/crates/test/tests/data/simple_table_uncommited/part-00077-2fcb1c7c-5390-48ee-93f6-0acf11199a0d-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00077-2fcb1c7c-5390-48ee-93f6-0acf11199a0d-c000.snappy.parquet deleted file mode 100644 index df7f737162902c4b54c83142f6af578074176961..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 429 zcmZ`$&r8EF7)|3CMZ{Z5138p{V}lx6=el)x@g|;zh!+t_mvk1>cIyvC#{LZt{TdWf&p}i&0PB!{hkAIQ2!4WrcN(0+(*`4LNN7jPU@ws zUZ3mvi_Rs%h;+`di!+5BOc`Z8n~@G3*{DmmEWr+TJ}u|V>e*KZ%LDqyoH7WEU_C-x zIYyMZgR)u(xs>26enN!xbw4flDPX`^aB zeO&WG=G#`LfZmOnvRFV>w8qH9Ko?}%=lg|xy54M_Hpso**`4Q=_ok~lD+@0aVIV@U ypR}1#o}9!gouqL*yhtLQM$vGnf+&b$JqqS=GLFYVq7xMkC}Z%tdkdi%eAG8iXjX&( diff --git a/crates/test/tests/data/simple_table_uncommited/part-00107-3f6c2aa0-fc28-4f4c-be15-135e15b398f4-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00107-3f6c2aa0-fc28-4f4c-be15-135e15b398f4-c000.snappy.parquet deleted file mode 100644 index ec63649165cdab46a6642fe3e32f8be13979b618..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 429 zcmZ`$&r8EF82y@!(SvwPX&{FZaBNUR>s+@EFW$t{5b+`+>5|T3+HU=!$k@N(!N1l_ zH+In+!u#I$z3;t{`So3h0aNVx7#YyRHg)X|{GKCnVDJZPrcN)BxjWWDf)Vj=PU@AY zUZ3m5i^>H?fV*eNMVSN#5lU#^rnpPSHuT7j#mGU_gD8k%H4YYWGKnWaq7oSn38m(B_ZGn1eAGAm!B#~8 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00112-07fd790a-11dc-4fde-9acd-623e740be992-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00112-07fd790a-11dc-4fde-9acd-623e740be992-c000.snappy.parquet deleted file mode 100644 index b5e5edad5e55405cda49241ab0bfa6bc71237653..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 429 zcmZ`$&r8EF7)|3CJq&Lt4dhS)jty#Po$J=&#hZ8f2XG%Om>7oH7WEU_C-R zIYyMZ!?Icmxsu>4enN!xZK4flDPX`^aB zd))9s=DSv=fZmUpvRFb@w8qH9Ko?}%=ZA%Sw%Kl}6HI5w!Eb*@{77jNQeh*fmWoYT-Bj(oVLnSUrDe^g zPa9T!>8U^z>8OP%wQHcx(gi`am2Mb_kKI$7c7*=us diff --git a/crates/test/tests/data/simple_table_uncommited/part-00121-d8bc3e53-d2f2-48ce-9909-78da7294ffbd-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00121-d8bc3e53-d2f2-48ce-9909-78da7294ffbd-c000.snappy.parquet deleted file mode 100644 index 960d2b6e47cb3b2079d3b15800927e18e342929a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 429 zcmZ`$&r8EF7)|3C1;txR138p{V}lx6=el)x@g|;zh!+t_mvk1>cIyvC#{LZt{TdWf&p}i&0PB!{hkAIQ2!4WrcN(0+(*`4LNN7jPU@ws zUZ3mvi_Rs%h;+`di!+5BOc`Z8n~@G3*{DmmEWr+TJ}u|V>e*KZ%LDqyoH7WEU_C-x zIYyMZgR)u(xs>26enN!xbw4flDPX`^aB zeO&WG=G#`LfZmOnvRFV>w8qH9Ko?}%=lg|xy54M_Hpso**`4Q=_ok~lD+@0aVIV@U ypR}1#o}9!gouqL*yhtLQM$vGnf+&b$JqqS=GLFYVq7xMkC}Z%tdkdi%eAG8u5LS%< diff --git a/crates/test/tests/data/simple_table_uncommited/part-00128-b31c3b81-24da-4a90-a8b4-578c9e9a218d-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00128-b31c3b81-24da-4a90-a8b4-578c9e9a218d-c000.snappy.parquet deleted file mode 100644 index 3b91e5be30c42fa560fdfce67f347a692072b1fb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 429 zcmZ`$&r8EF82uW@=s~=tG>}6HI5w!Eb*`?%i#PE!M7)Sdx}>w1wp)KFGWKtH@UJ!1 zja@XSm-oH*y%#dSz6&v6iaj4A1Nzveu8rXL9FPOUKUkwWx#+}4*4}~<@o$dnm8f2y z>&1)81xA2-r^rQ_1P2jH=)k79M$T>l&AL?$ugZ6xt42uC2=`&bnva!YGgdcMyJ(ot(oAVtv)SW@ z6(a9OnKblHrpsb!ZbhrLNVJ(6oATLyA)9Ts+ovtKx4XEryz<^uRcB@4g**&+=na!L w)6x^uSfqRoq|zuFk7W=AQLHAxB2F&j%OFvS3`c}g^SXNrU~WF@8}S)dJOBUy diff --git a/crates/test/tests/data/simple_table_uncommited/part-00140-e9b1971d-d708-43fb-b07f-975d2226b800-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00140-e9b1971d-d708-43fb-b07f-975d2226b800-c000.snappy.parquet deleted file mode 100644 index ea2985c2bc2c2730d8d4442a6eb4e120b8e51c10..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 429 zcmZ`$&r8EF7)|3CJ&3oI2689?#|AaD&UNeX;!Qja5icT=F6k_$?baU?8T&Up_}BVP zH+In+!u#I$z3;t{+0A`O5Mjjg2?o$5HgoL``aMVFp#C2$Or2d}xKFHugkb95p4LlQ zy}i`)SDj0O5$T*`7iS7Nm@>+GHX|K6vQd}rSb`nwd|J+z)w8dUmPhoDIb{$S!Fq&t za*QZ*2W7Poaw)-C{Dcto$_PlaYIP&3@?GS*kxDkwecbToW2yO+*G<*V8}9Qo(?->N z`n2JN%y+F!0lgnHWwC&&XpNDHfiB3j&kqavbhF()Z;^Yqx4XzIZ>_63D+@0aVIV@U ypR}1#o}9!gouqL*yi6jUM$vGnf+&b$JqqS=GLFYVq7xMkC}Z%t2MeJYeAG7xcvemT diff --git a/crates/test/tests/data/simple_table_uncommited/part-00143-03ceb88e-5283-4193-aa43-993cdf937fd3-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00143-03ceb88e-5283-4193-aa43-993cdf937fd3-c000.snappy.parquet deleted file mode 100644 index 25fe1a24f16f5e8a38ed3c836e4bb61f42ae8138..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 429 zcmZ`$&r8EF7)|3CJ*c;o2689?#|AaD&UNeX;!Qja5icT=F6k_$?baWPjQtxP{A>ND z8@p%@;eGG>-uGU}?D{Swh%n;$1Ow<2o4NK0`aK8ap#C2$Or2h0xR0#8gkb959M?-( zy*}6T7oAIj5$T*^7iS7Nm@>+GHX|K6vQd|AS%Mwxd|J+z)w8b-mIw5YIb{$S!Fq(Y za*QZ*2W7Poaw)-C{Dcto$_PlaYIP&3@?GS*kxDkweOUA6W2yO+*G<*V8}9Qo(?->N z`ncwW%(tyf0lgbDWwC&&XpNDHfiB3j&-V-YbiLU;ZIFArvpdf#?@d>ARu*0;!a#&x yKWQ_gJUNL~I!WVrc#%XpjiTXD1yK;idKAp#WE_u!L?BcUaLwMi&zW2QsGP}MD2_lSmKEVLG#AdF2ihj=lIjH{!3sWbT815r$FCm!vH^=o- zRs+@EFW$t{5b+`+>5|T3+HU=!$k@N(!N1l_ zH+In+!u#I$z3;t{`So3h0aNVx7#YyTHg)YD{GKCnVDJZPrh1pi+$YvSf)Vj=PV1GZ zUZ3m5i^>H?fIDZ%MVSN#5lU#^rnp1KHgw63#mGU_gD8k%H4YYWGKnWaq7oSn38m(B_ZGn1eAG962v(>7 diff --git a/crates/test/tests/data/simple_table_uncommited/part-00164-bf40481c-4afd-4c02-befa-90f056c2d77a-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00164-bf40481c-4afd-4c02-befa-90f056c2d77a-c000.snappy.parquet deleted file mode 100644 index c3cf2f57ca0058ef3d77ac1fa8c1e569a05480d2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 429 zcmZ`$&r8EF7)|3CJ&2xC8pxpp92?ZoI@hhki#PE!M7)Sdx}>w1wp)KFGWKtH@UQio zZtS8tg!jGgd*6E@v+KK%Ai{{}6AYkBZ06dh==U6ugZh84Fm-Z?;XbnV5`w9Jb6hWF z_4-`TUvw@BMx=9wU7RW8V9F@#*^G4P$VOedWeIk$^JzI>R?ogVSRT+n=9EET1nUvn z$}ytM9hB8V$fX2l@e@MSDEoIgGT*i`1@vyrl*Iz7qBTY)2D%{AKHo3o)AeTav_bCe&h9*~yfV2p6+}T4>rpU|lW{x_5}l}UKpBJA-CGFF;G@0)h_F_% diff --git a/crates/test/tests/data/simple_table_uncommited/part-00190-8ac0ae67-fb1d-461d-a3d3-8dc112766ff5-c000.snappy.parquet b/crates/test/tests/data/simple_table_uncommited/part-00190-8ac0ae67-fb1d-461d-a3d3-8dc112766ff5-c000.snappy.parquet deleted file mode 100644 index be5bf31a8b174d20f341bff6689da4770272c5c5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 429 zcmZ`$&r8EF7)|3CJ&1ToX&{FZaBNUR>s+@EFW$t{5b+`+>5|T3+HU=!$k@N(!N1mT zy0MGq5Z?E`?|tuu%&zZ3f(Rp?PcVQkv6*Y1qTh2s4(k8G!qmwnhWp6cO9-a^&2hby z)$4OTf6=)l7?I8yc5$YVgDIn|XEV~FBO7(;mL=H1&Zp&kSv~veV0l3Qm{SIU5v)gO zE60d3cTiRfA(s-I#ZL%PuZ(~+t5!FnD&Iw}8>wU?-G?=AK9-tKdEHd)yx~4iGi_AO zr;lr1$b8$%6wtddQx*%Tiq;sJ80dmb`+UEUPuH8x(+0V>JG=9|^4@e+XJz4qA`C?6 z^^-O;%9E2=rIR#{hZjkt( Date: Fri, 9 May 2025 17:27:57 -0700 Subject: [PATCH 3/8] Apply reformatting from cargo fmt. Signed-off-by: Corwin Joy --- crates/core/src/kernel/snapshot/log_segment.rs | 1 - crates/core/tests/time_travel.rs | 7 +++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/core/src/kernel/snapshot/log_segment.rs b/crates/core/src/kernel/snapshot/log_segment.rs index f333d4ee97..b6fa578c41 100644 --- a/crates/core/src/kernel/snapshot/log_segment.rs +++ b/crates/core/src/kernel/snapshot/log_segment.rs @@ -543,7 +543,6 @@ pub(super) async fn list_log_files( let mut commit_files = Vec::with_capacity(25); let mut checkpoint_files = Vec::with_capacity(10); - // QUESTION: Does this need to be filtered to exclude subdirs? for meta in fs_client .list_with_offset(Some(log_root), &start_from) diff --git a/crates/core/tests/time_travel.rs b/crates/core/tests/time_travel.rs index 1faaaf5f58..bb815b248b 100644 --- a/crates/core/tests/time_travel.rs +++ b/crates/core/tests/time_travel.rs @@ -6,7 +6,7 @@ use std::time::SystemTime; #[tokio::test] async fn time_travel_by_ds() { // test time travel on a table with an uncommited delta in a .tmp subfolder - + // git does not preserve mtime, so we need to manually set it in the test let log_dir = "../test/tests/data/simple_table/_delta_log"; let log_mtime_pair = vec![ @@ -16,7 +16,10 @@ async fn time_travel_by_ds() { ("00000000000000000003.json", "2020-05-04T22:47:31-07:00"), ("00000000000000000004.json", "2020-05-05T22:47:31-07:00"), // Final file is uncommitted by Spark and is in a .tmp subdir - (".tmp/00000000000000000005.json", "2020-05-06T22:47:31-07:00"), + ( + ".tmp/00000000000000000005.json", + "2020-05-06T22:47:31-07:00", + ), ]; for (fname, ds) in log_mtime_pair { let ts: SystemTime = ds_to_ts(ds).into(); From 7d4efb6f268edff2be01eb5800bc0432774e7008 Mon Sep 17 00:00:00 2001 From: Corwin Joy Date: Fri, 9 May 2025 17:33:39 -0700 Subject: [PATCH 4/8] Add uncommited delta_log entry Signed-off-by: Corwin Joy --- .../data/simple_table/_delta_log/.tmp/00000000000000000005.json | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 crates/test/tests/data/simple_table/_delta_log/.tmp/00000000000000000005.json diff --git a/crates/test/tests/data/simple_table/_delta_log/.tmp/00000000000000000005.json b/crates/test/tests/data/simple_table/_delta_log/.tmp/00000000000000000005.json new file mode 100644 index 0000000000..b24b1f9026 --- /dev/null +++ b/crates/test/tests/data/simple_table/_delta_log/.tmp/00000000000000000005.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1587968626637,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":4,"isBlindAppend":true}} +{"add":{"path":"part-00000-c1777d7d-89d9-4790-b38a-6ee7e24456b1-c001.snappy.parquet","partitionValues":{},"size":262,"modificationTime":1587968626600,"dataChange":true}} From c319ae437161201549d8d5b6d27b0defbe52e2bf Mon Sep 17 00:00:00 2001 From: Corwin Joy Date: Fri, 9 May 2025 17:48:34 -0700 Subject: [PATCH 5/8] Add parquet file linked to uncommited transaction. Signed-off-by: Corwin Joy --- ...9d9-4790-b38a-6ee7e24456b1-c001.snappy.parquet | Bin 0 -> 262 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 crates/test/tests/data/simple_table/part-00000-c1777d7d-89d9-4790-b38a-6ee7e24456b1-c001.snappy.parquet diff --git a/crates/test/tests/data/simple_table/part-00000-c1777d7d-89d9-4790-b38a-6ee7e24456b1-c001.snappy.parquet b/crates/test/tests/data/simple_table/part-00000-c1777d7d-89d9-4790-b38a-6ee7e24456b1-c001.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..37061709636c6f5050fdadc50e72d8549ec85e03 GIT binary patch literal 262 zcmXw#ze~h06vsnxytUFohY}Dxs3EbKyL59Cr{e|}IgX@xJ&Wm&OHxHj|8XayoWmR5 z`+R@E+db}C9Hqt6%8k;#A6@+hos1(CU)@AW)M|8%(p8!z>%pFg9A)6-PfFa`kl;B_ z;E5eRNe5nP>1FaVWB)Y*uCViVsy!}n(}K|sSFbbfW%r9(9nKe-8TwP$`pFonOrXIA z1^y4j)x0c+1trgyy1UNiA7EWO^f@Dp5|-bpsWm#6TcMk6BSd+xI5eD>rKX&60S~km VY9lsOfzoW9W*I{0JIpS8;0O1OPTT+h literal 0 HcmV?d00001 From 861c19f995d5f6fe4c01909ece5f74c741efbef5 Mon Sep 17 00:00:00 2001 From: Corwin Joy Date: Mon, 12 May 2025 12:53:16 -0700 Subject: [PATCH 6/8] Update crates/core/src/logstore/mod.rs Co-authored-by: Adam Reeve --- crates/core/src/logstore/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/core/src/logstore/mod.rs b/crates/core/src/logstore/mod.rs index 6c3cb8045b..694af2dc15 100644 --- a/crates/core/src/logstore/mod.rs +++ b/crates/core/src/logstore/mod.rs @@ -542,7 +542,7 @@ pub async fn get_latest_version( while let Some(obj_meta) = files.next().await { let obj_meta = obj_meta?; - let location_path: Path = obj_meta.location.clone(); + let location_path: &Path = &obj_meta.location; let parts: Vec<_> = location_path.prefix_match(prefix).unwrap().collect(); if parts.len() > 1 { // Per the spec, ignore any files in subdirectories. From e6b85e3cbbf7e314a6bfaeeaf36c201bbeab33ae Mon Sep 17 00:00:00 2001 From: Corwin Joy Date: Mon, 12 May 2025 12:56:10 -0700 Subject: [PATCH 7/8] Update crates/core/src/logstore/mod.rs Co-authored-by: Adam Reeve --- crates/core/src/logstore/mod.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/core/src/logstore/mod.rs b/crates/core/src/logstore/mod.rs index 694af2dc15..6bb88320a7 100644 --- a/crates/core/src/logstore/mod.rs +++ b/crates/core/src/logstore/mod.rs @@ -543,8 +543,15 @@ pub async fn get_latest_version( while let Some(obj_meta) = files.next().await { let obj_meta = obj_meta?; let location_path: &Path = &obj_meta.location; - let parts: Vec<_> = location_path.prefix_match(prefix).unwrap().collect(); - if parts.len() > 1 { + let part_count = location_path + .prefix_match(prefix) + .ok_or_else(|| { + DeltaTableError::generic(format!( + "Failed to match object location ({location_path}) with log store path" + )) + }) + .map(|parts| parts.count())?; + if part_count > 1 { // Per the spec, ignore any files in subdirectories. // Spark may create these as uncommited transactions which we don't want // From 84b44e7df3c36a868974c3a7bca1a3da47e0fafb Mon Sep 17 00:00:00 2001 From: Corwin Joy Date: Mon, 12 May 2025 13:23:05 -0700 Subject: [PATCH 8/8] Streamline path part count and apply to both locations. --- crates/core/src/logstore/mod.rs | 9 +-------- crates/core/src/table/mod.rs | 4 ++-- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/crates/core/src/logstore/mod.rs b/crates/core/src/logstore/mod.rs index 6bb88320a7..1f9f792ee9 100644 --- a/crates/core/src/logstore/mod.rs +++ b/crates/core/src/logstore/mod.rs @@ -543,14 +543,7 @@ pub async fn get_latest_version( while let Some(obj_meta) = files.next().await { let obj_meta = obj_meta?; let location_path: &Path = &obj_meta.location; - let part_count = location_path - .prefix_match(prefix) - .ok_or_else(|| { - DeltaTableError::generic(format!( - "Failed to match object location ({location_path}) with log store path" - )) - }) - .map(|parts| parts.count())?; + let part_count = location_path.prefix_match(prefix).unwrap().count(); if part_count > 1 { // Per the spec, ignore any files in subdirectories. // Spark may create these as uncommited transactions which we don't want diff --git a/crates/core/src/table/mod.rs b/crates/core/src/table/mod.rs index 54a186928d..31b1fb594d 100644 --- a/crates/core/src/table/mod.rs +++ b/crates/core/src/table/mod.rs @@ -572,8 +572,8 @@ impl DeltaTable { while let Some(obj_meta) = files.next().await { let obj_meta = obj_meta?; let location_path: Path = obj_meta.location.clone(); - let parts: Vec<_> = location_path.prefix_match(prefix).unwrap().collect(); - if parts.len() > 1 { + let part_count = location_path.prefix_match(prefix).unwrap().count(); + if part_count > 1 { // Per the spec, ignore any files in subdirectories. // Spark may create these as uncommited transactions which we don't want //