chore: enable dat testing with the existing code

prior to bringing kernel replay in, I would like "classic" delta-rs
parsing

Closes #863

Signed-off-by: R. Tyler Croy <[email protected]>

Author: roeap

.github/workflows/build.yml

@@ -96,7 +96,9 @@ jobs:
           override: true
 
       - name: Run tests
-        run: cargo test --verbose --features ${{ env.DEFAULT_FEATURES }}
+        run: |
+          gmake setup-dat
+          cargo test --verbose --features ${{ env.DEFAULT_FEATURES }}
 
   integration_test:
     name: Integration Tests
@@ -141,11 +143,13 @@ jobs:
 
       - name: Run tests with rustls (default)
         run: |
+          gmake setup-dat
           cargo test --features integration_test,${{ env.DEFAULT_FEATURES }}
 
       - name: Run tests with native-tls
         run: |
           cargo clean
+          gmake setup-dat
           cargo test --no-default-features --features integration_test,s3-native-tls,datafusion
 
   integration_test_lakefs:
@@ -176,5 +180,6 @@ jobs:
 
       - name: Run tests with rustls (default)
         run: |
+          gmake setup-dat
           cargo test --features integration_test_lakefs,lakefs,datafusion
 

.github/workflows/python_build.yml

@@ -73,9 +73,6 @@ jobs:
       - name: Build and install deltalake
         run: make develop
 
-      - name: Download Data Acceptance Tests (DAT) files
-        run: make setup-dat
-
       - name: Run tests
         run: uv run --no-sync pytest -m '((s3 or azure) and integration) or not integration and not benchmark' --doctest-modules
 
@@ -104,9 +101,6 @@ jobs:
       - name: Build and install deltalake
         run: make develop
 
-      - name: Download Data Acceptance Tests (DAT) files
-        run: make setup-dat
-
       - name: Run tests
         run: uv run --no-sync pytest -m '(lakefs and integration)' --doctest-modules
 
@@ -133,9 +127,6 @@ jobs:
       - name: Build and install deltalake
         run: make develop
 
-      - name: Download Data Acceptance Tests (DAT) files
-        run: make setup-dat
-
       - name: Run tests
         run: ../../delta-rs/.github/scripts/retry_integration_test.sh unitycatalog_databricks 5 10
 
@@ -162,9 +153,6 @@ jobs:
       - name: Build and install deltalake
         run: make develop
 
-      - name: Download Data Acceptance Tests (DAT) files
-        run: make setup-dat
-
       - name: Run tests
         run: ../../delta-rs/.github/scripts/retry_integration_test.sh unitycatalog_oss 5 10
 

crates/core/Cargo.toml

@@ -100,6 +100,7 @@ humantime = { version = "2.1.0" }
 [dev-dependencies]
 criterion = "0.5"
 ctor = "0"
+datatest-stable = "0.2"
 deltalake-test = { path = "../test", features = ["datafusion"] }
 dotenvy = "0"
 fs_extra = "1.2.0"
@@ -134,3 +135,7 @@ cloud = ["object_store/cloud"]
 
 # enable caching some file I/O operations when scanning delta logs
 delta-cache = ["foyer", "tempfile", "url/serde"]
+
+[[test]]
+name = "dat"
+harness = false

crates/core/tests/dat.rs

@@ -0,0 +1,52 @@
+use std::path::Path;
+
+use deltalake_core::DeltaTableBuilder;
+use deltalake_test::acceptance::read_dat_case;
+
+static SKIPPED_TESTS: &[&str; 4] = &[
+    "iceberg_compat_v1",
+    "column_mapping",
+    "check_constraints",
+    "deletion_vectors",
+];
+
+fn reader_test_eager(path: &Path) -> datatest_stable::Result<()> {
+    let root_dir = format!(
+        "{}/{}",
+        env!["CARGO_MANIFEST_DIR"],
+        path.parent().unwrap().to_str().unwrap()
+    );
+    for skipped in SKIPPED_TESTS {
+        if root_dir.ends_with(skipped) {
+            println!("Skipping test: {}", skipped);
+            return Ok(());
+        }
+    }
+
+    tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()?
+        .block_on(async {
+            let case = read_dat_case(root_dir).unwrap();
+
+            let table = DeltaTableBuilder::from_uri(case.table_root().unwrap())
+                .load()
+                .await
+                .expect("table");
+            let table_info = case.table_summary().expect("load summary");
+            let snapshot = table.snapshot().expect("Failed to load snapshot");
+            let protocol = table.protocol().expect("Failed to load protocol");
+            assert_eq!(snapshot.version() as u64, table_info.version);
+            assert_eq!(
+                (protocol.min_reader_version, protocol.min_writer_version),
+                (table_info.min_reader_version, table_info.min_writer_version)
+            );
+        });
+    Ok(())
+}
+
+datatest_stable::harness!(
+    reader_test_eager,
+    "../../dat/v0.0.3/reader_tests/generated/",
+    r"test_case_info\.json"
+);