feat: update delta_kernel to version 0.18.2 and refactor DeltaTable engine initialization (spiceai#9528)

Author: lukekim

Cargo.lock

@@ -164,7 +164,7 @@ datafusion-proto = { version = "51.0.0", features = ["json"] }
 datafusion-pruning = "51.0.0"
 datafusion-spark = "51.0.0"
 datafusion-table-providers = "0.1.0"  # Placeholder version; actual crate is provided via [patch.crates-io] from spiceai fork with DF51 support
-delta_kernel = { version = "0.17.0", features = [
+delta_kernel = { version = "0.18.2", features = [
     "default-engine-rustls",
     "arrow-57",
     "internal-api",
@@ -373,7 +373,7 @@ datafusion-table-providers = { git = "https://github.com/datafusion-contrib/data
 ballista-core = { git = "https://github.com/spiceai/datafusion-ballista.git", rev = "9298d4410b3cfde5639aa3d7bea8777942741573" } # spiceai-51
 ballista-executor = { git = "https://github.com/spiceai/datafusion-ballista.git", rev = "9298d4410b3cfde5639aa3d7bea8777942741573" } # spiceai-51
 ballista-scheduler = { git = "https://github.com/spiceai/datafusion-ballista.git", rev = "9298d4410b3cfde5639aa3d7bea8777942741573" } # spiceai-51
-delta_kernel = { git = "https://github.com/spiceai/delta-kernel-rs.git", rev = "4ce55c37be6f394675457545e806c503c813b34e" } # spiceai-0.17.0
+delta_kernel = { git = "https://github.com/spiceai/delta-kernel-rs.git", rev = "9d26fb3662d2edb8f382d4d0f7899c303549f0d0" } # spiceai-0.18.2
 
 # Patch duckdb for direct dependencies and spiceai_duckdb_fork for datafusion-table-providers
 duckdb = { git = "https://github.com/spiceai/duckdb-rs.git", rev = "f4096c7592ed46b9e68755a49252d69783dece96" } # spiceai-1.4.4

Cargo.toml

@@ -42,6 +42,7 @@ use datafusion::scalar::ScalarValue;
 use datafusion::sql::TableReference;
 use delta_kernel::engine::default::DefaultEngine;
 use delta_kernel::engine::default::executor::tokio::TokioBackgroundExecutor;
+use delta_kernel::engine::default::storage::store_from_url_opts;
 use delta_kernel::expressions::{BinaryExpressionOp, DecimalData, Expression, Scalar};
 use delta_kernel::scan::ScanBuilder;
 use delta_kernel::scan::state::{DvInfo, Stats};
@@ -196,18 +197,10 @@ impl DeltaTable {
         };
 
         let engine = match table_object_store {
-            Some(object_store) => Arc::new(DefaultEngine::new(
-                object_store.into(),
-                Arc::new(TokioBackgroundExecutor::default()),
+            Some(object_store) => Arc::new(DefaultEngine::new(object_store.into())),
+            None => Arc::new(DefaultEngine::new(
+                store_from_url_opts(&table_url, storage_options).map_err(handle_delta_error)?,
             )),
-            None => Arc::new(
-                DefaultEngine::try_new(
-                    &table_url,
-                    storage_options,
-                    Arc::new(TokioBackgroundExecutor::default()),
-                )
-                .map_err(handle_delta_error)?,
-            ),
         };
 
         let snapshot = Snapshot::builder_for(table_url.clone())

crates/data_components/src/delta_lake.rs

@@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
+#![expect(clippy::expect_used)]
 
 use arrow_flight::flight_service_server::{FlightService, FlightServiceServer};
 use arrow_flight::{

crates/flight_client/tests/cookie_middleware.rs

@@ -214,7 +214,6 @@ mod tests {
 
     #[test]
     fn test_u64_to_i64_telemetry_max_i64() {
-        #[expect(clippy::cast_sign_loss)]
         let max_i64_as_u64 = i64::MAX as u64;
         assert_eq!(u64_to_i64_telemetry(max_i64_as_u64, "test"), i64::MAX);
     }
@@ -223,7 +222,6 @@ mod tests {
     fn test_u64_to_i64_telemetry_overflow_clamps() {
         assert_eq!(u64_to_i64_telemetry(u64::MAX, "test"), i64::MAX);
 
-        #[expect(clippy::cast_sign_loss)]
         let just_over = (i64::MAX as u64) + 1;
         assert_eq!(u64_to_i64_telemetry(just_over, "test"), i64::MAX);
     }