Merge branch 'main' into refactor/writer-async

Author: rtyler

.github/scripts/retry_integration_test.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set +e
+TEST_NAME=$1
+MAX_RETRIES=$2
+RETRY_DELAY=$3
+ATTEMPT=1
+run_command() {
+  uv run --no-sync pytest -m "($TEST_NAME and integration)" --doctest-modules 2>&1
+}
+until [ $ATTEMPT -gt $MAX_RETRIES ]
+do
+  echo "Attempt $ATTEMPT"
+  OUTPUT=$(run_command)
+  EXIT_CODE=$?
+  echo "$OUTPUT"
+  if [ $EXIT_CODE -eq 0 ]; then
+    echo "Tests passed."
+    exit 0
+  fi
+  echo "Failed. Retrying."
+  ATTEMPT=$((ATTEMPT+1))
+  sleep $RETRY_DELAY
+done
+if [ $ATTEMPT -gt $MAX_RETRIES ]; then
+  echo "Tests failed after $MAX_RETRIES attempts."
+  exit 1
+fi

.github/workflows/python_build.yml

@@ -137,7 +137,7 @@ jobs:
         run: make setup-dat
 
       - name: Run tests
-        run: uv run --no-sync pytest -m '(unitycatalog_databricks and integration)' --doctest-modules
+        run: ../../delta-rs/.github/scripts/retry_integration_test.sh unitycatalog_databricks 5 10
 
   test-unitycatalog-oss:
     name: Python Build (Python 3.10 Unity Catalog Integration tests)
@@ -166,7 +166,7 @@ jobs:
         run: make setup-dat
 
       - name: Run tests
-        run: uv run --no-sync pytest -m '(unitycatalog_oss and integration)' --doctest-modules
+        run: ../../delta-rs/.github/scripts/retry_integration_test.sh unitycatalog_oss 5 10
 
   test-pyspark:
     name: PySpark Integration Tests
@@ -192,7 +192,6 @@ jobs:
       - name: Run tests
         run: make test-pyspark
 
-
   multi-python-running:
     name: Running with Python ${{ matrix.python-version }}
     runs-on: ubuntu-latest

CONTRIBUTING.md

@@ -15,26 +15,22 @@ If you want to claim an issue to work on, you can write the word `take` as a com
 ## Quick start
 
 - Install Rust, e.g. as described [here](https://doc.rust-lang.org/cargo/getting-started/installation.html)
-- Have a compatible Python version installed (check `python/pyproject.toml` for current requirement)
-- Create a Python virtual environment (required for development builds), e.g. as described [here](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/)
-    ```sh
-    python -m venv .venv
-    ```
+- Install the [uv Python package manager](https://docs.astral.sh/uv/getting-started/installation/).
 
-- Build the project for development (this requires an active virtual environment and will also install `deltalake` in that virtual environment. [Uv](https://github.com/astral-sh/uv) packet manager needs to be installed)
+- Build the project for development. This will install `deltalake` into the Python virtual environment managed by uv.
     ```sh
     cd python
     make develop
     ```
 
 - Run some Python code, e.g. to run a specific test
     ```sh
-    python -m pytest tests/test_writer.py -s -k "test_with_deltalake_schema"
+    uv run pytest tests/test_writer.py -s -k "test_with_deltalake_schema"
     ```
 
 - Run some Rust code, e.g. run an example
     ```sh
-    cd crates/deltalake
+    cd ../crates/deltalake
     cargo run --example basic_operations --features="datafusion"
     ```
 

python/Cargo.toml

@@ -43,11 +43,14 @@ tokio = { workspace = true, features = ["rt-multi-thread"] }
 
 deltalake-mount = { path = "../crates/mount" }
 
+# catalog-unity
+deltalake-catalog-unity = { path = "../crates/catalog-unity", features = ["aws", "azure", "gcp", "r2"] }
+
 # Non-unix or emscripten os
 [target.'cfg(any(not(target_family = "unix"), target_os = "emscripten"))'.dependencies]
 mimalloc = { version = "0.1", default-features = false }
 
-# Unix (excluding macOS & emscripten) → jemalloc 
+# Unix (excluding macOS & emscripten) → jemalloc
 [target.'cfg(all(target_family = "unix", not(target_os = "macos"), not(target_os = "emscripten")))'.dependencies]
 jemallocator = { version = "0.5", features = ["disable_initial_exec_tls", "background_threads"] }
 

python/Makefile

@@ -1,7 +1,5 @@
 .DEFAULT_GOAL := help
 
-VENV := venv
-MATURIN_VERSION := $(shell grep 'requires =' pyproject.toml | cut -d= -f2- | tr -d '[ "]')
 PACKAGE_VERSION := $(shell grep version Cargo.toml | head -n 1 | awk '{print $$3}' | tr -d '"' )
 DAT_VERSION := 0.0.2
 
@@ -96,7 +94,8 @@ build-docs: ## Build documentation with mkdocs
 clean: ## Run clean
 	$(warning --- Clean virtualenv and target directory ---)
 	cargo clean
-	rm -rf $(VENV)
+	# Remove uv's venv
+	rm -rf .venv
 	find . -type f -name '*.pyc' -delete
 
 .PHONY: help