Migrate to uv for python package management

.github/workflows/ci.yml

@@ -12,16 +12,16 @@ jobs:
         runs-on: ubuntu-latest
         steps:
             - name: checkout
-              uses: actions/checkout@v3
-            - name: Set up Python 3.9
-              uses: actions/setup-python@v4
+              uses: actions/checkout@v4
+            - name: Install uv
+              uses: astral-sh/setup-uv@v7
               with:
-                  python-version: 3.9
+                  python-version: "3.9"
+            - name: Install dependencies
+              run: uv sync --all-extras
             - name: Linting
-              run: |
-                  python -m pip install --upgrade pip
-                  if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi
-                  pre-commit run --all-files
+              run: uv run pre-commit run --all-files
+
     build-n-test:
         name: Build and test
         runs-on: ubuntu-latest
@@ -34,23 +34,14 @@ jobs:
               uses: actions/checkout@v4
               with:
                   fetch-depth: 0
-            - name: Set up Python ${{ matrix.python-version }} (minimum supported python version for deltaCAT is 3.9)
-              uses: actions/setup-python@v4
+            - name: Install uv
+              uses: astral-sh/setup-uv@v7
               with:
                   python-version: ${{ matrix.python-version }}
-            - name: Install pypa/build
-              run: >-
-                  python -m
-                  pip install
-                  build
-                  --user
             - name: Install dependencies
-              run: |
-                  python -m pip install --upgrade pip
-                  if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi
+              run: uv sync --all-extras
             - name: Run unit tests + benchmarks
-              run: >-
-                  python -m pytest -m "not integration" --benchmark-json output.json
+              run: uv run pytest -m "not integration" --benchmark-json output.json
             # Download previous benchmark result from cache (if exists)
             - name: Download previous benchmark data
               uses: actions/cache@v4

.github/workflows/publish-to-pypi.yml

@@ -8,32 +8,21 @@ jobs:
         runs-on: ubuntu-latest
         steps:
             - name: Checkout
-              uses: actions/checkout@main
+              uses: actions/checkout@v4
               with:
                   fetch-depth: 0
-            - name: Set up Python 3.9 (minimum supported python version for deltaCAT)
-              uses: actions/setup-python@v4
+            - name: Install uv
+              uses: astral-sh/setup-uv@v7
               with:
                   python-version: "3.9"
-            - name: Install pypa/build
-              run: >-
-                  python -m
-                  pip install
-                  build
-                  --user
             - name: Install dependencies
-              run: |
-                  python -m pip install --upgrade pip
-                  pip install pytest
-                  if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi
+              run: uv sync --all-extras
             - name: Run unit tests
-              run: >-
-                  python -m pytest -m "not integration"
+              run: uv run pytest -m "not integration"
             - name: Echo release tag
               run: echo ${{ github.ref_name }}
-            - name: Build a binary wheel and a source tarball
-              run: >-
-                  python setup.py sdist bdist_wheel
+            - name: Build
+              run: uv build
             - name: Publish distribution to PyPI
               if: startsWith(github.ref, 'refs/tags')
               uses: pypa/gh-action-pypi-publish@release/v1

.gitignore

@@ -1,3 +1,6 @@
+# uv
+.venv/
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

Makefile

@@ -1,42 +1,30 @@
-venv:
-	if [ ! -d "venv" ]; then \
-  		if [[ '$(shell uname -m)' == 'arm64' ]]; then \
-			/usr/bin/python3 -m venv venv; \
-		else \
-			python -m venv venv; \
-		fi \
-	fi
+.PHONY: install lint test unit-test build clean clean-build type-mappings benchmark benchmark-aws test-converter-integration test-integration-rebuild publish
 
-clean-build:
-	rm -rf dist
-	rm -rf build
-	rm -rf deltacat.egg-info
+install:
+	uv sync --all-extras
 
-clean-venv:
-	rm -rf venv
+clean-build:
+	rm -rf dist build deltacat.egg-info
 
-clean: clean-build clean-venv
+clean: clean-build
+	rm -rf .venv
 
-build: venv
-	venv/bin/python setup.py sdist bdist_wheel
+build:
+	uv build
 
 rebuild: clean-build build
 
 deploy-s3:
 	./dev/deploy/aws/scripts/s3-build-and-deploy.sh
 
-install: venv
-	venv/bin/pip install --upgrade pip
-	venv/bin/pip install -r dev-requirements.txt
-
 lint: install
-	venv/bin/pre-commit run --all-files
+	uv run pre-commit run --all-files
 
 test: install
-	venv/bin/pytest -m "not integration"
+	uv run pytest -m "not integration"
 
 unit-test: install
-	venv/bin/pytest -m "not integration and not benchmark"
+	uv run pytest -m "not integration and not benchmark"
 
 test-converter-integration: install
 	docker-compose -f dev/iceberg-integration/docker-compose-integration.yml kill
@@ -45,27 +33,27 @@ test-converter-integration: install
 	sleep 3
 	docker-compose -f dev/iceberg-integration/docker-compose-integration.yml exec -T spark-iceberg ipython ./provision.py
 	export SPARK_LOCAL_IP="127.0.0.1"
-	venv/bin/python -m pytest deltacat/tests/compute/converter/integration -vv
+	uv run pytest deltacat/tests/compute/converter/integration -vv
 
 test-integration-rebuild:
 	docker-compose -f dev/iceberg-integration/docker-compose-integration.yml kill
 	docker-compose -f dev/iceberg-integration/docker-compose-integration.yml rm -f
 	docker-compose -f dev/iceberg-integration/docker-compose-integration.yml build --no-cache
 
 benchmark-aws: install
-	venv/bin/pytest deltacat/benchmarking/benchmark_parquet_reads.py --benchmark-only --benchmark-group-by=group,param:name
+	uv run pytest deltacat/benchmarking/benchmark_parquet_reads.py --benchmark-only --benchmark-group-by=group,param:name
 
 benchmark: install
-	pytest -m benchmark deltacat/benchmarking
+	uv run pytest -m benchmark deltacat/benchmarking
 
 type-mappings: install
 	@echo "Generating type mappings..."
-	venv/bin/python deltacat/docs/autogen/schema/inference/generate_type_mappings.py
+	uv run python deltacat/docs/autogen/schema/inference/generate_type_mappings.py
 	@echo "Parsing type mappings to markdown..."
-	venv/bin/python deltacat/docs/autogen/schema/inference/parse_json_type_mappings.py generate_type_mappings_results.json
+	uv run python deltacat/docs/autogen/schema/inference/parse_json_type_mappings.py generate_type_mappings_results.json
 	@echo "Generating Python compatibility mapping..."
-	venv/bin/python deltacat/docs/autogen/schema/inference/parse_json_type_mappings.py generate_type_mappings_results.json --python
+	uv run python deltacat/docs/autogen/schema/inference/parse_json_type_mappings.py generate_type_mappings_results.json --python
 	@echo "Type mappings generation complete!"
 
-publish: test test-integration rebuild
-	twine upload dist/*
+publish: test build
+	uv publish

pyproject.toml

@@ -0,0 +1,88 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "deltacat"
+dynamic = ["version"]
+description = "DeltaCAT is a portable Pythonic Data Lakehouse powered by Ray."
+readme = "README.md"
+license = "Apache-2.0"
+requires-python = ">=3.9,<3.11"
+authors = [{ name = "Ray Team" }]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "aws-embedded-metrics==3.2.0",
+    "boto3~=1.34",
+    "google-cloud-storage",
+    "gcsfs==2025.3.2",
+    "daft==0.5.22",
+    "intervaltree==3.1.0",
+    "msgpack~=1.0.7",
+    "numpy==1.23.2",
+    "pandas==2.2.3",
+    "polars==1.28.1",
+    "pyarrow==16.0.0",
+    "pydantic!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<2.10.0",
+    "pymemcache==4.0.0",
+    "ray[default]==2.46.0",
+    "tenacity==8.2.3",
+    "typing-extensions>=4.6.1",
+    "redis==5.0.0",
+    "schedule==1.2.0",
+]
+
+[project.optional-dependencies]
+iceberg = [
+    "pyiceberg[glue]>=0.9.0",
+    "pyiceberg[hive]>=0.9.0",
+    "pyiceberg[sql-sqlite]>=0.9.0",
+]
+beam = ["apache-beam==2.65.0"]
+s3fs = ["s3fs==2025.3.2"]
+dev = [
+    "apache-beam==2.65.0",
+    "black==22.12.0",
+    "faker==33.1.0",
+    "faker-file>=0.5.0",
+    "flake8==5.0.4",
+    "ipywidgets==8.1.5",
+    "isort==5.10.1",
+    "memray==1.6.0; platform_system != 'Windows' and sys_platform != 'darwin' and platform_machine != 'arm64' and platform_machine != 'aarch64'",
+    "moto==4.1.12",
+    "Pillow==11.0.0",
+    "pre-commit==2.20.0",
+    "pyiceberg[glue]>=0.9.0",
+    "pyiceberg[hive]>=0.9.0",
+    "pyiceberg[sql-sqlite]>=0.9.0",
+    "pyspark==3.5.3",
+    "pytest==7.2.0",
+    "pytest-benchmark==4.0.0",
+    "pytest-cov==4.0.0",
+    "pytest-mock==3.14.0",
+    "requests-mock==1.11.0",
+    "sphinx==7.2.6",
+    "tabulate==0.9.0",
+    "torch==2.8.0",
+    "torchvision==0.23.0",
+    "transformers==4.48.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/ray-project/deltacat"
+
+[tool.hatch.version]
+path = "deltacat/__init__.py"
+
+[tool.hatch.build.targets.sdist]
+include = ["/deltacat"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["deltacat"]