Migrate to uv for python package management (#590)

* Migrate from pip/setup.py to uv for package management

- Replace setup.py with pyproject.toml using hatchling build backend
- Consolidate requirements.txt and dev-requirements.txt into pyproject.toml
- Update Makefile to use uv commands instead of venv/pip
- Update GitHub Actions workflows to use astral-sh/setup-uv@v4
- Remove MANIFEST.in (handled by hatch)
- Update .gitignore for uv artifacts

* update uv install action version

* Fix uv migration: add missing msgpack dep, commit lockfile

* update documentation and examples post migration

Author: JayThomason

.github/workflows/ci.yml

@@ -12,16 +12,16 @@ jobs:
         runs-on: ubuntu-latest
         steps:
             - name: checkout
-              uses: actions/checkout@v3
-            - name: Set up Python 3.9
-              uses: actions/setup-python@v4
+              uses: actions/checkout@v4
+            - name: Install uv
+              uses: astral-sh/setup-uv@v7
               with:
-                  python-version: 3.9
+                  python-version: "3.9"
+            - name: Install dependencies
+              run: uv sync --all-extras
             - name: Linting
-              run: |
-                  python -m pip install --upgrade pip
-                  if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi
-                  pre-commit run --all-files
+              run: uv run pre-commit run --all-files
+
     build-n-test:
         name: Build and test
         runs-on: ubuntu-latest
@@ -34,23 +34,14 @@ jobs:
               uses: actions/checkout@v4
               with:
                   fetch-depth: 0
-            - name: Set up Python ${{ matrix.python-version }} (minimum supported python version for deltaCAT is 3.9)
-              uses: actions/setup-python@v4
+            - name: Install uv
+              uses: astral-sh/setup-uv@v7
               with:
                   python-version: ${{ matrix.python-version }}
-            - name: Install pypa/build
-              run: >-
-                  python -m
-                  pip install
-                  build
-                  --user
             - name: Install dependencies
-              run: |
-                  python -m pip install --upgrade pip
-                  if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi
+              run: uv sync --all-extras
             - name: Run unit tests + benchmarks
-              run: >-
-                  python -m pytest -m "not integration" --benchmark-json output.json
+              run: uv run pytest -m "not integration" --benchmark-json output.json
             # Download previous benchmark result from cache (if exists)
             - name: Download previous benchmark data
               uses: actions/cache@v4

.github/workflows/publish-to-pypi.yml

@@ -8,32 +8,21 @@ jobs:
         runs-on: ubuntu-latest
         steps:
             - name: Checkout
-              uses: actions/checkout@main
+              uses: actions/checkout@v4
               with:
                   fetch-depth: 0
-            - name: Set up Python 3.9 (minimum supported python version for deltaCAT)
-              uses: actions/setup-python@v4
+            - name: Install uv
+              uses: astral-sh/setup-uv@v7
               with:
                   python-version: "3.9"
-            - name: Install pypa/build
-              run: >-
-                  python -m
-                  pip install
-                  build
-                  --user
             - name: Install dependencies
-              run: |
-                  python -m pip install --upgrade pip
-                  pip install pytest
-                  if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi
+              run: uv sync --all-extras
             - name: Run unit tests
-              run: >-
-                  python -m pytest -m "not integration"
+              run: uv run pytest -m "not integration"
             - name: Echo release tag
               run: echo ${{ github.ref_name }}
-            - name: Build a binary wheel and a source tarball
-              run: >-
-                  python setup.py sdist bdist_wheel
+            - name: Build
+              run: uv build
             - name: Publish distribution to PyPI
               if: startsWith(github.ref, 'refs/tags')
               uses: pypa/gh-action-pypi-publish@release/v1

.gitignore

@@ -1,3 +1,6 @@
+# uv
+.venv/
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

MANIFEST.in

@@ -1,42 +1,30 @@
-venv:
-	if [ ! -d "venv" ]; then \
-  		if [[ '$(shell uname -m)' == 'arm64' ]]; then \
-			/usr/bin/python3 -m venv venv; \
-		else \
-			python -m venv venv; \
-		fi \
-	fi
+.PHONY: install lint test unit-test build clean clean-build type-mappings benchmark benchmark-aws test-converter-integration test-integration-rebuild publish
 
-clean-build:
-	rm -rf dist
-	rm -rf build
-	rm -rf deltacat.egg-info
+install:
+	uv sync --all-extras
 
-clean-venv:
-	rm -rf venv
+clean-build:
+	rm -rf dist build deltacat.egg-info
 
-clean: clean-build clean-venv
+clean: clean-build
+	rm -rf .venv
 
-build: venv
-	venv/bin/python setup.py sdist bdist_wheel
+build:
+	uv build
 
 rebuild: clean-build build
 
 deploy-s3:
 	./dev/deploy/aws/scripts/s3-build-and-deploy.sh
 
-install: venv
-	venv/bin/pip install --upgrade pip
-	venv/bin/pip install -r dev-requirements.txt
-
 lint: install
-	venv/bin/pre-commit run --all-files
+	uv run pre-commit run --all-files
 
 test: install
-	venv/bin/pytest -m "not integration"
+	uv run pytest -m "not integration"
 
 unit-test: install
-	venv/bin/pytest -m "not integration and not benchmark"
+	uv run pytest -m "not integration and not benchmark"
 
 test-converter-integration: install
 	docker-compose -f dev/iceberg-integration/docker-compose-integration.yml kill
@@ -45,27 +33,27 @@ test-converter-integration: install
 	sleep 3
 	docker-compose -f dev/iceberg-integration/docker-compose-integration.yml exec -T spark-iceberg ipython ./provision.py
 	export SPARK_LOCAL_IP="127.0.0.1"
-	venv/bin/python -m pytest deltacat/tests/compute/converter/integration -vv
+	uv run pytest deltacat/tests/compute/converter/integration -vv
 
 test-integration-rebuild:
 	docker-compose -f dev/iceberg-integration/docker-compose-integration.yml kill
 	docker-compose -f dev/iceberg-integration/docker-compose-integration.yml rm -f
 	docker-compose -f dev/iceberg-integration/docker-compose-integration.yml build --no-cache
 
 benchmark-aws: install
-	venv/bin/pytest deltacat/benchmarking/benchmark_parquet_reads.py --benchmark-only --benchmark-group-by=group,param:name
+	uv run pytest deltacat/benchmarking/benchmark_parquet_reads.py --benchmark-only --benchmark-group-by=group,param:name
 
 benchmark: install
-	pytest -m benchmark deltacat/benchmarking
+	uv run pytest -m benchmark deltacat/benchmarking
 
 type-mappings: install
 	@echo "Generating type mappings..."
-	venv/bin/python deltacat/docs/autogen/schema/inference/generate_type_mappings.py
+	uv run python deltacat/docs/autogen/schema/inference/generate_type_mappings.py
 	@echo "Parsing type mappings to markdown..."
-	venv/bin/python deltacat/docs/autogen/schema/inference/parse_json_type_mappings.py generate_type_mappings_results.json
+	uv run python deltacat/docs/autogen/schema/inference/parse_json_type_mappings.py generate_type_mappings_results.json
 	@echo "Generating Python compatibility mapping..."
-	venv/bin/python deltacat/docs/autogen/schema/inference/parse_json_type_mappings.py generate_type_mappings_results.json --python
+	uv run python deltacat/docs/autogen/schema/inference/parse_json_type_mappings.py generate_type_mappings_results.json --python
 	@echo "Type mappings generation complete!"
 
-publish: test test-integration rebuild
-	twine upload dist/*
+publish: test build
+	uv publish

Makefile

@@ -20,19 +20,6 @@ make lint
 We use `make` to automate common development tasks. If you feel that any recurring development routine is
 missing from the current set of Makefile targets, please propose a new one!
 
-#### venv
-```shell
-make venv
-```
-Creates a virtual environment in the `venv` directory with all required development dependencies installed. You usually
-don't need to run this command directly, since it will be invoked automatically by any other target that needs it.
-
-#### clean-venv
-```shell
-make clean-venv
-```
-Removes all artifacts created by the `venv` Makefile target.
-
 #### build
 ```shell
 make build
@@ -55,7 +42,7 @@ Runs `clean-build` followed by `build`.
 ```shell
 make clean
 ```
-Removes all artifacts created by the `build` and `venv` Makefile targets.
+Removes all artifacts created by the `build` target and removes the virtual environment.
 
 #### deploy-s3
 ```shell
@@ -67,7 +54,7 @@ Builds and uploads a wheel to S3. See [Build and Deploy an S3 Wheel](#build-and-
 ```shell
 make install
 ```
-Installs all developer requirements from `dev-requirements.txt` in your virtual environment.
+Creates a virtual environment if it doesn't exist and installs all build and runtime dependencies from pyproject.toml.
 
 #### lint
 ```shell

README-development.md

@@ -1,4 +1,4 @@
-All the dependencies needed to test are in dev-requirements.txt
+All the dependencies needed to test are in pyproject.toml
 
 You can run these tests via:
 

deltacat/benchmarking/README.md

@@ -11,7 +11,7 @@ If you're working from DeltaCAT source code, you may want to install and activat
 make install
 
 # Activate the virtual environment
-source venv/bin/activate
+source .venv/bin/activate
 
 # Navigate to examples
 cd deltacat/examples/compactor

deltacat/examples/compactor/README.md

@@ -50,7 +50,7 @@ Run the following commands from the DeltaCAT project root directory
 make install
 
 # Activate virtual environment
-source venv/bin/activate
+source .venv/bin/activate
 
 # Navigate to example directory
 cd deltacat/examples/experimental/iceberg/converter/beam

deltacat/examples/experimental/iceberg/converter/beam/README.md

@@ -45,7 +45,7 @@ def run(warehouse="s3://my-bucket/my/key/prefix", **kwargs):
         2. retrieve the IAM Role AWS Credential and cache locally in ~/.aws/credentials
         3. run below command to execute the example
         ```
-        make venv && source venv/bin/activate
+        make install && source .venv/bin/activate
         python -m deltacat.examples.iceberg.iceberg_bucket_writer --warehouse=s3://<YOUR_S3_LOCATION>
         ```