Merge branch 'main' into alancleary/VCF-2/stats-improvements

Author: alancleary

.github/workflows/ci-linters.yml

@@ -0,0 +1,33 @@
+name: Run pre-commit
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+
+jobs:
+  lint:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout TileDB-VCF
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Restore pre-commit cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pre-commit
+          key: pre-commit-ubuntu-24.04-3.12-${{ hashFiles('.github/workflows/lint-python.yml', '.pre-commit-config.yaml') }}
+
+      - name: Install pre-commit
+        run: pip -v install pre-commit
+
+      - name: Log pip dependencies
+        run: pip list
+
+      - name: Run pre-commit hooks on all files
+        run: pre-commit run -a -v

.pre-commit-config.yaml

@@ -0,0 +1,13 @@
+repos:
+  - repo: https://github.com/pre-commit/mirrors-clang-format
+    rev: v15.0.7
+    hooks:
+    - id: clang-format
+      types_or: [c++, c]
+      files: 'libtiledbvcf/.*|apis/java/src/main/java/io/tiledb/libvcfnative/.*|apis/python/src/.*'
+      exclude: 'libtiledbvcf/external/.*'
+  - repo: https://github.com/psf/black
+    rev: 25.9.0
+    hooks:
+    - id: black
+      files: 'apis/python/src/.*'

CONTRIBUTING.md

@@ -18,3 +18,12 @@ Thanks for your interest in TileDB-VCF. The notes below give some pointers for f
 - Make changes locally, then rebuild as appropriate for the level of changes (e.g.: `make` for `libtilebvcf` or `python setup.py develop` for `apis/python`).
 - Make sure to run `make check`, or `pytest` to verify changes against tests (add new tests where applicable).
 - Please submit [pull requests](https://help.github.com/en/desktop/contributing-to-projects/creating-a-pull-request) against the default [`main` branch of TileDB-VCF](https://github.com/TileDB-Inc/TileDB-VCF/tree/main).
+- If you edit the Python files, please run the pre-commit hooks
+
+  ```sh
+  python -m venv ./pre-commit
+  source ./pre-commit/bin/activate
+  python -m pip -v install pre-commit
+  pre-commit run -a -v
+  deactivate
+  ```

Makefile

@@ -51,22 +51,6 @@ docs:
 	@rm tiledbvcf
 	quarto render --fail-if-warnings
 
-# format
-# -------------------------------------------------------------------
-.PHONY: check-format
-check-format:
-	@./ci/run-clang-format.sh . clang-format 0 \
-		`find libtiledbvcf/src -name "*.cc" -or -name "*.h"`
-	@./ci/run-clang-format.sh . clang-format 0 \
-		`find libtiledbvcf/test -name "*.cc" -or -name "*.h"`
-
-.PHONY: format
-format:
-	 @./ci/run-clang-format.sh . clang-format 1 \
-		`find libtiledbvcf/src -name "*.cc" -or -name "*.h"`
-	 @./ci/run-clang-format.sh . clang-format 1 \
-		`find libtiledbvcf/test -name "*.cc" -or -name "*.h"`
-
 # venv
 # -------------------------------------------------------------------
 .PHONY: venv
@@ -116,8 +100,6 @@ Rules:
   notebooks           Execute notebooks and update cell outputs
   docs                Render the documentation
   docker              Build and test docker images
-  check-format        Run C++ format check
-  format              Run C++ format
   venv                Create a virtual environment
   wheel               Build python wheel
   clean               Remove build artifacts

apis/java/src/main/java/io/tiledb/libvcfnative/README.md

@@ -5,14 +5,15 @@ To build the binding you must first compile the java file to a header file by us
 javac -cp .:commons-io-2.14.0.jar *.java -h .;
 ```
 
-Next, format the header and replace the old
+Next, replace the old header
 ```
-clang-format -i io_tiledb_libvcfnative_LibVCFNative.h;
 mv io_tiledb_libvcfnative_LibVCFNative.h LibVCFNative.h;
 ```
 
 Finally, remove all ```.class``` files 
 
 ```
 rm *.class;
-```
+```
+
+This process can be done autmatically using the `genearteJNI.sh` script.

apis/java/src/main/java/io/tiledb/libvcfnative/generateJNI.sh

@@ -1,4 +1,3 @@
 javac -cp .:commons-io-2.14.0.jar *.java -h .;
-clang-format -i io_tiledb_libvcfnative_LibVCFNative.h;
 mv io_tiledb_libvcfnative_LibVCFNative.h LibVCFNative.h;
-rm *.class;
+rm *.class;

apis/python/pyproject.toml

@@ -44,7 +44,7 @@ dependencies = ["pandas", "pyarrow>=14.0.1", "numpy"]
 
 [project.optional-dependencies]
 test = ["pytest", "tiledb"]
-dev = ["black", "tiledb-cloud", "pybind11[global]"]
+dev = ["tiledb-cloud", "pybind11[global]"]
 
 [project.urls]
 homepage = "https://tiledb.com"

ci/azure-linux_mac.yml

@@ -6,34 +6,6 @@ steps:
       addToPath: true
     displayName: 'Set Python Version'
 
-  - script: python -m pip install --upgrade black pylint
-    displayName: 'Install black (formatter) and pylint (linter)'
-
-  - script: black --check .
-    displayName: 'Test Formatting'
-
-  # pylint cannot lint all files in a directory, it requires an __init__.py file (https://github.com/PyCQA/pylint/issues/352)
-  # run pylint on all directories with an __init__.py file
-  - bash: |
-      set -eux pipefail
-      find . -name __init__.py | xargs dirname | xargs pylint --disable=all --enable=E1101
-    displayName: 'Check function name/signature mismatches'
-
-  - bash: |
-      set -e pipefail
-      sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 15CF4D18AF4F7421
-      sudo add-apt-repository 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main'
-      sudo apt-get install -y clang-format-15
-      src=$BUILD_REPOSITORY_LOCALPATH/libtiledbvcf
-      cd $BUILD_REPOSITORY_LOCALPATH
-      ci/run-clang-format.sh $src clang-format-15 0 \
-        $(find $src/src $src/test -name "*.cc" -or -name "*.c" -or -name "*.h")
-      apis=$BUILD_REPOSITORY_LOCALPATH/apis
-      ci/run-clang-format.sh $apis clang-format-15 0 \
-        $(find $apis -name "*.cc" -or -name "*.c" -or -name "*.h")
-    condition: eq(variables['Agent.OS'], 'Linux')
-    displayName: 'Check formatting (Linux only)'
-
   - bash: |
       set -e pipefail