Merge pull request #4 from jorgeMFS/main

Enhance VCFX with bug fixes, CI, Docker support & utilities

Author: jorgeMFS

.clang-format

@@ -0,0 +1,3 @@
+BasedOnStyle: LLVM
+IndentWidth: 4
+ColumnLimit: 120

.dockerignore

@@ -29,4 +29,4 @@ LICENSE
 # Docker files (not needed in the build context)
 Dockerfile
 docker-compose.yml
-.dockerignore 
+.dockerignore 

.github/workflows/build-test.yml

@@ -0,0 +1,99 @@
+name: Build and Test
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build-and-test:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Install dependencies (Linux)
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential cmake libz-dev
+
+      - name: Install dependencies (macOS)
+        if: runner.os == 'macOS'
+        run: |
+          brew update
+          brew install cmake zlib bash
+          echo "$(brew --prefix)/bin" >> $GITHUB_PATH
+
+      - name: Configure
+        run: cmake -S . -B build
+        shell: bash
+      - name: Build
+        run: cmake --build build --parallel
+        shell: bash
+
+      - name: Run tests
+        run: |
+          cd build
+          ctest --output-on-failure
+        shell: bash
+
+  python-wheels:
+    needs: build-and-test
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Install dependencies (Linux)
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential cmake libz-dev
+
+      - name: Install dependencies (macOS)
+        if: runner.os == 'macOS'
+        run: |
+          brew update
+          brew install cmake zlib bash
+          echo "$(brew --prefix)/bin" >> $GITHUB_PATH
+
+      - name: Build project
+        run: |
+          cmake -S . -B build -DPYTHON_BINDINGS=ON
+          cmake --build build --parallel
+          cmake --install build --prefix $PWD/install
+        shell: bash
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+
+      - name: Build wheel
+        run: |
+          python -m pip install --upgrade pip wheel
+          python -m pip wheel ./python -w dist
+        shell: bash
+
+      - name: Test Python wheel
+        run: |
+          python -m pip install dist/*.whl
+          export PATH="$PWD/install/bin:$PATH"
+          echo "$PWD/install/bin" >> $GITHUB_PATH
+          cat <<'EOF' | python -
+          import vcfx
+          print('version:', vcfx.get_version())
+          tools = vcfx.available_tools()
+          print('tools:', len(tools))
+          if tools:
+              vcfx.run_tool(tools[0], '--help', check=False)
+          EOF
+        shell: bash

.github/workflows/docker-publish.yml

@@ -57,4 +57,4 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           platforms: linux/amd64,linux/arm64
           cache-from: type=gha
-          cache-to: type=gha,mode=max 
+          cache-to: type=gha,mode=max

.github/workflows/docs.yml

@@ -29,40 +29,40 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      
+
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: '3.x'
-      
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install mkdocs-material pymdown-extensions
-      
+
       - name: Deploy to GitHub Pages
         run: |
           git config --global user.name "${GITHUB_ACTOR}"
           git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com"
           mkdocs gh-deploy --force
-  
+
   # Only for pull requests - just build to validate
   build:
     runs-on: ubuntu-latest
     if: github.event_name == 'pull_request'
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
-      
+
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: '3.x'
-      
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install mkdocs-material pymdown-extensions
-      
+
       - name: Build documentation
-        run: mkdocs build 
+        run: mkdocs build

.gitignore

@@ -41,4 +41,22 @@ Thumbs.db
 # Other
 tools.md
 prompt.md
-names.md
+names.md
+
+# Temporary outputs from genotype_query tests
+tests/tmp/genotype_query/
+tests/data/genotype_query/missing_malformed.vcf
+tests/data/genotype_query/multi_sample.vcf
+tests/data/genotype_query/single_sample.vcf
+tests/expected/genotype_query/missing_malformed_01.vcf
+tests/expected/genotype_query/multi_11_flexible.vcf
+tests/expected/genotype_query/multi_11_strict.vcf
+tests/expected/genotype_query/multi_12_flexible.vcf
+tests/expected/genotype_query/no_match.vcf
+tests/expected/genotype_query/single_sample_flex_01.vcf
+tests/expected/genotype_query/single_sample_strict_01.vcf
+
+# General temporary test output directories
+tests/tmp/
+tests/out/
+tmp/

CMakeLists.txt

@@ -6,17 +6,40 @@ set(VCFX_VERSION_MINOR 0)
 set(VCFX_VERSION_PATCH 2)
 set(VCFX_VERSION "${VCFX_VERSION_MAJOR}.${VCFX_VERSION_MINOR}.${VCFX_VERSION_PATCH}")
 
-project(VCFX 
+add_compile_definitions(VCFX_VERSION="${VCFX_VERSION}")
+
+project(VCFX
   VERSION ${VCFX_VERSION}
   DESCRIPTION "A Comprehensive VCF Manipulation Toolkit"
   LANGUAGES CXX
 )
 
+# Set a user-friendly default install prefix when none is provided.
+# This prevents installation attempts into system directories when
+# running without root privileges.
+if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
+    set(CMAKE_INSTALL_PREFIX "$ENV{HOME}/.local" CACHE PATH "Install prefix" FORCE)
+endif()
+
+include(GNUInstallDirs)
+
 # Optionally allow building for WebAssembly via Emscripten
 option(BUILD_WASM "Build with emscripten toolchain" OFF)
+option(PYTHON_BINDINGS "Build Python bindings" ON)
 
 if(BUILD_WASM)
-    set(CMAKE_TOOLCHAIN_FILE "/path/to/emscripten.cmake" CACHE FILEPATH "Emscripten toolchain" FORCE)
+    if(NOT CMAKE_TOOLCHAIN_FILE)
+        if(DEFINED ENV{EMSDK} AND EXISTS "$ENV{EMSDK}/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake")
+            set(CMAKE_TOOLCHAIN_FILE "$ENV{EMSDK}/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake" CACHE FILEPATH "Emscripten toolchain" FORCE)
+        elseif(DEFINED ENV{EMSCRIPTEN} AND EXISTS "$ENV{EMSCRIPTEN}/cmake/Modules/Platform/Emscripten.cmake")
+            set(CMAKE_TOOLCHAIN_FILE "$ENV{EMSCRIPTEN}/cmake/Modules/Platform/Emscripten.cmake" CACHE FILEPATH "Emscripten toolchain" FORCE)
+        endif()
+    endif()
+
+    if(NOT EXISTS "${CMAKE_TOOLCHAIN_FILE}")
+        message(FATAL_ERROR "Emscripten toolchain file not found. Please set CMAKE_TOOLCHAIN_FILE or EMSDK.")
+    endif()
+
     message(STATUS "Building for WebAssembly (Emscripten).")
 endif()
 
@@ -42,12 +65,14 @@ enable_testing()
 # Add top-level 'src' subdirectory, which in turn references each tool subdirectory
 add_subdirectory(src)
 
-# Add a tests subdir if you have tests
-# Comment out this line since we don't have a CMakeLists.txt file in the tests directory
-# add_subdirectory(tests)
+if(PYTHON_BINDINGS)
+  add_subdirectory(python)
+endif()
+
+# Add the test suite
+add_subdirectory(tests)
 
 # Installation configuration
-include(GNUInstallDirs)
 
 # Install header files
 install(FILES 

DOCKER.md

@@ -8,16 +8,16 @@ VCFX is available as a pre-built Docker image on GitHub Container Registry:
 
 ```bash
 # Pull the image (only needed once)
-docker pull ghcr.io/ieeta-pt/vcfx:latest
+docker pull ghcr.io/jorgemfs/vcfx:latest
 
 # Run a VCFX tool
-docker run --rm ghcr.io/ieeta-pt/vcfx:latest VCFX_tool_name [options]
+docker run --rm ghcr.io/jorgemfs/vcfx:latest VCFX_tool_name [options]
 
 # Mount a directory with your data
-docker run --rm -v /path/to/your/data:/data ghcr.io/ieeta-pt/vcfx:latest VCFX_tool_name [options]
+docker run --rm -v /path/to/your/data:/data ghcr.io/jorgemfs/vcfx:latest VCFX_tool_name [options]
 
 # Example: Process a VCF file (using tests/data/valid.vcf as an example)
-docker run --rm -v $(pwd)/tests/data:/data ghcr.io/ieeta-pt/vcfx:latest 'cat /data/valid.vcf | VCFX_allele_freq_calc > /data/output.tsv'
+docker run --rm -v $(pwd)/tests/data:/data ghcr.io/jorgemfs/vcfx:latest 'cat /data/valid.vcf | VCFX_allele_freq_calc > /data/output.tsv'
 ```
 
 Using the pre-built image is recommended for most users as it:
@@ -65,19 +65,19 @@ There are several ways to run VCFX tools with Docker:
 
 ```bash
 # With the pre-built image
-docker run --rm ghcr.io/ieeta-pt/vcfx:latest VCFX_tool_name [options]
+docker run --rm ghcr.io/jorgemfs/vcfx:latest VCFX_tool_name [options]
 
 # With a locally built image
 docker run --rm vcfx:local VCFX_tool_name [options]
 
 # Mount the tests/data directory to access test files
-docker run --rm -v $(pwd)/tests/data:/data ghcr.io/ieeta-pt/vcfx:latest VCFX_tool_name [options]
+docker run --rm -v $(pwd)/tests/data:/data ghcr.io/jorgemfs/vcfx:latest VCFX_tool_name [options]
 
 # Process files in the tests/data directory
-docker run --rm -v $(pwd)/tests/data:/data ghcr.io/ieeta-pt/vcfx:latest 'cat /data/valid.vcf | VCFX_validator'
+docker run --rm -v $(pwd)/tests/data:/data ghcr.io/jorgemfs/vcfx:latest 'cat /data/valid.vcf | VCFX_validator'
 
 # Example: Calculate allele frequencies for a VCF file
-docker run --rm -v $(pwd)/tests/data:/data ghcr.io/ieeta-pt/vcfx:latest 'cat /data/valid.vcf | VCFX_allele_freq_calc > /data/output.tsv'
+docker run --rm -v $(pwd)/tests/data:/data ghcr.io/jorgemfs/vcfx:latest 'cat /data/valid.vcf | VCFX_allele_freq_calc > /data/output.tsv'
 ```
 
 ### Using Docker Compose
@@ -98,7 +98,7 @@ docker-compose run --rm vcfx 'cat /data/valid.vcf | VCFX_allele_freq_calc > /dat
 When using Docker directly, you need to mount a directory to access your files:
 
 ```bash
-docker run --rm -v $(pwd)/tests/data:/data ghcr.io/ieeta-pt/vcfx:latest VCFX_tool_name [options]
+docker run --rm -v $(pwd)/tests/data:/data ghcr.io/jorgemfs/vcfx:latest VCFX_tool_name [options]
 ```
 
 When using Docker Compose, the `tests/data` directory is mounted by default:
@@ -115,7 +115,7 @@ You can modify the docker-compose.yml file to mount a different directory if nee
 You can create complex pipelines by chaining VCFX tools:
 
 ```bash
-docker run --rm -v $(pwd)/tests/data:/data ghcr.io/ieeta-pt/vcfx:latest 'cat /data/classifier_mixed.vcf | VCFX_variant_classifier --append-info | grep "VCF_CLASS=SNP" | VCFX_allele_freq_calc > /data/snp_frequencies.tsv'
+docker run --rm -v $(pwd)/tests/data:/data ghcr.io/jorgemfs/vcfx:latest 'cat /data/classifier_mixed.vcf | VCFX_variant_classifier --append-info | grep "VCF_CLASS=SNP" | VCFX_allele_freq_calc > /data/snp_frequencies.tsv'
 ```
 
 ### Creating Shell Scripts
@@ -126,7 +126,7 @@ For complex workflows, consider creating a shell script:
 #!/bin/bash
 # save as vcfx_workflow.sh
 
-docker run --rm -v $(pwd)/tests/data:/data ghcr.io/ieeta-pt/vcfx:latest 'cat /data/valid.vcf | \
+docker run --rm -v $(pwd)/tests/data:/data ghcr.io/jorgemfs/vcfx:latest 'cat /data/valid.vcf | \
   VCFX_validator | \
   VCFX_variant_classifier --append-info | \
   VCFX_allele_freq_calc > /data/pipeline_output.tsv'
@@ -147,7 +147,7 @@ If you encounter permission issues with files created in the container:
 
 ```bash
 # Run the container with your user ID
-docker run --rm -v $(pwd)/tests/data:/data -u $(id -u):$(id -g) ghcr.io/ieeta-pt/vcfx:latest VCFX_tool_name [options]
+docker run --rm -v $(pwd)/tests/data:/data -u $(id -u):$(id -g) ghcr.io/jorgemfs/vcfx:latest VCFX_tool_name [options]
 ```
 
 ### Container Not Finding Commands
@@ -156,5 +156,5 @@ If the container can't find VCFX commands, ensure they were properly built in th
 
 ```bash
 # List available VCFX tools in the container
-docker run --rm ghcr.io/ieeta-pt/vcfx:latest 'ls -1 /usr/local/bin/VCFX_*'
+docker run --rm ghcr.io/jorgemfs/vcfx:latest 'ls -1 /usr/local/bin/VCFX_*'
 ``` 

Dockerfile

@@ -9,6 +9,8 @@ RUN apt-get update && apt-get install -y \
     cmake \
     git \
     libz-dev \
+    python3 \
+    python3-dev \
     && rm -rf /var/lib/apt/lists/*
 
 # Create a working directory
@@ -46,14 +48,15 @@ COPY --from=builder /app/build/src /usr/local/bin/
 # Create a directory for data
 WORKDIR /data
 
-# Add the script that adds tools to PATH
+# Add the helper scripts
 COPY add_vcfx_tools_to_path.sh /usr/local/bin/
+COPY docker_entrypoint.sh /usr/local/bin/
 
-# Make the script executable
-RUN chmod +x /usr/local/bin/add_vcfx_tools_to_path.sh
+# Make them executable
+RUN chmod +x /usr/local/bin/add_vcfx_tools_to_path.sh /usr/local/bin/docker_entrypoint.sh
 
-# Set the entry point
-ENTRYPOINT ["/bin/bash", "-c"]
+# Use a custom entrypoint that sets up PATH for the tools
+ENTRYPOINT ["/usr/local/bin/docker_entrypoint.sh"]
 
 # Default command shows available tools
-CMD ["echo 'VCFX Toolkit is ready. Run any VCFX tool by name, for example:' && ls -1 /usr/local/bin/VCFX_* | xargs -n1 basename"] 
+CMD ["bash", "-c", "echo 'VCFX Toolkit is ready. Run any VCFX tool by name, for example:' && ls -1 /usr/local/bin/VCFX_* | xargs -n1 basename"]

PUBLISHING.md

@@ -26,8 +26,11 @@ This setup only needs to be done once. After this, the GitHub Actions workflow w
 
 Before pushing any changes, you should test the documentation site locally:
 
-1. Install the required dependencies:
+1. Install the required dependencies. It's best to use a Python virtual
+   environment because some systems mark the system Python as
+   "externally managed", which can interfere with installation:
    ```bash
+   python3 -m venv venv && source venv/bin/activate
    pip install mkdocs-material pymdown-extensions
    ```