Update TileDB Embedded to 2.28.1 (#826)

* Bump nightly build release branch to 2.28 (#819)

* fix img and academy links (#815)

Use TileDB Academy links for docs

* remove ENTRYPOINT for Nextflow compatibility (#816)

* remove ENTRYPOINT for Nextflow compatibility

* add tiledbvcf to test_cmd since no entrypoint

* Updated spdlog to version 1.15.3 (#821)

This required adding an include of fmt/ranges.h because of a bug in libfmt 11, which spdlog depends on.

* Revert "Updated spdlog to version 1.15.3 (#821)" (#824)

This reverts commit e4b00ac.

* Updated HTSlib to version 1.22.1 (#822)

* Updated HTSlib to version 1.22.1

This includes changing the CMake External Project URL hash for HTSlib to use a SHA256 checksum because GitHub now auto-generates SHA256 checksums for HTSlib release artifacts.

* Removed unused HTSlib CMake patch

* Updated Azure CI config to install curl prior to bcftools

bcftools 1.22 contains a bug fix that causes the build in this configuration to fail if curl is not present.

* Update to TileDB Embedded 2.28.1

---------

Co-authored-by: John Blischak <jdblischak@gmail.com>
Co-authored-by: Jeremy Leipzig <jeremy.leipzig@tiledb.com>

Author: 3 people

.azure-pipelines.yml

@@ -1,9 +1,9 @@
 trigger:
   branches:
     include:
-    - main
-    - release-*
-    - refs/tags/*
+      - main
+      - release-*
+      - refs/tags/*
 
 variables:
   isRefTag: $[startsWith(variables['Build.SourceBranch'], 'refs/tags')]
@@ -13,44 +13,44 @@ stages:
   - stage: CI
     condition: eq(variables.isRefTag, false)
     jobs:
-    - job: Build
-      strategy:
-        matrix:
-          linux:
-            imageName: 'ubuntu-24.04'
-            python.version: '3.x'
-            CXX: g++
-            BUILD_PYTHON_API: ON
-            DOWNLOAD_TILEDB_PREBUILT: ON
-            ARTIFACT_NAME: linux
-      pool:
-        vmImage: $(imageName)
-      steps:
-      - task: UsePythonVersion@0
-        inputs:
-          versionSpec: '$(python.version)'
-          architecture: 'x64'
-      - script: printenv
-        displayName: 'Print Env'
-      - template: ci/azure-linux_mac.yml
+      - job: Build
+        strategy:
+          matrix:
+            linux:
+              imageName: "ubuntu-24.04"
+              python.version: "3.x"
+              CXX: g++
+              BUILD_PYTHON_API: ON
+              DOWNLOAD_TILEDB_PREBUILT: ON
+              ARTIFACT_NAME: linux
+        pool:
+          vmImage: $(imageName)
+        steps:
+          - task: UsePythonVersion@0
+            inputs:
+              versionSpec: "$(python.version)"
+              architecture: "x64"
+          - script: printenv
+            displayName: "Print Env"
+          - template: ci/azure-linux_mac.yml
 
   - stage: Docker
     condition: or(succeeded(), eq(variables.isRefTag, true))
     jobs:
       - job: Build
         pool:
-          vmImage: 'ubuntu-24.04'
+          vmImage: "ubuntu-24.04"
         strategy:
           matrix:
             cli:
               dockerfile: docker/Dockerfile-cli
               repository: tiledb/tiledbvcf-cli
               context: .
-              test_cmd: version
+              test_cmd: tiledbvcf version
             python:
               dockerfile: docker/Dockerfile-py
               repository: tiledb/tiledbvcf-py
               context: .
               test_cmd: -c "import tiledbvcf; print(tiledbvcf.version)"
         steps:
-        - template: ci/build-images.yml
+          - template: ci/build-images.yml

.github/workflows/nightly.yml

@@ -24,7 +24,7 @@ jobs:
       matrix:
         os: [macos-13, ubuntu-24.04]
         branches:
-          - {libtiledb: release-2.27, tiledb-py: 0.33.3}
+          - {libtiledb: release-2.28, tiledb-py: 0.34.0}
           - {libtiledb: main, tiledb-py: main}
     env:
       MACOSX_DEPLOYMENT_TARGET: 10.15

README.md

@@ -1,4 +1,4 @@
-<a href="https://tiledb.com"><img src="https://github.com/TileDB-Inc/TileDB/raw/dev/doc/source/_static/tiledb-logo_color_no_margin_@4x.png" alt="TileDB logo" width="400"></a>
+<a href="https://tiledb.com"><img src="https://github.com/TileDB-Inc/TileDB/raw/main/doc/source/_static/tiledb-logo_color_no_margin_@4x.png" alt="TileDB logo" width="400"></a>
 
 [![Build Status](https://img.shields.io/azure-devops/build/tiledb-inc/836549eb-f74a-4986-a18f-7fbba6bbb5f0/8/main?label=Azure%20Pipelines&logo=azure-pipelines&style=flat-square)](https://dev.azure.com/TileDB-Inc/CI/_build/latest?definitionId=8&branchName=main)
 [![Docker-CLI](https://img.shields.io/static/v1?label=Docker&message=tiledbvcf-cli&color=099cec&logo=docker&style=flat-square)](https://hub.docker.com/repository/docker/tiledb/tiledbvcf-cli)
@@ -12,7 +12,7 @@ A C++ library for efficient storage and retrieval of genomic variant-call data u
 
 - Easily ingest large amounts of variant-call data at scale
 - Supports ingesting single sample VCF and BCF files
-- New samples are added *incrementally*, avoiding computationally expensive merging operations
+- New samples are added _incrementally_, avoiding computationally expensive merging operations
 - Allows for highly compressed storage using TileDB sparse arrays
 - Efficient, parallelized queries of variant data stored locally or remotely on S3
 - Export lossless VCF/BCF files or extract specific slices of a dataset
@@ -24,9 +24,9 @@ A C++ library for efficient storage and retrieval of genomic variant-call data u
 
 ## Quick Start
 
-The [documentation website][vcf] provides comprehensive usage examples but here are a few quick exercises to get you started.
+The [TileDB Academy][vcf] provides comprehensive usage examples but here are a few quick exercises to get you started.
 
-We'll use a dataset that includes 20 synthetic samples, each one containing over 20 million variants. We host a publicly accessible version of this dataset on S3, so if you have TileDB-VCF installed and you'd like to follow along just swap out the `uri`'s below for `s3://tiledb-inc-demo-data/tiledbvcf-arrays/v4/vcf-samples-20`. And if you *don't* have TileDB-VCF installed yet, you can use our [Docker images](docker/README.md) to test things out.
+We'll use a dataset that includes 20 synthetic samples, each one containing over 20 million variants. We host a publicly accessible version of this dataset on S3, so if you have TileDB-VCF installed and you'd like to follow along just swap out the `uri`'s below for `s3://tiledb-inc-demo-data/tiledbvcf-arrays/v4/vcf-samples-20`. And if you _don't_ have TileDB-VCF installed yet, you can use our [Docker images](docker/README.md) to test things out.
 
 ### CLI
 
@@ -84,30 +84,28 @@ returns results as a pandas `DataFrame`
 
 ## Want to Learn More?
 
-
-* [Blog "Population Genomics is a Data Management Problem"][blog]
-* [Check out the full documentation][vcf]
-  * [Why use TileDB-VCF?][docswhytile]
-  * [Data Model][docsdatamodel]
-  * [Installation][docsinstallation]
-  * [How To][docshowto]
-  * [Reference][docsreference]
-
+- [Blog "Population Genomics is a Data Management Problem"][blogdatamgmt]
+- [Blog "TileDB 101: Population Genomics"][blogpopgen101]
+- [Check out the TileDB Academy][vcf]
+  - [Data Model][docsdatamodel]
+  - [Installation][docsinstallation]
+  - [Tutorials][docstutorials]
+  - [API Reference][docsreference]
 
 # Code of Conduct
 
 All participants in TileDB spaces are expected to adhere to high standards of
 professionalism in all interactions. This repository is governed by the
 specific standards and reporting procedures detailed in depth in the
-[TileDB core repository Code Of Conduct](
-https://github.com/TileDB-Inc/TileDB/blob/dev/CODE_OF_CONDUCT.md).
+[TileDB core repository Code Of Conduct](https://github.com/TileDB-Inc/TileDB/blob/dev/CODE_OF_CONDUCT.md).
 
 <!-- links -->
+
 [tiledb]: https://github.com/TileDB-Inc/TileDB
-[vcf]: https://docs.tiledb.com/main/integrations-and-extensions/population-genomics
-[docswhytile]: https://docs.tiledb.com/main/integrations-and-extensions/genomics/population-genomics#why-use-tiledb-vcf
-[docsdatamodel]: https://docs.tiledb.com/main/integrations-and-extensions/population-genomics/data-model
-[docsinstallation]: https://docs.tiledb.com/main/integrations-and-extensions/genomics/population-genomics/installation
-[docshowto]: https://docs.tiledb.com/main/integrations-and-extensions/genomics/population-genomics/how-to
-[docsreference]: https://docs.tiledb.com/main/integrations-and-extensions/genomics/population-genomics/api-reference
-[blog]: https://tiledb.com/blog/population-genomics-is-a-data-management-problem
+[vcf]: https://cloud.tiledb.com/academy/structure/life-sciences/population-genomics/
+[docsdatamodel]: https://cloud.tiledb.com/academy/structure/life-sciences/population-genomics/foundation/data-model/
+[docsinstallation]: https://cloud.tiledb.com/academy/structure/life-sciences/population-genomics/quickstart/#installation
+[docstutorials]: https://cloud.tiledb.com/academy/structure/life-sciences/population-genomics/tutorials/
+[docsreference]: https://cloud.tiledb.com/academy/structure/life-sciences/population-genomics/api-reference/
+[blogdatamgmt]: https://tiledb.com/blog/population-genomics-is-a-data-management-problem
+[blogpopgen101]: https://tiledb.com/blog/tiledb-101-population-genomics

apis/java/src/main/java/io/tiledb/libvcfnative/NativeLibLoader.java

@@ -95,7 +95,7 @@ static void loadNativeTileDBVCFJNI() {
   /** Finds and loads native HTSlib. */
   static void loadNativeHTSLib() {
     String os = getOSClassifier();
-    String versionedLibName = os.startsWith("osx") ? "libhts.1.20.dylib" : "libhts.so.1.20";
+    String versionedLibName = os.startsWith("osx") ? "libhts.1.22.1.dylib" : "libhts.so.1.22.1";
     try {
       // Don't use name mapping to get the versioned htslib
       loadNativeLib(versionedLibName, false);

ci/azure-linux_mac.yml

@@ -39,11 +39,11 @@ steps:
 
       # Install htslib dependencies
       if [[ "$AGENT_OS" == "Linux" ]]; then
-          sudo apt-get update && sudo apt-get install -y automake autoconf libbz2-dev liblzma-dev
+          sudo apt-get update && sudo apt-get install -y automake autoconf libbz2-dev liblzma-dev libcurl4-openssl-dev
       fi
 
       # Install bcftools (only required for running the CLI tests)
-      version=1.20
+      version=1.22
       if [[ "$AGENT_OS" == "Linux" ]]; then
           pushd /tmp
           wget https://github.com/samtools/bcftools/releases/download/${version}/bcftools-${version}.tar.bz2

docker/Dockerfile-cli

@@ -53,4 +53,3 @@ COPY --from=builder /usr/local/include/* /usr/local/include/
 COPY --from=builder /usr/local/lib/* /usr/local/lib/
 
 WORKDIR /data
-ENTRYPOINT ["/usr/local/bin/tiledbvcf"]

libtiledbvcf/cmake/Modules/FindHTSlib.cmake

@@ -66,9 +66,9 @@ if (NOT HTSLIB_FOUND)
     message(STATUS "Adding HTSlib as an external project")
     # Use explicit soname to avoid having to use library symlinks (easier for embedding).
     if (APPLE)
-      set(EXTRA_LDFLAGS "-Wl,-install_name,@rpath/libhts.1.20.dylib")
+      set(EXTRA_LDFLAGS "-Wl,-install_name,@rpath/libhts.1.22.1.dylib")
     else()
-      set(EXTRA_LDFLAGS "-Wl,-soname,libhts.so.1.20")
+      set(EXTRA_LDFLAGS "-Wl,-soname,libhts.so.1.22.1")
     endif()
     SET(CFLAGS "")
     string( TOUPPER "${CMAKE_BUILD_TYPE}" BUILD_TYPE)
@@ -94,8 +94,8 @@ if (NOT HTSLIB_FOUND)
 
       ExternalProject_Add(ep_htslib
         PREFIX "externals"
-        URL https://github.com/TileDB-Inc/m2w64-htslib-build/releases/download/1.20-0/m2w64-htslib-1.20-0.tar.gz
-        URL_HASH SHA1=da39a3ee5e6b4b0d3255bfef95601890afd80709
+        URL "https://github.com/TileDB-Inc/m2w64-htslib-build/releases/download/1.22.1-0/m2w64-htslib-1.22.1-0.tar.gz"
+        URL_HASH SHA256=d6bce9d6dd7001450a1804255af2144b7b00fee1792ab6bed6991733601cc75f
         UPDATE_COMMAND ""
         CONFIGURE_COMMAND ""
         BUILD_COMMAND ""
@@ -122,8 +122,8 @@ if (NOT HTSLIB_FOUND)
 
       ExternalProject_Add(ep_htslib
         PREFIX "externals"
-        URL "https://github.com/samtools/htslib/releases/download/1.20/htslib-1.20.tar.bz2"
-        URL_HASH SHA1=da8bf95e4ae2404d1a48f88ece94ca25d69d6596
+        URL "https://github.com/samtools/htslib/releases/download/1.22.1/htslib-1.22.1.tar.bz2"
+        URL_HASH SHA256=3dfa6eeb71db719907fe3ef7c72cb2ec9965b20b58036547c858c89b58c342f7
         UPDATE_COMMAND ""
         CONFIGURE_COMMAND
             autoheader

libtiledbvcf/cmake/Modules/FindTileDB_EP.cmake

@@ -52,20 +52,20 @@ else()
     # Try to download prebuilt artifacts unless the user specifies to build from source
     if(DOWNLOAD_TILEDB_PREBUILT)
         if (WIN32) # Windows
-          SET(DOWNLOAD_URL "https://github.com/TileDB-Inc/TileDB/releases/download/2.28.0/tiledb-windows-x86_64-2.28.0-4764907.zip")
-          SET(DOWNLOAD_SHA1 "3629193fc9b8e8aecff4156a2a2183f1c4c1a0ee")
+          SET(DOWNLOAD_URL "https://github.com/TileDB-Inc/TileDB/releases/download/2.28.1/tiledb-windows-x86_64-2.28.1-d648231.zip")
+          SET(DOWNLOAD_SHA1 "ca6ccd339b6e3ce0ad1ff8c195a00412e3392178")
         elseif(APPLE) # OSX
 
           if (CMAKE_OSX_ARCHITECTURES STREQUAL x86_64 OR CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(AMD64|amd64)|(^i.86$)")
-            SET(DOWNLOAD_URL "https://github.com/TileDB-Inc/TileDB/releases/download/2.28.0/tiledb-macos-x86_64-2.28.0-4764907.tar.gz")
-            SET(DOWNLOAD_SHA1 "4e053376b707765eb223afd9261505315f217be0")
+            SET(DOWNLOAD_URL "https://github.com/TileDB-Inc/TileDB/releases/download/2.28.1/tiledb-macos-x86_64-2.28.1-d648231.tar.gz")
+            SET(DOWNLOAD_SHA1 "6191aca6e76a95a57288e9733f299c7b47e0d55e")
           elseif (CMAKE_OSX_ARCHITECTURES STREQUAL arm64 OR CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "^arm")
-            SET(DOWNLOAD_URL "https://github.com/TileDB-Inc/TileDB/releases/download/2.28.0/tiledb-macos-arm64-2.28.0-4764907.tar.gz")
-            SET(DOWNLOAD_SHA1 "de6f1c79351191702ebc08e15a7257ade8b40b92")
+            SET(DOWNLOAD_URL "https://github.com/TileDB-Inc/TileDB/releases/download/2.28.1/tiledb-macos-arm64-2.28.1-d648231.tar.gz")
+            SET(DOWNLOAD_SHA1 "ce14964eb027e1bab3704668208f5289c9fbca1d")
           endif()
         else() # Linux
-          SET(DOWNLOAD_URL "https://github.com/TileDB-Inc/TileDB/releases/download/2.28.0/tiledb-linux-x86_64-2.28.0-4764907.tar.gz")
-          SET(DOWNLOAD_SHA1 "cd65b2978920ecd5017b061e15f172048c081dd2")
+          SET(DOWNLOAD_URL "https://github.com/TileDB-Inc/TileDB/releases/download/2.28.1/tiledb-linux-x86_64-2.28.1-d648231.tar.gz")
+          SET(DOWNLOAD_SHA1 "f7c2c76bae0373cfc259b2a854b464abb873bbee")
         endif()
 
         ExternalProject_Add(ep_tiledb
@@ -87,8 +87,8 @@ else()
     else() # Build from source
         ExternalProject_Add(ep_tiledb
           PREFIX "externals"
-          URL "https://github.com/TileDB-Inc/TileDB/archive/2.28.0.zip"
-          URL_HASH SHA1=ec633e4440ff7d16f2668e37f24e80e3b7d168cd
+          URL "https://github.com/TileDB-Inc/TileDB/archive/2.28.1.zip"
+          URL_HASH SHA1=9118c60f166ebe62ac39ab7fc72c39403b5af086
           DOWNLOAD_NAME "tiledb.zip"
           CMAKE_ARGS
             -DCMAKE_INSTALL_PREFIX=${EP_INSTALL_PREFIX}