Fix segfaults from CRAN, including GHA to catch ASAN/UBSAN issue (#318)

* GHA to catch ASAN/UBSAN issue

* fix installation

* pushing further

* another attempt

* expect_no_error

* overhaul

* kitchen sink

* missed library(patrick)

* incidental numeric issue

* more kitchen sink turning off compiler optimization

* another pass

* whitespace yay YAML

* another pass

* attempt to resolve install issue

* another one

* ugh

* we seem pretty lost here...

* try string clang

* fix setup?

* trying to read straight from the CRAN manual

* go harder

* down the rabbit hole

* sure why not

* stick with R CMD check

* sneaky sneaky

* minimize

* stripping down

* did we over-do it? maybe

* partial restoration

* fix segfaults

* trigger only on limited PRs

* NEWS

Author: MichaelChirico

.github/workflows/R-CMD-check.yaml

@@ -5,6 +5,8 @@ on:
     branches: [main, master]
   pull_request:
     branches: [main, master]
+    paths:
+      - 'src/**'
 
 name: R-CMD-check.yaml
 
@@ -50,6 +52,36 @@ jobs:
           upload-snapshots: true
           build_args: 'c("--no-manual","--compact-vignettes=gs+qpdf")'
 
+  clang-ASAN:
+    name: "Minimal clang-ASAN"
+    runs-on: ubuntu-latest
+    container: rocker/r-devel-ubsan-clang
+    env:
+      _R_CHECK_TESTS_NLINES_: 0
+      ASAN_OPTIONS: 'detect_leaks=0:halt_on_error=1'
+      UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1'
+      CC: "clang -fsanitize=address,undefined -fno-omit-frame-pointer"
+      CXX: "clang++ -fsanitize=address,undefined -fno-omit-frame-pointer"
+      CFLAGS: "-g -O3"
+      CXXFLAGS: "-g -O3"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up ASAN
+        run: echo "LD_PRELOAD=$(clang -print-file-name=libclang_rt.asan-x86_64.so)" >> $GITHUB_ENV
+      - name: Install dependencies
+        run: |
+          apt-get update && apt-get install -y libuv1-dev
+          mkdir -p ~/.R
+          echo "CC=$CC" >> ~/.R/Makevars
+          echo "CXX=$CXX" >> ~/.R/Makevars
+          echo "CFLAGS=$CFLAGS" >> ~/.R/Makevars
+          echo "CXXFLAGS=$CXXFLAGS" >> ~/.R/Makevars
+          Rscript -e 'install.packages("remotes", repos="https://cloud.r-project.org"); remotes::install_deps(dependencies = TRUE, type = "source")'
+      - name: Build and Check
+        run: |
+          R CMD build .
+          R CMD check bit64_*.tar.gz --as-cran --no-manual --no-vignettes
+
   test-ancient:
     runs-on: ubuntu-latest
     steps:
@@ -75,16 +107,16 @@ jobs:
           for FILE in tests/testthat/test-*.R; do
             echo "----------------------------------------------------------------"
             echo "Running $FILE"
-            
+
             sed -E "s/(patrick|withr):://g" "$FILE" > "${FILE}_SHIMMED.R"
             Rscript -e "library(methods); suppressWarnings(suppressMessages(library(bit64))); source('tests/testthat/helper.R'); source('${FILE}_SHIMMED.R')"
             rm "${FILE}_SHIMMED.R"
-            
+
             # Capture failure
             if [ $? -ne 0 ]; then
               EXIT_CODE=1
             fi
-            
+
           done
 
           exit $EXIT_CODE

NEWS.md

@@ -1,5 +1,9 @@
 # bit64 4.8.99 (in development)
 
+## BUG FIXES
+
+1. Fix some potential segfaults from shellsort algorithms (#315). The culprit code has been faulty for many years, but only newly caught by CRAN's rigorous environment with extensive new test coverage.
+
 # bit64 4.8.0 (2025-04-19)
 
 ## NOTICE OF PLANNED BREAKING CHANGES

src/sort64.c

@@ -383,7 +383,8 @@ void ram_integer64_shellsort_asc(ValueT *data, IndexT l, IndexT r)
     IndexT i, j, h, lh, t, n=r-l+1;
     if (n < 2) return;
     for (t = 0; shellincs[t] > n; t++);
-    for (h = shellincs[t]; t < SHELLARRAYSIZE; h = shellincs[++t]){
+    for (; t < SHELLARRAYSIZE; t++){
+      h = shellincs[t];
       lh = l+h;
       for (i = lh; i <= r; i++) {
       MOVE(v, data[i])
@@ -402,7 +403,8 @@ void ram_integer64_shellsort_desc(ValueT *data, IndexT l, IndexT r)
     IndexT i, j, h, lh, t, n=r-l+1;
     if (n < 2) return;
     for (t = 0; shellincs[t] > n; t++);
-    for (h = shellincs[t]; t < SHELLARRAYSIZE; h = shellincs[++t]){
+    for (; t < SHELLARRAYSIZE; t++){
+      h = shellincs[t];
       lh = l+h;
       for (i = lh; i <= r; i++) {
       MOVE(v, data[i])
@@ -422,7 +424,8 @@ void ram_integer64_shellsortorder_asc(ValueT *data, IndexT *index, IndexT l, Ind
     IndexT vi, i, j, h, lh, t, n=r-l+1;
     if (n < 2) return;
     for (t = 0; shellincs[t] > n; t++);
-    for (h = shellincs[t]; t < SHELLARRAYSIZE; h = shellincs[++t]){
+    for (; t < SHELLARRAYSIZE; t++){
+      h = shellincs[t];
       lh = l+h;
       for (i = lh; i <= r; i++) {
       MOVE(vi, index[i])
@@ -444,7 +447,8 @@ void ram_integer64_shellsortorder_desc(ValueT *data, IndexT *index, IndexT l, In
     IndexT vi, i, j, h, lh, t, n=r-l+1;
     if (n < 2) return;
     for (t = 0; shellincs[t] > n; t++);
-    for (h = shellincs[t]; t < SHELLARRAYSIZE; h = shellincs[++t]){
+    for (; t < SHELLARRAYSIZE; t++){
+      h = shellincs[t];
       lh = l+h;
       for (i = lh; i <= r; i++) {
       MOVE(vi, index[i])
@@ -467,7 +471,8 @@ void ram_integer64_shellorder_asc(ValueT *data, IndexT *index, IndexT l, IndexT
     IndexT vi, i, j, h, lh, t, n=r-l+1;
     if (n < 2) return;
     for (t = 0; shellincs[t] > n; t++);
-    for (h = shellincs[t]; t < SHELLARRAYSIZE; h = shellincs[++t]){
+    for (; t < SHELLARRAYSIZE; t++){
+      h = shellincs[t];
       lh = l+h;
       for (i = lh; i <= r; i++) {
       MOVE(vi, index[i])
@@ -487,7 +492,8 @@ void ram_integer64_shellorder_desc(ValueT *data, IndexT *index, IndexT l, IndexT
     IndexT vi, i, j, h, lh, t, n=r-l+1;
     if (n < 2) return;
     for (t = 0; shellincs[t] > n; t++);
-    for (h = shellincs[t]; t < SHELLARRAYSIZE; h = shellincs[++t]){
+    for (; t < SHELLARRAYSIZE; t++){
+      h = shellincs[t];
       lh = l+h;
       for (i = lh; i <= r; i++) {
       MOVE(vi, index[i])