Exit early for trivial sort cases to avoid segfault (#229)

* initial fix

* more tests

* NEWS

Author: MichaelChirico

NEWS.md

@@ -64,6 +64,7 @@
 1. `orderfin(as.integer64(10:1), 1:3, 8:11)` enforces that `table` be sorted by `order` instead of segfaulting (#166).
 1. `ordertab()` no longer segfaults when `nunique` is smaller than the actual number of unique values (#168).
 1. `as.integer64.character` now returns `NA` for out of range values, with warning, e.g. `as.integer64("22222222222222222222")`. Thanks @hcirellu.
+1. `quicksort()` and others no longer segfault on trivial cases (e.g. sorting 0 or 1 item, #220).
 
 ## NOTES
 

src/sort64.c

@@ -379,6 +379,7 @@ void ram_integer64_shellsort_asc(ValueT *data, IndexT l, IndexT r)
 {
     ValueT v;
     IndexT i, j, h, lh, t, n=r-l+1;
+    if (n < 2) return;
     for (t = 0; shellincs[t] > n; t++);
     for (h = shellincs[t]; t < SHELLARRAYSIZE; h = shellincs[++t]){
       lh = l+h;
@@ -397,6 +398,7 @@ void ram_integer64_shellsort_desc(ValueT *data, IndexT l, IndexT r)
 {
     ValueT v;
     IndexT i, j, h, lh, t, n=r-l+1;
+    if (n < 2) return;
     for (t = 0; shellincs[t] > n; t++);
     for (h = shellincs[t]; t < SHELLARRAYSIZE; h = shellincs[++t]){
       lh = l+h;
@@ -416,6 +418,7 @@ void ram_integer64_shellsortorder_asc(ValueT *data, IndexT *index, IndexT l, Ind
 {
     ValueT v;
     IndexT vi, i, j, h, lh, t, n=r-l+1;
+    if (n < 2) return;
     for (t = 0; shellincs[t] > n; t++);
     for (h = shellincs[t]; t < SHELLARRAYSIZE; h = shellincs[++t]){
       lh = l+h;
@@ -437,6 +440,7 @@ void ram_integer64_shellsortorder_desc(ValueT *data, IndexT *index, IndexT l, In
 {
     ValueT v;
     IndexT vi, i, j, h, lh, t, n=r-l+1;
+    if (n < 2) return;
     for (t = 0; shellincs[t] > n; t++);
     for (h = shellincs[t]; t < SHELLARRAYSIZE; h = shellincs[++t]){
       lh = l+h;
@@ -459,6 +463,7 @@ void ram_integer64_shellorder_asc(ValueT *data, IndexT *index, IndexT l, IndexT
 {
     ValueT v;
     IndexT vi, i, j, h, lh, t, n=r-l+1;
+    if (n < 2) return;
     for (t = 0; shellincs[t] > n; t++);
     for (h = shellincs[t]; t < SHELLARRAYSIZE; h = shellincs[++t]){
       lh = l+h;
@@ -478,6 +483,7 @@ void ram_integer64_shellorder_desc(ValueT *data, IndexT *index, IndexT l, IndexT
 {
     ValueT v;
     IndexT vi, i, j, h, lh, t, n=r-l+1;
+    if (n < 2) return;
     for (t = 0; shellincs[t] > n; t++);
     for (h = shellincs[t]; t < SHELLARRAYSIZE; h = shellincs[++t]){
       lh = l+h;

tests/testthat/test-sort64.R

@@ -254,13 +254,22 @@ test_that("Shellsort direct invocation", {
 test_that("Corner cases for partitioning logic", {
   # Single element and empty vectors often trip up "do { ... } while" or sentinel loops
 
-  # TODO(#220): restore this.
   # Case 1: Empty
-  # x_empty = integer64()
-  # x = bit::clone(x_empty)
-  # # bit::quicksort returns the NA count (0L), and modifies 'x' in-place
-  # expect_identical(bit::quicksort(x), 0L) 
-  # expect_identical(x, x_empty)
+  x_empty = integer64()
+  x = bit::clone(x_empty)
+  # bit::quicksort returns the NA count (0L), and modifies 'x' in-place
+  expect_identical(bit::quicksort(x), 0L)
+  expect_identical(x, x_empty)
+  expect_identical(bit::quicksort(x, decreasing = TRUE), 0L)
+  expect_identical(x, x_empty)
+
+  # Add explicit tests for shellsort variants with empty vectors
+  expect_identical(bit::shellsort(x_empty), 0L)
+  expect_identical(bit::shellsort(x_empty, decreasing = TRUE), 0L)
+  expect_identical(bit::shellsortorder(x_empty, integer(0)), 0L)
+  expect_identical(bit::shellsortorder(x_empty, integer(0), decreasing = TRUE), 0L)
+  expect_identical(bit::shellorder(x_empty, integer(0)), 0L)
+  expect_identical(bit::shellorder(x_empty, integer(0), decreasing = TRUE), 0L)
 
   # Case 2: Single Element
   x_single = as.integer64(1L)