From feff9afaab7282d18b42a72153c13c98bf6d7978 Mon Sep 17 00:00:00 2001 From: Enrico Olivelli Date: Wed, 20 May 2026 18:10:38 +0200 Subject: [PATCH 1/2] =?UTF-8?q?issue=20#616:=20Phase=20B=20checkpoint=20gu?= =?UTF-8?q?ard=20=E2=80=94=20pre-track=20multipart=20files=20before=20uplo?= =?UTF-8?q?ad?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move trackProvisionalMultipartFile() calls in writeShardAsFusedPQSegment to BEFORE the corresponding writeMultipartIndexFile() call so that mid-flight upload failures (e.g. OutOfDirectMemoryError from Netty's direct-buffer pool while writing block 0) leave the partial multipart file registered in provisionalMultipartFiles. The outer Phase B catch arm's rollbackProvisionalArtefacts() then deletes the partial blocks instead of leaking them indefinitely — which previously poisoned every subsequent compaction cycle with NoSuchKeyException on the orphan (issue #616 reproduction). Also wrap each writeMultipartIndexFile() call in a narrow catch that logs SEVERE with the failing segUuid, segmentId and vector count before rethrowing. With PHASE_B_SEGMENT_PARALLELISM > 1 the outer "Phase B exception" log line previously hid shard identity; the new log names the victim segment so operators can identify which upload aborted from the IS log stream. The pre-track is correct under all branches: on Phase B + Phase C-prep success the tracker is cleared to null without deletion (existing behaviour at line 5760); on partial-write failure the rollback's deleteMultipartIndexFile call is a no-op against a missing key and the rollback path already swallows DataStorageManagerException / RuntimeException from such deletes. Co-Authored-By: Claude Sonnet 4.6 --- .../vector/PersistentVectorStore.java | 54 ++- ...sue616PhaseBPartialUploadRollbackTest.java | 417 ++++++++++++++++++ 2 files changed, 469 insertions(+), 2 deletions(-) create mode 100644 herddb-indexing-service/src/test/java/herddb/indexing/vector/Issue616PhaseBPartialUploadRollbackTest.java diff --git a/herddb-indexing-service/src/main/java/herddb/indexing/vector/PersistentVectorStore.java b/herddb-indexing-service/src/main/java/herddb/indexing/vector/PersistentVectorStore.java index 540a6c84..3f6de00f 100644 --- a/herddb-indexing-service/src/main/java/herddb/indexing/vector/PersistentVectorStore.java +++ b/herddb-indexing-service/src/main/java/herddb/indexing/vector/PersistentVectorStore.java @@ -6704,15 +6704,53 @@ private SegmentWriteResult writeShardAsFusedPQSegment( uploadingActive.incrementAndGet(); String graphFilePath; String segUuid = indexUUID + "_seg" + segmentId; + // Issue #616: register the segUuid in the provisional-multipart-files + // tracker BEFORE the upload starts, not after it returns. If the upload + // aborts mid-flight (e.g. OutOfDirectMemoryError from Netty's direct + // buffer pool while writing block 0), any partial S3 / file-server + // blocks were tracked-for-cleanup; the outer Phase B catch arm's + // rollbackProvisionalArtefacts() call then deletes them. Pre-fix the + // track happened only on successful return, leaking partial-upload + // blocks indefinitely — see the issue #616 reproduction. The pre-track + // is safe under success too: when Phase B + Phase C-prep complete the + // outer code clears provisionalMultipartFiles to null WITHOUT deleting + // anything (see the post-Phase-C-prep block in indexCheckpoint). It is + // also safe when the upload wrote zero bytes: rollback's + // deleteMultipartIndexFile call is a no-op against a missing key and + // the rollback path already swallows DataStorageManagerException / + // RuntimeException from such deletes (see rollbackProvisionalArtefacts). + trackProvisionalMultipartFile(segUuid, "graph"); try { graphFilePath = dataStorageManager.writeMultipartIndexFile( tableSpaceUUID, segUuid, "graph", tempFile, uploadBytesDone::addAndGet); + } catch (IOException | RuntimeException e) { + // Issue #616: log SEVERE with the failing segment identity BEFORE + // the exception propagates to awaitAllOrFirstFailure (which only + // surfaces the FIRST failure when shards run in parallel). Without + // this log the outer Phase B catch in doCheckpointFusedPQThreePhase + // prints "Phase B exception" with no shard identity — operators + // cannot tell which segment's upload aborted, which is the + // diagnostic gap the issue is trying to close. The broad + // RuntimeException catch is intentional: Netty wraps allocation + // failures (including OutOfDirectMemoryError-style conditions) + // and protocol errors into unchecked exceptions thrown out of the + // multipart writer, and DataStorageManagerException is itself a + // RuntimeException subclass. We re-throw unchanged so the existing + // Phase B failure handling (rollbackProvisionalArtefacts + + // recoverFromPhaseBFailure) runs exactly as before. + LOGGER.log(Level.SEVERE, + "checkpoint " + indexName + " Phase B: graph multipart" + + " write FAILED for segUuid=" + segUuid + + " (segmentId=" + segmentId + + ", vectors=" + shardSize + + ") — partial multipart state queued for rollback" + + " cleanup", e); + throw e; } finally { uploadingActive.decrementAndGet(); } - trackProvisionalMultipartFile(segUuid, "graph"); graphUploadDoneNanos = System.nanoTime(); ConcurrentHashMap ordinalToPk = buildOrdinalToPk(shard); @@ -6722,15 +6760,27 @@ private SegmentWriteResult writeShardAsFusedPQSegment( uploadBytesTotal.addAndGet(mapSize); uploadingActive.incrementAndGet(); String mapFilePath; + // Issue #616: pre-track the map multipart file for the same reason + // as the graph upload above. See the matching comment there. + trackProvisionalMultipartFile(segUuid, "map"); try { mapFilePath = dataStorageManager.writeMultipartIndexFile( tableSpaceUUID, segUuid, "map", mapFile, uploadBytesDone::addAndGet); + } catch (IOException | RuntimeException e) { + // Issue #616: see the matching catch on the graph upload above. + LOGGER.log(Level.SEVERE, + "checkpoint " + indexName + " Phase B: map multipart" + + " write FAILED for segUuid=" + segUuid + + " (segmentId=" + segmentId + + ", vectors=" + shardSize + + ") — partial multipart state queued for" + + " rollback cleanup", e); + throw e; } finally { uploadingActive.decrementAndGet(); } - trackProvisionalMultipartFile(segUuid, "map"); compactionNodesDone.addAndGet(shardSize); SegmentWriteResult swr = new SegmentWriteResult(segmentId, graphFilePath, graphSize, diff --git a/herddb-indexing-service/src/test/java/herddb/indexing/vector/Issue616PhaseBPartialUploadRollbackTest.java b/herddb-indexing-service/src/test/java/herddb/indexing/vector/Issue616PhaseBPartialUploadRollbackTest.java new file mode 100644 index 00000000..731fc138 --- /dev/null +++ b/herddb-indexing-service/src/test/java/herddb/indexing/vector/Issue616PhaseBPartialUploadRollbackTest.java @@ -0,0 +1,417 @@ +/* + Licensed to Diennea S.r.l. under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. Diennea S.r.l. licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/ + +package herddb.indexing.vector; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import herddb.core.MemoryManager; +import herddb.mem.MemoryDataStorageManager; +import herddb.storage.DataStorageManagerException; +import herddb.utils.Bytes; +import java.io.IOException; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +/** + * Regression test for issue #616 — Phase B checkpoint guard. + * + *

The pre-fix code in {@link PersistentVectorStore#writeShardAsFusedPQSegment} + * called {@code trackProvisionalMultipartFile(segUuid, "graph")} AFTER the + * graph multipart upload returned successfully (same for {@code "map"}). + * When an upload aborted mid-flight — e.g. an {@code OutOfDirectMemoryError} + * from Netty's direct-buffer pool while writing one of the file's blocks — + * control jumped out of the function via the thrown exception WITHOUT + * registering the partial file in {@code provisionalMultipartFiles}. The + * subsequent {@code rollbackProvisionalArtefacts()} call therefore had no + * record of the partial multipart file and never deleted it. The partial + * blocks were left orphaned in remote storage, which is exactly the failure + * mode the issue's reproduction describes (every later compaction cycle + * re-selected the orphan, failed with {@code NoSuchKeyException}, and looped + * indefinitely). + * + *

The fix moves both {@code trackProvisionalMultipartFile} calls to + * BEFORE the corresponding {@code writeMultipartIndexFile} call so any + * partial upload is eligible for rollback cleanup. + * + *

This test reproduces the partial-write scenario with an in-memory + * {@link MemoryDataStorageManager} subclass that: + *

    + *
  1. actually performs the write (so the partial blocks "leak" into + * the simulated remote storage), then
  2. + *
  3. throws an {@link IOException} to simulate the mid-flight abort.
  4. + *
+ * Under the pre-fix behaviour the simulated remote storage retains the + * partial file forever; under the post-fix behaviour the rollback path + * deletes it. The assertions below catch the regression. + */ +public class Issue616PhaseBPartialUploadRollbackTest { + + @Rule + public TemporaryFolder tmpFolder = new TemporaryFolder(); + + @Before + public void disableDeferral() { + // Allow tiny shards to seal so a single checkpoint() call produces + // an on-disk segment with real multipart files. + PersistentVectorStore.minLiveVectorsForCheckpoint = 0; + } + + @After + public void restoreDeferral() { + PersistentVectorStore.minLiveVectorsForCheckpoint = 50_000; + } + + private static float[] vec(Random rng, int dim) { + float[] v = new float[dim]; + for (int i = 0; i < dim; i++) { + v[i] = rng.nextFloat(); + } + return v; + } + + private PersistentVectorStore createStore(Path tmpDir, MemoryDataStorageManager dsm) { + MemoryManager mm = new MemoryManager(64 * 1024 * 1024, 0, 1024 * 1024, 1024 * 1024); + PersistentVectorStore store = new PersistentVectorStore( + "testidx", "testtable", "tstblspace", "vector_col", + tmpDir, dsm, mm, + 16, 100, 1.2f, 1.4f, true, 2_000_000_000L, 0, + Long.MAX_VALUE); + store.configureCompaction(Long.MAX_VALUE, 1L, Long.MAX_VALUE, 4, Integer.MAX_VALUE, 0); + return store; + } + + /** + * Phase B partial-upload failure must leave the simulated remote storage + * unchanged across the failing checkpoint: the pre-track + rollback + * pipeline deletes the partial file that the failing + * {@code writeMultipartIndexFile} call put into the storage map before + * throwing. + * + *

The test triggers a multi-segment Phase B and arms the DSM to fail + * on the second {@code writeMultipartIndexFile} call (the graph upload + * of the second segment). After the failing checkpoint: + *

    + *
  • Phase B must have failed (thrown or counter bumped).
  • + *
  • The set of multipart files in the DSM must equal the snapshot + * taken before the failing checkpoint — no partial leftover. + * This is the assertion that fails pre-fix.
  • + *
  • {@code this.segments} must not have grown: no half-written + * segment leaks into the registry.
  • + *
  • A SEVERE log record naming the failing {@code segUuid} must + * have been emitted so operators can identify the victim segment + * from the IS log stream.
  • + *
+ */ + @Test(timeout = 60_000) + public void phaseBPartialUploadRolledBack() throws Exception { + Path tmpDir = tmpFolder.newFolder("issue616-phaseB-partial").toPath(); + MidWriteFailingDsm dsm = new MidWriteFailingDsm(); + + // Capture SEVERE log lines emitted by PersistentVectorStore so we + // can assert the failing segment's UUID is named (issue #616 + // diagnostic gap — pre-fix the outer "Phase B exception" log line + // hides shard identity). + SevereCaptor captor = new SevereCaptor(); + Logger storeLogger = Logger.getLogger(PersistentVectorStore.class.getName()); + storeLogger.addHandler(captor); + Level previousLevel = storeLogger.getLevel(); + storeLogger.setLevel(Level.ALL); + + try (PersistentVectorStore store = createStore(tmpDir, dsm)) { + store.start(); + + // First, drive a successful baseline checkpoint with some vectors + // so the store has a clean known-good state and Phase B has + // exercised the success path at least once. We snapshot the + // DSM's multipart-files set after this baseline so the test + // can compare against it once the failing checkpoint completes. + Random rng = new Random(616); + int dim = 16; + for (int i = 0; i < 50; i++) { + store.addVector(Bytes.from_int(i), vec(rng, dim)); + } + store.checkpoint(); + + Set filesAfterBaseline = dsm.snapshotMultipartFiles(); + int segmentsAfterBaseline = store.getSegmentCount(); + + // Add more vectors so the next checkpoint will emit at least one + // new shard segment in Phase B. + for (int i = 50; i < 100; i++) { + store.addVector(Bytes.from_int(i), vec(rng, dim)); + } + + // Arm the DSM to fail mid-write on the next multipart write call. + // (Whichever call lands second under parallel Phase B execution + // is the victim — the test does not assume strict ordering.) + dsm.resetWriteTracking(); + dsm.failOnNextWrite(); + + // Trigger the checkpoint. The first writeMultipartIndexFile call + // succeeds; the second performs the put-then-throw simulation + // (so the partial multipart file appears in the DSM's storage + // map, just as a partial multipart upload would leave block 0..N + // in S3 / MinIO). + long failuresBefore = store.getConsecutiveCheckpointFailures(); + boolean threw = false; + try { + store.checkpoint(); + } catch (RuntimeException expected) { + // checkpoint() declares DataStorageManagerException (a + // RuntimeException via HerdDBInternalException); the + // Phase B IOException catch in checkpoint() wraps the + // IOException from our injected DSM into a + // DataStorageManagerException, which surfaces here. + threw = true; + } + + // (A) Phase B must have failed — surfaced by either a thrown + // exception OR the consecutiveCheckpointFailures counter + // advancing. If NEITHER fired, the failure injection did not + // actually trip the Phase B handler and the rest of the test + // is meaningless. + assertTrue("Phase B must have failed — either by throw" + + " (threw=" + threw + ") or by counter bump" + + " (before=" + failuresBefore + ", after=" + + store.getConsecutiveCheckpointFailures() + ")", + threw || store.getConsecutiveCheckpointFailures() > failuresBefore); + + // (B) THE FIX: the partial multipart file that the failing + // writeMultipartIndexFile call left in the DSM must have been + // deleted by the rollback path. Net effect: the set of files + // in the DSM after the failing checkpoint is equal to the + // baseline snapshot (no orphan, no leak). + // + // Pre-fix the failing segment's "graph" key remains in the + // DSM map: trackProvisionalMultipartFile was never called + // (the upload threw), so the rollback's tracker did not list + // it, and deleteMultipartIndexFile was never invoked for it. + // The remote storage would carry that orphan indefinitely, + // poisoning every subsequent compaction cycle (issue #616 + // reproduction). + // + // Post-fix the pre-track entry ensures the rollback knows + // about the partial file and deletes it. + Set filesAfterFailure = dsm.snapshotMultipartFiles(); + Set orphanedFiles = new HashSet<>(filesAfterFailure); + orphanedFiles.removeAll(filesAfterBaseline); + assertTrue("Phase B rollback must delete every partial multipart" + + " file from the failing checkpoint (issue #616)." + + " Orphaned: " + orphanedFiles, + orphanedFiles.isEmpty()); + + // The DSM must also have observed at least one delete call + // against the failing checkpoint's segUuid — confirming the + // rollback path traversed the (now correctly-populated) + // provisional tracker and reached the partial file. + assertTrue("Phase B rollback must call deleteMultipartIndexFile" + + " for the failing segment (issue #616). Deletes" + + " observed: " + dsm.deletedFiles, + dsm.deleteCalls.get() > 0); + + // (C) The on-disk segment registry must not have grown: no + // half-written segment from the failing Phase B can leak + // into this.segments. recoverFromPhaseBFailure leaves the + // registry untouched, and the failing Phase B never reached + // persistIndexStatusMultiSegment. + assertEquals("Failing Phase B must not grow the segment registry", + segmentsAfterBaseline, store.getSegmentCount()); + + // (D) The SEVERE log must name the failing segUuid so + // operators can identify which segment's upload aborted + // (issue #616 diagnostic gap). + assertTrue("SEVERE log must name the failing segUuid. Captured" + + " messages: " + captor.severeMessages, + captor.sawMessageContaining("Phase B: graph multipart write FAILED") + || captor.sawMessageContaining("Phase B: map multipart write FAILED")); + // The IS-internal segment naming convention is "_seg" + // — verify a UUID-shaped token is present in the SEVERE line. + assertTrue("SEVERE log must name a segUuid of the form '_seg'." + + " Captured messages: " + captor.severeMessages, + captor.sawMessageContaining("_seg")); + + // (E) Belt-and-suspenders: no SEVERE log should have leaked an + // "OK" / success indication for the failing segment. (Sanity: + // this catches a future refactor that accidentally promotes + // the success log to SEVERE.) + assertFalse("SEVERE log must not contain a successful-write line", + captor.sawMessageContaining("Phase B: shard segment")); + } finally { + storeLogger.removeHandler(captor); + storeLogger.setLevel(previousLevel); + } + } + + /** + * {@link MemoryDataStorageManager} subclass that: + *
    + *
  • counts {@code deleteMultipartIndexFile} calls so the test can + * assert the rollback path reached the partial file;
  • + *
  • simulates a mid-multipart-write abort: when armed, the next + * {@code writeMultipartIndexFile} call performs the underlying put + * AND THEN throws an {@link IOException}, mimicking the case where + * block N has already been transmitted to remote storage but + * block N+1's allocation aborts (the OOM scenario from the issue's + * reproduction). Without the "put then throw" simulation the test + * would not exercise the bug — the bug is precisely that a put + * that left bytes in the store was not tracked for cleanup.
  • + *
+ */ + private static final class MidWriteFailingDsm extends MemoryDataStorageManager { + + final AtomicInteger writeCalls = new AtomicInteger(); + final AtomicInteger deleteCalls = new AtomicInteger(); + final Set deletedFiles = + java.util.Collections.synchronizedSet(new java.util.LinkedHashSet<>()); + final AtomicBoolean armed = new AtomicBoolean(false); + + void resetWriteTracking() { + writeCalls.set(0); + deleteCalls.set(0); + deletedFiles.clear(); + } + + void failOnNextWrite() { + armed.set(true); + } + + Set snapshotMultipartFiles() { + // multipartFiles is the package-private map in MemoryDataStorageManager. + try { + java.lang.reflect.Field f = MemoryDataStorageManager.class + .getDeclaredField("multipartFiles"); + f.setAccessible(true); + @SuppressWarnings("unchecked") + java.util.Map map = (java.util.Map) f.get(this); + return new java.util.LinkedHashSet<>(map.keySet()); + } catch (ReflectiveOperationException e) { + throw new AssertionError( + "MemoryDataStorageManager.multipartFiles layout changed", e); + } + } + + @Override + public String writeMultipartIndexFile(String tableSpace, String uuid, String fileType, + java.nio.file.Path tempFile, + java.util.function.LongConsumer progress) + throws IOException, DataStorageManagerException { + writeCalls.incrementAndGet(); + // Compare-and-set so concurrent Phase B parallel writers race + // for the "victim" slot deterministically — exactly ONE write + // fails per arm, regardless of PHASE_B_SEGMENT_PARALLELISM. + boolean takeVictimSlot = armed.compareAndSet(true, false); + if (takeVictimSlot) { + // Simulate "block N has been put to remote storage but + // block N+1 aborts". We perform the actual put first so + // the in-memory map carries the file (this is the leak + // the rollback must clean up), then throw. + super.writeMultipartIndexFile(tableSpace, uuid, fileType, tempFile, progress); + throw new IOException("issue #616 test: simulated mid-multipart-write" + + " failure for " + tableSpace + "/" + uuid + "/" + fileType); + } + return super.writeMultipartIndexFile(tableSpace, uuid, fileType, tempFile, progress); + } + + @Override + public void deleteMultipartIndexFile(String tableSpace, String uuid, String fileType) + throws DataStorageManagerException { + deleteCalls.incrementAndGet(); + deletedFiles.add(tableSpace + "/" + uuid + "/" + fileType); + super.deleteMultipartIndexFile(tableSpace, uuid, fileType); + } + + // Pin to the reader-supplier path so test behaviour does not change + // if a future refactor flips the direct-download default (same + // pattern as PhaseCPrepRollbackSafetyTest). + @Override + public boolean supportsDirectMultipartDownload() { + return false; + } + + @Override + public void downloadMultipartIndexFile(String tableSpace, String uuid, String fileType, + long fileSize, java.nio.file.Path target) + throws IOException, DataStorageManagerException { + throw new UnsupportedOperationException("test pins reader-supplier path"); + } + } + + /** + * JUL handler that captures SEVERE log records emitted by + * {@link PersistentVectorStore}, resolving {0}-style parameter + * substitutions so assertions can grep the rendered text. + */ + private static final class SevereCaptor extends Handler { + final java.util.List severeMessages = + java.util.Collections.synchronizedList(new java.util.ArrayList<>()); + + @Override + public void publish(LogRecord record) { + if (record == null) { + return; + } + if (record.getLevel().intValue() >= Level.SEVERE.intValue()) { + String msg = record.getMessage(); + Object[] params = record.getParameters(); + if (msg != null && params != null) { + for (int i = 0; i < params.length; i++) { + msg = msg.replace("{" + i + "}", String.valueOf(params[i])); + } + } + severeMessages.add(msg); + } + } + + boolean sawMessageContaining(String needle) { + synchronized (severeMessages) { + for (String m : severeMessages) { + if (m != null && m.contains(needle)) { + return true; + } + } + return false; + } + } + + @Override + public void flush() { + // no-op + } + + @Override + public void close() { + // no-op + } + } +} From 7477de1bdeb1722f1cf894574f1aaef063e288cd Mon Sep 17 00:00:00 2001 From: Enrico Olivelli Date: Wed, 20 May 2026 18:35:22 +0200 Subject: [PATCH 2/2] review: cover Error path + tighten test + clarify sibling call sites MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the pr-reviewer findings on PR #618: 1. Broaden the inner SEVERE catch arms in writeShardAsFusedPQSegment from `catch (IOException | RuntimeException)` to `catch (Throwable)`. Netty's OutOfDirectMemoryError extends OutOfMemoryError extends Error — the exact failure mode cited in the issue's reproduction — and was previously skipped by the diagnostic log. We rethrow the original Throwable unchanged so the JVM's Error-handling semantics are preserved. 2. Broaden the outer Phase B catch in doCheckpointFusedPQThreePhase from `catch (IOException | RuntimeException)` to `catch (IOException | RuntimeException | Error)`. In the parallel Phase B path, awaitAllOrFirstFailure already wraps non-IOException/non-RuntimeException Throwables into an IOException, so the existing catch sufficed; but in the SERIAL path (shardTasks.size() == 1) an Error propagated directly, bypassing rollbackProvisionalArtefacts() and recoverFromPhaseBFailure(). Now Error drives the full rollback pipeline in both paths. 3. Add clarifying comments at the two sibling track-after-success sites the reviewer flagged: - writeStreamingCompactedSegment (compaction code path): provisionalMultipartFiles is null during compaction; orphan cleanup runs via VectorIndexCompactor.CompactionException.orphanPaths + runCompactionCycle's pendingDeletes pipeline — pre-track would double-clean. - writeOneSegmentData (reachable only via buildSegmentsInParallel, which has no in-tree caller — dormant code): documented so a future re-enablement adopts the same pre-track + Throwable-catch layout as the patched live path. 4. Tighten Issue616PhaseBPartialUploadRollbackTest: - Replace the "threw OR counter bumped" assertion with two separate assertions requiring both — under the fix both must always hold. - Drop the trivially-true assertion that the SEVERE log does not contain "Phase B: shard segment" (the success line is logged at INFO and the captor only records SEVERE+). - Add a new phaseBPartialUploadUnderErrorRolledBack test case that throws OutOfMemoryError (mirroring Netty's OutOfDirectMemoryError class hierarchy) and asserts the diagnostic SEVERE log fires, the orphan is cleaned up, and the segment registry is untouched. This case fails pre-fix on both the Throwable-catch and outer-Error gaps from items 1 and 2 above. 5. Move uploadingActive.incrementAndGet() inside the try block so a hypothetical throw from trackProvisionalMultipartFile cannot leak the active-upload counter (minor defensive cleanup). Co-Authored-By: Claude Sonnet 4.6 --- .../vector/PersistentVectorStore.java | 80 +++++-- ...sue616PhaseBPartialUploadRollbackTest.java | 202 ++++++++++++++++-- 2 files changed, 252 insertions(+), 30 deletions(-) diff --git a/herddb-indexing-service/src/main/java/herddb/indexing/vector/PersistentVectorStore.java b/herddb-indexing-service/src/main/java/herddb/indexing/vector/PersistentVectorStore.java index 3f6de00f..6e4eecc0 100644 --- a/herddb-indexing-service/src/main/java/herddb/indexing/vector/PersistentVectorStore.java +++ b/herddb-indexing-service/src/main/java/herddb/indexing/vector/PersistentVectorStore.java @@ -3535,6 +3535,20 @@ SegmentWriteResult writeStreamingCompactedSegment(Path graphTempFile, } finally { uploadingActive.decrementAndGet(); } + // Issue #616 review item: track-after-success here is INTENTIONAL and + // benign. writeStreamingCompactedSegment runs on the compaction code + // path, where provisionalMultipartFiles is NOT installed (only Phase B + // of indexCheckpoint installs it; see the only assignment at the top + // of doCheckpointFusedPQThreePhase). trackProvisionalMultipartFile is + // therefore a no-op on this path — both pre- and post-track placements + // are equivalent here. Compaction's own orphan-cleanup mechanism is + // independent: VectorIndexCompactor.rebuildSegmentStreaming wraps any + // failure from writeStreamingCompactedSegment into a CompactionException + // carrying a failureOrphans list (see VectorIndexCompactor.java around + // line 1330), which runCompactionCycle's catch arm then queues for the + // retention-aware pendingDeletes reaper. Do NOT pre-track here without + // first removing that orphan-list mechanism, or both pipelines would + // attempt to delete the same files. trackProvisionalMultipartFile(segUuid, "graph"); // Upload map @@ -3549,6 +3563,9 @@ SegmentWriteResult writeStreamingCompactedSegment(Path graphTempFile, } finally { uploadingActive.decrementAndGet(); } + // Issue #616 review item: see the matching comment on the graph + // upload above — track-after-success is intentional on the compaction + // path because provisionalMultipartFiles is not installed here. trackProvisionalMultipartFile(segUuid, "map"); compactionNodesDone.addAndGet(vectorCount); return new SegmentWriteResult(segmentId, @@ -5671,7 +5688,26 @@ private void doCheckpointFusedPQThreePhase(LogSequenceNumber sequenceNumber) if (hook != null) { hook.run(); } - } catch (IOException | RuntimeException e) { + } catch (IOException | RuntimeException | Error e) { + // Issue #616: include Error in the catch list so an Error thrown + // by writeShardAsFusedPQSegment (e.g. Netty's OutOfDirectMemoryError, + // which extends OutOfMemoryError extends Error — the failure mode + // cited in the issue's reproduction) still drives the rollback + // pipeline. In the parallel Phase B path, + // awaitAllOrFirstFailure already wraps a non-IOException / + // non-RuntimeException Throwable into an IOException; but in the + // serial path (shardTasks.size() == 1, which forces parallelism=1 + // at doCheckpointFusedPQPhaseB) the Error propagates directly, so + // without this broadening the rollback + recovery would be + // skipped and the partial multipart files queued by the issue + // #616 pre-track would not be deleted. + // + // The broad Error catch is bounded: we run a small, fixed set of + // safe operations (log line + atomic counter bumps + remote-file + // deletes + writeLock'd state restore), then rethrow the original + // Throwable unchanged so the JVM's normal Error-handling semantics + // (e.g. thread termination for VirtualMachineError) are preserved + // at the call site. LOGGER.log(Level.SEVERE, "checkpoint " + indexName + ": Phase B exception", e); rollbackProvisionalArtefacts(); consecutiveCheckpointFailures.incrementAndGet(); @@ -6701,7 +6737,6 @@ private SegmentWriteResult writeShardAsFusedPQSegment( // Upload graph file long graphSize = Files.size(tempFile); uploadBytesTotal.addAndGet(graphSize); - uploadingActive.incrementAndGet(); String graphFilePath; String segUuid = indexUUID + "_seg" + segmentId; // Issue #616: register the segUuid in the provisional-multipart-files @@ -6720,26 +6755,34 @@ private SegmentWriteResult writeShardAsFusedPQSegment( // the rollback path already swallows DataStorageManagerException / // RuntimeException from such deletes (see rollbackProvisionalArtefacts). trackProvisionalMultipartFile(segUuid, "graph"); + uploadingActive.incrementAndGet(); try { graphFilePath = dataStorageManager.writeMultipartIndexFile( tableSpaceUUID, segUuid, "graph", tempFile, uploadBytesDone::addAndGet); - } catch (IOException | RuntimeException e) { + } catch (Throwable e) { // Issue #616: log SEVERE with the failing segment identity BEFORE // the exception propagates to awaitAllOrFirstFailure (which only // surfaces the FIRST failure when shards run in parallel). Without // this log the outer Phase B catch in doCheckpointFusedPQThreePhase // prints "Phase B exception" with no shard identity — operators // cannot tell which segment's upload aborted, which is the - // diagnostic gap the issue is trying to close. The broad - // RuntimeException catch is intentional: Netty wraps allocation - // failures (including OutOfDirectMemoryError-style conditions) - // and protocol errors into unchecked exceptions thrown out of the - // multipart writer, and DataStorageManagerException is itself a - // RuntimeException subclass. We re-throw unchanged so the existing - // Phase B failure handling (rollbackProvisionalArtefacts + - // recoverFromPhaseBFailure) runs exactly as before. + // diagnostic gap the issue is trying to close. + // + // The broad Throwable catch is required and intentional: + // (a) Netty surfaces direct-buffer allocation failures as + // OutOfDirectMemoryError, which extends OutOfMemoryError + // (an Error, NOT a RuntimeException) — this is the exact + // motivating failure cited in issue #616's reproduction. + // A narrower catch would skip the diagnostic for the very + // case the issue is trying to fix. + // (b) DataStorageManagerException is a RuntimeException + // subclass; IOException is checked. Both are caught. + // (c) We re-throw the original Throwable unchanged so the + // outer Phase B failure handling (awaitAllOrFirstFailure + // wrap + rollbackProvisionalArtefacts + + // recoverFromPhaseBFailure) runs exactly as before. LOGGER.log(Level.SEVERE, "checkpoint " + indexName + " Phase B: graph multipart" + " write FAILED for segUuid=" + segUuid @@ -6758,18 +6801,19 @@ private SegmentWriteResult writeShardAsFusedPQSegment( try { long mapSize = Files.size(mapFile); uploadBytesTotal.addAndGet(mapSize); - uploadingActive.incrementAndGet(); String mapFilePath; // Issue #616: pre-track the map multipart file for the same reason // as the graph upload above. See the matching comment there. trackProvisionalMultipartFile(segUuid, "map"); + uploadingActive.incrementAndGet(); try { mapFilePath = dataStorageManager.writeMultipartIndexFile( tableSpaceUUID, segUuid, "map", mapFile, uploadBytesDone::addAndGet); - } catch (IOException | RuntimeException e) { - // Issue #616: see the matching catch on the graph upload above. + } catch (Throwable e) { + // Issue #616: see the matching catch on the graph upload + // above for why the catch is Throwable, not narrower. LOGGER.log(Level.SEVERE, "checkpoint " + indexName + " Phase B: map multipart" + " write FAILED for segUuid=" + segUuid @@ -7489,6 +7533,14 @@ private void loadFusedPQSegment(VectorSegment seg, Path mapFile, int dim, long s * Writes the graph and map data for one segment via the multipart API of * the underlying {@link herddb.storage.DataStorageManager}. */ + // Issue #616 review item: writeOneSegmentData is reachable only through + // buildSegmentsInParallel, which currently has no in-tree caller — this is + // dormant Phase B code preserved for the parallel-slice rebuild path. + // The track-after-success pattern below mirrors the pre-#616 form of + // writeShardAsFusedPQSegment; if this method is re-enabled it MUST adopt + // the same pre-track + Throwable-catch layout as the patched live path + // (search for "Issue #616:" in writeShardAsFusedPQSegment) or partial + // multipart uploads will again leak into remote storage on Phase B failure. private SegmentWriteResult writeOneSegmentData( SegmentSlice s, ConcurrentHashMap> partVectors, diff --git a/herddb-indexing-service/src/test/java/herddb/indexing/vector/Issue616PhaseBPartialUploadRollbackTest.java b/herddb-indexing-service/src/test/java/herddb/indexing/vector/Issue616PhaseBPartialUploadRollbackTest.java index 731fc138..e57e1b9f 100644 --- a/herddb-indexing-service/src/test/java/herddb/indexing/vector/Issue616PhaseBPartialUploadRollbackTest.java +++ b/herddb-indexing-service/src/test/java/herddb/indexing/vector/Issue616PhaseBPartialUploadRollbackTest.java @@ -20,7 +20,6 @@ package herddb.indexing.vector; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import herddb.core.MemoryManager; import herddb.mem.MemoryDataStorageManager; @@ -196,16 +195,20 @@ public void phaseBPartialUploadRolledBack() throws Exception { threw = true; } - // (A) Phase B must have failed — surfaced by either a thrown - // exception OR the consecutiveCheckpointFailures counter - // advancing. If NEITHER fired, the failure injection did not - // actually trip the Phase B handler and the rest of the test - // is meaningless. - assertTrue("Phase B must have failed — either by throw" - + " (threw=" + threw + ") or by counter bump" - + " (before=" + failuresBefore + ", after=" + // (A) Phase B must have failed — under the fix, BOTH a thrown + // exception (the original IOException wrapped as + // DataStorageManagerException by checkpoint()) AND the + // consecutiveCheckpointFailures counter bump must fire (the + // outer Phase B catch in doCheckpointFusedPQThreePhase + // increments the counter BEFORE rethrowing). Asserting both + // tightens the contract — a future refactor that swallows + // the throw or skips the counter bump is caught immediately. + assertTrue("Phase B must have surfaced as a thrown exception" + + " (threw=" + threw + ")", threw); + assertTrue("Phase B must have bumped the consecutive failure" + + " counter (before=" + failuresBefore + ", after=" + store.getConsecutiveCheckpointFailures() + ")", - threw || store.getConsecutiveCheckpointFailures() > failuresBefore); + store.getConsecutiveCheckpointFailures() > failuresBefore); // (B) THE FIX: the partial multipart file that the failing // writeMultipartIndexFile call left in the DSM must have been @@ -261,12 +264,106 @@ public void phaseBPartialUploadRolledBack() throws Exception { + " Captured messages: " + captor.severeMessages, captor.sawMessageContaining("_seg")); - // (E) Belt-and-suspenders: no SEVERE log should have leaked an - // "OK" / success indication for the failing segment. (Sanity: - // this catches a future refactor that accidentally promotes - // the success log to SEVERE.) - assertFalse("SEVERE log must not contain a successful-write line", - captor.sawMessageContaining("Phase B: shard segment")); + } finally { + storeLogger.removeHandler(captor); + storeLogger.setLevel(previousLevel); + } + } + + /** + * Issue #616 review item: the OOM scenario cited in the issue's + * reproduction surfaces as Netty's {@code OutOfDirectMemoryError}, + * which extends {@link java.lang.OutOfMemoryError} (an {@link Error}, + * NOT a {@link RuntimeException}). The inner SEVERE diagnostic catch + * in {@code writeShardAsFusedPQSegment} must catch {@link Throwable} + * — not just {@code IOException | RuntimeException} — so the failing + * segment's identity is logged in exactly the failure mode the issue + * is trying to make diagnosable. + * + *

In the default parallel Phase B path + * ({@code PHASE_B_SEGMENT_PARALLELISM=2}), {@code awaitAllOrFirstFailure} + * wraps a thrown {@link Throwable} that is neither {@link IOException} + * nor {@link RuntimeException} into an {@link IOException} with the + * original {@link Error} as the cause — so the outer Phase B catch + * arm still runs {@code rollbackProvisionalArtefacts()} + + * {@code recoverFromPhaseBFailure()} on Error. We assert that the + * partial multipart file is cleaned up AND the new SEVERE diagnostic + * line names the failing {@code segUuid}. + */ + @Test(timeout = 60_000) + public void phaseBPartialUploadUnderErrorRolledBack() throws Exception { + Path tmpDir = tmpFolder.newFolder("issue616-phaseB-partial-error").toPath(); + ErrorWriteFailingDsm dsm = new ErrorWriteFailingDsm(); + + SevereCaptor captor = new SevereCaptor(); + Logger storeLogger = Logger.getLogger(PersistentVectorStore.class.getName()); + storeLogger.addHandler(captor); + Level previousLevel = storeLogger.getLevel(); + storeLogger.setLevel(Level.ALL); + + try (PersistentVectorStore store = createStore(tmpDir, dsm)) { + store.start(); + + Random rng = new Random(616); + int dim = 16; + for (int i = 0; i < 50; i++) { + store.addVector(Bytes.from_int(i), vec(rng, dim)); + } + store.checkpoint(); + + Set filesAfterBaseline = dsm.snapshotMultipartFiles(); + int segmentsAfterBaseline = store.getSegmentCount(); + + for (int i = 50; i < 100; i++) { + store.addVector(Bytes.from_int(i), vec(rng, dim)); + } + + dsm.resetWriteTracking(); + dsm.failOnNextWriteWithError(); + + // We do NOT assert that checkpoint() throws an Error — the + // outer parallel-Phase-B path wraps it into an IOException + // (then DataStorageManagerException via checkpoint()). What + // matters is (i) the diagnostic SEVERE log fires (Throwable + // catch in the patched method), (ii) the rollback path runs + // (no orphan), and (iii) the segment registry is untouched. + long failuresBefore = store.getConsecutiveCheckpointFailures(); + boolean failed = false; + try { + store.checkpoint(); + } catch (Throwable expected) { + // Either a wrapped DataStorageManagerException (parallel + // path), or the raw Error (serial path) — both are + // acceptable evidence that Phase B failed. We catch + // Throwable here precisely because the Error path is + // exactly what we are testing. + failed = true; + } + + assertTrue("Phase B must have failed under Error injection", + failed || store.getConsecutiveCheckpointFailures() > failuresBefore); + + Set filesAfterFailure = dsm.snapshotMultipartFiles(); + Set orphanedFiles = new HashSet<>(filesAfterFailure); + orphanedFiles.removeAll(filesAfterBaseline); + assertTrue("Phase B rollback must delete every partial multipart" + + " file from the failing checkpoint, even when the" + + " injected failure is an Error (issue #616 review)." + + " Orphaned: " + orphanedFiles, + orphanedFiles.isEmpty()); + + assertEquals("Failing Phase B (Error path) must not grow the" + + " segment registry", + segmentsAfterBaseline, store.getSegmentCount()); + + // The diagnostic log MUST fire for the Error path — that is + // the gap the reviewer identified. Without the Throwable + // catch in writeShardAsFusedPQSegment, this assertion fails. + assertTrue("SEVERE log must name the failing segUuid even when" + + " the upload throws an Error (issue #616 review)." + + " Captured messages: " + captor.severeMessages, + captor.sawMessageContaining("Phase B: graph multipart write FAILED") + || captor.sawMessageContaining("Phase B: map multipart write FAILED")); } finally { storeLogger.removeHandler(captor); storeLogger.setLevel(previousLevel); @@ -367,6 +464,79 @@ public void downloadMultipartIndexFile(String tableSpace, String uuid, String fi } } + /** + * Variant of {@link MidWriteFailingDsm} that simulates the Netty + * {@code OutOfDirectMemoryError} scenario from the issue's reproduction: + * performs the underlying put (so the file appears in the simulated + * remote storage), then throws an {@link OutOfMemoryError}. The + * {@code OutOfMemoryError} hierarchy mirrors Netty's + * {@code io.netty.util.internal.OutOfDirectMemoryError extends + * java.lang.OutOfMemoryError extends VirtualMachineError extends Error}. + */ + private static final class ErrorWriteFailingDsm extends MemoryDataStorageManager { + + final AtomicInteger deleteCalls = new AtomicInteger(); + final AtomicBoolean armed = new AtomicBoolean(false); + + void resetWriteTracking() { + deleteCalls.set(0); + } + + void failOnNextWriteWithError() { + armed.set(true); + } + + Set snapshotMultipartFiles() { + try { + java.lang.reflect.Field f = MemoryDataStorageManager.class + .getDeclaredField("multipartFiles"); + f.setAccessible(true); + @SuppressWarnings("unchecked") + java.util.Map map = (java.util.Map) f.get(this); + return new java.util.LinkedHashSet<>(map.keySet()); + } catch (ReflectiveOperationException e) { + throw new AssertionError( + "MemoryDataStorageManager.multipartFiles layout changed", e); + } + } + + @Override + public String writeMultipartIndexFile(String tableSpace, String uuid, String fileType, + java.nio.file.Path tempFile, + java.util.function.LongConsumer progress) + throws IOException, DataStorageManagerException { + boolean takeVictimSlot = armed.compareAndSet(true, false); + if (takeVictimSlot) { + // Same put-then-throw simulation as MidWriteFailingDsm, + // but throws an Error instead of an IOException. + super.writeMultipartIndexFile(tableSpace, uuid, fileType, tempFile, progress); + throw new OutOfMemoryError("issue #616 test: simulated direct-buffer" + + " allocation failure for " + tableSpace + "/" + uuid + + "/" + fileType); + } + return super.writeMultipartIndexFile(tableSpace, uuid, fileType, tempFile, progress); + } + + @Override + public void deleteMultipartIndexFile(String tableSpace, String uuid, String fileType) + throws DataStorageManagerException { + deleteCalls.incrementAndGet(); + super.deleteMultipartIndexFile(tableSpace, uuid, fileType); + } + + @Override + public boolean supportsDirectMultipartDownload() { + return false; + } + + @Override + public void downloadMultipartIndexFile(String tableSpace, String uuid, String fileType, + long fileSize, java.nio.file.Path target) + throws IOException, DataStorageManagerException { + throw new UnsupportedOperationException("test pins reader-supplier path"); + } + } + /** * JUL handler that captures SEVERE log records emitted by * {@link PersistentVectorStore}, resolving {0}-style parameter