Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1665,6 +1665,23 @@ public int getLoadedIndexCount() {
return vectorStores.size();
}

/**
* Snapshot of every loaded {@link PersistentVectorStore}. Used by
* {@code GetEngineStats} (issue #640) to aggregate compaction state
* across all indexes without holding the {@code vectorStores} map
* lock. The returned list is a defensive copy: it is safe to iterate
* concurrently with engine mutations.
*/
public java.util.List<PersistentVectorStore> getPersistentVectorStoresSnapshot() {
java.util.List<PersistentVectorStore> out = new java.util.ArrayList<>(vectorStores.size());
for (AbstractVectorStore s : vectorStores.values()) {
if (s instanceof PersistentVectorStore) {
out.add((PersistentVectorStore) s);
}
}
return out;
}

public int getApplyQueueSize() {
ExecutorService[] workers = this.applyWorkers;
if (workers == null) {
Expand Down Expand Up @@ -4239,6 +4256,15 @@ public IndexDetails describeIndex(String tablespace, String table, String index)
d.compactionNodesTotal = pvs.getCompactionNodesTotal();
d.uploadBytesDone = pvs.getUploadBytesDone();
d.uploadBytesTotal = pvs.getUploadBytesTotal();
// Issue #640: real-time compaction observability fields.
d.compactionBatchesDone = pvs.getCompactionBatchesDone();
d.compactionBatchesTotal = pvs.getCompactionBatchesTotal();
d.compactionCycleId = pvs.getCompactionCycleId();
d.compactionInputSegmentCount = (int) Math.min(Integer.MAX_VALUE,
pvs.getCompactionInputSegmentCount());
d.compactionInputVectorCount = pvs.getCompactionInputVectorCount();
d.compactionElapsedMs = pvs.getCompactionElapsedMs();
d.compactionRunning = pvs.isCompactionRunning();
d.nextNodeId = pvs.getNextNodeId();
if (!"idle".equals(d.compactionPhase)) {
d.status = d.compactionPhase;
Expand Down Expand Up @@ -5783,6 +5809,14 @@ public static final class IndexDetails {
public long compactionNodesTotal;
public long uploadBytesDone;
public long uploadBytesTotal;
// Real-time compaction observability (issue #640).
public long compactionBatchesDone;
public long compactionBatchesTotal;
public long compactionCycleId;
public int compactionInputSegmentCount;
public long compactionInputVectorCount;
public long compactionElapsedMs;
public boolean compactionRunning;
// Global monotonic node-id counter (issue #256). Widened to long end-to-end;
// surfaced so clients and dashboards can observe the burn rate.
public long nextNodeId;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -375,7 +375,15 @@ public void describeIndex(DescribeIndexRequest request, StreamObserver<DescribeI
.setUploadBytesDone(details.uploadBytesDone)
.setUploadBytesTotal(details.uploadBytesTotal)
.setNextNodeId(details.nextNodeId)
.setNumShards(details.numShards);
.setNumShards(details.numShards)
// Issue #640: real-time compaction observability fields.
.setCompactionBatchesDone(details.compactionBatchesDone)
.setCompactionBatchesTotal(details.compactionBatchesTotal)
.setCompactionCycleId(details.compactionCycleId)
.setCompactionInputSegmentCount(details.compactionInputSegmentCount)
.setCompactionInputVectorCount(details.compactionInputVectorCount)
.setCompactionElapsedMs(details.compactionElapsedMs)
.setCompactionRunning(details.compactionRunning);
responseObserver.onNext(builder.build());
responseObserver.onCompleted();
} catch (RuntimeException e) {
Expand Down Expand Up @@ -480,6 +488,36 @@ public void getEngineStats(GetEngineStatsRequest request,
addInt64(b, "jvm_heap_max_bytes", heapMax);
addInt64(b, "jvm_heap_used_pct", heapPct);

// Issue #640: aggregate compaction state across all loaded
// PersistentVectorStores so a single top-level engine-stats call
// reveals whether any index is currently compacting, without
// requiring a describe-index per index.
//
// compaction_running : true iff at least one PVS has an active
// cycle (started-nanos != 0).
// compaction_phase : the most advanced phase string observed
// across all PVSs (priority:
// uploading-segment > compacting-graph >
// writing-graph > idle). "idle" when no
// store reports any activity.
boolean anyRunning = false;
String aggregatePhase = "idle";
int aggregateRank = 0;
for (herddb.indexing.vector.PersistentVectorStore pvs
: engine.getPersistentVectorStoresSnapshot()) {
if (pvs.isCompactionRunning()) {
anyRunning = true;
}
String ph = pvs.getCompactionPhase();
int rank = phaseRank(ph);
if (rank > aggregateRank) {
aggregateRank = rank;
aggregatePhase = ph;
}
}
addBool(b, "compaction_running", anyRunning);
addString(b, "compaction_phase", aggregatePhase);

responseObserver.onNext(b.build());
responseObserver.onCompleted();
} catch (RuntimeException e) {
Expand All @@ -505,6 +543,37 @@ private static void addBool(GetEngineStatsResponse.Builder b, String key, boolea
.build());
}

// Issue #640: string-valued engine-stats metric helper. Mirrors
// addInt64/addBool — used so the aggregated `compaction_phase` can be
// surfaced as a human-readable string rather than an opaque integer
// rank.
private static void addString(GetEngineStatsResponse.Builder b, String key, String value) {
b.addMetrics(MetricEntry.newBuilder()
.setKey(key)
.setValue(MetricValue.newBuilder()
.setStringValue(value == null ? "" : value)
.build())
.build());
}

/**
* Rank used to pick the "most advanced" compaction phase across all
* loaded stores in {@link #getEngineStats}. Matches the priority order
* of {@link herddb.indexing.vector.PersistentVectorStore#getCompactionPhase}.
*/
private static int phaseRank(String phase) {
if (phase == null) {
return 0;
}
switch (phase) {
case "uploading-segment": return 3;
case "compacting-graph": return 2;
case "writing-graph": return 1;
case "idle":
default: return 0;
}
}

@Override
public void getInstanceInfo(GetInstanceInfoRequest request,
StreamObserver<GetInstanceInfoResponse> responseObserver) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -538,6 +538,14 @@ private static Map<String, Object> describeIndexToMap(DescribeIndexResponse r) {
m.put("compaction_nodes_total", r.getCompactionNodesTotal());
m.put("upload_bytes_done", r.getUploadBytesDone());
m.put("upload_bytes_total", r.getUploadBytesTotal());
// Issue #640: real-time compaction observability fields.
m.put("compaction_batches_done", r.getCompactionBatchesDone());
m.put("compaction_batches_total", r.getCompactionBatchesTotal());
m.put("compaction_cycle_id", r.getCompactionCycleId());
m.put("compaction_input_segment_count", r.getCompactionInputSegmentCount());
m.put("compaction_input_vector_count", r.getCompactionInputVectorCount());
m.put("compaction_elapsed_ms", r.getCompactionElapsedMs());
m.put("compaction_running", r.getCompactionRunning());
m.put("num_shards", r.getNumShards());
return m;
}
Expand Down
Loading
Loading