Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions herddb-indexing-service/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,11 @@
<artifactId>herddb-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.github.jbellis</groupId>
<artifactId>jvector</artifactId>
<version>4.0.0-rc.9-herddb-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.herddb</groupId>
<artifactId>herddb-utils</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

package herddb.indexing;

import herddb.index.vector.AbstractVectorStore;
import herddb.indexing.vector.AbstractVectorStore;
import herddb.utils.Bytes;
import java.util.AbstractMap;
import java.util.ArrayList;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,11 @@
import herddb.codec.DataAccessorForFullRecord;
import herddb.core.MemoryManager;
import herddb.file.FileMetadataStorageManager;
import herddb.index.vector.AbstractVectorStore;
import herddb.index.vector.PersistentVectorStore;
import herddb.index.vector.ReadOnlyVectorStore;
import herddb.indexing.vector.AbstractVectorStore;
import herddb.indexing.vector.PersistentVectorStore;
import herddb.indexing.vector.ReadOnlyVectorStore;
import herddb.index.vector.VectorIndexManager;
import herddb.index.vector.VectorMemoryBudget;
import herddb.indexing.vector.VectorMemoryBudget;
import herddb.indexing.segment.SegmentRegistryClient;
import herddb.indexing.segment.SegmentRegistryException;
import herddb.indexing.segment.SegmentRegistryPublisher;
Expand Down Expand Up @@ -356,8 +356,8 @@ public enum EngineStatus { ACTIVE, JOINING, FAILED }
* One {@link herddb.indexing.segment.SegmentAssignmentWatcher} per vector store,
* keyed by {@link #storeKey}. Each watcher watches the ZK segment-registry subtree
* for that store's index and calls
* {@link herddb.index.vector.AbstractVectorStore#adoptExternalSegment} /
* {@link herddb.index.vector.AbstractVectorStore#dropSegmentByUuid} when the
* {@link herddb.indexing.vector.AbstractVectorStore#adoptExternalSegment} /
* {@link herddb.indexing.vector.AbstractVectorStore#dropSegmentByUuid} when the
* optimizer produces or deprecates segments (issue #514).
*
* <p>Populated by the {@code vectorStoreFactory} lambda and closed in
Expand Down Expand Up @@ -829,7 +829,7 @@ public void start() throws Exception {
.PROPERTY_VECTOR_INDEX_COMPACTION_STREAMING_ENABLED,
IndexingServerConfiguration
.PROPERTY_VECTOR_INDEX_COMPACTION_STREAMING_ENABLED_DEFAULT);
herddb.index.vector.PersistentVectorStore.setStreamingCompactionEnabled(
herddb.indexing.vector.PersistentVectorStore.setStreamingCompactionEnabled(
vectorCompactionStreamingEnabled);
LOGGER.log(Level.INFO,
"vector index compaction: tieredEnabled={0}, backpressureSegments={1}, "
Expand All @@ -849,7 +849,7 @@ public void start() throws Exception {
IndexingServerConfiguration.PROPERTY_VECTOR_SEARCH_ASYNC_PIPELINE_ENABLED,
Boolean.getBoolean(
IndexingServerConfiguration.SYSPROP_VECTOR_SEARCH_ASYNC_PIPELINE_ENABLED));
herddb.index.vector.PersistentVectorStore.setSearchAsyncPipelineEnabled(
herddb.indexing.vector.PersistentVectorStore.setSearchAsyncPipelineEnabled(
vectorSearchAsyncPipelineEnabled);
LOGGER.log(Level.INFO, "vector search async IO pipeline (issue #547): enabled={0}",
vectorSearchAsyncPipelineEnabled);
Expand Down Expand Up @@ -1021,7 +1021,7 @@ public void start() throws Exception {
// the initial scan fires.
SegmentRegistryClient watcherRegistry = this.segmentRegistry;
if (watcherRegistry != null && tableSpaceUUID != null) {
final herddb.index.vector.AbstractVectorStore finalStore = store;
final herddb.indexing.vector.AbstractVectorStore finalStore = store;
final String finalIndexUUID = autoIndexUUID;
final String finalIndexName = indexName;
herddb.indexing.segment.SegmentAssignmentWatcher watcher =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@
package herddb.indexing;

import herddb.codec.DataAccessorForFullRecord;
import herddb.index.vector.AbstractVectorStore;
import herddb.index.vector.PersistentVectorStore;
import herddb.indexing.vector.AbstractVectorStore;
import herddb.indexing.vector.PersistentVectorStore;
import herddb.index.vector.VectorIndexManager;
import herddb.log.LogSequenceNumber;
import herddb.model.Index;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

package herddb.indexing;

import herddb.index.vector.AbstractVectorStore;
import herddb.indexing.vector.AbstractVectorStore;
import java.nio.file.Path;
import java.util.Map;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
*/
package herddb.indexing.optimizer;

import herddb.index.vector.RemoteSegmentGraphMerger;
import herddb.indexing.vector.RemoteSegmentGraphMerger;
import herddb.indexing.segment.SegmentMetadata;
import herddb.indexing.segment.SegmentRegistryClient;
import herddb.indexing.segment.SegmentRegistryException;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -288,7 +288,7 @@ public synchronized void start() throws Exception {
boolean streamingEnabled = configuration.getBoolean(
OptimizerConfiguration.PROPERTY_MERGE_STREAMING_ENABLED,
OptimizerConfiguration.PROPERTY_MERGE_STREAMING_ENABLED_DEFAULT);
herddb.index.vector.PersistentVectorStore.setStreamingCompactionEnabled(streamingEnabled);
herddb.indexing.vector.PersistentVectorStore.setStreamingCompactionEnabled(streamingEnabled);
LOGGER.log(Level.INFO,
"optimizer-pod streaming compaction: enabled={0} (config key {1})",
new Object[]{streamingEnabled,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
/**
* Decides which {@code ACTIVE} segments to merge on a given tick.
*
* <p>Mirrors the logic of {@code herddb.index.vector.VectorIndexCompactor#chooseSegmentsToMerge}
* <p>Mirrors the logic of {@code herddb.indexing.vector.VectorIndexCompactor#chooseSegmentsToMerge}
* but operates on {@link VersionedSegmentMetadata} from the registry (no
* in-memory {@code VectorSegment}s, since the optimizer doesn't have those).
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
*/
package herddb.indexing.optimizer;

import herddb.index.vector.RemoteSegmentGraphMerger;
import herddb.indexing.vector.RemoteSegmentGraphMerger;
import herddb.indexing.segment.SegmentMetadata;
import herddb.indexing.segment.SegmentState;
import herddb.indexing.segment.TombstoneOverlay;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
*
* <p>Implementations:
* <ul>
* <li>The production implementation will reuse {@link herddb.index.vector.VectorIndexCompactor}
* <li>The production implementation will reuse {@link herddb.indexing.vector.VectorIndexCompactor}
* once that code path is extracted from PersistentVectorStore (out of
* scope for the initial step 5 MVP).</li>
* <li>{@link InMemorySegmentMerger} (in tests) performs no graph work but
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@
*/
package herddb.indexing.segment;

import herddb.index.vector.NewSegmentInfo;
import herddb.index.vector.SegmentPublisher;
import herddb.indexing.vector.NewSegmentInfo;
import herddb.indexing.vector.SegmentPublisher;
import java.util.List;
import java.util.Objects;
import java.util.function.LongSupplier;
Expand All @@ -29,12 +29,12 @@

/**
* Adapter that wires {@link SegmentPublisher} (called by
* {@link herddb.index.vector.PersistentVectorStore} after each successful
* {@link herddb.indexing.vector.PersistentVectorStore} after each successful
* checkpoint) to a {@link SegmentRegistryClient}, registering every freshly-emitted
* segment as ACTIVE owned by the local IS instance.
*
* <p>The UUID is taken from {@link NewSegmentInfo#getSegmentUuid()} — it must already
* have been stamped by {@link herddb.index.vector.PersistentVectorStore} during Phase B
* have been stamped by {@link herddb.indexing.vector.PersistentVectorStore} during Phase B
* BEFORE IndexStatus is persisted, so the same UUID survives a restart and we cannot
* double-register the same physical segment file (review item A2). The publisher never
* mints UUIDs of its own.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

*/

package herddb.index.vector;
package herddb.indexing.vector;

import herddb.storage.DataStorageManagerException;
import herddb.utils.Bytes;
Expand Down Expand Up @@ -119,7 +119,7 @@ public void forEachPrimaryKey(boolean includeOnDisk, Predicate<Bytes> visitor) {
* (e.g. {@link herddb.indexing.InMemoryVectorStore}). In that case the
* checkpoint-and-save path skips UUID embedding.
*
* <p>{@link herddb.index.vector.PersistentVectorStore} overrides this to
* <p>{@link herddb.indexing.vector.PersistentVectorStore} overrides this to
* return {@code getIndexUUID()}.
*/
public String getStoreUUID() {
Expand All @@ -136,7 +136,7 @@ public String getStoreUUID() {
* this method returns {@code false} immediately without loading it again.
*
* <p>The default implementation is a no-op (used by in-memory stores that have no
* persistent segments). {@link herddb.index.vector.PersistentVectorStore} overrides
* persistent segments). {@link herddb.indexing.vector.PersistentVectorStore} overrides
* this with a full implementation.
*
* @param segmentUuid the UUID of the segment (from the ZK registry)
Expand Down Expand Up @@ -170,7 +170,7 @@ public boolean adoptExternalSegment(String segmentUuid, long externalSegmentId,
* responsible for managing the lifecycle of files it produced.
*
* <p>The default implementation is a no-op.
* {@link herddb.index.vector.PersistentVectorStore} overrides this.
* {@link herddb.indexing.vector.PersistentVectorStore} overrides this.
*
* @param segmentUuid the UUID of the segment to remove
*/
Expand All @@ -187,7 +187,7 @@ public void dropSegmentByUuid(String segmentUuid) {
* handle IS-was-down-while-optimizer-deleted scenarios.
*
* <p>The default implementation is a no-op (non-persistent stores have no
* adopted segments). {@link herddb.index.vector.PersistentVectorStore} overrides.
* adopted segments). {@link herddb.indexing.vector.PersistentVectorStore} overrides.
*
* @param knownUuids segment UUIDs currently visible in the ZK registry
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
under the License.
*/

package herddb.index.vector;
package herddb.indexing.vector;

import herddb.core.ClockProPolicy;
import herddb.core.PageReplacementPolicy;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
under the License.
*/

package herddb.index.vector;
package herddb.indexing.vector;

import herddb.index.blink.BLink;
import herddb.utils.Bytes;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
under the License.
*/

package herddb.index.vector;
package herddb.indexing.vector;

import io.github.jbellis.jvector.graph.RandomAccessVectorValues;
import io.github.jbellis.jvector.vector.types.VectorFloat;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
under the License.
*/

package herddb.index.vector;
package herddb.indexing.vector;

import io.github.jbellis.jvector.graph.ImmutableGraphIndex;
import io.github.jbellis.jvector.graph.disk.OnDiskGraphIndexWriter;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
under the License.
*/

package herddb.index.vector;
package herddb.indexing.vector;

import io.github.jbellis.jvector.graph.RandomAccessVectorValues;
import io.github.jbellis.jvector.vector.VectorizationProvider;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
under the License.

*/
package herddb.index.vector;
package herddb.indexing.vector;

import herddb.log.LogSequenceNumber;
import java.util.List;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
under the License.
*/

package herddb.index.vector;
package herddb.indexing.vector;

import herddb.index.blink.BLink;
import herddb.utils.Bytes;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
under the License.
*/

package herddb.index.vector;
package herddb.indexing.vector;

import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import herddb.core.MemoryManager;
Expand Down Expand Up @@ -100,6 +100,7 @@
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Stream;
import herddb.index.vector.PinModeReaderSupplier;
import org.apache.bookkeeper.stats.OpStatsLogger;

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

*/

package herddb.index.vector;
package herddb.indexing.vector;

import herddb.core.MemoryManager;
import herddb.log.LogSequenceNumber;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
under the License.

*/
package herddb.index.vector;
package herddb.indexing.vector;

import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import herddb.storage.DataStorageManager;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
under the License.

*/
package herddb.index.vector;
package herddb.indexing.vector;

import java.util.List;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
under the License.
*/

package herddb.index.vector;
package herddb.indexing.vector;

import io.github.jbellis.jvector.disk.RandomAccessReader;
import io.github.jbellis.jvector.disk.ReaderSupplier;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@
under the License.
*/

package herddb.index.vector;
package herddb.indexing.vector;

import herddb.index.vector.PersistentVectorStore.PendingDelete;
import herddb.indexing.vector.PersistentVectorStore.PendingDelete;
import herddb.storage.DataStorageManagerException;
import herddb.utils.Bytes;
import io.github.jbellis.jvector.graph.GraphIndexBuilder;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

*/

package herddb.index.vector;
package herddb.indexing.vector;

/**
* Global memory budget shared across all vector stores.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
under the License.
*/

package herddb.index.vector;
package herddb.indexing.vector;

import herddb.index.blink.BLink;
import herddb.utils.Bytes;
Expand Down Expand Up @@ -73,7 +73,7 @@ static Bytes ordinalToBytes(int ordinal) {
/**
* Globally-unique identifier for this segment. Populated from the v4 IndexStatus
* on load. {@code null} only for segments that were written by a checkpoint that
* ran before a {@link herddb.index.vector.SegmentPublisher} was first attached to
* ran before a {@link herddb.indexing.vector.SegmentPublisher} was first attached to
* the store — i.e. the very first checkpoint after the external optimizer feature
* is enabled. Such segments will acquire a UUID on the next checkpoint that
* re-seals or re-writes them.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
under the License.
*/

package herddb.index.vector;
package herddb.indexing.vector;

import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.github.jbellis.jvector.vector.types.VectorFloat;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
under the License.
*/

package herddb.index.vector;
package herddb.indexing.vector;

import io.github.jbellis.jvector.graph.RandomAccessVectorValues;
import io.github.jbellis.jvector.vector.types.VectorFloat;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import herddb.index.vector.AbstractVectorStore;
import herddb.indexing.vector.AbstractVectorStore;
import herddb.utils.Bytes;
import java.util.List;
import java.util.Map;
Expand Down
Loading
Loading