Skip to content

Update to Lucene 10.3.2 #3082

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zdann15 wants to merge 4 commits into
base: master
Choose a base branch
from
+348 −236
Open

Update to Lucene 10.3.2 #3082

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a838786
Update to Lucene 10.1.0
zdann15 Dec 31, 2025
4a7bd48
Bump to Lucene 10.3.2
zdann15 Dec 31, 2025
0b6a2b7
Fix flaky test
zdann15 Dec 31, 2025
0aa1c21
Undo end to end test changes
zdann15 Jan 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
</developers>

<properties>
<lucene.version>9.9.1</lucene.version>
<lucene.version>10.3.2</lucene.version>
<java.version>21</java.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
Expand Down
27 changes: 14 additions & 13 deletions src/main/java/io/anserini/index/IndexFlatDenseVectors.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,17 @@

package io.anserini.index;

import io.anserini.collection.SourceDocument;
import io.anserini.collection.ParquetDenseVectorCollection;
import io.anserini.index.codecs.AnseriniLucene99FlatVectorFormat;
import io.anserini.index.codecs.AnseriniLucene99ScalarQuantizedVectorsFormat;
import io.anserini.index.generator.LuceneDocumentGenerator;
import io.anserini.index.generator.DenseVectorDocumentGenerator;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.codecs.KnnVectorsWriter;
import org.apache.lucene.codecs.lucene99.Lucene99Codec;
import org.apache.lucene.codecs.lucene103.Lucene103Codec;
import org.apache.lucene.index.ConcurrentMergeScheduler;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
Expand All @@ -40,10 +39,12 @@
import org.kohsuke.args4j.Option;
import org.kohsuke.args4j.ParserProperties;

import java.io.IOException;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import io.anserini.collection.ParquetDenseVectorCollection;
import io.anserini.collection.SourceDocument;
import io.anserini.index.codecs.AnseriniLucene99FlatVectorFormat;
import io.anserini.index.codecs.AnseriniLucene99ScalarQuantizedVectorsFormat;
import io.anserini.index.generator.DenseVectorDocumentGenerator;
import io.anserini.index.generator.LuceneDocumentGenerator;

public final class IndexFlatDenseVectors extends AbstractIndexer {
private static final Logger LOG = LogManager.getLogger(IndexFlatDenseVectors.class);
Expand Down Expand Up @@ -88,15 +89,15 @@ public IndexFlatDenseVectors(Args args) {

if (args.quantizeInt8) {
config = new IndexWriterConfig().setCodec(
new Lucene99Codec() {
new Lucene103Codec() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
return new DelegatingKnnVectorsFormat(new AnseriniLucene99ScalarQuantizedVectorsFormat(), 4096);
}
});
} else {
config = new IndexWriterConfig().setCodec(
new Lucene99Codec() {
new Lucene103Codec() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
return new DelegatingKnnVectorsFormat(new AnseriniLucene99FlatVectorFormat(), 4096);
Expand Down
21 changes: 11 additions & 10 deletions src/main/java/io/anserini/index/IndexHnswDenseVectors.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,17 @@

package io.anserini.index;

import io.anserini.collection.SourceDocument;
import io.anserini.index.generator.DenseVectorDocumentGenerator;
import io.anserini.index.generator.LuceneDocumentGenerator;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.codecs.KnnVectorsWriter;
import org.apache.lucene.codecs.lucene99.Lucene99Codec;
import org.apache.lucene.codecs.lucene103.Lucene103Codec;
import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
import org.apache.lucene.index.ConcurrentMergeScheduler;
Expand All @@ -41,10 +43,9 @@
import org.kohsuke.args4j.Option;
import org.kohsuke.args4j.ParserProperties;

import java.io.IOException;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import io.anserini.collection.SourceDocument;
import io.anserini.index.generator.DenseVectorDocumentGenerator;
import io.anserini.index.generator.LuceneDocumentGenerator;

public final class IndexHnswDenseVectors extends AbstractIndexer {
private static final Logger LOG = LogManager.getLogger(IndexHnswDenseVectors.class);
Expand Down Expand Up @@ -100,7 +101,7 @@ public IndexHnswDenseVectors(Args args) throws Exception {

if (args.quantizeInt8) {
config = new IndexWriterConfig().setCodec(
new Lucene99Codec() {
new Lucene103Codec() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
return new DelegatingKnnVectorsFormat(
Expand All @@ -109,7 +110,7 @@ public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
});
} else {
config = new IndexWriterConfig().setCodec(
new Lucene99Codec() {
new Lucene103Codec() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
return new DelegatingKnnVectorsFormat(
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/io/anserini/index/IndexReaderUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -837,7 +837,7 @@ public static Map<String, String> getFieldInfoDescription(IndexReader reader) {

FieldInfos fieldInfos = FieldInfos.getMergedFieldInfos(reader);
for (FieldInfo fi : fieldInfos) {
description.put(fi.name, "(" + "indexOption: " + fi.getIndexOptions() + ", hasVectors: " + fi.hasVectors() + ")");
description.put(fi.name, "(" + "indexOption: " + fi.getIndexOptions() + ", hasVectors: " + fi.hasTermVectors() + ")");
}

return description;
Expand Down
76 changes: 44 additions & 32 deletions src/main/java/io/anserini/index/codecs/AnseriniLucene99FlatVectorFormat.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,13 @@

package io.anserini.index.codecs;

import org.apache.lucene.codecs.FlatVectorsFormat;
import org.apache.lucene.codecs.FlatVectorsReader;
import org.apache.lucene.codecs.FlatVectorsWriter;
import java.io.IOException;

import org.apache.lucene.codecs.KnnFieldVectorsWriter;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.codecs.KnnVectorsWriter;
import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer;
import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.FieldInfo;
Expand All @@ -31,18 +31,18 @@
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.Sorter;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.KnnCollector;
import org.apache.lucene.search.VectorScorer;
import org.apache.lucene.search.AcceptDocs;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.hnsw.OrdinalTranslatedKnnCollector;
import org.apache.lucene.util.hnsw.RandomVectorScorer;

import java.io.IOException;

public class AnseriniLucene99FlatVectorFormat extends KnnVectorsFormat {

static final String NAME = "AnseriniLucene99FlatVectorFormat";

private final FlatVectorsFormat format = new Lucene99FlatVectorsFormat();
private final KnnVectorsFormat format = new Lucene99FlatVectorsFormat(new DefaultFlatVectorScorer());

/**
* Sole constructor
Expand All @@ -51,6 +51,11 @@ public AnseriniLucene99FlatVectorFormat() {
super(NAME);
}

@Override
public int getMaxDimensions(String fieldName) {
return format.getMaxDimensions(fieldName);
}

@Override
public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {
return new AnseriniLucene99FlatVectorWriter(format.fieldsWriter(state));
Expand All @@ -63,16 +68,16 @@ public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException

public static class AnseriniLucene99FlatVectorWriter extends KnnVectorsWriter {

private final FlatVectorsWriter writer;
private final KnnVectorsWriter writer;

public AnseriniLucene99FlatVectorWriter(FlatVectorsWriter writer) {
public AnseriniLucene99FlatVectorWriter(KnnVectorsWriter writer) {
super();
this.writer = writer;
}

@Override
public KnnFieldVectorsWriter<?> addField(FieldInfo fieldInfo) throws IOException {
return writer.addField(fieldInfo, null);
return writer.addField(fieldInfo);
}

@Override
Expand Down Expand Up @@ -103,9 +108,9 @@ public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOE

public static class AnseriniLucene99FlatVectorReader extends KnnVectorsReader {

private final FlatVectorsReader reader;
private final KnnVectorsReader reader;

public AnseriniLucene99FlatVectorReader(FlatVectorsReader reader) {
public AnseriniLucene99FlatVectorReader(KnnVectorsReader reader) {
super();
this.reader = reader;
}
Expand All @@ -126,35 +131,42 @@ public ByteVectorValues getByteVectorValues(String field) throws IOException {
}

@Override
public void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException {
collectAllMatchingDocs(knnCollector, acceptDocs, reader.getRandomVectorScorer(field, target));
}

private void collectAllMatchingDocs(KnnCollector knnCollector, Bits acceptDocs, RandomVectorScorer scorer) throws IOException {
OrdinalTranslatedKnnCollector collector = new OrdinalTranslatedKnnCollector(knnCollector, scorer::ordToDoc);
Bits acceptedOrds = scorer.getAcceptOrds(acceptDocs);
for (int i = 0; i < scorer.maxOrd(); i++) {
if (acceptedOrds == null || acceptedOrds.get(i)) {
collector.collect(i, scorer.score(i));
collector.incVisitedCount(1);
public void search(String field, float[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException {
FloatVectorValues vectors = reader.getFloatVectorValues(field);
if (vectors == null) {
return;
}
VectorScorer scorer = vectors.scorer(target);
DocIdSetIterator it = scorer.iterator();
Bits bits = acceptDocs == null ? null : acceptDocs.bits();
for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {
if (bits == null || bits.get(doc)) {
knnCollector.collect(doc, scorer.score());
}
knnCollector.incVisitedCount(1);
}
assert collector.earlyTerminated() == false;
}

@Override
public void search(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException {
collectAllMatchingDocs(knnCollector, acceptDocs, reader.getRandomVectorScorer(field, target));
public void search(String field, byte[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException {
ByteVectorValues vectors = reader.getByteVectorValues(field);
if (vectors == null) {
return;
}
VectorScorer scorer = vectors.scorer(target);
DocIdSetIterator it = scorer.iterator();
Bits bits = acceptDocs == null ? null : acceptDocs.bits();
for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {
if (bits == null || bits.get(doc)) {
knnCollector.collect(doc, scorer.score());
}
knnCollector.incVisitedCount(1);
}
}

@Override
public void close() throws IOException {
reader.close();
}

@Override
public long ramBytesUsed() {
return reader.ramBytesUsed();
}
}
}
}
Loading