Skip to content

Commit f1596cd

Browse files
yhmoyhmo
committed
Support minhash function
Signed-off-by: yhmo <yihua.mo@zilliz.com>
1 parent 00d5f35 commit f1596cd

7 files changed

Lines changed: 401 additions & 9 deletions

File tree

‎examples/src/main/java/io/milvus/v2/MinhashFunctionExample.java‎

Lines changed: 217 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package io.milvus.v2;
21+
22+
import com.google.gson.JsonObject;
23+
import io.milvus.common.clientenum.FunctionType;
24+
import io.milvus.v2.client.ConnectConfig;
25+
import io.milvus.v2.client.MilvusClientV2;
26+
import io.milvus.v2.common.ConsistencyLevel;
27+
import io.milvus.v2.common.DataType;
28+
import io.milvus.v2.common.IndexParam;
29+
import io.milvus.v2.service.collection.request.AddFieldReq;
30+
import io.milvus.v2.service.collection.request.CreateCollectionReq;
31+
import io.milvus.v2.service.collection.request.DropCollectionReq;
32+
import io.milvus.v2.service.vector.request.InsertReq;
33+
import io.milvus.v2.service.vector.request.QueryReq;
34+
import io.milvus.v2.service.vector.request.SearchReq;
35+
import io.milvus.v2.service.vector.request.data.EmbeddedText;
36+
import io.milvus.v2.service.vector.response.QueryResp;
37+
import io.milvus.v2.service.vector.response.SearchResp;
38+
39+
import java.util.ArrayList;
40+
import java.util.Arrays;
41+
import java.util.Collections;
42+
import java.util.HashSet;
43+
import java.util.List;
44+
import java.util.Set;
45+
46+
public class MinhashFunctionExample {
47+
private static final String COLLECTION_NAME = "java_sdk_example_minhash_v2";
48+
private static final String ID_FIELD = "id";
49+
private static final String TEXT_FIELD = "text";
50+
private static final String SIGNATURE_FIELD = "minhash_signature";
51+
private static final int NUM_HASHES = 16;
52+
private static final int SIGNATURE_DIM = NUM_HASHES * 32;
53+
54+
private static void createDedupCollection(MilvusClientV2 client) {
55+
client.dropCollection(DropCollectionReq.builder()
56+
.collectionName(COLLECTION_NAME)
57+
.build());
58+
59+
CreateCollectionReq.CollectionSchema schema = CreateCollectionReq.CollectionSchema.builder()
60+
.enableDynamicField(true)
61+
.build();
62+
schema.addField(AddFieldReq.builder()
63+
.fieldName(ID_FIELD)
64+
.dataType(DataType.Int64)
65+
.isPrimaryKey(true)
66+
.autoID(false)
67+
.build());
68+
schema.addField(AddFieldReq.builder()
69+
.fieldName(TEXT_FIELD)
70+
.dataType(DataType.VarChar)
71+
.maxLength(65535)
72+
.build());
73+
schema.addField(AddFieldReq.builder()
74+
.fieldName(SIGNATURE_FIELD)
75+
.dataType(DataType.BinaryVector)
76+
.dimension(SIGNATURE_DIM)
77+
.build());
78+
79+
schema.addFunction(CreateCollectionReq.Function.builder()
80+
.name("text_to_minhash")
81+
.functionType(FunctionType.MINHASH)
82+
.inputFieldNames(Collections.singletonList(TEXT_FIELD))
83+
.outputFieldNames(Collections.singletonList(SIGNATURE_FIELD))
84+
.param("num_hashes", String.valueOf(NUM_HASHES))
85+
.param("shingle_size", "3")
86+
.param("token_level", "word")
87+
.build());
88+
89+
List<IndexParam> indexes = new ArrayList<>();
90+
indexes.add(IndexParam.builder()
91+
.fieldName(SIGNATURE_FIELD)
92+
.indexType(IndexParam.IndexType.MINHASH_LSH)
93+
.metricType(IndexParam.MetricType.MHJACCARD)
94+
.extraParams(new java.util.HashMap<String, Object>() {{
95+
put("mh_lsh_band", 8);
96+
put("with_raw_data", true);
97+
}})
98+
.build());
99+
100+
client.createCollection(CreateCollectionReq.builder()
101+
.collectionName(COLLECTION_NAME)
102+
.collectionSchema(schema)
103+
.indexParams(indexes)
104+
.consistencyLevel(ConsistencyLevel.BOUNDED)
105+
.build());
106+
System.out.println("Collection created");
107+
}
108+
109+
private static List<String> insertTexts(MilvusClientV2 client) {
110+
List<String> texts = Arrays.asList(
111+
"The quick brown fox jumps over the lazy dog.",
112+
"A quick brown fox jumped over a lazy dog.",
113+
"The fast brown fox leaps over the sleepy dog.",
114+
"Machine learning is transforming artificial intelligence.",
115+
"Deep learning transforms artificial intelligence research.",
116+
"Completely unrelated text about cooking recipes.",
117+
"Completely unrelated text about cooking recipes!"
118+
);
119+
120+
List<JsonObject> rows = new ArrayList<>();
121+
for (int i = 0; i < texts.size(); i++) {
122+
JsonObject row = new JsonObject();
123+
row.addProperty(ID_FIELD, i + 1L);
124+
row.addProperty(TEXT_FIELD, texts.get(i));
125+
rows.add(row);
126+
}
127+
128+
client.insert(InsertReq.builder()
129+
.collectionName(COLLECTION_NAME)
130+
.data(rows)
131+
.build());
132+
133+
QueryResp countResp = client.query(QueryReq.builder()
134+
.collectionName(COLLECTION_NAME)
135+
.outputFields(Collections.singletonList("count(*)"))
136+
.consistencyLevel(ConsistencyLevel.STRONG)
137+
.build());
138+
System.out.printf("%d rows in collection%n", (long) countResp.getQueryResults().get(0).getEntity().get("count(*)"));
139+
return texts;
140+
}
141+
142+
private static SearchResp searchByText(MilvusClientV2 client, String text, int topK) {
143+
return client.search(SearchReq.builder()
144+
.collectionName(COLLECTION_NAME)
145+
.annsField(SIGNATURE_FIELD)
146+
.data(Collections.singletonList(new EmbeddedText(text)))
147+
.metricType(IndexParam.MetricType.MHJACCARD)
148+
.searchParams(new java.util.HashMap<String, Object>() {{
149+
put("mh_search_with_jaccard", true);
150+
put("refine_k", 50);
151+
}})
152+
.limit(topK)
153+
.outputFields(Arrays.asList(ID_FIELD, TEXT_FIELD))
154+
.build());
155+
}
156+
157+
private static void deduplicateTexts(MilvusClientV2 client, List<String> texts, double similarityThreshold, int topK) {
158+
Set<Long> uniqueIds = new HashSet<>();
159+
List<String> duplicates = new ArrayList<>();
160+
161+
for (int i = 0; i < texts.size(); i++) {
162+
long docId = i + 1L;
163+
String text = texts.get(i);
164+
SearchResp searchResp = searchByText(client, text, topK);
165+
List<SearchResp.SearchResult> hits = searchResp.getSearchResults().get(0);
166+
167+
boolean isDuplicate = false;
168+
for (SearchResp.SearchResult hit : hits) {
169+
long hitId = ((Number) hit.getEntity().get(ID_FIELD)).longValue();
170+
if (hitId == docId) {
171+
continue;
172+
}
173+
if (hit.getScore() >= similarityThreshold && hitId < docId) {
174+
duplicates.add(String.format("ID %d is duplicate of ID %d, similarity=%.4f", docId, hitId, hit.getScore()));
175+
isDuplicate = true;
176+
break;
177+
}
178+
}
179+
180+
if (!isDuplicate) {
181+
uniqueIds.add(docId);
182+
}
183+
}
184+
185+
System.out.println("\nUnique texts:");
186+
for (int i = 0; i < texts.size(); i++) {
187+
long id = i + 1L;
188+
if (uniqueIds.contains(id)) {
189+
System.out.println(" - " + texts.get(i));
190+
}
191+
}
192+
193+
System.out.println("\nDuplicates:");
194+
if (duplicates.isEmpty()) {
195+
System.out.println(" (none)");
196+
} else {
197+
for (String duplicate : duplicates) {
198+
System.out.println(" - " + duplicate);
199+
}
200+
}
201+
}
202+
203+
public static void main(String[] args) {
204+
MilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder()
205+
.uri("http://localhost:19530")
206+
.build());
207+
208+
createDedupCollection(client);
209+
List<String> texts = insertTexts(client);
210+
deduplicateTexts(client, texts, 0.8, 5);
211+
212+
client.dropCollection(DropCollectionReq.builder()
213+
.collectionName(COLLECTION_NAME)
214+
.build());
215+
client.close();
216+
}
217+
}

‎sdk-core/src/main/java/io/milvus/common/clientenum/FunctionType.java‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,11 @@
2121

2222
public enum FunctionType {
2323
UNKNOWN("Unknown", 0), // in milvus-proto, the name is "Unknown"
24-
BM25("BM25", 1), // Added missing name parameter
24+
BM25("BM25", 1),
2525
TEXTEMBEDDING("TextEmbedding", 2), // in milvus-proto, the name is "TextEmbedding"
26-
RERANK("RERANK", 3); // Added missing name parameter
26+
RERANK("Rerank", 3),
27+
MINHASH("MinHash", 4),
28+
MOLFINGERPRINT("MolFingerprint", 5);
2729

2830
private final String name;
2931
private final int code;
@@ -44,10 +46,19 @@ public String getName() {
4446

4547
public static FunctionType fromName(String name) {
4648
for (FunctionType type : FunctionType.values()) {
47-
if (type.getName().equals(name)) {
49+
if (type.getName().equals(name) || type.name().equals(name)) {
4850
return type;
4951
}
5052
}
51-
return null;
53+
return UNKNOWN;
54+
}
55+
56+
public static FunctionType fromCode(int code) {
57+
for (FunctionType type : FunctionType.values()) {
58+
if (type.getCode() == code) {
59+
return type;
60+
}
61+
}
62+
return UNKNOWN;
5263
}
5364
}

‎sdk-core/src/main/java/io/milvus/v2/utils/SchemaUtils.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -318,10 +318,13 @@ public static CreateCollectionReq.StructFieldSchema convertFromGrpcStructFieldSc
318318
}
319319

320320
public static CreateCollectionReq.Function convertFromGrpcFunction(FunctionSchema functionSchema) {
321+
io.milvus.common.clientenum.FunctionType functionType =
322+
io.milvus.common.clientenum.FunctionType.fromCode(functionSchema.getTypeValue());
323+
321324
CreateCollectionReq.Function.FunctionBuilder builder = CreateCollectionReq.Function.builder()
322325
.name(functionSchema.getName())
323326
.description(functionSchema.getDescription())
324-
.functionType(io.milvus.common.clientenum.FunctionType.fromName(functionSchema.getType().name()))
327+
.functionType(functionType)
325328
.inputFieldNames(functionSchema.getInputFieldNamesList().stream().collect(Collectors.toList()))
326329
.outputFieldNames(functionSchema.getOutputFieldNamesList().stream().collect(Collectors.toList()));
327330
List<KeyValuePair> pairs = functionSchema.getParamsList();

‎sdk-core/src/test/java/io/milvus/docker-compose-multi.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ services:
3131

3232
standalone:
3333
container_name: milvus-javasdk-standalone-1
34-
image: milvusdb/milvus:master-20260609-1853a148
34+
image: milvusdb/milvus:master-20260622-a6568ac2-amd64
3535
user: "0:0"
3636
command: ["milvus", "run", "standalone"]
3737
security_opt:
@@ -83,7 +83,7 @@ services:
8383

8484
standaloneslave:
8585
container_name: milvus-javasdk-standalone-2
86-
image: milvusdb/milvus:master-20260609-1853a148
86+
image: milvusdb/milvus:master-20260622-a6568ac2-amd64
8787
user: "0:0"
8888
command: ["milvus", "run", "standalone"]
8989
security_opt:

‎sdk-core/src/test/java/io/milvus/docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ services:
3434

3535
standalone:
3636
container_name: milvus-javasdk-standalone
37-
image: milvusdb/milvus:master-20260609-1853a148
37+
image: milvusdb/milvus:master-20260622-a6568ac2-amd64
3838
user: "0:0"
3939
command: ["milvus", "run", "standalone"]
4040
security_opt:

0 commit comments

Comments
 (0)