diff --git a/examples/src/main/java/io/milvus/v2/MinhashFunctionExample.java b/examples/src/main/java/io/milvus/v2/MinhashFunctionExample.java new file mode 100644 index 000000000..ebc5040fd --- /dev/null +++ b/examples/src/main/java/io/milvus/v2/MinhashFunctionExample.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.milvus.v2; + +import com.google.gson.JsonObject; +import io.milvus.common.clientenum.FunctionType; +import io.milvus.v2.client.ConnectConfig; +import io.milvus.v2.client.MilvusClientV2; +import io.milvus.v2.common.ConsistencyLevel; +import io.milvus.v2.common.DataType; +import io.milvus.v2.common.IndexParam; +import io.milvus.v2.service.collection.request.AddFieldReq; +import io.milvus.v2.service.collection.request.CreateCollectionReq; +import io.milvus.v2.service.collection.request.DropCollectionReq; +import io.milvus.v2.service.vector.request.InsertReq; +import io.milvus.v2.service.vector.request.QueryReq; +import io.milvus.v2.service.vector.request.SearchReq; +import io.milvus.v2.service.vector.request.data.EmbeddedText; +import io.milvus.v2.service.vector.response.QueryResp; +import io.milvus.v2.service.vector.response.SearchResp; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class MinhashFunctionExample { + private static final String COLLECTION_NAME = "java_sdk_example_minhash_v2"; + private static final String ID_FIELD = "id"; + private static final String TEXT_FIELD = "text"; + private static final String SIGNATURE_FIELD = "minhash_signature"; + private static final int NUM_HASHES = 16; + private static final int SIGNATURE_DIM = NUM_HASHES * 32; + + private static void createDedupCollection(MilvusClientV2 client) { + client.dropCollection(DropCollectionReq.builder() + .collectionName(COLLECTION_NAME) + .build()); + + CreateCollectionReq.CollectionSchema schema = CreateCollectionReq.CollectionSchema.builder() + .enableDynamicField(true) + .build(); + schema.addField(AddFieldReq.builder() + .fieldName(ID_FIELD) + .dataType(DataType.Int64) + .isPrimaryKey(true) + .autoID(false) + .build()); + schema.addField(AddFieldReq.builder() + .fieldName(TEXT_FIELD) + .dataType(DataType.VarChar) + .maxLength(65535) + .build()); + schema.addField(AddFieldReq.builder() + .fieldName(SIGNATURE_FIELD) + .dataType(DataType.BinaryVector) + .dimension(SIGNATURE_DIM) + .build()); + + schema.addFunction(CreateCollectionReq.Function.builder() + .name("text_to_minhash") + .functionType(FunctionType.MINHASH) + .inputFieldNames(Collections.singletonList(TEXT_FIELD)) + .outputFieldNames(Collections.singletonList(SIGNATURE_FIELD)) + .param("num_hashes", String.valueOf(NUM_HASHES)) + .param("shingle_size", "3") + .param("token_level", "word") + .build()); + + List indexes = new ArrayList<>(); + indexes.add(IndexParam.builder() + .fieldName(SIGNATURE_FIELD) + .indexType(IndexParam.IndexType.MINHASH_LSH) + .metricType(IndexParam.MetricType.MHJACCARD) + .extraParams(new java.util.HashMap() {{ + put("mh_lsh_band", 8); + put("with_raw_data", true); + }}) + .build()); + + client.createCollection(CreateCollectionReq.builder() + .collectionName(COLLECTION_NAME) + .collectionSchema(schema) + .indexParams(indexes) + .consistencyLevel(ConsistencyLevel.BOUNDED) + .build()); + System.out.println("Collection created"); + } + + private static List insertTexts(MilvusClientV2 client) { + List texts = Arrays.asList( + "The quick brown fox jumps over the lazy dog.", + "A quick brown fox jumped over a lazy dog.", + "The fast brown fox leaps over the sleepy dog.", + "Machine learning is transforming artificial intelligence.", + "Deep learning transforms artificial intelligence research.", + "Completely unrelated text about cooking recipes.", + "Completely unrelated text about cooking recipes!" + ); + + List rows = new ArrayList<>(); + for (int i = 0; i < texts.size(); i++) { + JsonObject row = new JsonObject(); + row.addProperty(ID_FIELD, i + 1L); + row.addProperty(TEXT_FIELD, texts.get(i)); + rows.add(row); + } + + client.insert(InsertReq.builder() + .collectionName(COLLECTION_NAME) + .data(rows) + .build()); + + QueryResp countResp = client.query(QueryReq.builder() + .collectionName(COLLECTION_NAME) + .outputFields(Collections.singletonList("count(*)")) + .consistencyLevel(ConsistencyLevel.STRONG) + .build()); + System.out.printf("%d rows in collection%n", (long) countResp.getQueryResults().get(0).getEntity().get("count(*)")); + return texts; + } + + private static SearchResp searchByText(MilvusClientV2 client, String text, int topK) { + return client.search(SearchReq.builder() + .collectionName(COLLECTION_NAME) + .annsField(SIGNATURE_FIELD) + .data(Collections.singletonList(new EmbeddedText(text))) + .metricType(IndexParam.MetricType.MHJACCARD) + .searchParams(new java.util.HashMap() {{ + put("mh_search_with_jaccard", true); + put("refine_k", 50); + }}) + .limit(topK) + .outputFields(Arrays.asList(ID_FIELD, TEXT_FIELD)) + .build()); + } + + private static void deduplicateTexts(MilvusClientV2 client, List texts, double similarityThreshold, int topK) { + Set uniqueIds = new HashSet<>(); + List duplicates = new ArrayList<>(); + + for (int i = 0; i < texts.size(); i++) { + long docId = i + 1L; + String text = texts.get(i); + SearchResp searchResp = searchByText(client, text, topK); + List hits = searchResp.getSearchResults().get(0); + + boolean isDuplicate = false; + for (SearchResp.SearchResult hit : hits) { + long hitId = ((Number) hit.getEntity().get(ID_FIELD)).longValue(); + if (hitId == docId) { + continue; + } + if (hit.getScore() >= similarityThreshold && hitId < docId) { + duplicates.add(String.format("ID %d is duplicate of ID %d, similarity=%.4f", docId, hitId, hit.getScore())); + isDuplicate = true; + break; + } + } + + if (!isDuplicate) { + uniqueIds.add(docId); + } + } + + System.out.println("\nUnique texts:"); + for (int i = 0; i < texts.size(); i++) { + long id = i + 1L; + if (uniqueIds.contains(id)) { + System.out.println(" - " + texts.get(i)); + } + } + + System.out.println("\nDuplicates:"); + if (duplicates.isEmpty()) { + System.out.println(" (none)"); + } else { + for (String duplicate : duplicates) { + System.out.println(" - " + duplicate); + } + } + } + + public static void main(String[] args) { + MilvusClientV2 client = new MilvusClientV2(ConnectConfig.builder() + .uri("http://localhost:19530") + .build()); + + createDedupCollection(client); + List texts = insertTexts(client); + deduplicateTexts(client, texts, 0.8, 5); + + client.dropCollection(DropCollectionReq.builder() + .collectionName(COLLECTION_NAME) + .build()); + client.close(); + } +} diff --git a/sdk-core/src/main/java/io/milvus/common/clientenum/FunctionType.java b/sdk-core/src/main/java/io/milvus/common/clientenum/FunctionType.java index 38eb5eb9c..9cfd0ae85 100644 --- a/sdk-core/src/main/java/io/milvus/common/clientenum/FunctionType.java +++ b/sdk-core/src/main/java/io/milvus/common/clientenum/FunctionType.java @@ -21,9 +21,11 @@ public enum FunctionType { UNKNOWN("Unknown", 0), // in milvus-proto, the name is "Unknown" - BM25("BM25", 1), // Added missing name parameter + BM25("BM25", 1), TEXTEMBEDDING("TextEmbedding", 2), // in milvus-proto, the name is "TextEmbedding" - RERANK("RERANK", 3); // Added missing name parameter + RERANK("Rerank", 3), + MINHASH("MinHash", 4), + MOLFINGERPRINT("MolFingerprint", 5); private final String name; private final int code; @@ -44,10 +46,19 @@ public String getName() { public static FunctionType fromName(String name) { for (FunctionType type : FunctionType.values()) { - if (type.getName().equals(name)) { + if (type.getName().equals(name) || type.name().equals(name)) { return type; } } - return null; + return UNKNOWN; + } + + public static FunctionType fromCode(int code) { + for (FunctionType type : FunctionType.values()) { + if (type.getCode() == code) { + return type; + } + } + return UNKNOWN; } } diff --git a/sdk-core/src/main/java/io/milvus/v2/utils/SchemaUtils.java b/sdk-core/src/main/java/io/milvus/v2/utils/SchemaUtils.java index 2a8000e9f..541a3a48a 100644 --- a/sdk-core/src/main/java/io/milvus/v2/utils/SchemaUtils.java +++ b/sdk-core/src/main/java/io/milvus/v2/utils/SchemaUtils.java @@ -318,10 +318,13 @@ public static CreateCollectionReq.StructFieldSchema convertFromGrpcStructFieldSc } public static CreateCollectionReq.Function convertFromGrpcFunction(FunctionSchema functionSchema) { + io.milvus.common.clientenum.FunctionType functionType = + io.milvus.common.clientenum.FunctionType.fromCode(functionSchema.getTypeValue()); + CreateCollectionReq.Function.FunctionBuilder builder = CreateCollectionReq.Function.builder() .name(functionSchema.getName()) .description(functionSchema.getDescription()) - .functionType(io.milvus.common.clientenum.FunctionType.fromName(functionSchema.getType().name())) + .functionType(functionType) .inputFieldNames(functionSchema.getInputFieldNamesList().stream().collect(Collectors.toList())) .outputFieldNames(functionSchema.getOutputFieldNamesList().stream().collect(Collectors.toList())); List pairs = functionSchema.getParamsList(); diff --git a/sdk-core/src/test/java/io/milvus/docker-compose-multi.yml b/sdk-core/src/test/java/io/milvus/docker-compose-multi.yml index bf4d10eef..ff98bccb2 100644 --- a/sdk-core/src/test/java/io/milvus/docker-compose-multi.yml +++ b/sdk-core/src/test/java/io/milvus/docker-compose-multi.yml @@ -31,7 +31,7 @@ services: standalone: container_name: milvus-javasdk-standalone-1 - image: milvusdb/milvus:master-20260609-1853a148 + image: milvusdb/milvus:master-20260622-a6568ac2-amd64 user: "0:0" command: ["milvus", "run", "standalone"] security_opt: @@ -83,7 +83,7 @@ services: standaloneslave: container_name: milvus-javasdk-standalone-2 - image: milvusdb/milvus:master-20260609-1853a148 + image: milvusdb/milvus:master-20260622-a6568ac2-amd64 user: "0:0" command: ["milvus", "run", "standalone"] security_opt: diff --git a/sdk-core/src/test/java/io/milvus/docker-compose.yml b/sdk-core/src/test/java/io/milvus/docker-compose.yml index e5434ef4d..71c0770f1 100644 --- a/sdk-core/src/test/java/io/milvus/docker-compose.yml +++ b/sdk-core/src/test/java/io/milvus/docker-compose.yml @@ -34,7 +34,7 @@ services: standalone: container_name: milvus-javasdk-standalone - image: milvusdb/milvus:master-20260609-1853a148 + image: milvusdb/milvus:master-20260622-a6568ac2-amd64 user: "0:0" command: ["milvus", "run", "standalone"] security_opt: diff --git a/sdk-core/src/test/java/io/milvus/v2/client/MilvusClientV2DockerTest.java b/sdk-core/src/test/java/io/milvus/v2/client/MilvusClientV2DockerTest.java index 72778e7c0..dbbb11966 100644 --- a/sdk-core/src/test/java/io/milvus/v2/client/MilvusClientV2DockerTest.java +++ b/sdk-core/src/test/java/io/milvus/v2/client/MilvusClientV2DockerTest.java @@ -3356,6 +3356,14 @@ public void run() { pool.removeConfig(dummyDb); Assertions.assertNull(pool.getConfig(dummyDb)); pool.close(); + + client.dropCollection(DropCollectionReq.builder() + .databaseName(dummyDb) + .collectionName(collectionName) + .build()); + client.dropDatabase(DropDatabaseReq.builder() + .databaseName(dummyDb) + .build()); } catch (Exception e) { System.out.println(e.getMessage()); Assertions.fail(e.getMessage()); @@ -3767,6 +3775,110 @@ void testDocInOut() { for (SearchResp.SearchResult result : firstResults) { System.out.println(result); } + + client.dropCollection(DropCollectionReq.builder().collectionName(randomCollectionName).build()); + } + + @Test + void testMinHashFunction() { + String randomCollectionName = generator.generate(10); + + CreateCollectionReq.CollectionSchema collectionSchema = CreateCollectionReq.CollectionSchema.builder() + .enableDynamicField(true) + .build(); + collectionSchema.addField(AddFieldReq.builder() + .fieldName("id") + .dataType(DataType.Int64) + .isPrimaryKey(Boolean.TRUE) + .autoID(Boolean.FALSE) + .build()); + collectionSchema.addField(AddFieldReq.builder() + .fieldName("text") + .dataType(DataType.VarChar) + .maxLength(65535) + .build()); + collectionSchema.addField(AddFieldReq.builder() + .fieldName("minhash_signature") + .dataType(DataType.BinaryVector) + .dimension(512) + .build()); + + collectionSchema.addFunction(CreateCollectionReq.Function.builder() + .name("text_to_minhash") + .description("desc minhash") + .functionType(FunctionType.MINHASH) + .inputFieldNames(Collections.singletonList("text")) + .outputFieldNames(Collections.singletonList("minhash_signature")) + .param("num_hashes", "16") + .param("shingle_size", "3") + .param("token_level", "word") + .build()); + + List indexParams = new ArrayList<>(); + indexParams.add(IndexParam.builder() + .fieldName("minhash_signature") + .indexType(IndexParam.IndexType.MINHASH_LSH) + .metricType(IndexParam.MetricType.MHJACCARD) + .extraParams(new HashMap() {{ + put("mh_lsh_band", 8); + put("with_raw_data", true); + }}) + .build()); + CreateCollectionReq requestCreate = CreateCollectionReq.builder() + .collectionName(randomCollectionName) + .collectionSchema(collectionSchema) + .indexParams(indexParams) + .build(); + client.createCollection(requestCreate); + + DescribeCollectionResp descResp = client.describeCollection(DescribeCollectionReq.builder() + .collectionName(randomCollectionName) + .build()); + List functions = descResp.getCollectionSchema().getFunctionList(); + Assertions.assertEquals(1, functions.size()); + Assertions.assertEquals(FunctionType.MINHASH, functions.get(0).getFunctionType()); + Assertions.assertEquals("minhash_signature", functions.get(0).getOutputFieldNames().get(0)); + + List texts = Arrays.asList( + "The quick brown fox jumps over the lazy dog.", + "A quick brown fox jumped over a lazy dog.", + "Machine learning is transforming artificial intelligence."); + List data = new ArrayList<>(); + for (int i = 0; i < texts.size(); i++) { + JsonObject row = new JsonObject(); + row.addProperty("id", i + 1); + row.addProperty("text", texts.get(i)); + data.add(row); + } + + InsertResp insertResp = client.insert(InsertReq.builder() + .collectionName(randomCollectionName) + .data(data) + .build()); + Assertions.assertEquals(3, insertResp.getInsertCnt()); + long rowCount = getRowCount("", randomCollectionName); + Assertions.assertEquals(texts.size(), rowCount); + + SearchResp searchResp = client.search(SearchReq.builder() + .collectionName(randomCollectionName) + .annsField("minhash_signature") + .data(Collections.singletonList(new EmbeddedText("The quick brown fox jumps over the lazy dog."))) + .limit(3) + .outputFields(Lists.newArrayList("id", "text")) + .metricType(IndexParam.MetricType.MHJACCARD) + .searchParams(new HashMap() {{ + put("mh_search_with_jaccard", true); + put("refine_k", 50); + }}) + .build()); + List> searchResults = searchResp.getSearchResults(); + Assertions.assertEquals(1, searchResults.size()); + Assertions.assertFalse(searchResults.get(0).isEmpty()); + Map firstEntity = searchResults.get(0).get(0).getEntity(); + Assertions.assertTrue(firstEntity.containsKey("id")); + Assertions.assertTrue(firstEntity.containsKey("text")); + + client.dropCollection(DropCollectionReq.builder().collectionName(randomCollectionName).build()); } @Test @@ -3866,6 +3978,8 @@ void testDynamicField() { fieldNames = descResp.getFieldNames(); Assertions.assertFalse(fieldNames.contains("flag")); Assertions.assertTrue(fieldNames.contains("text")); + + client.dropCollection(DropCollectionReq.builder().collectionName(collectionName).build()); } @Test @@ -3922,6 +4036,8 @@ void testFunctionFieldLifecycle() { Assertions.assertTrue(fieldNames.contains("text")); Assertions.assertFalse(fieldNames.contains("sparse")); Assertions.assertTrue(descResp.getCollectionSchema().getFunctionList().isEmpty()); + + client.dropCollection(DropCollectionReq.builder().collectionName(collectionName).build()); } @Test diff --git a/sdk-core/src/test/java/io/milvus/v2/utils/SchemaUtilsTest.java b/sdk-core/src/test/java/io/milvus/v2/utils/SchemaUtilsTest.java index 167007222..eac3437ed 100644 --- a/sdk-core/src/test/java/io/milvus/v2/utils/SchemaUtilsTest.java +++ b/sdk-core/src/test/java/io/milvus/v2/utils/SchemaUtilsTest.java @@ -157,7 +157,8 @@ void testConvertFromGrpcFunction() { CreateCollectionReq.Function func = SchemaUtils.convertFromGrpcFunction(functionSchema); Assertions.assertEquals(func.getName(), "abc"); Assertions.assertEquals(func.getDescription(), "xxx"); - Assertions.assertEquals(func.getFunctionType(), io.milvus.common.clientenum.FunctionType.fromName(type.name())); + Assertions.assertNotNull(func.getFunctionType()); + Assertions.assertEquals(func.getFunctionType().getCode(), type.getNumber()); Assertions.assertEquals(func.getInputFieldNames().size(), 1); Assertions.assertEquals(func.getInputFieldNames().get(0), "text"); Assertions.assertEquals(func.getOutputFieldNames().size(), 1); @@ -168,6 +169,50 @@ void testConvertFromGrpcFunction() { } } + @Test + void testConvertFromGrpcMinHashFunction() { + FunctionSchema functionSchema = FunctionSchema.newBuilder() + .setName("text_to_minhash") + .setDescription("minhash function") + .setType(FunctionType.MinHash) + .addInputFieldNames("text") + .addOutputFieldNames("minhash_signature") + .addParams(KeyValuePair.newBuilder().setKey("num_hashes").setValue("16").build()) + .build(); + + CreateCollectionReq.Function func = SchemaUtils.convertFromGrpcFunction(functionSchema); + Assertions.assertEquals(io.milvus.common.clientenum.FunctionType.MINHASH, func.getFunctionType()); + Assertions.assertEquals("16", func.getParams().get("num_hashes")); + } + + @Test + void testConvertToGrpcMinHashFunction() { + CreateCollectionReq.Function function = CreateCollectionReq.Function.builder() + .name("text_to_minhash") + .description("minhash function") + .functionType(io.milvus.common.clientenum.FunctionType.MINHASH) + .inputFieldNames(Collections.singletonList("text")) + .outputFieldNames(Collections.singletonList("minhash_signature")) + .param("num_hashes", "16") + .build(); + + FunctionSchema functionSchema = SchemaUtils.convertToGrpcFunction(function); + Assertions.assertEquals(FunctionType.MinHash, functionSchema.getType()); + Assertions.assertEquals("16", functionSchema.getParams(0).getValue()); + } + + @Test + void testConvertFromGrpcUnknownFunctionFallsBackToUnknown() { + FunctionSchema functionSchema = FunctionSchema.newBuilder() + .setName("unknown_func") + .setDescription("unknown function") + .setTypeValue(999) + .build(); + + CreateCollectionReq.Function func = SchemaUtils.convertFromGrpcFunction(functionSchema); + Assertions.assertEquals(io.milvus.common.clientenum.FunctionType.UNKNOWN, func.getFunctionType()); + } + @Test void testConvertToGrpcFunction() { for (io.milvus.common.clientenum.FunctionType type : io.milvus.common.clientenum.FunctionType.values()) {