+{"codeList":["def sentence_chunker(document, batch_size=10000):\n nlp = spacy.blank(\"en\")\n nlp.add_pipe(\"sentencizer\", config={\"punct_chars\": None})\n doc = nlp(document)\n\n docs = []\n for i in range(0, len(document), batch_size):\n batch = document[i : i + batch_size]\n docs.append(nlp(batch))\n\n doc = Doc.from_docs(docs)\n\n span_annotations = []\n chunks = []\n for i, sent in enumerate(doc.sents):\n span_annotations.append((sent.start, sent.end))\n chunks.append(sent.text)\n\n return chunks, span_annotations\n","def document_to_token_embeddings(model, tokenizer, document, batch_size=4096):\n tokenized_document = tokenizer(document, return_tensors=\"pt\")\n tokens = tokenized_document.tokens()\n\n outputs = []\n for i in range(0, len(tokens), batch_size):\n \n start = i\n end = min(i + batch_size, len(tokens))\n\n batch_inputs = {k: v[:, start:end] for k, v in tokenized_document.items()}\n\n with torch.no_grad():\n model_output = model(**batch_inputs)\n\n outputs.append(model_output.last_hidden_state)\n\n model_output = torch.cat(outputs, dim=1)\n return model_output\n","def late_chunking(token_embeddings, span_annotation, max_length=None):\n outputs = []\n for embeddings, annotations in zip(token_embeddings, span_annotation):\n if (\n max_length is not None\n ):\n annotations = [\n (start, min(end, max_length - 1))\n for (start, end) in annotations\n if start < (max_length - 1)\n ]\n pooled_embeddings = []\n for start, end in annotations:\n if (end - start) >= 1:\n pooled_embeddings.append(\n embeddings[start:end].sum(dim=0) / (end - start)\n )\n \n pooled_embeddings = [\n embedding.detach().cpu().numpy() for embedding in pooled_embeddings\n ]\n outputs.append(pooled_embeddings)\n\n return outputs\n","tokenizer = AutoTokenizer.from_pretrained('jinaai/jina-embeddings-v2-base-en', trust_remote_code=True)\nmodel = AutoModel.from_pretrained('jinaai/jina-embeddings-v2-base-en', trust_remote_code=True)\n\n# First chunk the text as normal, to obtain the beginning and end points of the chunks.\nchunks, span_annotations = sentence_chunker(document)\n# Then embed the full document.\ntoken_embeddings = document_to_token_embeddings(model, tokenizer, document)\n# Then perform the late chunking\nchunk_embeddings = late_chunking(token_embeddings, [span_annotations])[0]\n","Milvus 2.4.13 introduces dynamic replica load, allowing users to adjust the number of collection replicas without needing to release and reload the collection. This version also addresses several critical bugs related to bulk importing, expression parsing, load balancing, and failure recovery. Additionally, significant improvements have been made to MMAP resource usage and import performance, enhancing overall system efficiency. We highly recommend upgrading to this release for better performance and stability.\n","cos_sim = lambda x, y: np.dot(x, y) / (np.linalg.norm(x) * np.linalg.norm(y))\n\nmilvus_embedding = model.encode('milvus 2.4.13')\n\nfor chunk, late_chunking_embedding, traditional_embedding in zip(chunks, chunk_embeddings, embeddings_traditional_chunking):\n print(f'similarity_late_chunking(\"milvus 2.4.13\", \"{chunk}\")')\n print('late_chunking: ', cos_sim(milvus_embedding, late_chunking_embedding))\n print(f'similarity_traditional(\"milvus 2.4.13\", \"{chunk}\")')\n print('traditional_chunking: ', cos_sim(milvus_embedding, traditional_embeddings))\n","similarity_late_chunking(\"milvus 2.4.13\", \"Milvus 2.4.13 introduces dynamic replica load, allowing users to adjust the number of collection replicas without needing to release and reload the collection.\")\nlate_chunking: 0.8785206\nsimilarity_traditional(\"milvus 2.4.13\", \"Milvus 2.4.13 introduces dynamic replica load, allowing users to adjust the number of collection replicas without needing to release and reload the collection.\")\ntraditional_chunking: 0.8354263\n\nsimilarity_late_chunking(\"milvus 2.4.13\", \"This version also addresses several critical bugs related to bulk importing, expression parsing, load balancing, and failure recovery.\")\nlate_chunking: 0.84828955\nsimilarity_traditional(\"milvus 2.4.13\", \"This version also addresses several critical bugs related to bulk importing, expression parsing, load balancing, and failure recovery.\")\ntraditional_chunking: 0.7222632\n\nsimilarity_late_chunking(\"milvus 2.4.13\", \"Additionally, significant improvements have been made to MMAP resource usage and import performance, enhancing overall system efficiency.\")\nlate_chunking: 0.84942204\nsimilarity_traditional(\"milvus 2.4.13\", \"Additionally, significant improvements have been made to MMAP resource usage and import performance, enhancing overall system efficiency.\")\ntraditional_chunking: 0.6907381\n\nsimilarity_late_chunking(\"milvus 2.4.13\", \"We highly recommend upgrading to this release for better performance and stability.\")\nlate_chunking: 0.85431844\nsimilarity_traditional(\"milvus 2.4.13\", \"We highly recommend upgrading to this release for better performance and stability.\")\ntraditional_chunking: 0.71859795\n","batch_data=[]\nfor i in range(len(chunks)):\n data = {\n \"content\": chunks[i],\n \"embedding\": chunk_embeddings[i].tolist(),\n }\n\n batch_data.append(data)\n\nres = client.insert(\n collection_name=collection,\n data=batch_data,\n)\n","def late_chunking_query_by_milvus(query, top_k = 3):\n query_vector = model(**tokenizer(query, return_tensors=\"pt\")).last_hidden_state.mean(1).detach().cpu().numpy().flatten()\n\n res = client.search(\n collection_name=collection,\n data=[query_vector.tolist()],\n limit=top_k,\n output_fields=[\"id\", \"content\"],\n )\n\n return [item.get(\"entity\").get(\"content\") for items in res for item in items]\n\ndef late_chunking_query_by_cosine_sim(query, k = 3):\n cos_sim = lambda x, y: np.dot(x, y) / (np.linalg.norm(x) * np.linalg.norm(y))\n query_vector = model(**tokenizer(query, return_tensors=\"pt\")).last_hidden_state.mean(1).detach().cpu().numpy().flatten()\n\n results = np.empty(len(chunk_embeddings))\n for i, (chunk, embedding) in enumerate(zip(chunks, chunk_embeddings)):\n results[i] = cos_sim(query_vector, embedding)\n\n results_order = results.argsort()[::-1]\n return np.array(chunks)[results_order].tolist()[:k]\n","> late_chunking_query_by_milvus(\"What are new features in milvus 2.4.13\", 3)\n\n['\\n\\n### Features\\n\\n- Dynamic replica adjustment for loaded collections ([#36417](https://github.com/milvus-io/milvus/pull/36417))\\n- Sparse vector MMAP in growing segment types ([#36565](https://github.com/milvus-io/milvus/pull/36565))...\n","> late_chunking_query_by_cosine_sim(\"What are new features in milvus 2.4.13\", 3)\n\n['\\n\\n### Features\\n\\n- Dynamic replica adjustment for loaded collections ([#36417](https://github.com/milvus-io/milvus/pull/36417))\\n- Sparse vector MMAP in growing segment types (#36565)...\n"],"headingContent":"","anchorList":[{"label":"ما هو التقطيع المتأخر؟","href":"What-Is-Late-Chunking","type":2,"isActive":false},{"label":"اختبار التقطيع المتأخر","href":"Testing-Late-Chunking","type":2,"isActive":false},{"label":"الخلاصة","href":"Conclusion","type":2,"isActive":false}]}
0 commit comments