diff --git a/src/components/Chat.tsx b/src/components/Chat.tsx index f799fbdd..d31915b3 100644 --- a/src/components/Chat.tsx +++ b/src/components/Chat.tsx @@ -18,12 +18,6 @@ import { emojifyPrompt, fixGrammarSpellingSelectionPrompt, formatDateTime, - getFileContent, - getFileName, - getNotesFromPath, - getNotesFromTags, - getSendChatContextNotesPrompt, - getTagsFromNote, glossaryPrompt, removeUrlsFromSelectionPrompt, rewriteLongerSelectionPrompt, @@ -31,7 +25,6 @@ import { rewriteShorterSelectionPrompt, rewriteTweetSelectionPrompt, rewriteTweetThreadSelectionPrompt, - sendNotesContentPrompt, simplifyPrompt, summarizePrompt, tocPrompt, @@ -243,125 +236,6 @@ ${chatContent}`; } }; - const handleSendActiveNoteToPrompt = async () => { - if (!app) { - console.error("App instance is not available."); - return; - } - - let noteFiles: TFile[] = []; - if (debug) { - console.log("Chat note context path:", settings.chatNoteContextPath); - console.log("Chat note context tags:", settings.chatNoteContextTags); - } - if (settings.chatNoteContextPath) { - // Recursively get all note TFiles in the path - noteFiles = await getNotesFromPath(app.vault, settings.chatNoteContextPath); - } - if (settings.chatNoteContextTags?.length > 0) { - // Get all notes with the specified tags - // If path is provided, get all notes with the specified tags in the path - // If path is not provided, get all notes with the specified tags - noteFiles = await getNotesFromTags(app.vault, settings.chatNoteContextTags, noteFiles); - } - const file = app.workspace.getActiveFile(); - // If no note context provided, default to the active note - if (noteFiles.length === 0) { - if (!file) { - new Notice("No active note found."); - console.error("No active note found."); - return; - } - new Notice("No valid Chat context provided. Defaulting to the active note."); - noteFiles = [file]; - } - - const notes = []; - for (const file of noteFiles) { - // Get the content of the note - const content = await getFileContent(file, app.vault); - const tags = await getTagsFromNote(file, app.vault); - if (content) { - notes.push({ name: getFileName(file), content, tags }); - } - } - - // Send the content of the note to AI - const promptMessageHidden: ChatMessage = { - message: sendNotesContentPrompt(notes), - sender: USER_SENDER, - isVisible: false, - timestamp: formatDateTime(new Date()), - }; - - // Visible user message that is not sent to AI - // const sendNoteContentUserMessage = `Please read the following notes [[${activeNoteContent}]] and be ready to answer questions about it.`; - const sendNoteContentUserMessage = getSendChatContextNotesPrompt( - notes, - settings.chatNoteContextPath, - settings.chatNoteContextTags - ); - const promptMessageVisible: ChatMessage = { - message: sendNoteContentUserMessage, - sender: USER_SENDER, - isVisible: true, - timestamp: formatDateTime(new Date()), - }; - - addMessage(promptMessageVisible); - addMessage(promptMessageHidden); - - setLoading(true); - await getAIResponse( - promptMessageHidden, - chainManager, - addMessage, - setCurrentAiMessage, - setAbortController, - { debug } - ); - setLoading(false); - }; - - const forceRebuildActiveNoteContext = async () => { - if (!app) { - console.error("App instance is not available."); - return; - } - - const file = app.workspace.getActiveFile(); - if (!file) { - new Notice("No active note found."); - console.error("No active note found."); - return; - } - const noteContent = await getFileContent(file, app.vault); - const noteName = getFileName(file); - if (!noteContent) { - new Notice("No note content found."); - console.error("No note content found."); - return; - } - - const fileMetadata = app.metadataCache.getFileCache(file); - const noteFile = { - path: file.path, - basename: file.basename, - mtime: file.stat.mtime, - content: noteContent, - metadata: fileMetadata?.frontmatter ?? {}, - }; - await chainManager.indexFile(noteFile); - const activeNoteOnMessage: ChatMessage = { - sender: AI_SENDER, - message: `Indexing [[${noteName}]]...\n\n Please switch to "QA" in Mode Selection to ask questions about it.`, - isVisible: true, - timestamp: formatDateTime(new Date()), - }; - - addMessage(activeNoteOnMessage); - }; - const refreshVaultContext = async () => { if (!app) { console.error("App instance is not available."); @@ -676,9 +550,10 @@ ${chatContent}`; clearCurrentAiMessage(); }} onSaveAsNote={() => handleSaveAsNote(true)} - onSendActiveNoteToPrompt={handleSendActiveNoteToPrompt} - onForceRebuildActiveNoteContext={forceRebuildActiveNoteContext} onRefreshVaultContext={refreshVaultContext} + onFindSimilarNotes={(content, activeFilePath) => + plugin.findSimilarNotes(content, activeFilePath) + } addMessage={addMessage} settings={settings} vault={app.vault} diff --git a/src/components/ChatComponents/ChatIcons.tsx b/src/components/ChatComponents/ChatIcons.tsx index 61c78c14..72eb1aae 100644 --- a/src/components/ChatComponents/ChatIcons.tsx +++ b/src/components/ChatComponents/ChatIcons.tsx @@ -1,4 +1,5 @@ import { CustomModel, SetChainOptions } from "@/aiParams"; +import { SimilarNotesModal } from "@/components/SimilarNotesModal"; import { AI_SENDER, VAULT_VECTOR_STORE_STRATEGY } from "@/constants"; import { CustomError } from "@/error"; import { CopilotSettings } from "@/settings/SettingsPage"; @@ -9,9 +10,9 @@ import React, { useEffect, useState } from "react"; import { ChainType } from "@/chainFactory"; import { + ConnectionIcon, RefreshIcon, SaveAsNoteIcon, - SendActiveNoteToPromptIcon, UseActiveNoteAsContextIcon, } from "@/components/Icons"; import { stringToChainType } from "@/utils"; @@ -23,9 +24,8 @@ interface ChatIconsProps { setCurrentChain: (chain: ChainType, options?: SetChainOptions) => void; onNewChat: (openNote: boolean) => void; onSaveAsNote: () => void; - onSendActiveNoteToPrompt: () => void; - onForceRebuildActiveNoteContext: () => void; onRefreshVaultContext: () => void; + onFindSimilarNotes: (content: string, activeFilePath: string) => Promise; addMessage: (message: ChatMessage) => void; settings: CopilotSettings; vault: Vault; @@ -40,9 +40,8 @@ const ChatIcons: React.FC = ({ setCurrentChain, onNewChat, onSaveAsNote, - onSendActiveNoteToPrompt, - onForceRebuildActiveNoteContext, onRefreshVaultContext, + onFindSimilarNotes, addMessage, settings, vault, @@ -75,7 +74,7 @@ const ChatIcons: React.FC = ({ } const activeNoteOnMessage: ChatMessage = { sender: AI_SENDER, - message: `OK Feel free to ask me questions about your vault: **${app.vault.getName()}**. \n\nIf you have *NEVER* as your auto-index strategy, you must click the *Refresh Index* button below, or run Copilot command: *Index vault for QA* first before you proceed!\n\nPlease note that this is a retrieval-based QA. Specific questions are encouraged. For generic questions like 'give me a summary', 'brainstorm based on the content', Chat mode with *Send Note to Prompt* button used with a *long context model* is a more suitable choice.`, + message: `OK Feel free to ask me questions about your vault: **${app.vault.getName()}**. \n\nIf you have *NEVER* as your auto-index strategy, you must click the *Refresh Index* button below, or run Copilot command: *Index vault for QA* first before you proceed!\n\nPlease note that this is a retrieval-based QA. Specific questions are encouraged. For generic questions like 'give me a summary', 'brainstorm based on the content', Chat mode with direct \`[[note title]]\` mention is a more suitable choice.`, isVisible: true, timestamp: formatDateTime(new Date()), }; @@ -100,6 +99,18 @@ const ChatIcons: React.FC = ({ handleChainSelection(); }, [selectedChain]); + const handleFindSimilarNotes = async () => { + const activeFile = app.workspace.getActiveFile(); + if (!activeFile) { + new Notice("No active file"); + return; + } + + const activeNoteContent = await app.vault.cachedRead(activeFile); + const similarChunks = await onFindSimilarNotes(activeNoteContent, activeFile.path); + new SimilarNotesModal(app, similarChunks).open(); + }; + return (
@@ -142,34 +153,26 @@ const ChatIcons: React.FC = ({ onChange={handleChainChange} > - + Mode Selection
- {selectedChain === "llm_chain" && ( - - )} {selectedChain === "vault_qa" && ( - + <> + + + )} ); diff --git a/src/components/ChatNoteContextModal.tsx b/src/components/ChatNoteContextModal.tsx deleted file mode 100644 index 462bbebe..00000000 --- a/src/components/ChatNoteContextModal.tsx +++ /dev/null @@ -1,87 +0,0 @@ -import { CopilotSettings } from "@/settings/SettingsPage"; -import { App, Modal } from "obsidian"; - -export class ChatNoteContextModal extends Modal { - private settings: CopilotSettings; - private onSubmit: (path: string, tags: string[]) => void; - - constructor( - app: App, - settings: CopilotSettings, - onSubmit: (path: string, tags: string[]) => void - ) { - super(app); - this.settings = settings; - this.onSubmit = onSubmit; - } - - onOpen() { - const formContainer = this.contentEl.createEl("div", { cls: "copilot-command-modal" }); - const pathContainer = formContainer.createEl("div", { cls: "copilot-command-input-container" }); - - pathContainer.createEl("h3", { text: "Filter by Folder Path", cls: "copilot-command-header" }); - const descFragment = createFragment((frag) => { - frag.appendText("All notes under the path will be sent to the prompt when the "); - frag.createEl("strong", { text: "Send Note(s) to Prompt" }); - frag.appendText(" button is clicked in Chat mode. "); - frag.appendText("If none provided, "); - frag.createEl("strong", { text: "default context is the active note" }); - }); - pathContainer.appendChild(descFragment); - - const pathField = pathContainer.createEl("input", { - type: "text", - cls: "copilot-command-input", - value: this.settings.chatNoteContextPath, - }); - pathField.setAttribute("name", "folderPath"); - - pathContainer.createEl("h3", { text: "Filter by Tags", cls: "copilot-command-header" }); - const descTagsFragment = createFragment((frag) => { - frag.createEl("strong", { - text: "Only tags in note property are used, tags in note content are not used.", - }); - frag.createEl("p", { - text: "All notes under the path above are further filtered by the specified tags. If no path is provided, only tags are used. Multiple tags should be separated by commas. ", - }); - frag.createEl("strong", { text: "Tags function as an OR filter, " }); - frag.appendText( - " any note that matches one of the tags will be sent to the prompt when button is clicked in Chat mode." - ); - }); - pathContainer.appendChild(descTagsFragment); - - const tagsField = pathContainer.createEl("input", { - type: "text", - cls: "copilot-command-input", - value: this.settings.chatNoteContextTags.join(","), - }); - tagsField.setAttribute("name", "tags"); - - const submitButtonContainer = formContainer.createEl("div", { - cls: "copilot-command-save-btn-container", - }); - const submitButton = submitButtonContainer.createEl("button", { - text: "Submit", - cls: "copilot-command-save-btn", - }); - - submitButton.addEventListener("click", () => { - // Remove the leading slash if it exists - let pathValue = pathField.value; - if (pathValue.startsWith("/") && pathValue.length > 1) { - pathValue = pathValue.slice(1); - } - - const tagsValue = tagsField.value - .split(",") - .map((tag) => tag.trim()) - .map((tag) => tag.toLowerCase()) - .map((tag) => tag.replace("#", "")) - .filter((tag) => tag !== ""); - - this.onSubmit(pathValue, tagsValue); - this.close(); - }); - } -} diff --git a/src/components/Icons.tsx b/src/components/Icons.tsx index 53a191cd..9a2ac161 100644 --- a/src/components/Icons.tsx +++ b/src/components/Icons.tsx @@ -220,6 +220,27 @@ export const DeleteIcon = () => ( ); +export const ConnectionIcon: React.FC = ({ className }) => ( + + + + + + + +); + export const InsertIcon: React.FC = ({ className }) => ( { + const itemEl = containerEl.createEl("div", { cls: "similar-note-item" }); + + // Create a collapsible section + const collapseEl = itemEl.createEl("details"); + const summaryEl = collapseEl.createEl("summary"); + + // Create a clickable title + const titleEl = summaryEl.createEl("a", { + text: `${item.chunk.metadata.title} (Score: ${item.score.toFixed(2)})`, + cls: "similar-note-title", + }); + + titleEl.addEventListener("click", (event) => { + event.preventDefault(); + this.navigateToNote(item.chunk.metadata.path); + }); + + // Create the content (initially hidden) + const contentEl = collapseEl.createEl("p"); + contentEl.setText(this.cleanChunkContent(item.chunk.pageContent)); + }); + } + + onClose() { + const { contentEl } = this; + contentEl.empty(); + } + + private navigateToNote(path: string) { + const file = this.app.vault.getAbstractFileByPath(path); + if (file instanceof TFile) { + const leaf = this.app.workspace.getLeaf(false); + if (leaf) { + leaf.openFile(file).then(() => { + this.close(); + }); + } + } + } + + private cleanChunkContent(content: string): string { + // Remove the "[[title]] --- " part at the beginning of the chunk + return content.replace(/^\[\[.*?\]\]\s*---\s*/, ""); + } +} diff --git a/src/main.ts b/src/main.ts index fe684189..ebcbeb15 100644 --- a/src/main.ts +++ b/src/main.ts @@ -6,10 +6,10 @@ import { parseChatContent, updateChatMemory } from "@/chatUtils"; import { registerBuiltInCommands } from "@/commands"; import { AddPromptModal } from "@/components/AddPromptModal"; import { AdhocPromptModal } from "@/components/AdhocPromptModal"; -import { ChatNoteContextModal } from "@/components/ChatNoteContextModal"; import CopilotView from "@/components/CopilotView"; import { ListPromptModal } from "@/components/ListPromptModal"; import { LoadChatHistoryModal } from "@/components/LoadChatHistoryModal"; +import { SimilarNotesModal } from "@/components/SimilarNotesModal"; import { BUILTIN_CHAT_MODELS, BUILTIN_EMBEDDING_MODELS, @@ -23,10 +23,13 @@ import { CustomPromptProcessor } from "@/customPromptProcessor"; import EncryptionService from "@/encryptionService"; import { CustomError } from "@/error"; import { TimestampUsageStrategy } from "@/promptUsageStrategy"; +import { HybridRetriever } from "@/search/hybridRetriever"; import { CopilotSettings, CopilotSettingTab } from "@/settings/SettingsPage"; import SharedState from "@/sharedState"; import { getAllNotesContent, sanitizeSettings } from "@/utils"; import VectorDBManager from "@/vectorDBManager"; +import { Embeddings } from "@langchain/core/embeddings"; +import { search } from "@orama/orama"; import { Editor, MarkdownView, @@ -308,19 +311,6 @@ export default class CopilotPlugin extends Plugin { }, }); - this.addCommand({ - id: "set-chat-note-context", - name: "Set note context for Chat mode", - callback: async () => { - new ChatNoteContextModal(this.app, this.settings, async (path: string, tags: string[]) => { - // Store the path in the plugin's settings, default to empty string - this.settings.chatNoteContextPath = path; - this.settings.chatNoteContextTags = tags; - await this.saveSettings(); - }).open(); - }, - }); - this.addCommand({ id: "load-copilot-chat-conversation", name: "Load Copilot Chat conversation", @@ -329,6 +319,22 @@ export default class CopilotPlugin extends Plugin { }, }); + this.addCommand({ + id: "find-similar-notes", + name: "Find similar notes to active note", + callback: async () => { + const activeFile = this.app.workspace.getActiveFile(); + if (!activeFile) { + new Notice("No active file"); + return; + } + + const activeNoteContent = await this.app.vault.cachedRead(activeFile); + const similarChunks = await this.findSimilarNotes(activeNoteContent, activeFile.path); + new SimilarNotesModal(this.app, similarChunks).open(); + }, + }); + this.registerEvent( this.app.vault.on("delete", async (file) => { await this.vectorStoreManager.removeDocs(file.path); @@ -659,4 +665,44 @@ export default class CopilotPlugin extends Plugin { copilotView.updateView(); } } + + async findSimilarNotes(content: string, activeFilePath: string): Promise { + // Wait for the VectorStoreManager to initialize + await this.vectorStoreManager.waitForInitialization(); + + const db = this.vectorStoreManager.getDb(); + + // Check if the index is empty + const singleDoc = await search(db, { + term: "", + limit: 1, + }); + + if (singleDoc.hits.length === 0) { + // Index is empty, trigger indexing + new Notice("Index does not exist, indexing vault for similarity search..."); + await this.vectorStoreManager.indexVaultToVectorStore(); + } + + const hybridRetriever = new HybridRetriever( + db, + this.app.vault, + this.chainManager.chatModelManager.getChatModel(), + this.vectorStoreManager.getEmbeddingsManager().getEmbeddingsAPI() as Embeddings, + { + minSimilarityScore: 0.3, + maxK: 20, + }, + this.settings.debug + ); + + const similarDocs = await hybridRetriever.getRelevantDocuments(content, { runName: "no_hyde" }); + return similarDocs + .filter((doc) => doc.metadata.path !== activeFilePath) + .map((doc) => ({ + chunk: doc, + score: doc.metadata.score || 0, + })) + .sort((a, b) => b.score - a.score); + } } diff --git a/src/search/hybridRetriever.ts b/src/search/hybridRetriever.ts index bc4f70cc..26d805bc 100644 --- a/src/search/hybridRetriever.ts +++ b/src/search/hybridRetriever.ts @@ -1,5 +1,6 @@ import { extractNoteTitles, getNoteFileFromTitle } from "@/utils"; import VectorDBManager from "@/vectorDBManager"; +import { BaseCallbackConfig } from "@langchain/core/callbacks/manager"; import { Document } from "@langchain/core/documents"; import { Embeddings } from "@langchain/core/embeddings"; import { BaseLanguageModel } from "@langchain/core/language_models/base"; @@ -33,13 +34,17 @@ export class HybridRetriever extends BaseRetriever { ); } - async getRelevantDocuments(query: string): Promise { + async getRelevantDocuments(query: string, config?: BaseCallbackConfig): Promise { // Extract note titles wrapped in [[]] from the query const noteTitles = extractNoteTitles(query); // Retrieve chunks for explicitly mentioned note titles const explicitChunks = await this.getExplicitChunks(noteTitles); - // Generate a hypothetical answer passage - const rewrittenQuery = await this.rewriteQuery(query); + let rewrittenQuery = query; + if (config?.runName !== "no_hyde") { + // Use config to determine if HyDE should be used + // Generate a hypothetical answer passage + rewrittenQuery = await this.rewriteQuery(query); + } // Perform vector similarity search using ScoreThresholdRetriever const oramaChunks = await this.getOramaChunks(rewrittenQuery); @@ -56,13 +61,15 @@ export class HybridRetriever extends BaseRetriever { } if (this.debug) { + console.log("*** HYBRID RETRIEVER DEBUG INFO: ***"); + + if (config?.runName !== "no_hyde") { + console.log("\nOriginal Query: ", query); + console.log("\nRewritten Query: ", rewrittenQuery); + } + console.log( - "*** HyDE HYBRID RETRIEVER DEBUG INFO: ***", - "\nOriginal Query: ", - query, - "\n\nRewritten Query: ", - rewrittenQuery, - "\n\nNote Titles extracted: ", + "\nNote Titles extracted: ", noteTitles, "\n\nExplicit Chunks:", explicitChunks, diff --git a/src/settings/components/QASettings.tsx b/src/settings/components/QASettings.tsx index 976d8edd..a246ba63 100644 --- a/src/settings/components/QASettings.tsx +++ b/src/settings/components/QASettings.tsx @@ -112,7 +112,7 @@ const QASettings: React.FC = ({
summary { + list-style: none; +} + +.similar-note-item details > summary::before { + content: "▶"; + display: inline-block; + width: 20px; + transition: transform 0.3s; +} + +.similar-note-item details[open] > summary::before { + transform: rotate(90deg); +} + +.similar-note-item details > p { + margin-left: 20px; + margin-top: 5px; +}