Merge pull request #24 from glorat/qna

feat: multidoc qna

Author: glorat

quasar.config.js

@@ -10,7 +10,17 @@
 
 
 const { configure } = require('quasar/wrappers');
+const {defaultUnstructuredUrl} = require(__dirname + '/src/staticConfig');
+const {reduce} = require("lodash");
 
+const pathsToProxy = ['/general']; // unstructured endpoint
+const proxyTarget = defaultUnstructuredUrl
+
+const proxyProxyConfig = reduce(
+  pathsToProxy,
+  (obj, path) => ({ ...obj, [path]: { target: proxyTarget, changeOrigin:true, secure:false } }),
+  {}
+);
 
 module.exports = configure(function (/* ctx */) {
   return {
@@ -88,7 +98,8 @@ module.exports = configure(function (/* ctx */) {
     // Full list of options: https://v2.quasar.dev/quasar-cli-vite/quasar-config-js#devServer
     devServer: {
       // https: true
-      open: true // opens browser window automatically
+      open: false, // opens browser window automatically
+      proxy: proxyProxyConfig
     },
 
     // https://v2.quasar.dev/quasar-cli-vite/quasar-config-js#framework

src/components/ChatEntry.vue

@@ -0,0 +1,48 @@
+<template>
+  <q-toolbar class="bg-grey-3 text-black row">
+    <q-btn @click="onReset" flat class="q-ml-sm" icon="refresh" color="primary"/>
+    <q-input rounded outlined dense class="WAL__field q-mr-sm" bg-color="white"
+             :model-value="modelValue"
+             @update:model-value="onModelValueUpdate"
+             @keydown.enter="sendMessage"
+             placeholder="Type your message"/>
+    <audio-transcriber @message="onAudioMessage"></audio-transcriber>
+    <q-btn disable round flat :icon="isVolumeOn ? matVolumeUp : matVolumeOff"/>
+    <q-btn @click="sendMessage" flat class="q-ml-sm" icon="send" color="primary"/>
+  </q-toolbar>
+</template>
+
+<script setup lang="ts">
+import {ref} from 'vue';
+import {matVolumeOff, matVolumeUp} from '@quasar/extras/material-icons';
+import AudioTranscriber from 'components/AudioTranscriber.vue';
+
+const isVolumeOn = ref(true);
+const emit = defineEmits(['message', 'update:modelValue', 'reset']);
+
+defineProps({
+  modelValue: {
+    type: String,
+  },
+});
+
+const onReset = () => {
+  emit('reset')
+}
+
+const sendMessage = async () => {
+  emit('message');
+};
+
+const onAudioMessage = (msg: string) => {
+  emit('update:modelValue', msg)
+};
+
+const onModelValueUpdate = (msg:string) => {
+  emit('update:modelValue', msg)
+}
+</script>
+
+<style scoped>
+/* Component styles */
+</style>

src/components/MultiFileManager.vue

@@ -33,6 +33,7 @@ import {createVectorStoreFromLargeContent} from 'src/lib/ai/largeDocQna'
 import {fileToText} from 'src/lib/ai/unstructured'
 import {DocumentInfo, useMultiFileStore} from 'stores/multiFileStore'
 import {matCloudUpload} from '@quasar/extras/material-icons'
+import {RecursiveCharacterTextSplitter} from "langchain/text_splitter";
 
 defineProps({
   loading:Boolean
@@ -90,42 +91,14 @@ const onFileChange = (newFiles: File[]) => {
   // Clear the files variable after populating uploadedFiles
   files.value = [];
 
-  processNextDocument()
+  multiFileStore.processNextDocument()
 };
 
 const processingDocuments = computed(() => {
   return multiFileStore.processing
   // return uploadedFiles.value.some(file => includes([ 'processing', 'parsing'], file.status));
 });
 
-const processNextDocument = async () => {
-  const pendingDocument = multiFileStore.documentInfo.find(file => file.status === 'pending');
-
-  if (pendingDocument) {
-    // Set the status to 'parsing'
-    pendingDocument.status = 'parsing';
-
-    try {
-      // Simulating asynchronous processing with a timeout
-      await new Promise(resolve => setTimeout(resolve, 2000));
-      const text = await fileToText(pendingDocument.file)
-      pendingDocument.status = 'processing'
-      const vectorStore = await createVectorStoreFromLargeContent(text, (p)=>{pendingDocument.progress=p})
-      // Important to markRaw to avoid proxying the insides
-      pendingDocument.vectors = markRaw(vectorStore)
-      // Update the status to 'ready' on successful processing
-      pendingDocument.status = 'ready';
-    } catch (error) {
-      // Set the status to 'error' on processing failure
-      pendingDocument.status = 'error';
-      console.error('Error occurred during document processing:', error);
-    }
-
-    // Call the processNextDocument function recursively to process the next document
-    await processNextDocument();
-  }
-};
-
 </script>
 
 

src/components/SettingsConfigEditor.vue

@@ -81,6 +81,7 @@ import {ref, onMounted} from 'vue'
 import {Dialog, Notify} from 'quasar'
 import {applyAiUserSettings, Config, getOpenAIAPI, getSettingsFromLocalStorage} from 'src/lib/ai/config'
 import {matRefresh, matSave, matShare} from '@quasar/extras/material-icons';
+import {merge} from "lodash";
 
 const settings = ref({
   server: 'openai',
@@ -98,7 +99,7 @@ onMounted(() => {
 function loadSettingsFromLocalStorage() {
   const savedSettings = getSettingsFromLocalStorage()
   if (savedSettings) {
-    settings.value = savedSettings
+    settings.value = merge(settings.value, savedSettings)
   }
 }
 

src/lib/ai/answer.ts

@@ -4,10 +4,11 @@ import {logger} from './logger'
 import {QnaStorage} from 'src/lib/ai/largeDocQna'
 import {answerMe, createEmbedding} from 'src/lib/ai/openaiFacade'
 import {VectorStore} from 'langchain/vectorstores';
+import {MemoryVectorStore} from "langchain/vectorstores/memory";
 
 
-export async function performQna2(question:string, db: VectorStore, prompt?: string): Promise<string|undefined> {
-  const similarities = await db.similaritySearch(question, 10)
+export async function performQna2(question:string, db: VectorStore, filter:(d:any)=>boolean = ()=>true, prompt?: string): Promise<string|undefined> {
+  const similarities = await db.similaritySearch(question, 10, filter)
 
   const contexts = []
   let contextLength = 0

src/lib/ai/config.ts

@@ -1,9 +1,10 @@
 import {Configuration, ConfigurationParameters, OpenAIApi} from 'openai'
-
+import {OpenAI, OpenAIChat} from 'langchain/llms/openai'
 import FormData from 'form-data'
 import ChatGPTClient from 'src/lib/ai/ChatGPTClient'
+import {defaultUnstructuredUrl} from '../myfirebase'
 
-type OpenAIEngine = 'text-ada-001' |  'text-davinci-003'
+type OpenAIEngine = 'text-ada-001' | 'text-davinci-003'
 
 const engineId: OpenAIEngine = 'text-davinci-003'
 export const Config = {
@@ -14,24 +15,25 @@ export const Config = {
 
 class CustomFormData extends FormData {
   getHeaders(): Record<string, string> {
-    return {};
+    return {}
   }
 }
 
 
 let aiUserSettings: Partial<AiUserSettings> = {
   server: 'openai',
-  openaiSettings: {apiKey: process.env.OPENAPI_KEY??''},
+  openaiSettings: {apiKey: process.env.OPENAPI_KEY ?? ''},
   unstructuredSettings: {endpoint: process.env.UNSTRUCTURED_URL ?? ''},
 }
+
 export interface AiUserSettings {
   server: 'openai' | 'azure' | 'hosted'
-  azureSettings: {apiKey:string, basePath:string}, // only applicable if server is 'azure
-  openaiSettings: {apiKey: string}, // only applicable if server is 'openai'
-  unstructuredSettings: {apiKey?: string, endpoint: string},
+  azureSettings: { apiKey: string, basePath: string }, // only applicable if server is 'azure
+  openaiSettings: { apiKey: string }, // only applicable if server is 'openai'
+  unstructuredSettings: { apiKey?: string, endpoint: string },
 }
 
-export function applyAiUserSettings(settings:AiUserSettings) {
+export function applyAiUserSettings(settings: AiUserSettings) {
   aiUserSettings = {...settings}
 }
 
@@ -60,8 +62,9 @@ export function getLangchainConfig() {
     return {
       azureOpenAIApiKey: aiUserSettings.azureSettings?.apiKey,
       azureOpenAIApiInstanceName: 'kevin-test-openai-1', // FIXME: configurable
-      azureOpenAIApiDeploymentName: Config.embedModel,
-      azureOpenAIApiVersion: '2023-03-15-preview'
+      azureOpenAIApiVersion: '2023-03-15-preview',
+      azureOpenAIApiDeploymentName: Config.chatModel.replaceAll(/\./g, ''),
+      azureOpenAIApiEmbeddingsDeploymentName: Config.embedModel.replaceAll(/\./g, ''),
     }
   } else {
     throw new Error('unsupported')
@@ -73,7 +76,7 @@ export function getOpenAIConfig(deployment?: string) {
   const params = {...getOpenAIParams()}
   if (params.basePath) {
     if (deployment) {
-      const d2 = deployment.replaceAll(/\./g,'')
+      const d2 = deployment.replaceAll(/\./g, '')
       params.basePath = `${params.basePath}/openai/deployments/${d2}`
       params.baseOptions = {
         headers: {'api-key': params.apiKey},
@@ -91,7 +94,7 @@ export function getOpenAIConfig(deployment?: string) {
 
 export const getOpenAIAPI = (deployment?: string) => {
   const config = getOpenAIConfig(deployment)
-  return new OpenAIApi(config);
+  return new OpenAIApi(config)
 }
 
 export function getSettingsFromLocalStorage() {
@@ -105,27 +108,36 @@ export function getSettingsFromLocalStorage() {
 }
 
 
-
-function getChatGPTClientOptions(clientOptions?:any) {
+function getChatGPTClientOptions(clientOptions?: any) {
   const cfg = getOpenAIConfig(Config.chatModel)
   if (cfg.basePath) {
     // Set up azure mode
     // https://kevin-test-openai-1.openai.azure.com//openai/deployments/gpt-35-turbo/chat/completions?api-version=2023-03-15-preview
     const opts = {azure: true, reverseProxyUrl: `${cfg.basePath}/chat/completions?api-version=2023-03-15-preview`}
-    return {...(clientOptions??{}), ...opts}
+    return {...(clientOptions ?? {}), ...opts}
   } else {
     return clientOptions ?? {}
   }
 
 }
 
-export function getChatGPTClient(cache:any) {
+export function getChatGPTClient(cache: any) {
   const clientOptions = getChatGPTClientOptions()
   const client = new ChatGPTClient(getOpenAIParams().apiKey, clientOptions, cache)
   return client
 }
 
 export function getUnstructuredEndpoint() {
-  const endpoint = aiUserSettings?.unstructuredSettings?.endpoint ?? ''
+  let endpoint = aiUserSettings?.unstructuredSettings?.endpoint
+  if (!endpoint) {
+    endpoint = process.env.UNSTRUCTURED_URL
+  }
+  if (!endpoint) {
+    endpoint = defaultUnstructuredUrl
+  }
   return `${endpoint}/general/v0/general`
 }
+
+export function getOpenAIChat() {
+  return new OpenAIChat(getLangchainConfig())
+}

src/lib/ai/langchainWrapper.ts

@@ -0,0 +1,6 @@
+import {useQnaStore} from '../../stores/qnaStore'
+
+
+export const performVectorStoreQnaDirect = async (args: {question:string}) => {
+  return useQnaStore().performVectorStoreQnaDirect(args)
+}

src/lib/ai/openaiFacade.ts

@@ -9,6 +9,7 @@ import {
 } from 'src/lib/ai/openaiWrapper'
 import {CreateImageRequest, ImagesResponse} from 'openai'
 import {getOpenAIAPI} from 'src/lib/ai/config'
+import {performVectorStoreQnaDirect} from "src/lib/ai/langchainWrapper";
 
 const FUNCTIONS_REGION = 'asia-northeast1'
 

src/lib/ai/unstructured.ts

@@ -1,6 +1,7 @@
 import axios from 'axios'
 import {getUnstructuredEndpoint} from 'src/lib/ai/config'
 
+// const FormData = require('form-data');
 export interface UnstructuredMetadata {
   filetype: string
   page_number: number
@@ -19,29 +20,54 @@ export interface UnstructuredElement {
 
 export async function fileToText(file:File) {
   const elems = await fileToPartitions(file)
+  return partitionsToText(elems)
+}
+
+export async function anyBufferToText(buffer: any, name:string) {
+  const form = new FormData();
+  form.append('files', buffer, name);
+
+  const response = await fetch(getUnstructuredEndpoint(),
+    {method: 'POST', body: form})
+
+  // const response = await axios.post(
+  //   getUnstructuredEndpoint(),
+  //   form
+  // );
+   if (response.ok) {
+    const json = await response.json()
+
+    return partitionsToText(json)
+  } else {
+    const text = await response.json()
+    throw new Error(text)
+  }
+
+}
+
+
+function partitionsToText(elemsOrig: Partial<UnstructuredElement>[]) {
+  const elems = elemsOrig.filter((el:any) => typeof el?.text === 'string')
+  if (!Array.isArray(elems)) {
+    throw new Error(
+      `Expected partitioning request to return an array, but got ${elems}`
+    );
+  }
   const texts:string[] = elems.map(x => x.text!)
   // TODO: Consider determine elem type to use '\n' or '\n\n' instead of ' '
   return texts.join(' ')
 }
 
 export async function fileToPartitions(file:File):Promise<Partial<UnstructuredElement>[]> {
-  const url = getUnstructuredEndpoint()
-
   const formData = new FormData();
   formData.append('files', file, file.name);
+  const url = getUnstructuredEndpoint()
   const response = await axios.post(url, formData, {
     headers: {
       'Accept': 'application/json',
       'Content-Type': 'multipart/form-data',
     },
   });
-
-  const elems = response.data.filter((el:any) => typeof el?.text === 'string')
-  if (!Array.isArray(elems)) {
-    throw new Error(
-      `Expected partitioning request to return an array, but got ${elems}`
-    );
-  }
-  return elems
+  return response.data
 
 }