@nk/rag multi text format (#175)

Author: NorbertKlockiewicz

database/exportImportRepository.ts

@@ -1,6 +1,6 @@
 import { SQLiteDatabase } from 'expo-sqlite';
 import { getChatMessages, Message } from './chatRepository';
-import * as FileSystem from 'expo-file-system';
+import { File, Paths } from 'expo-file-system';
 import * as Sharing from 'expo-sharing';
 import * as DocumentPicker from 'expo-document-picker';
 import { Alert } from 'react-native';
@@ -20,11 +20,9 @@ export const exportChatRoom = async (
     });
 
     const fileName = `chat-${Date.now()}.json`;
-    const fileUri = `${FileSystem.documentDirectory}${fileName}`;
-
-    await FileSystem.writeAsStringAsync(fileUri, jsonData, {
-      encoding: FileSystem.EncodingType.UTF8,
-    });
+    const file = new File(Paths.document, fileName);
+    await file.write(jsonData);
+    const fileUri = file.uri;
 
     if (await Sharing.isAvailableAsync()) {
       await Sharing.shareAsync(fileUri, {
@@ -54,7 +52,8 @@ export async function importChatRoom(): Promise<
     });
     if (result.canceled || !result.assets[0]?.uri) return;
     const uri = result.assets[0].uri;
-    const fileContent = await FileSystem.readAsStringAsync(uri);
+    const file = new File(uri);
+    const fileContent = await file.text();
 
     const chatData = JSON.parse(fileContent);
     const chat = {
@@ -67,7 +66,7 @@ export async function importChatRoom(): Promise<
     };
     return chat;
   } catch (error) {
-    console.error('❌ Failed to import chat room JSON:', error);
+    console.error('Failed to import chat room JSON:', error);
     return;
   }
 }

hooks/useSourceUpload.ts

@@ -16,7 +16,7 @@ export const useSourceUpload = () => {
 
   const uploadSource = useCallback(async () => {
     const pickedFileResult = await DocumentPicker.getDocumentAsync({
-      type: 'application/pdf',
+      type: ['application/pdf', 'text/plain', 'text/markdown', 'text/html'],
       copyToCacheDirectory: true,
     });
 
@@ -41,15 +41,15 @@ export const useSourceUpload = () => {
         });
       } else if (result.isEmpty) {
         warningSheetRef.current?.present({
-          title: "Can't read PDF",
-          subtitle: `The PDF "${newSource.name}" appears to be empty or unreadable. It won't be added to your sources.`,
+          title: "Can't read document",
+          subtitle: `The document "${newSource.name}" appears to be empty or unreadable. It won't be added to your sources.`,
           buttonTitle: 'OK',
           onConfirm: () => {},
         });
       } else {
         warningSheetRef.current?.present({
-          title: "Can't read PDF",
-          subtitle: `There was an error processing the PDF "${newSource.name}". Please try again.`,
+          title: "Can't read document",
+          subtitle: `There was an error processing the document "${newSource.name}". Please try again.`,
           buttonTitle: 'OK',
           onConfirm: () => {},
         });

store/sourceStore.ts

@@ -10,7 +10,7 @@ import {
 import { SQLiteDatabase } from 'expo-sqlite';
 import { OPSQLiteVectorStore } from '@react-native-rag/op-sqlite';
 import { RecursiveCharacterTextSplitter } from 'react-native-rag';
-import { readPDF } from 'react-native-pdfium';
+import { readDocumentText } from '../utils/fileReaders';
 import { useLLMStore } from './llmStore';
 
 interface SourceStore {
@@ -56,13 +56,12 @@ export const useSourceStore = create<SourceStore>((set, get) => ({
     const db = get().db;
     if (!db) return { success: false };
 
-    const normalizedUri = sourceUri.replace('file://', '');
     const tempId = -Date.now();
 
     set({ isReading: true });
 
     try {
-      const sourceTextContent = await readPDF(normalizedUri);
+      const sourceTextContent = await readDocumentText(sourceUri, source.type);
 
       if (!sourceTextContent || sourceTextContent.trim().length === 0) {
         set({ isReading: false });

utils/fileReaders.ts

@@ -0,0 +1,49 @@
+import { readPDF } from 'react-native-pdfium';
+import { File } from 'expo-file-system';
+
+/**
+ * Reads text content from various file formats
+ * @param filePath - The path to the file
+ * @param fileType - The file extension (pdf, txt, md, html, csv, etc.)
+ * @returns The extracted text content
+ */
+export async function readDocumentText(
+  filePath: string,
+  fileType: string
+): Promise<string> {
+  const lowerFileType = fileType.toLowerCase();
+
+  switch (lowerFileType) {
+    case 'pdf':
+      // PDF reader needs path without file:// prefix
+      const normalizedPath = filePath.replace('file://', '');
+      return await readPDF(normalizedPath);
+
+    case 'txt':
+    case 'md':
+    case 'markdown':
+      const textFile = new File(filePath);
+      return await textFile.text();
+
+    case 'html':
+    case 'htm':
+      const htmlFile = new File(filePath);
+      const htmlContent = await htmlFile.text();
+      // Basic HTML tag stripping - removes all HTML tags
+      return htmlContent
+        .replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '') // Remove script tags
+        .replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, '') // Remove style tags
+        .replace(/<[^>]+>/g, ' ') // Remove all HTML tags
+        .replace(/&nbsp;/g, ' ')
+        .replace(/&amp;/g, '&')
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        .replace(/&quot;/g, '"')
+        .replace(/&#39;/g, "'")
+        .replace(/\s+/g, ' ') // Normalize whitespace
+        .trim();
+
+    default:
+      throw new Error(`Unsupported file type: ${fileType}`);
+  }
+}