update to support multiple providers

Author: mrlynn

.github/workflows/test.yml

@@ -7,7 +7,8 @@ jobs:
     runs-on: ubuntu-latest
     env:
       MONGODB_URI: ${{ secrets.MONGODB_URI }}
-      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      EMBEDDING_PROVIDER: ${{ secrets.EMBEDDING_PROVIDER }}
+      EMBEDDING_API_KEY: ${{ secrets.EMBEDDING_API_KEY }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
@@ -22,9 +23,10 @@ jobs:
 
       - name: Debug Environment Variables
         run: |
-          echo "OPENAI_API_KEY length: ${#OPENAI_API_KEY}"
+          echo "Embedding Provider: $EMBEDDING_PROVIDER"
+          echo "API Key Length: ${#EMBEDDING_API_KEY}"
         env:
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          EMBEDDING_API_KEY: ${{ secrets.EMBEDDING_API_KEY }}
 
       - name: Run tests
         run: npm test

examples/basic-usage.js

@@ -1,3 +1,4 @@
+// examples/basic-usage.js
 import { MongoRAG } from '../src/index.js';
 import dotenv from 'dotenv';
 
@@ -7,26 +8,21 @@ dotenv.config();
 // Debug loading of environment variables
 console.log('Environment variables loaded:', {
   mongoUri: process.env.MONGODB_URI ? 'Set' : 'Not set',
-  openaiKey: process.env.OPENAI_API_KEY ? 'Set' : 'Not set'
+  provider: process.env.EMBEDDING_PROVIDER || 'openai',
+  apiKey: process.env.EMBEDDING_API_KEY ? 'Set' : 'Not set'
 });
 
 // Sample documents
 const documents = [
   {
     id: 'doc1',
-    content: 'MongoDB Atlas Vector Search enables semantic search capabilities through the storage and querying of vector embeddings.',
-    metadata: {
-      source: 'documentation',
-      category: 'features'
-    }
+    content: 'MongoDB Atlas Vector Search enables semantic search capabilities through vector embeddings.',
+    metadata: { source: 'docs', category: 'features' }
   },
   {
     id: 'doc2',
-    content: 'Vector similarity search uses mathematical representations of content to find related items, making it ideal for semantic search applications.',
-    metadata: {
-      source: 'tutorial',
-      category: 'concepts'
-    }
+    content: 'Vector similarity search uses mathematical representations to find related items.',
+    metadata: { source: 'tutorial', category: 'concepts' }
   }
 ];
 
@@ -38,15 +34,14 @@ async function runExample() {
     database: 'ragtest',
     collection: 'documents',
     embedding: {
-      provider: 'openai',
-      apiKey: process.env.OPENAI_API_KEY,
+      provider: process.env.EMBEDDING_PROVIDER || 'openai',
+      apiKey: process.env.EMBEDDING_API_KEY,
       dimensions: 1536,
-      model: 'text-embedding-3-small'
+      model: process.env.EMBEDDING_MODEL || 'text-embedding-3-small'
     }
   });
 
   try {
-    console.log('Attempting to connect to MongoDB...');
     await rag.connect();
     console.log('Successfully connected and initialized indexes');
 
@@ -56,13 +51,13 @@ async function runExample() {
         console.log(`Progress: ${progress.percent}%`);
       }
     });
-    
+
     console.log(`Ingested ${ingestResult.processed} documents`);
 
     console.log('\nPerforming search...');
     const searchQuery = 'What is vector similarity search?';
     console.log(`Query: "${searchQuery}"`);
-    
+
     const results = await rag.search(searchQuery, {
       maxResults: 2,
       includeMetadata: true
@@ -83,4 +78,4 @@ async function runExample() {
 }
 
 // Run the example
-runExample().catch(console.error);
+runExample().catch(console.error);

src/core/MongoRAG.js

@@ -192,22 +192,29 @@ class MongoRAG {
     async _initializeEmbeddingProvider() {
         if (!this.embeddingProvider) {
             const { provider, apiKey, ...options } = this.config.embedding;
-
-            console.log('Initializing embedding provider with API key present:', Boolean(apiKey));
-
+    
+            console.log(`Initializing embedding provider: ${provider}`);
+    
             switch (provider) {
                 case 'openai':
                     const OpenAIEmbeddingProvider = (await import('../providers/OpenAIEmbeddingProvider.js')).default;
-                    this.embeddingProvider = new OpenAIEmbeddingProvider({
-                        apiKey: apiKey, // Pass the API key directly
-                        ...options
-                    });
+                    this.embeddingProvider = new OpenAIEmbeddingProvider({ apiKey, ...options });
+                    break;
+                case 'anthropic':
+                    const AnthropicEmbeddingProvider = (await import('../providers/AnthropicEmbeddingProvider.js')).default;
+                    this.embeddingProvider = new AnthropicEmbeddingProvider({ apiKey, ...options });
+                    break;
+                case 'deepseek':
+                    const DeepSeekEmbeddingProvider = (await import('../providers/DeepSeekEmbeddingProvider.js')).default;
+                    this.embeddingProvider = new DeepSeekEmbeddingProvider({ apiKey, ...options });
                     break;
                 default:
                     throw new Error(`Unknown embedding provider: ${provider}`);
             }
         }
     }
+    
+
 }
 
 export default MongoRAG;

src/providers/AnthropicEmbeddingProvider.js

@@ -0,0 +1,49 @@
+// src/providers/AnthropicEmbeddingProvider.js
+import BaseEmbeddingProvider from './BaseEmbeddingProvider.js';
+import axios from 'axios';
+import debug from 'debug';
+
+const log = debug('mongodb-rag:embedding:anthropic');
+
+class AnthropicEmbeddingProvider extends BaseEmbeddingProvider {
+  constructor(options = {}) {
+    super(options);
+    
+    if (!options.apiKey) {
+      throw new Error('Anthropic API key is required');
+    }
+
+    this.apiKey = options.apiKey;
+    this.model = options.model || 'claude-3';
+    this.client = axios.create({
+      baseURL: 'https://api.anthropic.com/v1',
+      headers: {
+        'x-api-key': this.apiKey,
+        'Content-Type': 'application/json'
+      }
+    });
+
+    log('Anthropic embedding provider initialized');
+  }
+
+  async _embedBatch(texts) {
+    try {
+      log(`Getting embeddings for batch of ${texts.length} texts`);
+      
+      const response = await this.client.post('/embeddings', {
+        model: this.model,
+        input: texts
+      });
+
+      const embeddings = response.data.data.map(item => item.embedding);
+      return embeddings;
+    } catch (error) {
+      if (error.response?.data) {
+        throw new Error(`Anthropic API error: ${error.response.data.error.message}`);
+      }
+      throw error;
+    }
+  }
+}
+
+export default AnthropicEmbeddingProvider;