refactor cli commands to individual files

Author: mrlynn

bin/commands/config/clear-config.js

@@ -0,0 +1,14 @@
+// bin/commands/config/clear-config.js
+import fs from 'fs';
+import chalk from 'chalk';
+
+export async function clearConfig(configPath) {
+  if (fs.existsSync(configPath)) {
+    fs.unlinkSync(configPath);
+    console.log(chalk.green("🗑️ Configuration deleted successfully."));
+    return true;
+  } else {
+    console.warn(chalk.yellow("⚠️ No configuration file found."));
+    return false;
+  }
+}

bin/commands/config/config-commands.js

@@ -0,0 +1,61 @@
+// tests/commands/config/config-commands.test.js
+import { jest, describe, expect, test, beforeEach } from '@jest/globals';
+import chalk from 'chalk';
+
+// Create mock fs functions
+const mockFsImpl = {
+  existsSync: jest.fn(),
+  readFileSync: jest.fn(),
+  writeFileSync: jest.fn(),
+  unlinkSync: jest.fn()
+};
+
+// Mock fs module
+await jest.unstable_mockModule('fs', () => mockFsImpl);
+
+// Import after mocking
+const { showConfig } = await import('../../../bin/commands/config/show-config.js');
+const { clearConfig } = await import('../../../bin/commands/config/clear-config.js');
+
+describe('Config Commands', () => {
+  const mockConfig = {
+    mongoUrl: 'mongodb+srv://test:test@cluster.mongodb.net',
+    database: 'test_db',
+    collection: 'test_collection',
+    embedding: {
+      provider: 'openai',
+      apiKey: 'test-key'
+    }
+  };
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    mockFsImpl.existsSync.mockReturnValue(true);
+    mockFsImpl.readFileSync.mockReturnValue(JSON.stringify(mockConfig));
+  });
+
+  test('showConfig displays configuration correctly', async () => {
+    mockFsImpl.existsSync.mockReturnValue(true);
+    mockFsImpl.readFileSync.mockReturnValue(JSON.stringify(mockConfig));
+    
+    const result = await showConfig('test-path');
+    expect(result).toEqual(mockConfig);
+    expect(mockFsImpl.readFileSync).toHaveBeenCalledWith('test-path', 'utf-8');
+  });
+
+  test('clearConfig removes configuration file', async () => {
+    mockFsImpl.existsSync.mockReturnValue(true);
+    
+    const result = await clearConfig('test-path');
+    expect(result).toBe(true);
+    expect(mockFsImpl.unlinkSync).toHaveBeenCalledWith('test-path');
+  });
+
+  test('clearConfig handles missing file', async () => {
+    mockFsImpl.existsSync.mockReturnValue(false);
+    
+    const result = await clearConfig('test-path');
+    expect(result).toBe(false);
+    expect(mockFsImpl.unlinkSync).not.toHaveBeenCalled();
+  });
+});

bin/commands/config/edit-config.js

@@ -0,0 +1,19 @@
+// bin/commands/config/edit-config.js
+import fs from 'fs';
+import chalk from 'chalk';
+import { promptForConfigEdits } from '../../utils/prompts.js';
+
+export async function editConfig(configPath) {
+  if (!fs.existsSync(configPath)) {
+    console.warn(chalk.yellow("⚠️ No configuration found. Run 'npx mongodb-rag init' first."));
+    return;
+  }
+
+  const config = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
+  const updatedConfig = await promptForConfigEdits(config);
+  
+  fs.writeFileSync(configPath, JSON.stringify(updatedConfig, null, 2));
+  console.log(chalk.green("✅ Configuration updated successfully!"));
+  
+  return updatedConfig;
+}

bin/commands/config/index.js

@@ -0,0 +1,6 @@
+// bin/commands/config/index.js
+export { showConfig } from './show-config.js';
+export { editConfig } from './edit-config.js';
+export { clearConfig } from './clear-config.js';
+export { resetConfig } from './reset-config.js';
+export { setIndexName } from './set-index-name.js';

bin/commands/config/reset-config.js

@@ -0,0 +1,14 @@
+// bin/commands/config/reset-config.js
+import fs from 'fs';
+import chalk from 'chalk';
+import { init } from '../init.js';
+
+export async function resetConfig(configPath) {
+  console.log(chalk.cyan("🔄 Resetting configuration...\n"));
+  
+  if (fs.existsSync(configPath)) {
+    fs.unlinkSync(configPath);
+  }
+  
+  return await init(configPath);
+}

bin/commands/config/set-index-name.js

@@ -0,0 +1,20 @@
+// bin/commands/config/set-index-name.js
+import fs from 'fs';
+import chalk from 'chalk';
+import { promptForIndexName } from '../../utils/prompts.js';
+
+export async function setIndexName(configPath) {
+  if (!fs.existsSync(configPath)) {
+    console.warn(chalk.yellow("⚠️ No configuration found. Run 'npx mongodb-rag init' first."));
+    return;
+  }
+
+  const config = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
+  const indexName = await promptForIndexName(config.indexName || 'vector_index');
+  
+  config.indexName = indexName;
+  fs.writeFileSync(configPath, JSON.stringify(config, null, 2));
+  console.log(chalk.green(`✅ Vector Index Name updated to "${indexName}"`));
+  
+  return config;
+}

bin/commands/config/show-config.js

@@ -0,0 +1,17 @@
+// bin/commands/config/show-config.js
+import fs from 'fs';
+import debug from 'debug';
+
+const log = debug('mongodb-rag:cli:config');
+
+export async function showConfig(configPath = '.mongodb-rag.json') {
+  try {
+    if (!fs.existsSync(configPath)) {
+      throw new Error(`Configuration file not found: ${configPath}`);
+    }
+    const content = fs.readFileSync(configPath, 'utf-8');
+    return JSON.parse(content);
+  } catch (error) {
+    throw new Error(`Failed to show config: ${error.message}`);
+  }
+}

bin/commands/data/generate-embedding.js

@@ -0,0 +1,55 @@
+// bin/commands/data/generate-embedding.js
+import chalk from 'chalk';
+import { isConfigValid } from '../../utils/validation.js';
+import MongoRAG from '../../../src/core/MongoRAG.js';
+
+export async function generateEmbedding(config, text) {
+  const isDevelopment = process.env.NODE_ENV === 'development' || process.env.NODE_ENV === 'test';
+
+  // Clean up any duplicate configuration
+  const cleanConfig = {
+    ...config,
+    // Remove any root-level embedding properties
+    provider: undefined,
+    apiKey: undefined,
+    model: undefined,
+    dimensions: undefined,
+    baseUrl: undefined
+  };
+
+  if (!isConfigValid(cleanConfig)) {
+    throw new Error("Invalid configuration. Please run 'npx mongodb-rag init' to set up your configuration properly.");
+  }
+
+  const rag = new MongoRAG(cleanConfig);
+  try {
+    if (isDevelopment) {
+      console.log(chalk.blue('🔄 Getting embedding for text:'), text);
+    }
+
+    await rag.connect();
+    const embedding = await rag.createEmbedding(text);
+    
+    if (isDevelopment) {
+      console.log(chalk.cyan("🔢 Generated Embedding:"), embedding);
+      console.log(chalk.green(`✅ Successfully generated ${embedding.length}-dimensional embedding`));
+    } else {
+      console.log(chalk.green('✅ Successfully generated embedding'));
+    }
+    
+    return embedding;
+  } catch (error) {
+    console.error(chalk.red("❌ Error generating embedding:"), error.message);
+    
+    if (config.embedding.provider === 'ollama') {
+      console.log(chalk.yellow('\nTroubleshooting:'));
+      console.log(chalk.cyan('1. Ensure Ollama is running'));
+      console.log(chalk.cyan(`2. Check if Ollama is accessible at ${config.embedding.baseUrl}`));
+      console.log(chalk.cyan('3. Check if the model is installed with `ollama list`'));
+    }
+    
+    throw error;
+  } finally {
+    await rag.client?.close();
+  }
+}

bin/commands/data/index.js

@@ -0,0 +1,4 @@
+// bin/commands/data/index.js
+export { searchDocuments } from './search.js';
+export { ingestData } from './ingest.js';
+export { generateEmbedding } from './generate-embedding.js';

bin/commands/data/ingest.js

@@ -0,0 +1,126 @@
+// bin/commands/data/ingest.js
+import fs from 'fs';
+import path from 'path';
+import chalk from 'chalk';
+import { isConfigValid } from '../../utils/validation.js';
+import MongoRAG from '../../../src/core/MongoRAG.js';
+import { parseDocument } from '../../utils/document-parsers.js';
+import { DocumentChunker } from '../../utils/chunking.js';
+
+export async function ingestData(config, options) {
+  if (!isConfigValid(config)) {
+    throw new Error("Configuration missing. Run 'npx mongodb-rag init' first.");
+  }
+
+  try {
+    let documents = [];
+    const isDevelopment = process.env.NODE_ENV === 'development' || process.env.NODE_ENV === 'test';
+
+    // Handle directory ingestion
+    if (options.directory) {
+      if (isDevelopment) console.log(chalk.blue(`📂 Processing directory: ${options.directory}`));
+      documents = await processDirectory(options.directory, options);
+    }
+    // Handle single file ingestion
+    else if (options.file) {
+      if (isDevelopment) console.log(chalk.blue(`📄 Processing file: ${options.file}`));
+      documents = await processFile(options.file, options);
+    }
+    else {
+      throw new Error("Either --file or --directory option must be specified");
+    }
+
+    // Initialize chunker if chunking is enabled
+    if (options.chunkSize) {
+      const chunker = new DocumentChunker({
+        chunkSize: options.chunkSize,
+        chunkOverlap: options.chunkOverlap,
+        method: options.chunkMethod
+      });
+
+      // Chunk each document
+      const chunkedDocs = [];
+      for (const doc of documents) {
+        if (isDevelopment) {
+          console.log(chalk.blue(`📄 Chunking document: ${doc.metadata.filename}`));
+        }
+        const chunks = chunker.chunkDocument(doc);
+        chunkedDocs.push(...chunks);
+
+        if (isDevelopment) {
+          console.log(chalk.green(`✅ Created ${chunks.length} chunks`));
+        }
+      }
+      documents = chunkedDocs;
+    }
+
+    if (isDevelopment) {
+      console.log(chalk.blue(`📊 Found ${documents.length} documents to process`));
+    }
+
+    const rag = new MongoRAG(config);
+    await rag.connect();
+
+    const result = await rag.ingestBatch(documents, {
+      database: options.database,
+      collection: options.collection
+    });
+
+    console.log(chalk.green(`✅ Successfully ingested ${result.processed} documents!`));
+    return result;
+  } catch (error) {
+    console.error(chalk.red('❌ Ingestion failed:'), error.message);
+    throw error;
+  }
+}
+
+async function processDirectory(dirPath, options) {
+  const documents = [];
+  const files = fs.readdirSync(dirPath);
+
+  for (const file of files) {
+    const filePath = path.join(dirPath, file);
+    const stat = fs.statSync(filePath);
+
+    if (stat.isDirectory() && options.recursive) {
+      const subDocs = await processDirectory(filePath, options);
+      documents.push(...subDocs);
+    } else if (stat.isFile()) {
+      const docs = await processFile(filePath, options);
+      documents.push(...docs);
+    }
+  }
+
+  return documents;
+}
+
+async function processFile(filePath, options) {
+  const ext = path.extname(filePath).toLowerCase();
+  const isDevelopment = process.env.NODE_ENV === 'development' || process.env.NODE_ENV === 'test';
+  
+  try {
+    // If it's a JSON file, parse it directly
+    if (ext === '.json') {
+      const content = fs.readFileSync(filePath, 'utf-8');
+      const data = JSON.parse(content);
+      return Array.isArray(data) ? data : [data];
+    }
+
+    // For other file types, use the document parser
+    const doc = await parseDocument(filePath, null, options);
+    
+    if (isDevelopment) {
+      console.log(chalk.blue(`📄 Processed ${filePath}`));
+      if (doc.metadata.processingFailed) {
+        console.warn(chalk.yellow(`⚠️ Warning: ${doc.metadata.error}`));
+      }
+    }
+    
+    return [doc];
+  } catch (error) {
+    if (isDevelopment) {
+      console.error(chalk.red(`❌ Failed to process ${filePath}:`), error.message);
+    }
+    return [];  // Skip failed files
+  }
+}