|
45 | 45 | * --retry-attempts=N Max retry attempts for failed files (default: 3) |
46 | 46 | * --retry-delay=N Delay between retries in ms (default: 1000) |
47 | 47 | * --dry-run Don't actually load files, just validate |
| 48 | + * --skip-embed Skip automatic embedding generation after import (embeddings run by default) |
48 | 49 | * --hours=N For S3: import vCons modified in last N hours (default: 24) |
49 | 50 | * --prefix=PREFIX For S3: filter objects by prefix (optional) |
50 | 51 | * --sync Enable continuous sync mode (checks for new vCons periodically) |
|
128 | 129 | */ |
129 | 130 |
|
130 | 131 | import { readdir, readFile, stat, writeFile, readFile as readFileSync, mkdtemp, rm, unlink } from 'fs/promises'; |
131 | | -import { join } from 'path'; |
| 132 | +import { join, dirname } from 'path'; |
| 133 | +import { spawn } from 'child_process'; |
132 | 134 | import { tmpdir } from 'os'; |
133 | 135 | import dotenv from 'dotenv'; |
134 | 136 | import { S3Client, ListObjectsV2Command, GetObjectCommand } from '@aws-sdk/client-s3'; |
@@ -1286,6 +1288,7 @@ async function main() { |
1286 | 1288 | const retryAttempts = parseInt(args.find(arg => arg.startsWith('--retry-attempts='))?.split('=')[1] || '3'); |
1287 | 1289 | const retryDelay = parseInt(args.find(arg => arg.startsWith('--retry-delay='))?.split('=')[1] || '1000'); |
1288 | 1290 | const dryRun = args.includes('--dry-run'); |
| 1291 | + const skipEmbed = args.includes('--skip-embed'); |
1289 | 1292 | const hours = parseInt(args.find(arg => arg.startsWith('--hours='))?.split('=')[1] || '24'); |
1290 | 1293 | const prefix = args.find(arg => arg.startsWith('--prefix='))?.split('=')[1] || undefined; |
1291 | 1294 | const sync = args.includes('--sync'); |
@@ -1383,11 +1386,38 @@ async function main() { |
1383 | 1386 | } |
1384 | 1387 |
|
1385 | 1388 | await loadVConsFromDirectory(directoryPath, options); |
1386 | | - |
| 1389 | + |
1387 | 1390 | // If sync mode is enabled, start continuous syncing |
1388 | 1391 | if (sync && !dryRun) { |
1389 | 1392 | await startSyncMode(directoryPath, options); |
1390 | 1393 | } |
| 1394 | + |
| 1395 | + // Generate embeddings for newly imported vCons (default behaviour; skip with --skip-embed) |
| 1396 | + if (!dryRun && !skipEmbed) { |
| 1397 | + console.log('\n' + '='.repeat(60)); |
| 1398 | + console.log('🔍 Running embeddings for newly imported vCons...'); |
| 1399 | + console.log(' (use --skip-embed to disable)'); |
| 1400 | + console.log('='.repeat(60) + '\n'); |
| 1401 | + |
| 1402 | + const embedScript = join(dirname(process.argv[1]), 'embed-vcons.ts'); |
| 1403 | + |
| 1404 | + await new Promise<void>((resolve) => { |
| 1405 | + const child = spawn( |
| 1406 | + 'npx', ['tsx', embedScript, '--continuous', '--limit=500'], |
| 1407 | + { stdio: 'inherit', env: process.env } |
| 1408 | + ); |
| 1409 | + child.on('close', (code) => { |
| 1410 | + if (code !== 0) { |
| 1411 | + console.warn(`\n⚠️ embed-vcons.ts exited with code ${code} — embeddings may be incomplete`); |
| 1412 | + } |
| 1413 | + resolve(); |
| 1414 | + }); |
| 1415 | + child.on('error', (err) => { |
| 1416 | + console.warn(`\n⚠️ Failed to run embed-vcons.ts: ${err.message}`); |
| 1417 | + resolve(); |
| 1418 | + }); |
| 1419 | + }); |
| 1420 | + } |
1391 | 1421 | } |
1392 | 1422 |
|
1393 | 1423 | /** |
|
0 commit comments