Skip to content

Commit e8f9c4f

Browse files
howethomasclaude
andcommitted
feat: run embeddings automatically after import
Add --skip-embed flag to load-legacy-vcons.ts; embed-vcons.ts now runs by default after every successful import so embeddings stay in sync. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 0ba1f42 commit e8f9c4f

1 file changed

Lines changed: 32 additions & 2 deletions

File tree

scripts/load-legacy-vcons.ts

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
* --retry-attempts=N Max retry attempts for failed files (default: 3)
4646
* --retry-delay=N Delay between retries in ms (default: 1000)
4747
* --dry-run Don't actually load files, just validate
48+
* --skip-embed Skip automatic embedding generation after import (embeddings run by default)
4849
* --hours=N For S3: import vCons modified in last N hours (default: 24)
4950
* --prefix=PREFIX For S3: filter objects by prefix (optional)
5051
* --sync Enable continuous sync mode (checks for new vCons periodically)
@@ -128,7 +129,8 @@
128129
*/
129130

130131
import { readdir, readFile, stat, writeFile, readFile as readFileSync, mkdtemp, rm, unlink } from 'fs/promises';
131-
import { join } from 'path';
132+
import { join, dirname } from 'path';
133+
import { spawn } from 'child_process';
132134
import { tmpdir } from 'os';
133135
import dotenv from 'dotenv';
134136
import { S3Client, ListObjectsV2Command, GetObjectCommand } from '@aws-sdk/client-s3';
@@ -1286,6 +1288,7 @@ async function main() {
12861288
const retryAttempts = parseInt(args.find(arg => arg.startsWith('--retry-attempts='))?.split('=')[1] || '3');
12871289
const retryDelay = parseInt(args.find(arg => arg.startsWith('--retry-delay='))?.split('=')[1] || '1000');
12881290
const dryRun = args.includes('--dry-run');
1291+
const skipEmbed = args.includes('--skip-embed');
12891292
const hours = parseInt(args.find(arg => arg.startsWith('--hours='))?.split('=')[1] || '24');
12901293
const prefix = args.find(arg => arg.startsWith('--prefix='))?.split('=')[1] || undefined;
12911294
const sync = args.includes('--sync');
@@ -1383,11 +1386,38 @@ async function main() {
13831386
}
13841387

13851388
await loadVConsFromDirectory(directoryPath, options);
1386-
1389+
13871390
// If sync mode is enabled, start continuous syncing
13881391
if (sync && !dryRun) {
13891392
await startSyncMode(directoryPath, options);
13901393
}
1394+
1395+
// Generate embeddings for newly imported vCons (default behaviour; skip with --skip-embed)
1396+
if (!dryRun && !skipEmbed) {
1397+
console.log('\n' + '='.repeat(60));
1398+
console.log('🔍 Running embeddings for newly imported vCons...');
1399+
console.log(' (use --skip-embed to disable)');
1400+
console.log('='.repeat(60) + '\n');
1401+
1402+
const embedScript = join(dirname(process.argv[1]), 'embed-vcons.ts');
1403+
1404+
await new Promise<void>((resolve) => {
1405+
const child = spawn(
1406+
'npx', ['tsx', embedScript, '--continuous', '--limit=500'],
1407+
{ stdio: 'inherit', env: process.env }
1408+
);
1409+
child.on('close', (code) => {
1410+
if (code !== 0) {
1411+
console.warn(`\n⚠️ embed-vcons.ts exited with code ${code} — embeddings may be incomplete`);
1412+
}
1413+
resolve();
1414+
});
1415+
child.on('error', (err) => {
1416+
console.warn(`\n⚠️ Failed to run embed-vcons.ts: ${err.message}`);
1417+
resolve();
1418+
});
1419+
});
1420+
}
13911421
}
13921422

13931423
/**

0 commit comments

Comments
 (0)