Remove create_embedding_pipeline

Author: jeremy-rifkin

indexes/cppref/generate-cppref-embeddings.ts

@@ -1,5 +1,5 @@
 import * as fs from "fs";
-import { create_embedding_pipeline, generate_embedding, EMBEDDING_MODEL } from "../../src/utils/embeddings.js";
+import { get_or_create_embedding_pipeline, generate_embedding, EMBEDDING_MODEL } from "../../src/utils/embeddings.js";
 
 const INDEX_DIR = "indexes/cppref";
 
@@ -47,7 +47,7 @@ function create_cppref_embedding_content(entry: CpprefEntry): string {
     console.log(`Loaded ${all_entries.length} cppref entries`);
 
     console.log("Loading embedding model (this may take a while on first run)...");
-    const extractor = await create_embedding_pipeline();
+    const extractor = await get_or_create_embedding_pipeline();
 
     console.log("Generating embeddings...");
     const embeddings: Record<string, number[]> = {};

indexes/man7/generate-man7-embeddings.ts

@@ -1,5 +1,5 @@
 import * as fs from "fs";
-import { create_embedding_pipeline, generate_embedding, EMBEDDING_MODEL } from "../../src/utils/embeddings.js";
+import { get_or_create_embedding_pipeline, generate_embedding, EMBEDDING_MODEL } from "../../src/utils/embeddings.js";
 
 const INDEX_DIR = "indexes/man7";
 
@@ -26,7 +26,7 @@ function create_man7_embedding_content(entry: Man7Entry): string {
     console.log(`Loaded ${index_data.length} man7 entries`);
 
     console.log("Loading embedding model (this may take a while on first run)...");
-    const extractor = await create_embedding_pipeline();
+    const extractor = await get_or_create_embedding_pipeline();
 
     console.log("Generating embeddings...");
     const embeddings: Record<string, number[]> = {};

indexes/wiki/generate-wiki-embeddings.ts

@@ -7,7 +7,7 @@ import {
     create_embedding_content,
 } from "../../src/modules/wheatley/components/wiki.js";
 import { load_wiki_web_articles } from "../../src/modules/wheatley/wiki-article-loader.js";
-import { create_embedding_pipeline, generate_embedding, EMBEDDING_MODEL } from "../../src/utils/embeddings.js";
+import { get_or_create_embedding_pipeline, generate_embedding, EMBEDDING_MODEL } from "../../src/utils/embeddings.js";
 
 const INDEX_DIR = "indexes/wiki";
 
@@ -43,7 +43,7 @@ const INDEX_DIR = "indexes/wiki";
     console.log(`Total articles: ${Object.keys(articles).length}`);
 
     console.log("Loading embedding model (this may take a while on first run)...");
-    const extractor = await create_embedding_pipeline();
+    const extractor = await get_or_create_embedding_pipeline();
 
     console.log("Generating embeddings...");
     const embeddings: Record<string, number[]> = {};

src/utils/embeddings.ts

@@ -12,10 +12,6 @@ export async function get_or_create_embedding_pipeline(): Promise<FeatureExtract
     return shared_pipeline;
 }
 
-export async function create_embedding_pipeline() {
-    return await pipeline("feature-extraction", EMBEDDING_MODEL);
-}
-
 export async function generate_embedding(text: string, extractor: FeatureExtractionPipeline): Promise<number[]> {
     const output = await extractor(text, { pooling: "mean", normalize: true });
     return Array.from(output.data);