refactor: code simplification phase 1 - centralize config, stateless pipeline, extract templates

ingesters/__tests__/IngesterFactory.test.ts

@@ -58,7 +58,7 @@ describe('IngesterFactory', () => {
       expect(() => {
         // @ts-ignore - Testing with invalid source
         IngesterFactory.createIngester('unknown_source');
-      }).toThrow('Unsupported source: unknown_source');
+      }).toThrow("Source 'unknown_source' not found in sources.json config");
     });
   });
 

ingesters/__tests__/sourceConfig.test.ts

@@ -0,0 +1,54 @@
+import { describe, it, expect, beforeEach } from 'bun:test';
+import {
+  clearConfigCache,
+  getAvailableSourcesFromConfig,
+  getBookConfig,
+  getSourceConfig,
+  loadSourcesConfig,
+} from '../src/utils/sourceConfig';
+import { DocumentSource } from '../src/types';
+
+describe('sourceConfig', () => {
+  beforeEach(() => {
+    clearConfigCache();
+  });
+
+  it('loads sources.json and exposes known sources', () => {
+    const config = loadSourcesConfig();
+    const cairoConfig = config.sources[DocumentSource.CAIRO_BOOK];
+
+    expect(cairoConfig).toBeDefined();
+    expect(cairoConfig.name).toBeTruthy();
+    expect(cairoConfig.config.fileExtensions).toBeInstanceOf(Array);
+  });
+
+  it('returns per-source config and book config helpers', () => {
+    const config = loadSourcesConfig();
+    const sourceConfig = getSourceConfig(DocumentSource.CAIRO_BOOK);
+    const bookConfig = getBookConfig(DocumentSource.CAIRO_BOOK);
+
+    expect(sourceConfig).toEqual(config.sources[DocumentSource.CAIRO_BOOK]);
+    expect(bookConfig).toEqual(
+      config.sources[DocumentSource.CAIRO_BOOK].config,
+    );
+  });
+
+  it('returns all available sources from config', () => {
+    const sources = getAvailableSourcesFromConfig().slice().sort();
+    const expected = Object.values(DocumentSource).slice().sort();
+
+    expect(sources).toEqual(expected);
+  });
+
+  it('caches config between loads and can be cleared', () => {
+    const first = loadSourcesConfig();
+    const second = loadSourcesConfig();
+
+    expect(first).toBe(second);
+
+    clearConfigCache();
+    const third = loadSourcesConfig();
+
+    expect(third).not.toBe(first);
+  });
+});

ingesters/config/sources.json

@@ -0,0 +1,155 @@
+{
+  "$schema": "./sources.schema.json",
+  "sources": {
+    "cairo_book": {
+      "name": "Cairo Book",
+      "description": "The official Cairo programming language book",
+      "ingesterClass": "CairoBookIngester",
+      "config": {
+        "repoOwner": "cairo-book",
+        "repoName": "cairo-book",
+        "fileExtensions": [".md"],
+        "chunkSize": 4096,
+        "chunkOverlap": 512,
+        "baseUrl": "https://book.cairo-lang.org",
+        "urlSuffix": ".html",
+        "useUrlMapping": true
+      }
+    },
+    "starknet_docs": {
+      "name": "Starknet Docs",
+      "description": "Official Starknet documentation",
+      "ingesterClass": "StarknetDocsIngester",
+      "config": {
+        "repoOwner": "starknet-io",
+        "repoName": "starknet-docs",
+        "fileExtensions": [".mdx"],
+        "chunkSize": 4096,
+        "chunkOverlap": 512,
+        "baseUrl": "https://docs.starknet.io",
+        "urlSuffix": "",
+        "useUrlMapping": true
+      }
+    },
+    "starknet_foundry": {
+      "name": "Starknet Foundry",
+      "description": "Starknet Foundry testing framework documentation",
+      "ingesterClass": "StarknetFoundryIngester",
+      "config": {
+        "repoOwner": "foundry-rs",
+        "repoName": "starknet-foundry",
+        "fileExtensions": [".md"],
+        "chunkSize": 4096,
+        "chunkOverlap": 512,
+        "baseUrl": "https://foundry-rs.github.io/starknet-foundry",
+        "urlSuffix": ".html",
+        "useUrlMapping": true
+      }
+    },
+    "cairo_by_example": {
+      "name": "Cairo By Example",
+      "description": "Learn Cairo through practical examples",
+      "ingesterClass": "CairoByExampleIngester",
+      "config": {
+        "repoOwner": "NethermindEth",
+        "repoName": "StarknetByExample",
+        "fileExtensions": [".md"],
+        "chunkSize": 4096,
+        "chunkOverlap": 512,
+        "baseUrl": "https://starknet-by-example.voyager.online",
+        "urlSuffix": ".html",
+        "useUrlMapping": true
+      }
+    },
+    "openzeppelin_docs": {
+      "name": "OpenZeppelin Docs",
+      "description": "OpenZeppelin Cairo contracts documentation",
+      "ingesterClass": "OpenZeppelinDocsIngester",
+      "config": {
+        "repoOwner": "OpenZeppelin",
+        "repoName": "cairo-contracts",
+        "fileExtensions": [".adoc"],
+        "chunkSize": 4096,
+        "chunkOverlap": 512,
+        "baseUrl": "https://docs.openzeppelin.com/contracts-cairo",
+        "urlSuffix": "",
+        "useUrlMapping": false
+      }
+    },
+    "corelib_docs": {
+      "name": "Core Library Docs",
+      "description": "Cairo core library documentation",
+      "ingesterClass": "CoreLibDocsIngester",
+      "config": {
+        "repoOwner": "starkware-libs",
+        "repoName": "cairo",
+        "fileExtensions": [".md"],
+        "chunkSize": 4096,
+        "chunkOverlap": 512,
+        "baseUrl": "https://docs.cairo-lang.org/core",
+        "urlSuffix": ".html",
+        "useUrlMapping": true
+      }
+    },
+    "scarb_docs": {
+      "name": "Scarb Docs",
+      "description": "Scarb package manager documentation",
+      "ingesterClass": "ScarbDocsIngester",
+      "config": {
+        "repoOwner": "software-mansion",
+        "repoName": "scarb",
+        "fileExtensions": [".md"],
+        "chunkSize": 4096,
+        "chunkOverlap": 512,
+        "baseUrl": "https://docs.swmansion.com/scarb/docs",
+        "urlSuffix": ".html",
+        "useUrlMapping": true
+      }
+    },
+    "starknet_js": {
+      "name": "Starknet.js",
+      "description": "Starknet JavaScript SDK documentation",
+      "ingesterClass": "StarknetJSIngester",
+      "config": {
+        "repoOwner": "starknet-io",
+        "repoName": "starknet.js",
+        "fileExtensions": [".md"],
+        "chunkSize": 4096,
+        "chunkOverlap": 512,
+        "baseUrl": "https://www.starknetjs.com",
+        "urlSuffix": "",
+        "useUrlMapping": true
+      }
+    },
+    "starknet_blog": {
+      "name": "Starknet Blog",
+      "description": "Official Starknet blog posts and articles",
+      "ingesterClass": "StarknetBlogIngester",
+      "config": {
+        "repoOwner": "starknet-io",
+        "repoName": "starknet-blog",
+        "fileExtensions": [".md"],
+        "chunkSize": 4096,
+        "chunkOverlap": 512,
+        "baseUrl": "https://starknet.io/blog",
+        "urlSuffix": "",
+        "useUrlMapping": false
+      }
+    },
+    "dojo_docs": {
+      "name": "Dojo Docs",
+      "description": "Dojo game engine documentation",
+      "ingesterClass": "DojoDocsIngester",
+      "config": {
+        "repoOwner": "dojoengine",
+        "repoName": "book",
+        "fileExtensions": [".mdx", ".md"],
+        "chunkSize": 4096,
+        "chunkOverlap": 512,
+        "baseUrl": "https://book.dojoengine.org",
+        "urlSuffix": "",
+        "useUrlMapping": true
+      }
+    }
+  }
+}

ingesters/config/sources.schema.json

@@ -0,0 +1,79 @@
+{
+  "$schema": "https://json-schema.org/draft-07/schema#",
+  "title": "Cairo Coder Sources Config",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["sources"],
+  "properties": {
+    "sources": {
+      "type": "object",
+      "additionalProperties": {
+        "$ref": "#/definitions/sourceConfig"
+      }
+    }
+  },
+  "definitions": {
+    "sourceConfig": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["name", "description", "ingesterClass", "config"],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "description": {
+          "type": "string"
+        },
+        "ingesterClass": {
+          "type": "string"
+        },
+        "config": {
+          "$ref": "#/definitions/bookConfig"
+        }
+      }
+    },
+    "bookConfig": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "repoOwner",
+        "repoName",
+        "fileExtensions",
+        "chunkSize",
+        "chunkOverlap",
+        "baseUrl",
+        "urlSuffix",
+        "useUrlMapping"
+      ],
+      "properties": {
+        "repoOwner": {
+          "type": "string"
+        },
+        "repoName": {
+          "type": "string"
+        },
+        "fileExtensions": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "chunkSize": {
+          "type": "integer"
+        },
+        "chunkOverlap": {
+          "type": "integer"
+        },
+        "baseUrl": {
+          "type": "string"
+        },
+        "urlSuffix": {
+          "type": "string"
+        },
+        "useUrlMapping": {
+          "type": "boolean"
+        }
+      }
+    }
+  }
+}