feat(ingesters): add cairo skills config

feat(python): add cairo skills source metadata types

Add CAIRO_SKILLS query resource description

feat: add async unique-id fetch helper for retriever

docs: add research context for Step 4 SkillsIngester implementation

Captures codebase patterns, type definitions, frontmatter parsing approaches,
GitHub API call patterns, and open questions (fullContent field gap, class name
discrepancy) for the cairo_skills SkillsIngester.

feat(ingesters): add cairo skills github ingester and chunks

ingesters: guard cairo skill markdown traversal depth

ingesters: document frontmatter parser scope

ingesters: fetch cairo skill files in parallel batches

ingesters: remove unused skills barrel export

ingesters: expose skills to subclasses for test setup

ingesters: align cairo skills config owner and repo

docs: add context document for CairoSkillsIngester TypeScript tests

Summarizes all 11 RFC test cases, their coverage status (all passing),
test patterns used (Bun test runner, vi mocking, TestCairoSkillsIngester
subclass pattern), and key type references for implementation.

feat(ingesters): register SkillsIngester for cairo_skills

feat(ingesters): register SkillsIngester for cairo_skills

feat(rag): expand cairo skill chunks to full documents

docs: add context document for integration tests (cairo skills expansion)

Captures fixture structure, data-flow trace, mock setup patterns, and
per-test implementation guides for the 3 RFC integration test cases.

test: add cairo skills integration coverage

chore: clean up review findings before merge

- Remove trivial enum identity assertion from types.test.ts
- Make skillsConfig.test.ts resilient to config changes (no hardcoded
  length/IDs, validate structure only)
- Remove low-value resource description substring test
- Move json import to module level in rag_pipeline.py
- Remove agent planning doc artifacts from branch

unslop

Author: enitrat

bun.lock

@@ -14,21 +14,12 @@ type SkillsConfigFile = {
 const skillsConfigPath = join(import.meta.dir, '..', 'config', 'skills.json');
 
 describe('skills config', () => {
-  it('matches the RFC Step 3 contract', () => {
+  it('should contain at least one skill with valid structure', () => {
     const raw = readFileSync(skillsConfigPath, 'utf8');
     const parsed = JSON.parse(raw) as SkillsConfigFile;
 
     expect(Array.isArray(parsed.skills)).toBe(true);
-    expect(parsed.skills).toHaveLength(4);
-
-    const expectedIds = [
-      'benchmarking-cairo',
-      'cairo-coding',
-      'avnu',
-      'starknet-defi',
-    ];
-
-    expect(parsed.skills.map((skill) => skill.id)).toEqual(expectedIds);
+    expect(parsed.skills.length).toBeGreaterThan(0);
 
     for (const skill of parsed.skills) {
       expect(typeof skill.id).toBe('string');
@@ -39,23 +30,22 @@ describe('skills config', () => {
       expect(url.protocol).toBe('https:');
       expect(url.hostname).toBe('github.com');
     }
+  });
 
-    const benchmarkingCairoUrl = parsed.skills.find(
-      (skill) => skill.id === 'benchmarking-cairo',
-    )?.url;
-    expect(benchmarkingCairoUrl).toBeDefined();
-    expect(benchmarkingCairoUrl as string).toContain('/tree/');
-
-    const cairoCodingUrl = parsed.skills.find(
-      (skill) => skill.id === 'cairo-coding',
-    )?.url;
-    expect(cairoCodingUrl).toBeDefined();
-    expect(cairoCodingUrl as string).toContain('/tree/');
-
-    const starknetDefiUrl = parsed.skills.find(
-      (skill) => skill.id === 'starknet-defi',
-    )?.url;
-    expect(starknetDefiUrl).toBeDefined();
-    expect(starknetDefiUrl as string).toMatch(/\/blob\/[0-9a-f]{40}\//);
+  it('should have unique skill ids', () => {
+    const raw = readFileSync(skillsConfigPath, 'utf8');
+    const parsed = JSON.parse(raw) as SkillsConfigFile;
+
+    const ids = parsed.skills.map((skill) => skill.id);
+    expect(new Set(ids).size).toBe(ids.length);
+  });
+
+  it('should use /tree/ or /blob/ GitHub URL formats', () => {
+    const raw = readFileSync(skillsConfigPath, 'utf8');
+    const parsed = JSON.parse(raw) as SkillsConfigFile;
+
+    for (const skill of parsed.skills) {
+      expect(skill.url).toMatch(/\/(tree|blob)\//);
+    }
   });
 });

ingesters/__tests__/skillsConfig.test.ts

@@ -2,10 +2,6 @@ import { describe, expect, it } from 'bun:test';
 import { DocumentSource, type BookChunk } from '../src/types';
 
 describe('types', () => {
-  it('exposes cairo_skills document source', () => {
-    expect(String(DocumentSource.CAIRO_SKILLS)).toBe('cairo_skills');
-  });
-
   it('supports optional skillId on BookChunk', () => {
     const withoutSkillId: BookChunk = {
       name: 'example',

ingesters/__tests__/types.test.ts

@@ -150,6 +150,21 @@
         "urlSuffix": "",
         "useUrlMapping": true
       }
+    },
+    "cairo_skills": {
+      "name": "Cairo Skills",
+      "description": "Curated Cairo ecosystem skills for all-or-nothing retrieval",
+      "ingesterClass": "SkillsIngester",
+      "config": {
+        "repoOwner": "",
+        "repoName": "",
+        "fileExtensions": [".md"],
+        "chunkSize": 4096,
+        "chunkOverlap": 512,
+        "baseUrl": "",
+        "urlSuffix": "",
+        "useUrlMapping": false
+      }
     }
   }
 }

ingesters/config/sources.json

@@ -10,7 +10,7 @@ import { ScarbDocsIngester } from './ingesters/ScarbDocsIngester';
 import { StarknetJSIngester } from './ingesters/StarknetJSIngester';
 import { StarknetBlogIngester } from './ingesters/StarknetBlogIngester';
 import { DojoDocsIngester } from './ingesters/DojoDocsIngester';
-import { CairoSkillsIngester } from './ingesters/CairoSkillsIngester';
+import { SkillsIngester } from './ingesters/SkillsIngester';
 import {
   getAvailableSourcesFromConfig,
   getSourceConfig,
@@ -34,7 +34,7 @@ const INGESTER_CLASSES: Record<string, new () => BaseIngester> = {
   StarknetJSIngester,
   StarknetBlogIngester,
   DojoDocsIngester,
-  CairoSkillsIngester,
+  SkillsIngester,
 };
 
 /**

ingesters/src/IngesterFactory.ts

@@ -0,0 +1,31 @@
+import { IngesterFactory } from '../IngesterFactory';
+import { DocumentSource } from '../types';
+import { SkillsIngester } from '../ingesters/SkillsIngester';
+import { getSourceConfig } from '../utils/sourceConfig';
+
+describe('IngesterFactory cairo_skills wiring', () => {
+  it('loads cairo_skills source config with SkillsIngester metadata', () => {
+    const sourceConfig = getSourceConfig(DocumentSource.CAIRO_SKILLS);
+
+    expect(sourceConfig.name).toBe('Cairo Skills');
+    expect(sourceConfig.ingesterClass).toBe('SkillsIngester');
+    expect(sourceConfig.config).toEqual({
+      repoOwner: '',
+      repoName: '',
+      fileExtensions: ['.md'],
+      chunkSize: 4096,
+      chunkOverlap: 512,
+      baseUrl: '',
+      urlSuffix: '',
+      useUrlMapping: false,
+    });
+  });
+
+  it('creates a SkillsIngester for cairo_skills', () => {
+    const ingester = IngesterFactory.createIngester(
+      DocumentSource.CAIRO_SKILLS,
+    );
+
+    expect(ingester).toBeInstanceOf(SkillsIngester);
+  });
+});

ingesters/src/__tests__/IngesterFactory.skills.test.ts

@@ -0,0 +1 @@
+export { CairoSkillsIngester as SkillsIngester } from './CairoSkillsIngester';

ingesters/src/ingesters/SkillsIngester.ts

@@ -7,8 +7,7 @@
   "dependencies": {
     "@ai-sdk/anthropic": "^3.0.44",
     "ai": "^6.0.86",
-    "smithers-orchestrator": "^0.6.0",
-    "takopi-smithers": "github:evmts/takopi-smithers",
+    "smithers-orchestrator": "^0.9.0",
     "zod": "^4.3.6"
   },
   "patchedDependencies": {

package.json

@@ -7,6 +7,7 @@
 
 import asyncio
 import contextlib
+import json
 import os
 from collections.abc import AsyncGenerator
 from dataclasses import dataclass
@@ -139,6 +140,8 @@ async def _aprocess_query_and_retrieve_docs(
             )
             # documents already contains all retrieved docs, no action needed
 
+        documents = await self._expand_skill_documents(documents)
+
         # Ensure Grok summary is present and first in order (for generation context)
         if grok_summary_doc is not None:
             if grok_summary_doc in documents:
@@ -150,6 +153,84 @@ async def _aprocess_query_and_retrieve_docs(
 
         return processed_query, documents, grok_citations
 
+    async def _expand_skill_documents(self, documents: list[Document]) -> list[Document]:
+        """
+        Replace skill chunks with full skill documents when available.
+
+        If a full document row cannot be fetched for a skill, keep that skill's
+        original chunks to degrade gracefully.
+        """
+        skill_chunks = [
+            document
+            for document in documents
+            if document.metadata.get("source") == DocumentSource.CAIRO_SKILLS
+            and document.metadata.get("skillId")
+        ]
+        if not skill_chunks:
+            return documents
+
+        skill_ids = list(dict.fromkeys(doc.metadata["skillId"] for doc in skill_chunks))
+        unique_ids = [f"skill-{skill_id}-full" for skill_id in skill_ids]
+
+        try:
+            rows = await self.document_retriever.vector_db.afetch_by_unique_ids(unique_ids)
+        except Exception as e:
+            logger.warning(
+                "_expand_skill_documents: failed to fetch full rows, keeping original chunks",
+                error=str(e),
+                exc_info=True,
+            )
+            return documents
+
+        full_documents_by_skill_id: dict[str, Document] = {}
+        for row in rows:
+            metadata: Any = row.get("metadata", {})
+            if isinstance(metadata, str):
+                try:
+                    metadata = json.loads(metadata)
+                except Exception:
+                    logger.warning(
+                        "_expand_skill_documents: unable to decode metadata json, skipping row"
+                    )
+                    continue
+
+            if not isinstance(metadata, dict):
+                continue
+
+            skill_id = metadata.get("skillId")
+            full_content = metadata.get("fullContent")
+            if skill_id and full_content:
+                full_documents_by_skill_id[skill_id] = Document(
+                    page_content=full_content,
+                    metadata=metadata,
+                )
+
+        result_documents = [
+            document
+            for document in documents
+            if document.metadata.get("source") != DocumentSource.CAIRO_SKILLS
+        ]
+
+        found_skill_ids = set(full_documents_by_skill_id)
+        for skill_id in skill_ids:
+            if skill_id not in found_skill_ids:
+                original_chunks = [
+                    document
+                    for document in skill_chunks
+                    if document.metadata.get("skillId") == skill_id
+                ]
+                result_documents.extend(original_chunks)
+                logger.warning(
+                    "_expand_skill_documents: no full document found, keeping chunks",
+                    skill_id=skill_id,
+                )
+
+        for skill_id in skill_ids:
+            if skill_id in full_documents_by_skill_id:
+                result_documents.append(full_documents_by_skill_id[skill_id])
+
+        return result_documents
+
     @traceable(name="RagPipeline", run_type="chain")
     async def aforward(
         self,