feat(catalog): compact catalog index for large apps to shrink the cold prompt (#294) (#306)

generatePromptCatalog() switches to a compact index (id + title + one-line
summary + map layer ids) above catalog_index_threshold datasets (default 8),
instead of front-loading every dataset's full description + read_parquet paths
+ provider + about-url. Those all arrive via get_schema(dataset_id) — a call
the model already makes before every query — so the cold prompt shrinks with
zero added round-trips. Small apps keep the full front-load (byte cost is
negligible there).

Measured on the real geo-agent-template catalog (27 datasets loaded):
catalog block ~9.8k -> ~2.0k tokens (79.5% smaller). Headless A/B on nimbus
qwen, CD-16 question: turn-1 cold prompt 29.5k -> 20.3k tok (-31%), total
prefill 526k -> 140k tok (3.8x), 77s -> 40s (1.9x). The compact run navigated
correctly via list_datasets -> get_schema -> query, confirming that dropping
front-loaded paths is safe because get_schema supplies them.

Part 1 of #294 (compact catalog index). Part 2 (gate map-tool schemas to
relevant turns) is separate. Configurable via catalog_index_threshold.

Refs #294

Author: cboettig

app/dataset-catalog.js

@@ -635,10 +635,29 @@ export class DatasetCatalog {
     /**
      * Generate a text summary of all datasets for injection into the LLM system prompt.
      *
-     * Includes paths and map layer IDs — the "table of contents" for the app.
-     * Column schemas are NOT included here; the model calls get_schema for those.
+     * Two renderings (#294): a **full** front-load (title + Collection ID + full
+     * description + provider + `read_parquet` paths + map layers) for small apps,
+     * and a **compact index** (id + title + one-line summary + layer ids) above a
+     * dataset-count threshold. The compact form drops the description, paths,
+     * provider, and about-url — all of which arrive via `get_schema(dataset_id)`,
+     * a call the model already makes before every query — so a large catalog's
+     * ~34k-token cold prompt shrinks with zero added round-trips. Column schemas
+     * are never front-loaded in either mode (they come from get_schema).
+     *
+     * @param {Object} [options]
+     * @param {number} [options.compactAbove=8] - Use the compact index when the
+     *   catalog has more than this many datasets. Small apps stay on the full
+     *   front-load where the byte cost is negligible.
      */
-    generatePromptCatalog() {
+    generatePromptCatalog(options = {}) {
+        const compactAbove = options.compactAbove ?? 8;
+        return this.datasets.size > compactAbove
+            ? this._renderCompactCatalog()
+            : this._renderFullCatalog();
+    }
+
+    /** Full front-load: every dataset's description, paths, provider, map layers. @private */
+    _renderFullCatalog() {
         const preamble = 'The following datasets are pre-loaded for this app. Paths are shown below — use them directly in SQL. Call `get_schema(dataset_id)` before your first SQL query against a dataset to get column names and coded values.\n';
         const sections = [preamble];
 
@@ -694,6 +713,62 @@ export class DatasetCatalog {
         return sections.join('\n---\n\n');
     }
 
+    /**
+     * Compact index for large catalogs (#294): one line per dataset —
+     * `id — title — one-line summary` (+ map layer ids for map datasets). The
+     * full description, `read_parquet` paths, provider, and about-url are
+     * intentionally omitted; they arrive via `get_schema(dataset_id)`, which the
+     * model calls before querying. This reinforces the get-schema-first flow
+     * (AGENTS.md: never guess S3 paths) while cutting the cold-prompt bytes.
+     * @private
+     */
+    _renderCompactCatalog() {
+        const preamble = 'The datasets pre-loaded for this app are indexed below as `id` — title — summary. '
+            + 'Before querying a dataset, call `get_schema(dataset_id)`: it returns the `read_parquet()` path, '
+            + 'column names, types, coded values, and the full description. '
+            + 'Use `browse_stac_catalog` / `get_stac_details` only for datasets not in this list.\n';
+        const lines = [preamble];
+
+        for (const ds of this.datasets.values()) {
+            const isParentContainer = ds.columns.length === 0 && ds.childIds.length > 0;
+            if (isParentContainer) {
+                const listed = ds.childIds.slice(0, 20).join(', ');
+                const more = ds.childIds.length > 20
+                    ? ` (+${ds.childIds.length - 20} more via get_stac_details("${ds.id}"))`
+                    : '';
+                lines.push(`- \`${ds.id}\` — **${ds.title}** — container; sub-datasets: ${listed}${more}`);
+                continue;
+            }
+
+            const summary = this._oneLine(ds.description);
+            let line = `- \`${ds.id}\` — **${ds.title}**${summary ? ` — ${summary}` : ''}`;
+            if (ds.mapLayers.length > 0) {
+                const layers = ds.mapLayers
+                    .map(ml => `\`${ds.id}/${ml.assetId}\` (${ml.layerType})`)
+                    .join(', ');
+                line += `\n  map layers: ${layers}`;
+            }
+            lines.push(line);
+        }
+
+        return lines.join('\n');
+    }
+
+    /**
+     * A compact one-line stand-in for a dataset description: the first sentence
+     * if it's a reasonable length, else the collapsed text, capped. Full text is
+     * available via get_schema, so this only needs to help the model *select* a
+     * dataset. @private
+     */
+    _oneLine(text, cap = 160) {
+        if (!text) return '';
+        const collapsed = text.replace(/\s+/g, ' ').trim();
+        const sentence = collapsed.match(/^(.{12,}?[.!?])(\s|$)/);
+        let s = sentence ? sentence[1] : collapsed;
+        if (s.length > cap) s = s.slice(0, cap - 1).trimEnd() + '…';
+        return s;
+    }
+
     /**
      * Render SQL asset paths only (no columns) for the system prompt.
      * Uses the client-direct parquetAssets extracted during load().

app/main.js

@@ -360,7 +360,9 @@ async function main() {
 
     /* ── 6. Build system prompt ────────────────────────────────────────── */
     const basePrompt = await basePromptP;   // fetch was kicked off at step 1c
-    const catalogText = catalog.generatePromptCatalog();
+    // Large catalogs (#294) switch to a compact index to shrink the cold prompt;
+    // tune or disable the threshold with `catalog_index_threshold` (Infinity = always full).
+    const catalogText = catalog.generatePromptCatalog({ compactAbove: appConfig.catalog_index_threshold ?? 8 });
     let systemPrompt = basePrompt + '\n\n' + catalogText;
 
     // Read server-provided prompt (if any)

docs/guide/configuration.md

@@ -18,6 +18,7 @@ Client apps configure GLEN via `layers-input.json`. All fields except `catalog`
 | `auto_approve` | No | Start with remote tool calls auto-approved (no confirmation prompt). Default: `true`. |
 | `max_tool_calls` | No | Remote queries in auto-approve mode before the agent pauses at a checkpoint. Default: `15`. |
 | `max_tool_calls_manual` | No | Remote queries in manual mode before a checkpoint. Default: `100`. |
+| `catalog_index_threshold` | No | Dataset count above which the front-loaded catalog switches to a compact index (id + title + one-line summary + layer ids) to shrink the cold prompt; full descriptions/paths then arrive on demand via `get_schema`. Default: `8`. Set very high (e.g. `9999`) to always front-load the full catalog. |
 | `links` | No | Optional links shown in the chat UI — see below. |
 
 ## View

test/dataset-catalog.test.js

@@ -220,6 +220,82 @@ describe('DatasetCatalog.generatePromptCatalog', () => {
     });
 });
 
+describe('DatasetCatalog.generatePromptCatalog compact index (#294)', () => {
+    // Register `n` realistic leaf datasets (long description, paths, a map layer).
+    const bigCatalog = (n) => {
+        const cat = new DatasetCatalog();
+        for (let i = 0; i < n; i++) {
+            cat.datasets.set(`ds${i}`, {
+                id: `ds${i}`,
+                title: `Dataset ${i}`,
+                description: `Land cover classification for region ${i}. Derived from Sentinel-2 imagery `
+                    + `at 10m resolution, updated annually. Includes forest, cropland, wetland, and urban `
+                    + `classes with per-pixel confidence scores and a detailed methodology appendix.`,
+                provider: `Provider ${i}`,
+                columns: [{ name: 'h3', type: 'string' }],
+                childIds: [],
+                mapLayers: [{ assetId: 'pmtiles', title: `Layer ${i}`, layerType: 'vector' }],
+                parquetAssets: [{ title: `Parquet ${i}`, s3Path: `s3://bucket/ds${i}/data.parquet` }],
+                aboutUrl: `https://example.org/ds${i}`,
+            });
+        }
+        return cat;
+    };
+
+    it('stays full at/below the threshold and switches to compact above it', () => {
+        expect(bigCatalog(8).generatePromptCatalog()).toContain('**Collection ID:**'); // full
+        expect(bigCatalog(9).generatePromptCatalog()).not.toContain('**Collection ID:**'); // compact
+    });
+
+    it('compact index keeps id/title/summary/layer but drops description, paths, provider, about-url', () => {
+        const out = bigCatalog(9).generatePromptCatalog();
+        expect(out).toContain('`ds0`');
+        expect(out).toContain('**Dataset 0**');
+        expect(out).toContain('Land cover classification for region 0.'); // one-line summary (first sentence)
+        expect(out).toContain('`ds0/pmtiles` (vector)');                   // layer id + type for map tools
+        expect(out).toContain('get_schema(dataset_id)');                   // preamble points to the deferred call
+        // Dropped — all available via get_schema:
+        expect(out).not.toContain('s3://bucket'); // the read_parquet path itself
+        expect(out).not.toContain('**Provider:**');
+        expect(out).not.toContain('per-pixel confidence scores'); // full description tail is gone
+        expect(out).not.toContain('https://example.org');
+    });
+
+    it('honors an explicit compactAbove override', () => {
+        expect(bigCatalog(4).generatePromptCatalog({ compactAbove: 3 })).not.toContain('**Collection ID:**');
+        expect(bigCatalog(40).generatePromptCatalog({ compactAbove: Infinity })).toContain('**Collection ID:**');
+    });
+
+    it('renders parent containers compactly (directory line, no full description)', () => {
+        const cat = bigCatalog(9);
+        cat.datasets.set('parent', {
+            id: 'parent', title: 'Watersheds', description: 'A very long container description that should not appear.',
+            provider: 'L', columns: [], childIds: ['a', 'b', 'c'], mapLayers: [], parquetAssets: [],
+        });
+        const out = cat.generatePromptCatalog();
+        expect(out).toContain('`parent` — **Watersheds** — container; sub-datasets: a, b, c');
+        expect(out).not.toContain('should not appear');
+    });
+
+    it('is dramatically smaller than the full front-load for the same catalog', () => {
+        const cat = bigCatalog(40);
+        const full = cat.generatePromptCatalog({ compactAbove: Infinity });
+        const compact = cat.generatePromptCatalog({ compactAbove: 8 });
+        // The whole point of #294: a large catalog's front-load shrinks by a lot.
+        expect(compact.length).toBeLessThan(full.length * 0.4);
+    });
+
+    it('_oneLine picks the first sentence and caps long single sentences', () => {
+        const cat = new DatasetCatalog();
+        expect(cat._oneLine('First sentence here. Second sentence.')).toBe('First sentence here.');
+        expect(cat._oneLine('')).toBe('');
+        const long = 'x'.repeat(300);
+        const out = cat._oneLine(long);
+        expect(out.length).toBeLessThanOrEqual(160);
+        expect(out.endsWith('…')).toBe(true);
+    });
+});
+
 describe('DatasetCatalog.load (mocked fetch)', () => {
     let originalFetch;
     beforeEach(() => {