Skip to content

Commit 1ceecf0

Browse files
Benoit Aubuchonclaude
Benoit Aubuchon
and
claude
authored
feat: ClickHouse Dictionary SDK support (TS + Python + E2E) (#4022)
## Summary Recovery PR restoring commits from PRs #3887, #3888, #3889 that were accidentally dropped when a force-push rewound the integration branch before #3886 squash-merged to main. **What's included:** - **TypeScript SDK** (originally PR #3887): `OlapDictionary` class, `olapDictionary.ts`, `sqlHelpers.ts` integration, infra map round-trip tests - **Python SDK** (originally PR #3888): `olap_dictionary.py`, full serialization/deserialization, `dmv2_serializer` integration tests, `SecretStr` regression - **E2E tests + docs** (originally PR #3889): `dictionary.test.ts` E2E suite, `olap-dictionary.mdx` documentation, test templates for both TS and Python **Fix included:** - `mask_dict_credentials()` now correctly destructures `ExternalDictionarySourceWrapper` to reach `ExternalDictionarySource` — this was broken during conflict resolution when cherry-picking commits that pre-dated the wrapper refactor. ## Test plan - [ ] CI green on all jobs - [ ] `cargo check --workspace` passes locally (verified before push) - [ ] `cargo clippy --all-targets -- -D warnings` passes (no warnings) - [ ] Python SDK tests: `cd packages/py-moose-lib && pytest tests/test_olap_dictionary.py` - [ ] TS SDK tests: `cd packages/ts-moose-lib && pnpm test --grep "OlapDictionary"` - [ ] E2E test: `cd apps/framework-cli-e2e && pnpm test --grep "dictionary"` Closes the remaining work from the 6-PR dictionary stack. 🤖 Generated with [Claude Code](https://claude.com/claude-code) <!-- CURSOR_SUMMARY --> --- > [!NOTE] > **Medium Risk** > Adds a new cross-language `OlapDictionary` resource with new JSON/proto shapes and registry/serialization paths, which can affect planning/migration and infra drift detection if shapes diverge. Risk is mitigated by extensive unit, integration, and E2E coverage but still touches core infra map comparisons and credential masking. > > **Overview** > Adds a new first-class `OlapDictionary` resource across the stack: TypeScript SDK gains `OlapDictionary` (layouts, lifetime, external sources, SQL helper methods), registry/infra-map emission, and `sql` interpolation support; Python SDK adds the matching `olap_dictionary` module, registry accessors, and infra-map serialization (including external-source secret handling and lifetime/layout/column encoding). > > Updates the Rust CLI dictionary infrastructure model to wrap external sources in `ExternalDictionarySourceWrapper` to avoid serde tag collisions, and adjusts drift detection + credential masking code paths to match the new nested JSON shape. Rounds out the feature with new E2E coverage for dictionary creation and `moose plan --json` change detection, plus new ClickHouse dictionary documentation and navigation entry. > > <sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit 506ba8f. Bugbot is set up for automated code reviews on this repo. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup> <!-- /CURSOR_SUMMARY --> --------- Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent cb63c58 commit 1ceecf0

27 files changed

Lines changed: 6545 additions & 73 deletions

File tree

‎apps/framework-cli-e2e/test/dictionary.test.ts‎

Lines changed: 251 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,251 @@
1+
/// <reference types="node" />
2+
/// <reference types="mocha" />
3+
/// <reference types="chai" />
4+
/**
5+
* E2E tests for OlapDictionary functionality.
6+
*
7+
* Tests verify that:
8+
* - Dictionaries defined in TypeScript code appear in moose plan output
9+
* - Moose creates dictionaries in ClickHouse when running in prod mode
10+
*
11+
* Uses moose prod with Docker infrastructure and moose plan --json to
12+
* inspect what operations would be generated for dictionary changes.
13+
*/
14+
15+
import { spawn } from "child_process";
16+
import { expect } from "chai";
17+
import * as fs from "fs";
18+
import * as path from "path";
19+
import { promisify } from "util";
20+
import { createClient } from "@clickhouse/client";
21+
22+
import {
23+
TIMEOUTS,
24+
CLICKHOUSE_CONFIG,
25+
SERVER_CONFIG,
26+
TEST_ADMIN_BEARER_TOKEN,
27+
} from "./constants";
28+
29+
import {
30+
waitForServerStart,
31+
waitForInfrastructureReady,
32+
createTempTestDirectory,
33+
cleanupTestSuite,
34+
performGlobalCleanup,
35+
hasDictionaryAdded,
36+
runMoosePlanJson,
37+
} from "./utils";
38+
39+
const execAsync = promisify(require("child_process").exec);
40+
41+
const CLI_PATH = path.resolve(__dirname, "../../../target/debug/moose-cli");
42+
// Use the staged template (with shared files injected by package-templates.js pretest)
43+
const TEMPLATE_SOURCE_DIR = path.resolve(
44+
__dirname,
45+
"../../../template-packages/_staging_typescript-tests",
46+
);
47+
48+
const TEST_ENV = {
49+
...process.env,
50+
TEST_AWS_ACCESS_KEY_ID: "test-access-key",
51+
TEST_AWS_SECRET_ACCESS_KEY: "test-secret-key",
52+
MOOSE_DEV__SUPPRESS_DEV_SETUP_PROMPT: "true",
53+
MOOSE_ADMIN_TOKEN: TEST_ADMIN_BEARER_TOKEN,
54+
};
55+
56+
/**
57+
* Sets up a fresh isolated test environment for a single dictionary test.
58+
* Each test gets its own temp directory, moose prod process, and Docker containers.
59+
*/
60+
async function setupTestEnvironment(testName: string) {
61+
const uniqueName = `ts-dict-${testName
62+
.replace(/[^a-z0-9-]/gi, "-")
63+
.toLowerCase()
64+
.slice(0, 28)}`;
65+
const testProjectDir = createTempTestDirectory(uniqueName);
66+
const projectName = path.basename(testProjectDir).toLowerCase();
67+
68+
console.log(`\n=== Setting up isolated environment for: ${testName} ===`);
69+
console.log(`Project name: ${projectName}`);
70+
console.log(`Test directory: ${testProjectDir}`);
71+
72+
// Copy template to temp directory
73+
console.log("Copying typescript-tests template...");
74+
fs.cpSync(TEMPLATE_SOURCE_DIR, testProjectDir, { recursive: true });
75+
console.log("✓ Template copied");
76+
77+
// Update package.json name to ensure unique Docker project name
78+
const packageJsonPath = path.join(testProjectDir, "package.json");
79+
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf-8"));
80+
packageJson.name = projectName;
81+
fs.writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2));
82+
console.log(`✓ Updated package.json name to: ${projectName}`);
83+
84+
// Install dependencies
85+
console.log("Installing dependencies...");
86+
await execAsync("npm install", { cwd: testProjectDir });
87+
console.log("✓ Dependencies installed");
88+
89+
// Start moose prod with Docker infrastructure
90+
console.log("Starting moose prod with Docker infrastructure...");
91+
const mooseProcess = spawn(
92+
CLI_PATH,
93+
["prod", "--start-include-dependencies"],
94+
{
95+
stdio: "pipe",
96+
cwd: testProjectDir,
97+
env: TEST_ENV,
98+
},
99+
);
100+
101+
await waitForServerStart(
102+
mooseProcess,
103+
TIMEOUTS.SERVER_STARTUP_MS,
104+
"production mode",
105+
SERVER_CONFIG.url,
106+
);
107+
console.log("✓ Moose prod started");
108+
109+
// Wait for infrastructure to be fully ready
110+
await new Promise((resolve) => setTimeout(resolve, 5000));
111+
await waitForInfrastructureReady(TIMEOUTS.SERVER_STARTUP_MS);
112+
console.log("✓ Infrastructure ready");
113+
114+
const client = createClient(CLICKHOUSE_CONFIG);
115+
116+
console.log(`=== Environment ready for: ${testName} ===\n`);
117+
118+
const cleanup = async () => {
119+
console.log(`\n=== Cleaning up environment for: ${testName} ===`);
120+
if (client) {
121+
await client.close();
122+
console.log(`✓ ClickHouse client closed for: ${testName}`);
123+
}
124+
await cleanupTestSuite(mooseProcess, testProjectDir, projectName, {
125+
logPrefix: testName,
126+
});
127+
console.log(`✓ Cleanup complete for: ${testName}\n`);
128+
};
129+
130+
return { mooseProcess, testProjectDir, client, cleanup };
131+
}
132+
133+
// Global setup - clean Docker state from previous runs
134+
before(async function () {
135+
this.timeout(TIMEOUTS.GLOBAL_CLEANUP_MS);
136+
console.log(
137+
"Running global setup for dictionary tests - cleaning Docker state from previous runs...",
138+
);
139+
await performGlobalCleanup();
140+
});
141+
142+
describe("OlapDictionary Tests", function () {
143+
before(async function () {
144+
console.log("\n=== Dictionary Tests - Starting ===");
145+
console.log("Each test will run in its own isolated environment\n");
146+
});
147+
148+
describe("dictionary created by moose prod", function () {
149+
it("should create the dictionary in ClickHouse when moose prod starts", async function () {
150+
this.timeout(TIMEOUTS.TEST_SETUP_MS + TIMEOUTS.MIGRATION_MS);
151+
152+
const { client, cleanup } = await setupTestEnvironment("dict-created");
153+
154+
try {
155+
console.log(
156+
"\n--- Verifying 'dict_index_test_lookup' exists in ClickHouse ---",
157+
);
158+
159+
// The template pre-includes dictionaryTests.ts which defines
160+
// dict_index_test_lookup. Moose prod auto-migrates it on startup.
161+
const result = await client.query({
162+
query: `
163+
SELECT count() AS cnt
164+
FROM system.dictionaries
165+
WHERE database = '${CLICKHOUSE_CONFIG.database}' AND name = 'dict_index_test_lookup'
166+
`,
167+
format: "JSONEachRow",
168+
});
169+
const rows = await result.json<{ cnt: string }>();
170+
expect(rows[0].cnt).to.equal("1");
171+
172+
console.log(
173+
"✓ Dictionary 'dict_index_test_lookup' exists in ClickHouse",
174+
);
175+
} finally {
176+
await cleanup();
177+
}
178+
});
179+
});
180+
181+
describe("dictionary plan generation", function () {
182+
it("should generate an OlapDictionary Added entry when a new dictionary is defined", async function () {
183+
this.timeout(TIMEOUTS.TEST_SETUP_MS + TIMEOUTS.MIGRATION_MS);
184+
185+
const { testProjectDir, cleanup } =
186+
await setupTestEnvironment("dict-plan-added");
187+
188+
try {
189+
console.log(
190+
"\n--- Testing plan shows Added for a new OlapDictionary ---",
191+
);
192+
193+
// Write a new dictionary file that is NOT pre-included in the template.
194+
// This dictionary therefore does not exist in ClickHouse yet, so
195+
// moose plan should report it as Added.
196+
const dictFilePath = path.join(
197+
testProjectDir,
198+
"src",
199+
"views",
200+
"newDictTest.ts",
201+
);
202+
fs.writeFileSync(
203+
dictFilePath,
204+
`
205+
import { OlapDictionary, UInt64 } from "@514labs/moose-lib";
206+
import { IndexTestTable } from "../ingest/models";
207+
208+
interface IndexTestLookup2 {
209+
u64: UInt64;
210+
i32: number;
211+
s: string;
212+
}
213+
214+
export const newIndexTestLookupDict = new OlapDictionary<IndexTestLookup2>(
215+
"dict_new_index_test_lookup",
216+
{
217+
sourceTable: IndexTestTable,
218+
primaryKey: ["u64"],
219+
layout: { type: "HASHED" },
220+
lifetime: 7200,
221+
},
222+
);
223+
`.trim(),
224+
);
225+
226+
console.log("✓ Added new dictionary file 'newDictTest.ts'");
227+
228+
// Export the new file from index.ts so moose can discover it.
229+
// Moose uses index.ts as the entry point for TypeScript resource discovery;
230+
// files not transitively reachable from it are invisible to the planner.
231+
const indexPath = path.join(testProjectDir, "src", "index.ts");
232+
fs.appendFileSync(
233+
indexPath,
234+
'\nexport * from "./views/newDictTest";\n',
235+
);
236+
console.log("✓ Exported 'newDictTest' from src/index.ts");
237+
238+
const plan = await runMoosePlanJson(testProjectDir);
239+
240+
const hasDict = hasDictionaryAdded(plan, "dict_new_index_test_lookup");
241+
expect(hasDict).to.be.true;
242+
243+
console.log(
244+
"✓ Plan contains OlapDictionary.Added for 'dict_new_index_test_lookup'",
245+
);
246+
} finally {
247+
await cleanup();
248+
}
249+
});
250+
});
251+
});

‎apps/framework-cli-e2e/test/utils/plan-utils.ts‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,54 @@ export function hasMvUpdated(plan: PlanOutput, mvName: string): boolean {
159159
});
160160
}
161161

162+
/**
163+
* Check if a dictionary was added (Created)
164+
*/
165+
export function hasDictionaryAdded(
166+
plan: PlanOutput,
167+
dictName: string,
168+
): boolean {
169+
if (!plan.changes?.olap_changes) return false;
170+
return plan.changes.olap_changes.some((change) => {
171+
const dictChange = change.OlapDictionary;
172+
if (!dictChange?.Added) return false;
173+
return dictChange.Added.name === dictName;
174+
});
175+
}
176+
177+
/**
178+
* Check if a dictionary was removed (Dropped)
179+
*/
180+
export function hasDictionaryRemoved(
181+
plan: PlanOutput,
182+
dictName: string,
183+
): boolean {
184+
if (!plan.changes?.olap_changes) return false;
185+
return plan.changes.olap_changes.some((change) => {
186+
const dictChange = change.OlapDictionary;
187+
if (!dictChange?.Removed) return false;
188+
return dictChange.Removed.name === dictName;
189+
});
190+
}
191+
192+
/**
193+
* Check if a dictionary was updated (layout, lifetime, source, or attribute change)
194+
*/
195+
export function hasDictionaryUpdated(
196+
plan: PlanOutput,
197+
dictName: string,
198+
): boolean {
199+
if (!plan.changes?.olap_changes) return false;
200+
return plan.changes.olap_changes.some((change) => {
201+
const dictChange = change.OlapDictionary;
202+
if (!dictChange?.Updated) return false;
203+
return (
204+
dictChange.Updated.before?.name === dictName ||
205+
dictChange.Updated.after?.name === dictName
206+
);
207+
});
208+
}
209+
162210
/**
163211
* Get all table changes for a specific table
164212
* Compares by table ID (includes database) for unambiguous identification

‎apps/framework-cli/src/cli/routines/migrate.rs‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ fn load_migration_files(db_name: &str) -> Result<MigrationFiles> {
116116
/// username change produces visible drift.
117117
fn strip_dict_metadata(dicts: &HashMap<String, OlapDictionary>) -> HashMap<String, OlapDictionary> {
118118
use crate::infrastructure::olap::clickhouse::dictionary::{
119-
DictionarySource, ExternalDictionarySource,
119+
DictionarySource, ExternalDictionarySource, ExternalDictionarySourceWrapper,
120120
};
121121
use crate::utilities::secrets::CREDENTIAL_PLACEHOLDER;
122122

@@ -130,8 +130,11 @@ fn strip_dict_metadata(dicts: &HashMap<String, OlapDictionary>) -> HashMap<Strin
130130
// (password = real value). Usernames are stored in plain-text in the JSON
131131
// by mask_credentials_for_json_export, so they must not be normalized here —
132132
// a username change must surface as drift.
133-
if let DictionarySource::External(ref mut ext) = dict.source {
134-
match ext {
133+
if let DictionarySource::External(ExternalDictionarySourceWrapper {
134+
ref mut external_source,
135+
}) = dict.source
136+
{
137+
match external_source {
135138
ExternalDictionarySource::ClickHouse(s) => {
136139
s.password = CREDENTIAL_PLACEHOLDER.to_string();
137140
}
@@ -2214,10 +2217,11 @@ mod tests {
22142217
fn make_external_ch_dict(name: &str, user: &str, password: &str) -> OlapDictionary {
22152218
use crate::infrastructure::olap::clickhouse::dictionary::{
22162219
DictionaryClickHouseSource, DictionarySource, ExternalDictionarySource,
2220+
ExternalDictionarySourceWrapper,
22172221
};
22182222
let mut dict = create_test_dict(name);
2219-
dict.source = DictionarySource::External(ExternalDictionarySource::ClickHouse(
2220-
DictionaryClickHouseSource {
2223+
dict.source = DictionarySource::External(ExternalDictionarySourceWrapper {
2224+
external_source: ExternalDictionarySource::ClickHouse(DictionaryClickHouseSource {
22212225
host: "remotehost".to_string(),
22222226
port: 9000,
22232227
user: user.to_string(),
@@ -2227,8 +2231,8 @@ mod tests {
22272231
query: None,
22282232
where_clause: None,
22292233
invalidate_query: None,
2230-
},
2231-
));
2234+
}),
2235+
});
22322236
dict
22332237
}
22342238

0 commit comments

Comments
 (0)