feat: instrument telemetry for create command (CLI + TUI) (#1202)

Hweinstock · web-flow · commit bcedddec962b · 2026-05-11T17:40:38.000-04:00
Add telemetry recording to the create command in both CLI and TUI paths:
- CLI: wrap handleCreateCLI with runCliCommand to emit CreateAttrs on success/failure
- TUI: wrap useCreateFlow's run() with withCommandRunTelemetry
- Add telemetry assertions to existing integration tests (frameworks + edge cases)
diff --git a/integ-tests/create-edge-cases.test.ts b/integ-tests/create-edge-cases.test.ts
@@ -1,5 +1,6 @@
 /* eslint-disable security/detect-non-literal-fs-filename */
 import { exists, prereqs, runCLI } from '../src/test-utils/index.js';
+import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
 import { randomUUID } from 'node:crypto';
 import { mkdir, rm } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
@@ -9,18 +10,21 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest';
 describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create edge cases', () => {
   let testDir: string;
 
+  const telemetry = createTelemetryHelper();
+
   beforeAll(async () => {
     testDir = join(tmpdir(), `agentcore-integ-edge-${randomUUID()}`);
     await mkdir(testDir, { recursive: true });
   });
 
   afterAll(async () => {
+    telemetry.destroy();
     await rm(testDir, { recursive: true, force: true });
   });
 
   describe('reserved names', () => {
     it('rejects reserved name "Test"', async () => {
-      const result = await runCLI(['create', '--name', 'Test', '--json'], testDir);
+      const result = await runCLI(['create', '--name', 'Test', '--json'], testDir, { env: telemetry.env });
 
       expect(result.exitCode).toBe(1);
       const json = JSON.parse(result.stdout);
@@ -30,6 +34,11 @@ describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create edge cases',
         json.error.toLowerCase().includes('reserved') || json.error.toLowerCase().includes('conflict'),
         `Error should mention reserved/conflict: ${json.error}`
       ).toBeTruthy();
+
+      telemetry.assertMetricEmitted({
+        command: 'create',
+        exit_reason: 'failure',
+      });
     });
 
     it('rejects reserved name "bedrock"', async () => {
@@ -121,12 +130,21 @@ describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create edge cases',
   describe('flag interactions', () => {
     it('--defaults creates project with default settings', async () => {
       const name = `Def${Date.now().toString().slice(-6)}`;
-      const result = await runCLI(['create', '--name', name, '--defaults', '--json'], testDir);
+      const result = await runCLI(['create', '--name', name, '--defaults', '--json'], testDir, { env: telemetry.env });
 
       expect(result.exitCode, `stderr: ${result.stderr}`).toBe(0);
       const json = JSON.parse(result.stdout);
       expect(json.success).toBe(true);
       expect(json.projectPath).toBeTruthy();
+
+      telemetry.assertMetricEmitted({
+        command: 'create',
+        exit_reason: 'success',
+        language: 'python',
+        framework: 'strands',
+        model_provider: 'bedrock',
+        has_agent: 'true',
+      });
     });
 
     it('--dry-run shows what would be created without writing files', async () => {
diff --git a/integ-tests/create-frameworks.test.ts b/integ-tests/create-frameworks.test.ts
@@ -1,4 +1,5 @@
 import { exists, prereqs, readProjectConfig, runCLI } from '../src/test-utils/index.js';
+import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
 import { randomUUID } from 'node:crypto';
 import { mkdir, readFile, rm } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
@@ -8,12 +9,15 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest';
 describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create with different frameworks', () => {
   let testDir: string;
 
+  const telemetry = createTelemetryHelper();
+
   beforeAll(async () => {
     testDir = join(tmpdir(), `agentcore-integ-frameworks-${randomUUID()}`);
     await mkdir(testDir, { recursive: true });
   });
 
   afterAll(async () => {
+    telemetry.destroy();
     await rm(testDir, { recursive: true, force: true });
   });
 
@@ -34,7 +38,8 @@ describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create with differen
         'none',
         '--json',
       ],
-      testDir
+      testDir,
+      { env: telemetry.env }
     );
 
     expect(result.exitCode, `stderr: ${result.stderr}`).toBe(0);
@@ -60,6 +65,15 @@ describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create with differen
     expect(agents).toBeDefined();
     expect(agents.length).toBe(1);
     expect(agents[0]!.name).toBe(agentName);
+
+    telemetry.assertMetricEmitted({
+      command: 'create',
+      exit_reason: 'success',
+      language: 'python',
+      framework: 'langchain_langgraph',
+      model_provider: 'bedrock',
+      has_agent: 'true',
+    });
   });
 
   it('creates GoogleADK project with Gemini provider', async () => {
diff --git a/src/cli/commands/create/command.tsx b/src/cli/commands/create/command.tsx
@@ -9,6 +9,18 @@ import type {
 } from '../../../schema';
 import { LIFECYCLE_TIMEOUT_MAX, LIFECYCLE_TIMEOUT_MIN } from '../../../schema';
 import { getErrorMessage } from '../../errors';
+import { runCliCommand } from '../../telemetry/cli-command-run.js';
+import {
+  AgentType,
+  Build,
+  Framework,
+  Language,
+  Memory,
+  ModelProvider as ModelProviderEnum,
+  NetworkMode as NetworkModeEnum,
+  Protocol,
+  standardize,
+} from '../../telemetry/schemas/common-shapes.js';
 import { COMMAND_DESCRIPTIONS } from '../../tui/copy';
 import { requireTTY } from '../../tui/guards';
 import { CreateScreen } from '../../tui/screens/create';
@@ -79,18 +91,17 @@ async function handleCreateCLI(options: CreateOptions): Promise<void> {
   const name = options.name ?? options.projectName;
   const projectName = options.projectName ?? name;
 
-  const validation = validateCreateOptions(options, cwd);
-  if (!validation.valid) {
-    if (options.json) {
-      console.log(JSON.stringify({ success: false, error: validation.error }));
-    } else {
-      console.error(validation.error);
-    }
-    process.exit(1);
-  }
-
-  // Handle dry-run mode
+  // Handle dry-run mode (no telemetry for dry-run)
   if (options.dryRun) {
+    const validation = validateCreateOptions(options, cwd);
+    if (!validation.valid) {
+      if (options.json) {
+        console.log(JSON.stringify({ success: false, error: validation.error }));
+      } else {
+        console.error(validation.error);
+      }
+      process.exit(1);
+    }
     const result = getDryRunInfo({ name: name!, projectName, cwd, language: options.language });
     if (options.json) {
       console.log(JSON.stringify(result));
@@ -103,74 +114,92 @@ async function handleCreateCLI(options: CreateOptions): Promise<void> {
     process.exit(0);
   }
 
-  const green = '\x1b[32m';
-  const reset = '\x1b[0m';
+  await runCliCommand('create', !!options.json, async () => {
+    const validation = validateCreateOptions(options, cwd);
+    if (!validation.valid) {
+      throw new Error(validation.error);
+    }
+    const green = '\x1b[32m';
+    const reset = '\x1b[0m';
 
-  // Progress callback for real-time output
-  const onProgress: ProgressCallback | undefined = options.json
-    ? undefined
-    : (step, status) => {
-        if (status === 'done') {
-          console.log(`${green}[done]${reset}  ${step}`);
-        } else if (status === 'error') {
-          console.log(`\x1b[31m[error]${reset} ${step}`);
-        }
-        // 'start' is silent - we only show when done
-      };
+    // Progress callback for real-time output
+    const onProgress: ProgressCallback | undefined = options.json
+      ? undefined
+      : (step, status) => {
+          if (status === 'done') {
+            console.log(`${green}[done]${reset}  ${step}`);
+          } else if (status === 'error') {
+            console.log(`\x1b[31m[error]${reset} ${step}`);
+          }
+          // 'start' is silent - we only show when done
+        };
 
-  // Commander.js --no-agent sets agent=false, not noAgent=true
-  const skipAgent = options.agent === false;
+    // Commander.js --no-agent sets agent=false, not noAgent=true
+    const skipAgent = options.agent === false;
 
-  const result = skipAgent
-    ? await createProject({
-        name: projectName!,
-        cwd,
-        skipGit: options.skipGit,
-        skipInstall: options.skipInstall,
-        onProgress,
-      })
-    : await createProjectWithAgent({
-        name: name!,
-        projectName,
-        cwd,
-        type: options.type as 'create' | 'import' | undefined,
-        buildType: (options.build as BuildType) ?? 'CodeZip',
-        language: (options.language as TargetLanguage) ?? (options.type === 'import' ? 'Python' : undefined),
-        framework: options.framework as SDKFramework | undefined,
-        modelProvider: options.modelProvider as ModelProvider | undefined,
-        apiKey: options.apiKey,
-        memory: (options.memory as 'none' | 'shortTerm' | 'longAndShortTerm') ?? 'none',
-        protocol: options.protocol as ProtocolMode | undefined,
-        agentId: options.agentId,
-        agentAliasId: options.agentAliasId,
-        region: options.region,
-        networkMode: options.networkMode as NetworkMode | undefined,
-        subnets: parseCommaSeparatedList(options.subnets),
-        securityGroups: parseCommaSeparatedList(options.securityGroups),
-        idleTimeout: options.idleTimeout ? Number(options.idleTimeout) : undefined,
-        maxLifetime: options.maxLifetime ? Number(options.maxLifetime) : undefined,
-        sessionStorageMountPath: options.sessionStorageMountPath,
-        withConfigBundle: options.withConfigBundle,
-        skipGit: options.skipGit,
-        skipInstall: options.skipInstall,
-        skipPythonSetup: options.skipPythonSetup,
-        onProgress,
-      });
+    const result = skipAgent
+      ? await createProject({
+          name: projectName!,
+          cwd,
+          skipGit: options.skipGit,
+          skipInstall: options.skipInstall,
+          onProgress,
+        })
+      : await createProjectWithAgent({
+          name: name!,
+          projectName,
+          cwd,
+          type: options.type as 'create' | 'import' | undefined,
+          buildType: (options.build as BuildType) ?? 'CodeZip',
+          language: (options.language as TargetLanguage) ?? (options.type === 'import' ? 'Python' : undefined),
+          framework: options.framework as SDKFramework | undefined,
+          modelProvider: options.modelProvider as ModelProvider | undefined,
+          apiKey: options.apiKey,
+          memory: (options.memory as 'none' | 'shortTerm' | 'longAndShortTerm') ?? 'none',
+          protocol: options.protocol as ProtocolMode | undefined,
+          agentId: options.agentId,
+          agentAliasId: options.agentAliasId,
+          region: options.region,
+          networkMode: options.networkMode as NetworkMode | undefined,
+          subnets: parseCommaSeparatedList(options.subnets),
+          securityGroups: parseCommaSeparatedList(options.securityGroups),
+          idleTimeout: options.idleTimeout ? Number(options.idleTimeout) : undefined,
+          maxLifetime: options.maxLifetime ? Number(options.maxLifetime) : undefined,
+          sessionStorageMountPath: options.sessionStorageMountPath,
+          withConfigBundle: options.withConfigBundle,
+          skipGit: options.skipGit,
+          skipInstall: options.skipInstall,
+          skipPythonSetup: options.skipPythonSetup,
+          onProgress,
+        });
 
-  if (options.json) {
-    console.log(JSON.stringify(result));
-  } else if (result.success) {
-    printCreateSummary(projectName!, result.agentName, options.language, options.framework);
-    if (options.skipInstall) {
-      console.log(
-        "\nDependency installation was skipped. Run 'npm install' in agentcore/cdk/ and 'uv sync' in your agent directory manually."
-      );
+    if (!result.success) {
+      throw new Error(result.error);
+    }
+
+    if (options.json) {
+      console.log(JSON.stringify(result));
+    } else {
+      printCreateSummary(projectName!, result.agentName, options.language, options.framework);
+      if (options.skipInstall) {
+        console.log(
+          "\nDependency installation was skipped. Run 'npm install' in agentcore/cdk/ and 'uv sync' in your agent directory manually."
+        );
+      }
     }
-  } else {
-    console.error(result.error);
-  }
 
-  process.exit(result.success ? 0 : 1);
+    return {
+      language: standardize(Language, options.language),
+      framework: standardize(Framework, options.framework),
+      model_provider: standardize(ModelProviderEnum, options.modelProvider),
+      memory: standardize(Memory, options.memory ?? 'none'),
+      protocol: standardize(Protocol, options.protocol ?? 'http'),
+      build: standardize(Build, options.build ?? 'codezip'),
+      agent_type: standardize(AgentType, options.type ?? 'create'),
+      network_mode: standardize(NetworkModeEnum, options.networkMode ?? 'public'),
+      has_agent: options.agent !== false,
+    };
+  });
 }
 
 export const registerCreate = (program: Command) => {
diff --git a/src/cli/telemetry/schemas/command-run.ts b/src/cli/telemetry/schemas/command-run.ts
@@ -41,7 +41,7 @@ const CreateAttrs = safeSchema({
   memory: Memory,
   protocol: Protocol,
   build: Build,
-  agent_type: z.enum(['create', 'import']),
+  agent_type: AgentType,
   network_mode: NetworkMode,
   has_agent: z.boolean(),
 });
diff --git a/src/cli/tui/screens/create/useCreateFlow.ts b/src/cli/tui/screens/create/useCreateFlow.ts
