Merge pull request #27 from camunda/feat/consolidate-endpoint-metadata

feat(metadata): consolidate endpoint metadata into spec-metadata.json (closes #21)

Author: jwulf

README.md

@@ -17,7 +17,7 @@ This utility solves all of these problems and produces three outputs:
 
 1. **Bundled spec** (`rest-api.bundle.json`) — A single, clean OpenAPI 3 JSON file with all schemas as proper `#/components/schemas/...` refs
 2. **Metadata IR** (`spec-metadata.json`) — A structured intermediate representation of domain-specific information extracted from the spec
-3. **Endpoint map** (`endpoint-map.json`) — A mapping of each API operation (method + path) to its source YAML file, useful for tracing endpoints back to their origin
+3. **Endpoint map** (`endpoint-map.json`) — _Deprecated, removed in 3.0.0._ Each `OperationSummary` in `spec-metadata.json` now carries `sourceFile` directly, so consumers no longer need to read this file or join on `"METHOD /path"`. Still emitted (with a deprecation warning) when `--output-endpoint-map` is set, for one minor cycle.
 
 ## Installation
 
@@ -103,7 +103,7 @@ camunda-schema-bundler --version
 | `--entry-file <name>` | Entry YAML file name (default: `rest-api.yaml`) |
 | `--output-spec <path>` | Output path for the bundled JSON spec |
 | `--output-metadata <path>` | Output path for the metadata IR JSON |
-| `--output-endpoint-map <path>` | Output path for the endpoint map JSON (method + path → source file) |
+| `--output-endpoint-map <path>` | _Deprecated, removed in 3.0.0._ Output path for the endpoint map JSON (method + path → source file). Use `OperationSummary.sourceFile` in `spec-metadata.json` instead. |
 | `--deref-path-local` | Inline remaining path-local `$ref`s (needed for Microsoft.OpenApi) |
 | `--allow-like-refs` | Don't fail on surviving path-local `$like` refs |
 | **General** | |
@@ -259,19 +259,30 @@ Operations marked with `x-eventually-consistent: true`:
 
 ### Operation Summaries
 
-Every operation with metadata about body shape and union variants:
+Every operation with metadata about body shape, union variants, source file, and response shape:
 
 ```json
 {
   "operationId": "createProcessInstance",
   "path": "/process-instances",
   "method": "post",
+  "sourceFile": "process-instance/process-instance.yaml",
   "eventuallyConsistent": false,
   "hasRequestBody": true,
-  "requestBodyUnion": false
+  "requestBodyUnion": false,
+  "requestBodyContentTypes": ["application/json"],
+  "requestBodySchemaRef": "CreateProcessInstanceRequest",
+  "successStatus": 200,
+  "successResponseSchemaRef": "CreateProcessInstanceResult",
+  "vendorExtensions": { "x-ergonomic-helper": "createProcessInstanceFromBpmnFile" }
 }
 ```
 
+The `sourceFile`, `requestBodyContentTypes`, `requestBodySchemaRef`,
+`successResponseSchemaRef`, `successStatus`, and `vendorExtensions` fields
+were added in `spec-metadata.json` schemaVersion `2.0.0`. Together they
+remove the need to read `endpoint-map.json` and join on `"METHOD /path"`.
+
 ## Per-SDK Configuration
 
 | SDK | `--deref-path-local` | Notes |

src/bundle.ts

@@ -294,6 +294,10 @@ export async function bundle(options: BundleOptions): Promise<BundleResult> {
     'get', 'put', 'post', 'delete', 'options', 'head', 'patch', 'trace',
   ]);
   const endpointMap: Record<string, string> = {};
+  // Per-operation source file map keyed by `${methodLower} ${path}` (e.g.
+  // "get /process-instances"). Used to populate `OperationSummary.sourceFile`
+  // in the metadata IR. See https://github.com/camunda/camunda-schema-bundler/issues/21
+  const sourceFileByOp = new Map<string, string>();
   const opsSeen = new Set<string>();
   // `bundled.paths` is constant for the whole run; resolve it once and
   // pre-compute the set of bundled (path, method) pairs so the per-file
@@ -355,6 +359,7 @@ export async function bundle(options: BundleOptions): Promise<BundleResult> {
             if (opsSeen.has(op)) continue;
             opsSeen.add(op);
             endpointMap[op] = relFile;
+            sourceFileByOp.set(`${key} ${apiPath}`, relFile);
           }
         }
       }
@@ -610,7 +615,7 @@ export async function bundle(options: BundleOptions): Promise<BundleResult> {
   // ── Step 6: Extract metadata IR ───────────────────────────────────────────
 
   const specHash = hashDirectoryTree(options.specDir);
-  const metadata = extractMetadata(bundled, schemas, specHash);
+  const metadata = extractMetadata(bundled, schemas, specHash, sourceFileByOp);
 
   // ── Step 7: Write outputs ─────────────────────────────────────────────────
 
@@ -635,6 +640,13 @@ export async function bundle(options: BundleOptions): Promise<BundleResult> {
   }
 
   if (options.outputEndpointMap) {
+    stats.endpointMapDeprecated = true;
+    console.warn(
+      '[camunda-schema-bundler] WARNING: endpoint-map.json is deprecated and ' +
+        'will be removed in 3.0.0. The same per-operation source file is now ' +
+        'available as `sourceFile` on each entry in `spec-metadata.json`\'s ' +
+        '`operations[]`. See https://github.com/camunda/camunda-schema-bundler/issues/21'
+    );
     const dir = path.dirname(options.outputEndpointMap);
     fs.mkdirSync(dir, { recursive: true });
     fs.writeFileSync(

src/cli.ts

@@ -138,7 +138,7 @@ Bundle options:
   --entry-file <name>       Entry YAML file name (default: rest-api.yaml)
   --output-spec <path>      Output path for bundled JSON spec
   --output-metadata <path>  Output path for metadata IR JSON
-  --output-endpoint-map <path>  Output path for endpoint map JSON
+  --output-endpoint-map <path>  Output path for endpoint map JSON [DEPRECATED — removed in 3.0.0; use OperationSummary.sourceFile in spec-metadata.json]
   --deref-path-local        Inline remaining path-local $refs
   --allow-like-refs         Don't fail on surviving path-local $like refs
   --help, -h                Show this help

src/metadata.ts

@@ -31,17 +31,21 @@ const HTTP_METHODS = ['get', 'post', 'put', 'patch', 'delete', 'head', 'options'
 export function extractMetadata(
   spec: Record<string, unknown>,
   schemas: Record<string, unknown>,
-  specHash: string
+  specHash: string,
+  sourceFileByOp?: Map<string, string>
 ): SpecMetadata {
   const semanticKeys = extractSemanticKeys(schemas);
   const unions = extractUnions(schemas);
   const arrays = extractArraySchemas(schemas);
-  const { eventuallyConsistentOps, operations } = extractOperations(spec);
+  const { eventuallyConsistentOps, operations } = extractOperations(
+    spec,
+    sourceFileByOp
+  );
   const deprecatedEnumMembers = extractDeprecatedEnumMembers(schemas);
   const semanticProviders = extractSemanticProviders(schemas);
 
   return {
-    schemaVersion: '1.0.0',
+    schemaVersion: '2.0.0',
     specHash,
     semanticKeys: semanticKeys.sort((a, b) => a.name.localeCompare(b.name)),
     unions: unions.sort((a, b) => a.name.localeCompare(b.name)),
@@ -317,7 +321,10 @@ function extractSemanticProviders(
   return results;
 }
 
-function extractOperations(spec: Record<string, unknown>): {
+function extractOperations(
+  spec: Record<string, unknown>,
+  sourceFileByOp?: Map<string, string>
+): {
   eventuallyConsistentOps: EventuallyConsistentOp[];
   operations: OperationSummary[];
 } {
@@ -332,6 +339,16 @@ function extractOperations(spec: Record<string, unknown>): {
       | Record<string, unknown>
       | undefined
   ) ?? {};
+  const componentRequestBodies = (
+    (spec['components'] as Record<string, unknown> | undefined)?.[
+      'requestBodies'
+    ] as Record<string, unknown> | undefined
+  ) ?? {};
+  const componentResponses = (
+    (spec['components'] as Record<string, unknown> | undefined)?.[
+      'responses'
+    ] as Record<string, unknown> | undefined
+  ) ?? {};
 
   for (const [pathStr, pathItem] of Object.entries(paths)) {
     if (!pathItem || typeof pathItem !== 'object') continue;
@@ -365,11 +382,33 @@ function extractOperations(spec: Record<string, unknown>): {
       let requestBodyUnion = false;
       const requestBodyUnionRefs: string[] = [];
       let optionalTenantIdInBody = false;
+      const requestBodyContentTypes: string[] = [];
+      let requestBodySchemaRef: string | undefined;
 
       if (hasRequestBody) {
-        const rb = op['requestBody'] as Record<string, unknown>;
-        const content = rb['content'] as Record<string, unknown> | undefined;
+        // Resolve `requestBody` itself if it points at
+        // `#/components/requestBodies/...` so that the rest of this block
+        // sees the actual content map.
+        const rb = resolveComponentRef(
+          op['requestBody'] as Record<string, unknown>,
+          componentRequestBodies
+        );
+        const content = rb?.['content'] as Record<string, unknown> | undefined;
         if (content) {
+          for (const ct of Object.keys(content)) {
+            requestBodyContentTypes.push(ct);
+          }
+          // First content entry whose schema is a $ref wins. Inline schemas
+          // do not block later $ref entries.
+          for (const mediaObj of Object.values(content)) {
+            const media = mediaObj as Record<string, unknown> | undefined;
+            const rawSchema = media?.['schema'] as Record<string, unknown> | undefined;
+            const ref = rawSchema?.['$ref'];
+            if (typeof ref === 'string') {
+              requestBodySchemaRef = ref.split('/').pop();
+              break;
+            }
+          }
           // Check all JSON-like content types
           for (const [contentType, mediaObj] of Object.entries(content)) {
             if (!/json|octet|multipart|text\//i.test(contentType)) continue;
@@ -408,6 +447,58 @@ function extractOperations(spec: Record<string, unknown>): {
 
       const bodyOnly = hasRequestBody && pathParams.length === 0 && queryParams.length === 0;
 
+      // Success response: pick the lowest 2xx status, then look for an
+      // application/json schema $ref under it.
+      let successStatus: number | undefined;
+      let successResponseSchemaRef: string | undefined;
+      const responses = op['responses'] as Record<string, unknown> | undefined;
+      if (responses) {
+        const statuses = Object.keys(responses)
+          .map((k) => ({ key: k, num: Number(k) }))
+          .filter(({ num }) => Number.isInteger(num) && num >= 200 && num < 300)
+          .sort((a, b) => a.num - b.num);
+        if (statuses.length > 0) {
+          successStatus = statuses[0].num;
+          // Resolve `responses[<status>]` if it points at
+          // `#/components/responses/...` so a factored-out response still
+          // exposes its content schema $ref.
+          const resp = resolveComponentRef(
+            responses[statuses[0].key] as Record<string, unknown> | undefined,
+            componentResponses
+          );
+          const respContent = resp?.['content'] as
+            | Record<string, unknown>
+            | undefined;
+          if (respContent) {
+            const json = respContent['application/json'] as
+              | Record<string, unknown>
+              | undefined;
+            const schema = json?.['schema'] as
+              | Record<string, unknown>
+              | undefined;
+            const ref = schema?.['$ref'];
+            if (typeof ref === 'string') {
+              successResponseSchemaRef = ref.split('/').pop();
+            }
+          }
+        }
+      }
+
+      // Vendor extensions: pass through every x-* key on the operation.
+      // Deep-clone object/array values so the metadata IR stays independent
+      // of the bundled spec (mutating one must not affect the other).
+      let vendorExtensions: Record<string, unknown> | undefined;
+      for (const k of Object.keys(op)) {
+        if (k.startsWith('x-')) {
+          if (!vendorExtensions) vendorExtensions = {};
+          const v = op[k];
+          vendorExtensions[k] =
+            v !== null && typeof v === 'object' ? structuredClone(v) : v;
+        }
+      }
+
+      const sourceFile = sourceFileByOp?.get(`${method} ${pathStr}`) ?? '';
+
       operations.push({
         operationId,
         path: pathStr,
@@ -423,6 +514,12 @@ function extractOperations(spec: Record<string, unknown>): {
         queryParams,
         requestBodyUnionRefs,
         optionalTenantIdInBody,
+        sourceFile,
+        requestBodyContentTypes,
+        requestBodySchemaRef,
+        successResponseSchemaRef,
+        successStatus,
+        vendorExtensions,
       });
 
       if (eventuallyConsistent) {
@@ -451,6 +548,23 @@ function resolveSchemaRef(
   return schema;
 }
 
+/**
+ * Resolve a `$ref` that targets a `#/components/<bucket>/<name>` entry
+ * (e.g. `requestBodies`, `responses`). Returns the original object when it
+ * is not a `$ref`, or `undefined` when the ref cannot be resolved against
+ * the supplied component bucket.
+ */
+function resolveComponentRef(
+  obj: Record<string, unknown> | undefined,
+  componentBucket: Record<string, unknown>
+): Record<string, unknown> | undefined {
+  if (!obj) return undefined;
+  const ref = obj['$ref'];
+  if (typeof ref !== 'string') return obj;
+  const name = ref.split('/').pop()!;
+  return componentBucket[name] as Record<string, unknown> | undefined;
+}
+
 /** Check if a resolved schema has an optional tenantId property. */
 function hasOptionalTenantId(schema: Record<string, unknown>): boolean {
   if (schema['type'] !== 'object') return false;

src/types.ts

@@ -26,7 +26,13 @@ export interface FetchAndBundleOptions {
   /** Output path for the metadata IR JSON. */
   outputMetadata?: string;
 
-  /** Output path for the endpoint map JSON (method + path → source file). */
+  /**
+   * Output path for the endpoint map JSON (method + path → source file).
+   *
+   * @deprecated Use `OperationSummary.sourceFile` in `spec-metadata.json`.
+   *   `endpoint-map.json` will be removed in 3.0.0. See
+   *   https://github.com/camunda/camunda-schema-bundler/issues/21
+   */
   outputEndpointMap?: string;
 
   /** Manual ref overrides. */
@@ -55,7 +61,13 @@ export interface BundleOptions {
   /** Output path for the metadata IR JSON. */
   outputMetadata?: string;
 
-  /** Output path for the endpoint map JSON (method + path → source file). */
+  /**
+   * Output path for the endpoint map JSON (method + path → source file).
+   *
+   * @deprecated Use `OperationSummary.sourceFile` in `spec-metadata.json`.
+   *   `endpoint-map.json` will be removed in 3.0.0. See
+   *   https://github.com/camunda/camunda-schema-bundler/issues/21
+   */
   outputEndpointMap?: string;
 
   /**
@@ -103,6 +115,14 @@ export interface BundleStats {
   freshDedupCount: number;
   dereferencedPathLocalRefCount: number;
   pathLocalLikeRefCount: number;
+
+  /**
+   * `true` when the caller requested `outputEndpointMap` (or `--output-endpoint-map`).
+   * `endpoint-map.json` is deprecated and slated for removal in 3.0.0; the same
+   * information is now carried per-operation on `OperationSummary` (`sourceFile`)
+   * in `spec-metadata.json`. See https://github.com/camunda/camunda-schema-bundler/issues/21
+   */
+  endpointMapDeprecated?: boolean;
 }
 
 // ── Metadata IR ──────────────────────────────────────────────────────────────
@@ -222,6 +242,48 @@ export interface OperationSummary {
 
   /** Whether the (resolved) request body has an optional tenantId property. */
   optionalTenantIdInBody: boolean;
+
+  /**
+   * Source YAML file the operation came from, relative to the spec dir.
+   * Replaces the join previously required against `endpoint-map.json`.
+   * Empty string when the source file is unknown (e.g. when `extractMetadata`
+   * is called directly without a source-file map).
+   */
+  sourceFile: string;
+
+  /**
+   * Content-type keys declared on `requestBody.content` (e.g.
+   * `['application/json']`, `['multipart/form-data']`). Empty when there is
+   * no request body.
+   */
+  requestBodyContentTypes: string[];
+
+  /**
+   * Component schema name referenced by the request body (`$ref` short name,
+   * no `#/components/schemas/` prefix). Resolved by scanning every entry under
+   * `requestBody.content` and returning the first one whose `schema` is a
+   * `$ref` — inline schemas are skipped, so a later media type with a `$ref`
+   * can win over an earlier one with an inline schema. `undefined` when no
+   * content entry has a `$ref` schema, or when there is no request body.
+   */
+  requestBodySchemaRef?: string;
+
+  /**
+   * Component schema name referenced by the chosen 2xx response's
+   * `application/json` content. `undefined` when the success response is
+   * empty, inline, or non-JSON.
+   */
+  successResponseSchemaRef?: string;
+
+  /** The chosen 2xx status code (200/201/204/…), if any. */
+  successStatus?: number;
+
+  /**
+   * Pass-through of all `x-*` keys declared on the operation. Promoted
+   * extensions (`x-eventually-consistent`) keep their first-class fields and
+   * are also included here for completeness.
+   */
+  vendorExtensions?: Record<string, unknown>;
 }
 
 export interface OperationQueryParam {