Update change to use x-operation-group only

lucy66hw · lucy66hw · commit 5c0022977bda · 2026-01-27T22:26:19.000Z
Signed-off-by: xil &lt;fridalu66@gmail.com&gt;
diff --git a/tools/proto-convert/src/Filter.ts b/tools/proto-convert/src/Filter.ts
@@ -34,23 +34,24 @@ function traverse_and_enqueue(node: any, queue: string[], visited: Set<string>,
 }
 
 /**
- * Filters an OpenAPI spec to include only specified paths and their referenced components.
+ * Filters an OpenAPI spec to include only paths matching specified x-operation-groups.
  * Schemas in the excluded set are skipped.
  */
 export default class Filter {
   protected input: Record<string, any>
   protected output: Record<string, any>
-  protected targetPathsMap: Map<string, Set<string> | null>  // path -> operation-groups (null means all operations)
+  protected targetGroups: Set<string>  // operation groups to include
   protected excludedSchemas: Set<string>
   paths: Record<string, Record<string, OpenAPIV3.PathItemObject>> = {} // namespace -> path -> path_item_object
 
-  constructor(input: Record<string, any>, targetPathsMap: Map<string, Set<string> | null>, excludedSchemas: Set<string> = new Set()) {
+  constructor(input: Record<string, any>, targetGroups: Set<string>, excludedSchemas: Set<string> = new Set()) {
     this.input = input;
-    this.targetPathsMap = targetPathsMap;
+    this.targetGroups = targetGroups;
     this.excludedSchemas = excludedSchemas;
     if (this.excludedSchemas.size > 0) {
       logger.info(`Loaded ${this.excludedSchemas.size} excluded schemas: ${Array.from(this.excludedSchemas).join(', ')}`);
     }
+    logger.info(`Filtering for operation groups: ${Array.from(targetGroups).join(', ')}`);
     this.output = {
       openapi: '3.1.0',
       info: {},
@@ -68,28 +69,20 @@ export default class Filter {
   filter(): OpenAPIV3.Document {
     this.output.info = this.input.info;
 
-    for (const [targetPath, targetGroups] of this.targetPathsMap) {
-      if (this.input.paths[targetPath] === undefined) {
-        logger.error(`Path not found in spec: ${targetPath}`);
-        continue;
-      }
-
-      const pathItem = this.input.paths[targetPath];
+    for (const path in this.input.paths) {
+      const pathItem = this.input.paths[path];
+      if (!pathItem) continue;
 
-      if (targetGroups === null) {
-        this.output.paths[targetPath] = pathItem;
-      } else {
-        const filteredPathItem: any = {};
-        for (const method of ['get', 'post', 'put', 'delete', 'head'] as const) {
-          const operation = pathItem?.[method];
-          if (operation && operation['x-operation-group'] && targetGroups.has(operation['x-operation-group'])) {
-            filteredPathItem[method] = operation;
-          }
-        }
-        if (Object.keys(filteredPathItem).length > 0) {
-          this.output.paths[targetPath] = filteredPathItem;
+      const filteredPathItem: any = {};
+      for (const method of ['get', 'post', 'put', 'delete', 'head'] as const) {
+        const operation = pathItem?.[method];
+        if (operation && operation['x-operation-group'] && this.targetGroups.has(operation['x-operation-group'])) {
+          filteredPathItem[method] = operation;
         }
       }
+      if (Object.keys(filteredPathItem).length > 0) {
+        this.output.paths[path] = filteredPathItem;
+      }
     }
 
     this.mergeOperationsByGroup(this.output.paths as OpenAPIV3.PathsObject);
diff --git a/tools/proto-convert/src/PreProcessing.ts b/tools/proto-convert/src/PreProcessing.ts
@@ -1,8 +1,6 @@
 import { Command, Option } from '@commander-js/extra-typings';
-import { read_yaml, write_yaml, parsePathsConfig } from './utils/helper';
+import { read_yaml, write_yaml, parseOperationGroupsConfig } from './utils/helper';
 
-// Path config type: path -> { x-operation-group: string[] }
-type PathConfig = Record<string, { 'x-operation-group'?: string[] } | null>;
 import Filter from './Filter';
 import { Sanitizer } from './Sanitizer';
 import logger from './utils/logger';
@@ -13,11 +11,11 @@ import {GlobalParameterConsolidator} from "./GlobalParamWrapper";
 import {OpenSearchVersionExtractor} from "./OpenSearchVersionExtractor";
 
 // Load config from spec-filter.yaml
-const config = read_yaml<{ paths?: PathConfig; excluded_schemas?: string[] }>(
+const config = read_yaml<{ 'x-operation-groups'?: string[]; excluded_schemas?: string[] }>(
   path.join(__dirname, 'config', 'spec-filter.yaml')
 );
 
-const target_paths_map = parsePathsConfig(config.paths);
+const target_groups = parseOperationGroupsConfig(config['x-operation-groups']);
 const excluded_schemas = new Set(config.excluded_schemas ?? []);
 
 const command = new Command()
@@ -40,10 +38,10 @@ type PreprocessingOpts = {
 const opts = command.opts() as PreprocessingOpts;
 
 try {
-  const pathsList = Array.from(target_paths_map.keys());
-  logger.info(`PreProcessing ${pathsList.join(', ')} into ${opts.output} ...`)
+  const groupsList = Array.from(target_groups);
+  logger.info(`PreProcessing operation groups [${groupsList.join(', ')}] into ${opts.output} ...`)
   const original_spec = read_yaml(opts.input)
-  const filtered_spec = new Filter(original_spec, target_paths_map, excluded_schemas).filter();
+  const filtered_spec = new Filter(original_spec, target_groups, excluded_schemas).filter();
   const version_processed_spec = new OpenSearchVersionExtractor(filtered_spec).process(opts.opensearchVersion);
   const sanitized_spec = new Sanitizer(version_processed_spec).sanitize();
   const consolidated_spec = new GlobalParameterConsolidator(sanitized_spec).consolidate();
diff --git a/tools/proto-convert/src/config/spec-filter.yaml b/tools/proto-convert/src/config/spec-filter.yaml
@@ -1,17 +1,7 @@
-# Target API paths to include in proto generation
-# Format:
-#   path:
-#     x-operation-group:    # optional, list of operation groups to include
-#       - group1
-#       - group2
-# If x-operation-group is not specified, all operations on that path are included
-paths:
-  /{index}/_bulk:
-    x-operation-group:
-    - bulk
-  /{index}/_search:
-    x-operation-group:
-    - search
+# Target operation groups to include in proto generation
+x-operation-groups:
+  - bulk
+  - search
 
 # Schemas to exclude from proto generation
 # These schemas and their nested dependencies will not be included
diff --git a/tools/proto-convert/src/utils/helper.ts b/tools/proto-convert/src/utils/helper.ts
@@ -218,19 +218,13 @@ export function is_simple_ref(schema: any): boolean {
 }
 
 /**
- * Convert paths config to Map<path, Set<operation-groups>>
- * @param paths - The path configuration from spec-filter.yaml
- * @returns Map where key is path and value is Set of operation groups (null means all operations)
+ * Parse x-operation-groups config from spec-filter.yaml
+ * @param groups - Array of operation group names
+ * @returns Set of operation group names
  */
-export function parsePathsConfig(paths: Record<string, { 'x-operation-group'?: string[] } | null> | undefined): Map<string, Set<string> | null> {
-    const result = new Map<string, Set<string> | null>();
-    if (!paths) {
-        result.set('/_search', null);
-        return result;
+export function parseOperationGroupsConfig(groups: string[] | undefined): Set<string> {
+    if (!groups || groups.length === 0) {
+        return new Set(['search']); // default
     }
-    for (const [p, config] of Object.entries(paths)) {
-        const groups = config?.['x-operation-group'];
-        result.set(p, groups && groups.length > 0 ? new Set(groups) : null);
-    }
-    return result;
+    return new Set(groups);
 }
diff --git a/tools/proto-convert/test/Filter.test.ts b/tools/proto-convert/test/Filter.test.ts
@@ -27,25 +27,25 @@ describe('Filter', () => {
     responses: responses ?? { '200': { description: 'OK' } }
   } as any);
 
-  describe('path filtering', () => {
-    it('should include all operations when targetGroups is null', () => {
+  describe('operation group filtering', () => {
+    it('should include operations matching target groups', () => {
       const spec = createSpec({
         '/pets': {
           get: createOperation('pets.list', 'listPets'),
           post: createOperation('pets.create', 'createPet')
         }
       });
 
-      const pathsMap = new Map<string, Set<string> | null>([['/pets', null]]);
-      const filter = new Filter(spec, pathsMap);
+      const targetGroups = new Set(['pets.list']);
+      const filter = new Filter(spec, targetGroups);
       const result = filter.filter();
 
       expect(result.paths['/pets']).toBeDefined();
       expect((result.paths['/pets'] as any).get).toBeDefined();
-      expect((result.paths['/pets'] as any).post).toBeDefined();
+      expect((result.paths['/pets'] as any).post).toBeUndefined();
     });
 
-    it('should filter operations by x-operation-group', () => {
+    it('should include multiple operation groups', () => {
       const spec = createSpec({
         '/pets': {
           get: createOperation('pets.list', 'listPets'),
@@ -54,45 +54,44 @@ describe('Filter', () => {
         }
       });
 
-      const pathsMap = new Map<string, Set<string> | null>([
-        ['/pets', new Set(['pets.list', 'pets.create'])]
-      ]);
-      const filter = new Filter(spec, pathsMap);
+      const targetGroups = new Set(['pets.list', 'pets.create']);
+      const filter = new Filter(spec, targetGroups);
       const result = filter.filter();
 
       expect((result.paths['/pets'] as any).get).toBeDefined();
       expect((result.paths['/pets'] as any).post).toBeDefined();
       expect((result.paths['/pets'] as any).delete).toBeUndefined();
     });
 
-    it('should skip paths not in targetPathsMap', () => {
+    it('should discover paths automatically based on operation group', () => {
       const spec = createSpec({
         '/pets': { get: createOperation('pets.list', 'listPets') },
+        '/pets/{id}': { get: createOperation('pets.list', 'getPet') },
         '/users': { get: createOperation('users.list', 'listUsers') }
       });
 
-      const pathsMap = new Map<string, Set<string> | null>([['/pets', null]]);
-      const filter = new Filter(spec, pathsMap);
+      const targetGroups = new Set(['pets.list']);
+      const filter = new Filter(spec, targetGroups);
       const result = filter.filter();
 
-      expect(result.paths['/pets']).toBeDefined();
+      // Should discover both /pets and /pets/{id} since both have pets.list group
+      // But they get merged, so only one remains
+      expect(Object.keys(result.paths).length).toBe(1);
       expect(result.paths['/users']).toBeUndefined();
     });
 
-    it('should handle path not found in spec gracefully', () => {
+    it('should skip paths with no matching operation group', () => {
       const spec = createSpec({
-        '/pets': { get: createOperation('pets.list', 'listPets') }
+        '/pets': { get: createOperation('pets.list', 'listPets') },
+        '/users': { get: createOperation('users.list', 'listUsers') }
       });
 
-      const pathsMap = new Map<string, Set<string> | null>([
-        ['/pets', null],
-        ['/nonexistent', null]
-      ]);
-      const filter = new Filter(spec, pathsMap);
+      const targetGroups = new Set(['pets.list']);
+      const filter = new Filter(spec, targetGroups);
       const result = filter.filter();
 
       expect(result.paths['/pets']).toBeDefined();
-      expect(result.paths['/nonexistent']).toBeUndefined();
+      expect(result.paths['/users']).toBeUndefined();
     });
   });
 
@@ -112,11 +111,8 @@ describe('Filter', () => {
         }
       });
 
-      const pathsMap = new Map<string, Set<string> | null>([
-        ['/pets', null],
-        ['/pets/{petId}', null]
-      ]);
-      const filter = new Filter(spec, pathsMap);
+      const targetGroups = new Set(['pets.list']);
+      const filter = new Filter(spec, targetGroups);
       const result = filter.filter();
 
       // Should merge to first path with merged parameters
@@ -143,11 +139,8 @@ describe('Filter', () => {
         }
       });
 
-      const pathsMap = new Map<string, Set<string> | null>([
-        ['/pets', null],
-        ['/pets/{id}', null]
-      ]);
-      const filter = new Filter(spec, pathsMap);
+      const targetGroups = new Set(['pets.list']);
+      const filter = new Filter(spec, targetGroups);
       const result = filter.filter();
 
       const operation = (result.paths['/pets'] as any)?.get;
@@ -174,11 +167,8 @@ describe('Filter', () => {
         }
       });
 
-      const pathsMap = new Map<string, Set<string> | null>([
-        ['/pets', null],
-        ['/pets/{id}', null]
-      ]);
-      const filter = new Filter(spec, pathsMap);
+      const targetGroups = new Set(['pets.list']);
+      const filter = new Filter(spec, targetGroups);
       const result = filter.filter();
 
       const operation = (result.paths['/pets'] as any)?.get;
@@ -192,8 +182,8 @@ describe('Filter', () => {
         }
       });
 
-      const pathsMap = new Map<string, Set<string> | null>([['/pets', null]]);
-      const filter = new Filter(spec, pathsMap);
+      const targetGroups = new Set(['pets.list']);
+      const filter = new Filter(spec, targetGroups);
       const result = filter.filter();
 
       const operation = (result.paths['/pets'] as any)?.get;
@@ -214,11 +204,8 @@ describe('Filter', () => {
         }
       });
 
-      const pathsMap = new Map<string, Set<string> | null>([
-        ['/pets', null],
-        ['/pets/{id}', null]
-      ]);
-      const filter = new Filter(spec, pathsMap);
+      const targetGroups = new Set(['pets.create']);
+      const filter = new Filter(spec, targetGroups);
 
       expect(() => filter.filter()).toThrow(/inconsistent requestBody/);
     });
@@ -235,11 +222,8 @@ describe('Filter', () => {
         }
       });
 
-      const pathsMap = new Map<string, Set<string> | null>([
-        ['/pets', null],
-        ['/pets/{id}', null]
-      ]);
-      const filter = new Filter(spec, pathsMap);
+      const targetGroups = new Set(['pets.list']);
+      const filter = new Filter(spec, targetGroups);
 
       expect(() => filter.filter()).toThrow(/inconsistent responses/);
     });
@@ -257,11 +241,8 @@ describe('Filter', () => {
         }
       });
 
-      const pathsMap = new Map<string, Set<string> | null>([
-        ['/pets', null],
-        ['/pets/{id}', null]
-      ]);
-      const filter = new Filter(spec, pathsMap);
+      const targetGroups = new Set(['pets.create']);
+      const filter = new Filter(spec, targetGroups);
 
       expect(() => filter.filter()).not.toThrow();
     });
@@ -276,11 +257,8 @@ describe('Filter', () => {
         }
       });
 
-      const pathsMap = new Map<string, Set<string> | null>([
-        ['/pets', null],
-        ['/pets/{id}', null]
-      ]);
-      const filter = new Filter(spec, pathsMap);
+      const targetGroups = new Set(['pets.list']);
+      const filter = new Filter(spec, targetGroups);
 
       expect(() => filter.filter()).not.toThrow();
     });
@@ -317,9 +295,9 @@ describe('Filter', () => {
         parameters: {}
       });
 
-      const pathsMap = new Map<string, Set<string> | null>([['/pets', null]]);
+      const targetGroups = new Set(['pets.list']);
       const excludedSchemas = new Set(['ExcludedSchema']);
-      const filter = new Filter(spec, pathsMap, excludedSchemas);
+      const filter = new Filter(spec, targetGroups, excludedSchemas);
       const result = filter.filter();
 
       expect(result.components?.schemas?.['PetList']).toBeDefined();
@@ -351,8 +329,8 @@ describe('Filter', () => {
         schemas: {}
       });
 
-      const pathsMap = new Map<string, Set<string> | null>([['/pets', null]]);
-      const filter = new Filter(spec, pathsMap);
+      const targetGroups = new Set(['pets.list']);
+      const filter = new Filter(spec, targetGroups);
       const result = filter.filter();
 
       expect(result.components?.parameters?.['limit']).toBeDefined();
@@ -396,8 +374,8 @@ describe('Filter', () => {
         responses: {}
       });
 
-      const pathsMap = new Map<string, Set<string> | null>([['/pets', null]]);
-      const filter = new Filter(spec, pathsMap);
+      const targetGroups = new Set(['pets.list']);
+      const filter = new Filter(spec, targetGroups);
       const result = filter.filter();
 
       expect(result.components?.schemas?.['PetList']).toBeDefined();
diff --git a/tools/proto-convert/test/PreProcessing.test.ts b/tools/proto-convert/test/PreProcessing.test.ts
