Skip to content

Commit 64afcbc

Browse files
lucy66hwlucy66hw
committed
Merge branch 'verb_check'
2 parents 5146af3 + 5c00229 commit 64afcbc

6 files changed

Lines changed: 95 additions & 190 deletions

File tree

‎tools/proto-convert/src/Filter.ts‎

Lines changed: 16 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -34,23 +34,24 @@ function traverse_and_enqueue(node: any, queue: string[], visited: Set<string>,
3434
}
3535

3636
/**
37-
* Filters an OpenAPI spec to include only specified paths and their referenced components.
37+
* Filters an OpenAPI spec to include only paths matching specified x-operation-groups.
3838
* Schemas in the excluded set are skipped.
3939
*/
4040
export default class Filter {
4141
protected input: Record<string, any>
4242
protected output: Record<string, any>
43-
protected targetPathsMap: Map<string, Set<string> | null> // path -> operation-groups (null means all operations)
43+
protected targetGroups: Set<string> // operation groups to include
4444
protected excludedSchemas: Set<string>
4545
paths: Record<string, Record<string, OpenAPIV3.PathItemObject>> = {} // namespace -> path -> path_item_object
4646

47-
constructor(input: Record<string, any>, targetPathsMap: Map<string, Set<string> | null>, excludedSchemas: Set<string> = new Set()) {
47+
constructor(input: Record<string, any>, targetGroups: Set<string>, excludedSchemas: Set<string> = new Set()) {
4848
this.input = input;
49-
this.targetPathsMap = targetPathsMap;
49+
this.targetGroups = targetGroups;
5050
this.excludedSchemas = excludedSchemas;
5151
if (this.excludedSchemas.size > 0) {
5252
logger.info(`Loaded ${this.excludedSchemas.size} excluded schemas: ${Array.from(this.excludedSchemas).join(', ')}`);
5353
}
54+
logger.info(`Filtering for operation groups: ${Array.from(targetGroups).join(', ')}`);
5455
this.output = {
5556
openapi: '3.1.0',
5657
info: {},
@@ -68,28 +69,20 @@ export default class Filter {
6869
filter(): OpenAPIV3.Document {
6970
this.output.info = this.input.info;
7071

71-
for (const [targetPath, targetGroups] of this.targetPathsMap) {
72-
if (this.input.paths[targetPath] === undefined) {
73-
logger.error(`Path not found in spec: ${targetPath}`);
74-
continue;
75-
}
76-
77-
const pathItem = this.input.paths[targetPath];
72+
for (const path in this.input.paths) {
73+
const pathItem = this.input.paths[path];
74+
if (!pathItem) continue;
7875

79-
if (targetGroups === null) {
80-
this.output.paths[targetPath] = pathItem;
81-
} else {
82-
const filteredPathItem: any = {};
83-
for (const method of ['get', 'post', 'put', 'delete', 'head'] as const) {
84-
const operation = pathItem?.[method];
85-
if (operation && operation['x-operation-group'] && targetGroups.has(operation['x-operation-group'])) {
86-
filteredPathItem[method] = operation;
87-
}
88-
}
89-
if (Object.keys(filteredPathItem).length > 0) {
90-
this.output.paths[targetPath] = filteredPathItem;
76+
const filteredPathItem: any = {};
77+
for (const method of ['get', 'post', 'put', 'delete', 'head'] as const) {
78+
const operation = pathItem?.[method];
79+
if (operation && operation['x-operation-group'] && this.targetGroups.has(operation['x-operation-group'])) {
80+
filteredPathItem[method] = operation;
9181
}
9282
}
83+
if (Object.keys(filteredPathItem).length > 0) {
84+
this.output.paths[path] = filteredPathItem;
85+
}
9386
}
9487

9588
this.mergeOperationsByGroup(this.output.paths as OpenAPIV3.PathsObject);

‎tools/proto-convert/src/PreProcessing.ts‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
import { Command, Option } from '@commander-js/extra-typings';
2-
import { read_yaml, write_yaml, parsePathsConfig } from './utils/helper';
2+
import { read_yaml, write_yaml, parseOperationGroupsConfig } from './utils/helper';
33

4-
// Path config type: path -> { x-operation-group: string[] }
5-
type PathConfig = Record<string, { 'x-operation-group'?: string[] } | null>;
64
import Filter from './Filter';
75
import { Sanitizer } from './Sanitizer';
86
import logger from './utils/logger';
@@ -13,11 +11,11 @@ import {GlobalParameterConsolidator} from "./GlobalParamWrapper";
1311
import {OpenSearchVersionExtractor} from "./OpenSearchVersionExtractor";
1412

1513
// Load config from spec-filter.yaml
16-
const config = read_yaml<{ paths?: PathConfig; excluded_schemas?: string[] }>(
14+
const config = read_yaml<{ 'x-operation-groups'?: string[]; excluded_schemas?: string[] }>(
1715
path.join(__dirname, 'config', 'spec-filter.yaml')
1816
);
1917

20-
const target_paths_map = parsePathsConfig(config.paths);
18+
const target_groups = parseOperationGroupsConfig(config['x-operation-groups']);
2119
const excluded_schemas = new Set(config.excluded_schemas ?? []);
2220

2321
const command = new Command()
@@ -40,10 +38,10 @@ type PreprocessingOpts = {
4038
const opts = command.opts() as PreprocessingOpts;
4139

4240
try {
43-
const pathsList = Array.from(target_paths_map.keys());
44-
logger.info(`PreProcessing ${pathsList.join(', ')} into ${opts.output} ...`)
41+
const groupsList = Array.from(target_groups);
42+
logger.info(`PreProcessing operation groups [${groupsList.join(', ')}] into ${opts.output} ...`)
4543
const original_spec = read_yaml(opts.input)
46-
const filtered_spec = new Filter(original_spec, target_paths_map, excluded_schemas).filter();
44+
const filtered_spec = new Filter(original_spec, target_groups, excluded_schemas).filter();
4745
const version_processed_spec = new OpenSearchVersionExtractor(filtered_spec).process(opts.opensearchVersion);
4846
const sanitized_spec = new Sanitizer(version_processed_spec).sanitize();
4947
const consolidated_spec = new GlobalParameterConsolidator(sanitized_spec).consolidate();

‎tools/proto-convert/src/config/spec-filter.yaml‎

Lines changed: 4 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,7 @@
1-
# Target API paths to include in proto generation
2-
# Format:
3-
# path:
4-
# x-operation-group: # optional, list of operation groups to include
5-
# - group1
6-
# - group2
7-
# If x-operation-group is not specified, all operations on that path are included
8-
paths:
9-
/{index}/_bulk:
10-
x-operation-group:
11-
- bulk
12-
/{index}/_search:
13-
x-operation-group:
14-
- search
1+
# Target operation groups to include in proto generation
2+
x-operation-groups:
3+
- bulk
4+
- search
155

166
# Schemas to exclude from proto generation
177
# These schemas and their nested dependencies will not be included

‎tools/proto-convert/src/utils/helper.ts‎

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -218,19 +218,13 @@ export function is_simple_ref(schema: any): boolean {
218218
}
219219

220220
/**
221-
* Convert paths config to Map<path, Set<operation-groups>>
222-
* @param paths - The path configuration from spec-filter.yaml
223-
* @returns Map where key is path and value is Set of operation groups (null means all operations)
221+
* Parse x-operation-groups config from spec-filter.yaml
222+
* @param groups - Array of operation group names
223+
* @returns Set of operation group names
224224
*/
225-
export function parsePathsConfig(paths: Record<string, { 'x-operation-group'?: string[] } | null> | undefined): Map<string, Set<string> | null> {
226-
const result = new Map<string, Set<string> | null>();
227-
if (!paths) {
228-
result.set('/_search', null);
229-
return result;
225+
export function parseOperationGroupsConfig(groups: string[] | undefined): Set<string> {
226+
if (!groups || groups.length === 0) {
227+
return new Set(['search']); // default
230228
}
231-
for (const [p, config] of Object.entries(paths)) {
232-
const groups = config?.['x-operation-group'];
233-
result.set(p, groups && groups.length > 0 ? new Set(groups) : null);
234-
}
235-
return result;
229+
return new Set(groups);
236230
}

‎tools/proto-convert/test/Filter.test.ts‎

Lines changed: 43 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -27,25 +27,25 @@ describe('Filter', () => {
2727
responses: responses ?? { '200': { description: 'OK' } }
2828
} as any);
2929

30-
describe('path filtering', () => {
31-
it('should include all operations when targetGroups is null', () => {
30+
describe('operation group filtering', () => {
31+
it('should include operations matching target groups', () => {
3232
const spec = createSpec({
3333
'/pets': {
3434
get: createOperation('pets.list', 'listPets'),
3535
post: createOperation('pets.create', 'createPet')
3636
}
3737
});
3838

39-
const pathsMap = new Map<string, Set<string> | null>([['/pets', null]]);
40-
const filter = new Filter(spec, pathsMap);
39+
const targetGroups = new Set(['pets.list']);
40+
const filter = new Filter(spec, targetGroups);
4141
const result = filter.filter();
4242

4343
expect(result.paths['/pets']).toBeDefined();
4444
expect((result.paths['/pets'] as any).get).toBeDefined();
45-
expect((result.paths['/pets'] as any).post).toBeDefined();
45+
expect((result.paths['/pets'] as any).post).toBeUndefined();
4646
});
4747

48-
it('should filter operations by x-operation-group', () => {
48+
it('should include multiple operation groups', () => {
4949
const spec = createSpec({
5050
'/pets': {
5151
get: createOperation('pets.list', 'listPets'),
@@ -54,45 +54,44 @@ describe('Filter', () => {
5454
}
5555
});
5656

57-
const pathsMap = new Map<string, Set<string> | null>([
58-
['/pets', new Set(['pets.list', 'pets.create'])]
59-
]);
60-
const filter = new Filter(spec, pathsMap);
57+
const targetGroups = new Set(['pets.list', 'pets.create']);
58+
const filter = new Filter(spec, targetGroups);
6159
const result = filter.filter();
6260

6361
expect((result.paths['/pets'] as any).get).toBeDefined();
6462
expect((result.paths['/pets'] as any).post).toBeDefined();
6563
expect((result.paths['/pets'] as any).delete).toBeUndefined();
6664
});
6765

68-
it('should skip paths not in targetPathsMap', () => {
66+
it('should discover paths automatically based on operation group', () => {
6967
const spec = createSpec({
7068
'/pets': { get: createOperation('pets.list', 'listPets') },
69+
'/pets/{id}': { get: createOperation('pets.list', 'getPet') },
7170
'/users': { get: createOperation('users.list', 'listUsers') }
7271
});
7372

74-
const pathsMap = new Map<string, Set<string> | null>([['/pets', null]]);
75-
const filter = new Filter(spec, pathsMap);
73+
const targetGroups = new Set(['pets.list']);
74+
const filter = new Filter(spec, targetGroups);
7675
const result = filter.filter();
7776

78-
expect(result.paths['/pets']).toBeDefined();
77+
// Should discover both /pets and /pets/{id} since both have pets.list group
78+
// But they get merged, so only one remains
79+
expect(Object.keys(result.paths).length).toBe(1);
7980
expect(result.paths['/users']).toBeUndefined();
8081
});
8182

82-
it('should handle path not found in spec gracefully', () => {
83+
it('should skip paths with no matching operation group', () => {
8384
const spec = createSpec({
84-
'/pets': { get: createOperation('pets.list', 'listPets') }
85+
'/pets': { get: createOperation('pets.list', 'listPets') },
86+
'/users': { get: createOperation('users.list', 'listUsers') }
8587
});
8688

87-
const pathsMap = new Map<string, Set<string> | null>([
88-
['/pets', null],
89-
['/nonexistent', null]
90-
]);
91-
const filter = new Filter(spec, pathsMap);
89+
const targetGroups = new Set(['pets.list']);
90+
const filter = new Filter(spec, targetGroups);
9291
const result = filter.filter();
9392

9493
expect(result.paths['/pets']).toBeDefined();
95-
expect(result.paths['/nonexistent']).toBeUndefined();
94+
expect(result.paths['/users']).toBeUndefined();
9695
});
9796
});
9897

@@ -112,11 +111,8 @@ describe('Filter', () => {
112111
}
113112
});
114113

115-
const pathsMap = new Map<string, Set<string> | null>([
116-
['/pets', null],
117-
['/pets/{petId}', null]
118-
]);
119-
const filter = new Filter(spec, pathsMap);
114+
const targetGroups = new Set(['pets.list']);
115+
const filter = new Filter(spec, targetGroups);
120116
const result = filter.filter();
121117

122118
// Should merge to first path with merged parameters
@@ -143,11 +139,8 @@ describe('Filter', () => {
143139
}
144140
});
145141

146-
const pathsMap = new Map<string, Set<string> | null>([
147-
['/pets', null],
148-
['/pets/{id}', null]
149-
]);
150-
const filter = new Filter(spec, pathsMap);
142+
const targetGroups = new Set(['pets.list']);
143+
const filter = new Filter(spec, targetGroups);
151144
const result = filter.filter();
152145

153146
const operation = (result.paths['/pets'] as any)?.get;
@@ -174,11 +167,8 @@ describe('Filter', () => {
174167
}
175168
});
176169

177-
const pathsMap = new Map<string, Set<string> | null>([
178-
['/pets', null],
179-
['/pets/{id}', null]
180-
]);
181-
const filter = new Filter(spec, pathsMap);
170+
const targetGroups = new Set(['pets.list']);
171+
const filter = new Filter(spec, targetGroups);
182172
const result = filter.filter();
183173

184174
const operation = (result.paths['/pets'] as any)?.get;
@@ -192,8 +182,8 @@ describe('Filter', () => {
192182
}
193183
});
194184

195-
const pathsMap = new Map<string, Set<string> | null>([['/pets', null]]);
196-
const filter = new Filter(spec, pathsMap);
185+
const targetGroups = new Set(['pets.list']);
186+
const filter = new Filter(spec, targetGroups);
197187
const result = filter.filter();
198188

199189
const operation = (result.paths['/pets'] as any)?.get;
@@ -214,11 +204,8 @@ describe('Filter', () => {
214204
}
215205
});
216206

217-
const pathsMap = new Map<string, Set<string> | null>([
218-
['/pets', null],
219-
['/pets/{id}', null]
220-
]);
221-
const filter = new Filter(spec, pathsMap);
207+
const targetGroups = new Set(['pets.create']);
208+
const filter = new Filter(spec, targetGroups);
222209

223210
expect(() => filter.filter()).toThrow(/inconsistent requestBody/);
224211
});
@@ -235,11 +222,8 @@ describe('Filter', () => {
235222
}
236223
});
237224

238-
const pathsMap = new Map<string, Set<string> | null>([
239-
['/pets', null],
240-
['/pets/{id}', null]
241-
]);
242-
const filter = new Filter(spec, pathsMap);
225+
const targetGroups = new Set(['pets.list']);
226+
const filter = new Filter(spec, targetGroups);
243227

244228
expect(() => filter.filter()).toThrow(/inconsistent responses/);
245229
});
@@ -257,11 +241,8 @@ describe('Filter', () => {
257241
}
258242
});
259243

260-
const pathsMap = new Map<string, Set<string> | null>([
261-
['/pets', null],
262-
['/pets/{id}', null]
263-
]);
264-
const filter = new Filter(spec, pathsMap);
244+
const targetGroups = new Set(['pets.create']);
245+
const filter = new Filter(spec, targetGroups);
265246

266247
expect(() => filter.filter()).not.toThrow();
267248
});
@@ -276,11 +257,8 @@ describe('Filter', () => {
276257
}
277258
});
278259

279-
const pathsMap = new Map<string, Set<string> | null>([
280-
['/pets', null],
281-
['/pets/{id}', null]
282-
]);
283-
const filter = new Filter(spec, pathsMap);
260+
const targetGroups = new Set(['pets.list']);
261+
const filter = new Filter(spec, targetGroups);
284262

285263
expect(() => filter.filter()).not.toThrow();
286264
});
@@ -317,9 +295,9 @@ describe('Filter', () => {
317295
parameters: {}
318296
});
319297

320-
const pathsMap = new Map<string, Set<string> | null>([['/pets', null]]);
298+
const targetGroups = new Set(['pets.list']);
321299
const excludedSchemas = new Set(['ExcludedSchema']);
322-
const filter = new Filter(spec, pathsMap, excludedSchemas);
300+
const filter = new Filter(spec, targetGroups, excludedSchemas);
323301
const result = filter.filter();
324302

325303
expect(result.components?.schemas?.['PetList']).toBeDefined();
@@ -351,8 +329,8 @@ describe('Filter', () => {
351329
schemas: {}
352330
});
353331

354-
const pathsMap = new Map<string, Set<string> | null>([['/pets', null]]);
355-
const filter = new Filter(spec, pathsMap);
332+
const targetGroups = new Set(['pets.list']);
333+
const filter = new Filter(spec, targetGroups);
356334
const result = filter.filter();
357335

358336
expect(result.components?.parameters?.['limit']).toBeDefined();
@@ -396,8 +374,8 @@ describe('Filter', () => {
396374
responses: {}
397375
});
398376

399-
const pathsMap = new Map<string, Set<string> | null>([['/pets', null]]);
400-
const filter = new Filter(spec, pathsMap);
377+
const targetGroups = new Set(['pets.list']);
378+
const filter = new Filter(spec, targetGroups);
401379
const result = filter.filter();
402380

403381
expect(result.components?.schemas?.['PetList']).toBeDefined();

0 commit comments

Comments
 (0)