Skip to content

Commit 45d908e

Browse files
committed
Add format back
Signed-off-by: xil <fridalu66@gmail.com> address type mismatch Signed-off-by: xil <fridalu66@gmail.com>
1 parent f13b2a4 commit 45d908e

6 files changed

Lines changed: 276 additions & 132 deletions

File tree

.github/workflows/convert-proto.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,10 @@ jobs:
121121
run: |
122122
java -jar cloned-repo/modules/openapi-generator-cli/target/openapi-generator-cli.jar generate -c tools/proto-convert/src/config/protobuf-generator-config.yaml
123123
124+
- name: Reformat proto files
125+
run: |
126+
buf format -w protos/generated
127+
124128
- name: Post Process Protobuf
125129
run: npm run postprocessing
126130

tools/proto-convert/src/Filter.ts

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -38,20 +38,20 @@ function traverse_and_enqueue(node: any, queue: string[], visited: Set<string>,
3838
* Schemas in the excluded set are skipped.
3939
*/
4040
export default class Filter {
41-
protected _spec: Record<string, any>
42-
protected sourceSpec: Record<string, any>
41+
protected input: Record<string, any>
42+
protected output: Record<string, any>
4343
protected targetPaths: string[]
4444
protected excludedSchemas: Set<string>
4545
paths: Record<string, Record<string, OpenAPIV3.PathItemObject>> = {} // namespace -> path -> path_item_object
4646

47-
constructor(sourceSpec: Record<string, any>, targetPaths: string[], excludedSchemas: Set<string> = new Set()) {
48-
this.sourceSpec = sourceSpec;
47+
constructor(input: Record<string, any>, targetPaths: string[], excludedSchemas: Set<string> = new Set()) {
48+
this.input = input;
4949
this.targetPaths = targetPaths;
5050
this.excludedSchemas = excludedSchemas;
5151
if (this.excludedSchemas.size > 0) {
5252
logger.info(`Loaded ${this.excludedSchemas.size} excluded schemas: ${Array.from(this.excludedSchemas).join(', ')}`);
5353
}
54-
this._spec = {
54+
this.output = {
5555
openapi: '3.1.0',
5656
info: {},
5757
paths: {},
@@ -66,20 +66,20 @@ export default class Filter {
6666

6767

6868
filter(): OpenAPIV3.Document {
69-
this._spec.info = this.sourceSpec.info;
69+
this.output.info = this.input.info;
7070
for (const p of this.targetPaths) {
71-
if (this.sourceSpec.paths[p] === undefined) {
71+
if (this.input.paths[p] === undefined) {
7272
logger.error(`Path not found in spec: ${p}`);
7373
continue;
7474
}
75-
this._spec.paths[p] = this.sourceSpec.paths[p];
75+
this.output.paths[p] = this.input.paths[p];
7676
}
77-
this.filter_by_max_parameters(this._spec.paths as OpenAPIV3.PathsObject);
77+
this.filter_by_max_parameters(this.output.paths as OpenAPIV3.PathsObject);
7878
const queue: string[] = [];
7979
const visited: Set<string> = new Set();
8080

8181
// collect all components that are referenced by the paths
82-
traverse_and_enqueue(this._spec.paths, queue, visited, this.excludedSchemas);
82+
traverse_and_enqueue(this.output.paths, queue, visited, this.excludedSchemas);
8383
while (queue.length > 0) {
8484
const ref_str = queue.shift();
8585
if (ref_str == null || ref_str == "") continue;
@@ -88,17 +88,17 @@ export default class Filter {
8888
const sub_component = parts[2];
8989
const key = parts[3];
9090

91-
if (this._spec.components[sub_component as keyof typeof this._spec.components] == null) {
92-
this._spec.components[sub_component] = {};
91+
if (this.output.components[sub_component as keyof typeof this.output.components] == null) {
92+
this.output.components[sub_component] = {};
9393
}
94-
if (this._spec.components[sub_component][key] == null) {
95-
if (this.sourceSpec.components != null && this.sourceSpec.components[sub_component] != null && this.sourceSpec.components[sub_component][key] != null) {
96-
this._spec.components[sub_component][key] = this.sourceSpec.components[sub_component][key];
97-
traverse_and_enqueue(this._spec.components[sub_component][key], queue, visited, this.excludedSchemas);
94+
if (this.output.components[sub_component][key] == null) {
95+
if (this.input.components != null && this.input.components[sub_component] != null && this.input.components[sub_component][key] != null) {
96+
this.output.components[sub_component][key] = this.input.components[sub_component][key];
97+
traverse_and_enqueue(this.output.components[sub_component][key], queue, visited, this.excludedSchemas);
9898
}
9999
}
100100
}
101-
return this._spec as OpenAPIV3.Document;
101+
return this.output as OpenAPIV3.Document;
102102
}
103103

104104
filter_by_max_parameters(paths: OpenAPIV3.PathsObject): void {
@@ -160,6 +160,6 @@ export default class Filter {
160160
new_paths[max_path] = { ...new_paths[max_path], head: max_path_item.head };
161161
}
162162
}
163-
this._spec.paths = new_paths;
163+
this.output.paths = new_paths;
164164
}
165165
}

tools/proto-convert/src/postprocessing/CleanupUnusedMessages.ts

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,7 @@
11
/**
2-
* Cleanup Unused Messages Script
32
*
43
* Given root protobuf messages, removes all messages NOT referenced
54
* (directly or indirectly) by any root message.
6-
*
7-
* Uses parser + writer for reliable handling of nested structures.
85
*/
96

107
import { existsSync, readFileSync, writeFileSync } from 'fs';
@@ -143,7 +140,6 @@ if (require.main === module) {
143140
process.exit(1);
144141
}
145142

146-
// Parse the proto file
147143
const parsed = parseProtoFile(opts.input);
148144

149145
// Verify root messages exist

tools/proto-convert/src/postprocessing/CompatibilityMerger.ts

Lines changed: 84 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,6 @@
11
/**
22
* Merger module: Compare and merge messages/enums for backward compatibility.
33
*
4-
* Rules:
5-
* - If modifier changed (optional added/removed) → ERROR
6-
* - If type changed → ERROR
7-
* - If field only in source but not upcoming → mark [deprecated = true]
8-
* - If field only in upcoming but not source → add at end with new field number
94
*/
105

116
import {
@@ -16,91 +11,144 @@ import {
1611
ProtoOneof
1712
} from './types';
1813

14+
/**
15+
* Extract base name
16+
*/
17+
function getBaseName(fieldName: string): string {
18+
const match = fieldName.match(/^(.+?)_(\d+)$/);
19+
return match ? match[1] : fieldName;
20+
}
21+
22+
/**
23+
* Get the current version suffix from field name
24+
*/
25+
function getFieldVersion(fieldName: string): number {
26+
const match = fieldName.match(/_(\d+)$/);
27+
return match ? parseInt(match[1], 10) : 0;
28+
}
29+
30+
/**
31+
* Check if a field is already deprecated.
32+
*/
33+
function isDeprecated(field: ProtoField): boolean {
34+
return field.options?.some(opt => opt.name === 'deprecated' && opt.value === 'true') ?? false;
35+
}
36+
37+
/**
38+
* Check if two fields are compatible (same type and modifier).
39+
*/
40+
function fieldsMatch(a: ProtoField, b: ProtoField): boolean {
41+
return a.type === b.type && (a.modifier || '') === (b.modifier || '');
42+
}
43+
1944
/**
2045
* Merge a source message with an upcoming message.
21-
* Errors are pushed to the errors array.
46+
* Errors are pushed to the errors array (currently unused but kept for future).
2247
*/
2348
export function mergeMessage(
2449
sourceMsg: ProtoMessage,
2550
upcomingMsg: ProtoMessage,
26-
errors: string[]
51+
_errors: string[]
2752
): ProtoMessage {
28-
const sourceFieldMap = new Map(sourceMsg.fields.map(f => [f.name, f]));
29-
const upcomingFieldMap = new Map(upcomingMsg.fields.map(f => [f.name, f]));
53+
const upcomingByName = new Map(upcomingMsg.fields.map(f => [f.name, f]));
3054

3155
let maxFieldNumber = 0;
3256
const mergedFields: ProtoField[] = [];
3357

3458
for (const sourceField of sourceMsg.fields) {
3559
maxFieldNumber = Math.max(maxFieldNumber, sourceField.number);
3660

37-
const upcomingField = upcomingFieldMap.get(sourceField.name);
61+
// Skip deprecated fields
62+
if (isDeprecated(sourceField)) {
63+
mergedFields.push(sourceField);
64+
continue;
65+
}
66+
67+
const baseName = getBaseName(sourceField.name);
68+
const upcomingField = upcomingByName.get(baseName);
3869

3970
if (upcomingField) {
40-
checkFieldCompatibility(sourceMsg.name, sourceField, upcomingField, errors);
71+
upcomingByName.delete(baseName);
72+
if (fieldsMatch(sourceField, upcomingField)) {
4173
mergedFields.push(sourceField);
74+
} else {
75+
mergedFields.push(markDeprecated(sourceMsg.name, sourceField));
76+
const currentVersion = getFieldVersion(sourceField.name);
77+
const newName = `${baseName}_${currentVersion + 1}`;
78+
upcomingByName.set(newName, { ...upcomingField, name: newName });
79+
}
4280
} else {
4381
mergedFields.push(markDeprecated(sourceMsg.name, sourceField));
4482
}
4583
}
4684

47-
// Merge oneofs
4885
let mergedOneofs: ProtoOneof[] | undefined;
86+
const oneofMaps: Map<string, Map<string, ProtoField>> = new Map();
87+
4988
if (sourceMsg.oneofs) {
5089
const upcomingOneofMap = new Map(
5190
(upcomingMsg.oneofs || []).map(o => [o.name, o])
5291
);
5392

5493
mergedOneofs = [];
5594
for (const sourceOneof of sourceMsg.oneofs) {
56-
const sourceOneofFieldMap = new Map(sourceOneof.fields.map(f => [f.name, f]));
5795
const upcomingOneof = upcomingOneofMap.get(sourceOneof.name);
58-
const upcomingOneofFieldMap = new Map(
96+
const upcomingOneofByName = new Map(
5997
(upcomingOneof?.fields || []).map(f => [f.name, f])
6098
);
6199

62100
const mergedOneofFields: ProtoField[] = [];
63101

64102
for (const sourceField of sourceOneof.fields) {
65103
maxFieldNumber = Math.max(maxFieldNumber, sourceField.number);
104+
// Skip deprecated fields
105+
if (isDeprecated(sourceField)) {
106+
mergedOneofFields.push(sourceField);
107+
continue;
108+
}
109+
110+
const baseName = getBaseName(sourceField.name);
111+
const upcomingField = upcomingOneofByName.get(baseName);
66112

67-
const upcomingField = upcomingOneofFieldMap.get(sourceField.name);
68113
if (upcomingField) {
69-
checkFieldCompatibility(sourceMsg.name, sourceField, upcomingField, errors);
114+
upcomingOneofByName.delete(baseName);
115+
if (fieldsMatch(sourceField, upcomingField)) {
70116
mergedOneofFields.push(sourceField);
117+
} else {
118+
mergedOneofFields.push(markDeprecated(sourceMsg.name, sourceField));
119+
const currentVersion = getFieldVersion(sourceField.name);
120+
const newName = `${baseName}_${currentVersion + 1}`;
121+
upcomingOneofByName.set(newName, { ...upcomingField, name: newName });
122+
}
71123
} else {
72124
mergedOneofFields.push(markDeprecated(sourceMsg.name, sourceField));
73125
}
74126
}
75127

76-
// Add new oneof fields from upcoming
77-
if (upcomingOneof) {
78-
for (const upcomingField of upcomingOneof.fields) {
79-
if (!sourceOneofFieldMap.has(upcomingField.name)) {
80-
maxFieldNumber++;
81-
mergedOneofFields.push({
82-
...upcomingField,
83-
number: maxFieldNumber
84-
});
85-
}
86-
}
87-
}
88-
89128
mergedOneofs.push({
90129
...sourceOneof,
91130
fields: mergedOneofFields
92131
});
132+
oneofMaps.set(sourceOneof.name, upcomingOneofByName);
93133
}
94134
}
95135

96-
// Fields only in upcoming - add at the end with new field numbers
97-
for (const [, upcomingField] of upcomingFieldMap) {
98-
if (!sourceFieldMap.has(upcomingField.name)) {
136+
// Assign field max number to remaining fields.
137+
for (const field of upcomingByName.values()) {
138+
maxFieldNumber++;
139+
mergedFields.push({ ...field, number: maxFieldNumber });
140+
}
141+
142+
// Assign field max number to remaining oneof fields.
143+
if (mergedOneofs) {
144+
for (const oneof of mergedOneofs) {
145+
const remaining = oneofMaps.get(oneof.name);
146+
if (remaining) {
147+
for (const field of remaining.values()) {
99148
maxFieldNumber++;
100-
mergedFields.push({
101-
...upcomingField,
102-
number: maxFieldNumber
103-
});
149+
oneof.fields.push({ ...field, number: maxFieldNumber });
150+
}
151+
}
104152
}
105153
}
106154

@@ -111,34 +159,6 @@ export function mergeMessage(
111159
};
112160
}
113161

114-
/**
115-
* Check field compatibility between source and upcoming.
116-
*/
117-
function checkFieldCompatibility(
118-
msgName: string,
119-
sourceField: ProtoField,
120-
upcomingField: ProtoField,
121-
errors: string[]
122-
): void {
123-
// Check if modifier changed
124-
const sourceModifier = sourceField.modifier || '';
125-
const upcomingModifier = upcomingField.modifier || '';
126-
if (sourceModifier !== upcomingModifier) {
127-
errors.push(
128-
`${msgName}.${sourceField.name}: MODIFIER CHANGED - ` +
129-
`"${sourceModifier || '(none)'}" → "${upcomingModifier || '(none)'}"`
130-
);
131-
}
132-
133-
// TODO: add support for type change
134-
if (sourceField.type !== upcomingField.type) {
135-
errors.push(
136-
`${msgName}.${sourceField.name}: TYPE CHANGED - ` +
137-
`"${sourceField.type}" → "${upcomingField.type}"`
138-
);
139-
}
140-
}
141-
142162
/**
143163
* Mark a field as deprecated if not already.
144164
*/

tools/proto-convert/src/postprocessing/types.ts

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
/**
22
* Internal types for proto processing.
3-
* These types provide a simplified, template-ready representation
4-
* of protobuf messages and enums.
53
*/
64

75
export interface FieldOption {

0 commit comments

Comments
 (0)