Skip to content

Commit c1df177

Browse files
lucy66hwlucy66hw
committed
simplify
Signed-off-by: xil <fridalu66@gmail.com> ignore CLI code Signed-off-by: xil <fridalu66@gmail.com>
1 parent 4cdf7fb commit c1df177

9 files changed

Lines changed: 774 additions & 165 deletions

File tree

‎protos/generated/services/default_service.proto‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,6 @@ option java_multiple_files = true;
1212
option java_outer_classname = "CommonProto";
1313
option java_package = "org.opensearch.protobufs";
1414

15-
service DefaultService {
16-
rpc Bulk (BulkRequest) returns (BulkResponse);
17-
18-
rpc Search (SearchRequest) returns (SearchResponse);
19-
20-
}
21-
2215
message BulkRequest {
2316
repeated BulkRequestBody bulk_request_body = 1;
2417

‎tools/proto-convert/src/postprocessing/BackwardCompatibleWriter.ts‎

Lines changed: 62 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
1-
import { existsSync, readFileSync, writeFileSync } from 'fs';
2-
import { join } from 'path';
1+
import { existsSync } from 'fs';
32
import { Command, Option } from '@commander-js/extra-typings';
43
import {
54
ProtoMessage,
@@ -8,39 +7,9 @@ import {
87
} from './types';
98
import { parseProtoFile } from './parser';
109
import { mergeMessage, mergeEnum } from './CompatibilityMerger';
11-
import { generateMessage, generateEnum } from './writer';
10+
import { writeProtoFile, CUSTOM_MESSAGE_NAMES, CUSTOM_ENUM_NAMES } from './writer';
1211
import logger from '../utils/logger';
1312

14-
const TEMPLATE_DIR = join(__dirname, '../config/protobuf-schema-template');
15-
16-
// Load fixed header and custom messages from templates
17-
const PROTO_HEADER = readFileSync(join(TEMPLATE_DIR, 'partial_header.mustache'), 'utf-8');
18-
const CUSTOM_MESSAGES = readFileSync(join(TEMPLATE_DIR, 'custom_message.mustache'), 'utf-8');
19-
20-
// ==================== CLI ====================
21-
22-
const command = new Command()
23-
.description('Merge incoming proto files into existing proto while maintaining backward compatibility.')
24-
.addOption(new Option('-e, --existing <path>', 'existing proto file (source of truth)').default('protos/schemas/common.proto'))
25-
.addOption(new Option('-i, --incoming <paths>', 'incoming proto files (comma-separated)')
26-
.argParser((val: string) => val.split(',').map(s => s.trim()))
27-
.default(['protos/generated/models/aggregated_models.proto', 'protos/generated/services/default_service.proto']))
28-
.addOption(new Option('-o, --output <path>', 'output proto file').default('protos/schemas/common.proto'))
29-
.allowExcessArguments(false)
30-
.parse();
31-
32-
type BackwardCompatOpts = {
33-
existing: string;
34-
incoming: string[];
35-
output: string;
36-
};
37-
38-
const opts = command.opts() as BackwardCompatOpts;
39-
40-
// Messages defined in custom_message.mustache - skip and use template instead
41-
const CUSTOM_MESSAGE_NAMES = new Set(['ObjectMap', 'GeneralNumber']);
42-
const CUSTOM_ENUM_NAMES = new Set(['NullValue']);
43-
4413
export class BackwardCompatibleWriter {
4514
private existingMessages: ProtoMessage[];
4615
private existingEnums: ProtoEnum[];
@@ -78,63 +47,55 @@ export class BackwardCompatibleWriter {
7847
}
7948

8049
process(): void {
81-
const outputParts: string[] = [];
82-
83-
// Use fixed header from template
84-
outputParts.push(PROTO_HEADER.trim());
50+
const finalMessages: ProtoMessage[] = [];
51+
const finalEnums: ProtoEnum[] = [];
8552

86-
// Process messages
53+
// Process existing messages (merge with incoming if present)
8754
for (const existingMsg of this.existingMessages) {
8855
if (CUSTOM_MESSAGE_NAMES.has(existingMsg.name)) {
8956
this.incomingMessageMap.delete(existingMsg.name);
9057
continue;
9158
}
9259

9360
const incomingMsg = this.incomingMessageMap.get(existingMsg.name);
94-
9561
if (incomingMsg) {
96-
const mergedMsg = mergeMessage(existingMsg, incomingMsg, this.errors);
97-
outputParts.push(generateMessage(mergedMsg));
62+
finalMessages.push(mergeMessage(existingMsg, incomingMsg, this.errors));
9863
this.incomingMessageMap.delete(existingMsg.name);
9964
} else {
100-
outputParts.push(generateMessage(existingMsg));
65+
finalMessages.push(existingMsg);
10166
}
10267
}
10368

104-
// Process enums
69+
// Process existing enums (merge with incoming if present)
10570
for (const existingEnum of this.existingEnums) {
10671
if (CUSTOM_ENUM_NAMES.has(existingEnum.name)) {
10772
this.incomingEnumMap.delete(existingEnum.name);
10873
continue;
10974
}
11075

11176
const incomingEnum = this.incomingEnumMap.get(existingEnum.name);
112-
11377
if (incomingEnum) {
114-
const mergedEnum = mergeEnum(existingEnum, incomingEnum);
115-
outputParts.push(generateEnum(mergedEnum));
78+
finalEnums.push(mergeEnum(existingEnum, incomingEnum));
11679
this.incomingEnumMap.delete(existingEnum.name);
11780
} else {
118-
outputParts.push(generateEnum(existingEnum));
81+
finalEnums.push(existingEnum);
11982
}
12083
}
12184

122-
// Append new messages from incoming proto files
85+
// Add new messages from incoming (not in existing)
12386
for (const [, msg] of this.incomingMessageMap) {
124-
outputParts.push('');
125-
outputParts.push(generateMessage(msg));
87+
if (!CUSTOM_MESSAGE_NAMES.has(msg.name)) {
88+
finalMessages.push(msg);
89+
}
12690
}
12791

128-
// Append new enums from incoming proto files
92+
// Add new enums from incoming (not in existing)
12993
for (const [, protoEnum] of this.incomingEnumMap) {
130-
outputParts.push('');
131-
outputParts.push(generateEnum(protoEnum));
94+
if (!CUSTOM_ENUM_NAMES.has(protoEnum.name)) {
95+
finalEnums.push(protoEnum);
96+
}
13297
}
13398

134-
// Append custom messages from template (ObjectMap, GeneralNumber, NullValue)
135-
outputParts.push('');
136-
outputParts.push(CUSTOM_MESSAGES.trim());
137-
13899
// Check for errors before writing
139100
if (this.errors.length > 0) {
140101
logger.error('Backward compatibility errors:');
@@ -146,36 +107,56 @@ export class BackwardCompatibleWriter {
146107
);
147108
}
148109

149-
writeFileSync(this.outputPath, outputParts.join('\n'));
110+
// Write output using shared function
111+
writeProtoFile(finalMessages, finalEnums, this.outputPath);
150112
logger.info(`Updated: ${this.outputPath}`);
151113
}
152114
}
153115

154-
// ==================== RUN ====================
155-
156-
if (!existsSync(opts.existing)) {
157-
logger.error(`Existing file not found: ${opts.existing}`);
158-
process.exit(1);
159-
}
116+
// ==================== CLI ====================
160117

161-
const existingIncoming = opts.incoming.filter(p => existsSync(p));
162-
if (existingIncoming.length === 0) {
163-
logger.error(`No incoming proto files found.`);
164-
process.exit(1);
165-
}
118+
/* istanbul ignore next -- CLI entry point */
119+
if (require.main === module) {
120+
const command = new Command()
121+
.description('Merge incoming proto files into existing proto while maintaining backward compatibility.')
122+
.addOption(new Option('-e, --existing <path>', 'existing proto file (source of truth)').default('protos/schemas/common.proto'))
123+
.addOption(new Option('-i, --incoming <paths>', 'incoming proto files (comma-separated)')
124+
.argParser((val: string) => val.split(',').map(s => s.trim()))
125+
.default(['protos/generated/models/aggregated_models.proto', 'protos/generated/services/default_service.proto']))
126+
.addOption(new Option('-o, --output <path>', 'output proto file').default('protos/schemas/common.proto'))
127+
.allowExcessArguments(false)
128+
.parse();
129+
130+
type BackwardCompatOpts = {
131+
existing: string;
132+
incoming: string[];
133+
output: string;
134+
};
135+
136+
const opts = command.opts() as BackwardCompatOpts;
137+
138+
if (!existsSync(opts.existing)) {
139+
logger.error(`Existing file not found: ${opts.existing}`);
140+
process.exit(1);
141+
}
166142

167-
try {
168-
const writer = new BackwardCompatibleWriter(
169-
opts.existing,
170-
opts.incoming,
171-
opts.output
172-
);
173-
writer.process();
174-
} catch (error) {
175-
if (error instanceof BackwardCompatibilityError) {
143+
const existingIncoming = opts.incoming.filter(p => existsSync(p));
144+
if (existingIncoming.length === 0) {
145+
logger.error(`No incoming proto files found.`);
176146
process.exit(1);
177147
}
178-
throw error;
179-
}
180148

181-
export { BackwardCompatibilityError };
149+
try {
150+
const writer = new BackwardCompatibleWriter(
151+
opts.existing,
152+
opts.incoming,
153+
opts.output
154+
);
155+
writer.process();
156+
} catch (error) {
157+
if (error instanceof BackwardCompatibilityError) {
158+
process.exit(1);
159+
}
160+
throw error;
161+
}
162+
}

‎tools/proto-convert/src/postprocessing/CleanupUnusedMessages.ts‎

Lines changed: 43 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -4,24 +4,13 @@
44
* (directly or indirectly) by any root message.
55
*/
66

7-
import { existsSync, readFileSync, writeFileSync } from 'fs';
8-
import { join } from 'path';
7+
import { existsSync } from 'fs';
98
import { Command, Option } from '@commander-js/extra-typings';
109
import { parseProtoFile } from './parser';
11-
import { generateMessage, generateEnum } from './writer';
10+
import { writeProtoFile, CUSTOM_MESSAGE_NAMES, CUSTOM_ENUM_NAMES } from './writer';
1211
import { ProtoMessage, ProtoEnum } from './types';
1312
import logger from '../utils/logger';
1413

15-
const TEMPLATE_DIR = join(__dirname, '../config/protobuf-schema-template');
16-
17-
// Load fixed header and custom messages from templates
18-
const PROTO_HEADER = readFileSync(join(TEMPLATE_DIR, 'partial_header.mustache'), 'utf-8');
19-
const CUSTOM_MESSAGES = readFileSync(join(TEMPLATE_DIR, 'custom_message.mustache'), 'utf-8');
20-
21-
// Custom messages/enums defined in template - always included at end
22-
const CUSTOM_MESSAGE_NAMES = new Set(['ObjectMap', 'GeneralNumber']);
23-
const CUSTOM_ENUM_NAMES = new Set(['NullValue']);
24-
2514
export function isBuiltInType(type: string): boolean {
2615
const builtIns = new Set([
2716
'double', 'float', 'int32', 'int64', 'uint32', 'uint64',
@@ -131,45 +120,30 @@ export function extractRootsFromServices(servicePath: string): string[] {
131120
return Array.from(roots);
132121
}
133122

134-
// ==================== CLI ====================
135-
136-
if (require.main === module) {
137-
const command = new Command()
138-
.description('Remove unused messages and enums from a proto file.')
139-
.addOption(new Option('-i, --input <path>', 'input proto file').default('protos/generated/models/aggregated_models.proto'))
140-
.addOption(new Option('-o, --output <path>', 'output proto file (defaults to input)'))
141-
.addOption(new Option('-s, --service <path>', 'service proto file to auto-detect roots')
142-
.default('protos/generated/services/default_service.proto'))
143-
.addOption(new Option('-r, --roots <names>', 'root message names (comma-separated, overrides --service)')
144-
.argParser((val: string) => val.split(',').map(s => s.trim())))
145-
.allowExcessArguments(false)
146-
.parse();
147-
148-
type CleanupOpts = {
149-
input: string;
150-
output?: string;
151-
service: string;
152-
roots?: string[];
153-
};
154-
155-
const opts = command.opts() as CleanupOpts;
123+
export type CleanupOptions = {
124+
input: string;
125+
output?: string;
126+
service?: string;
127+
roots?: string[];
128+
};
156129

130+
/**
131+
* Clean up unused messages and enums from a proto file.
132+
* Returns the number of removed messages and enums.
133+
*/
134+
export function cleanupUnusedMessages(opts: CleanupOptions): { removedMessages: number; removedEnums: number } {
157135
if (!existsSync(opts.input)) {
158-
logger.error(`Input file not found: ${opts.input}`);
159-
process.exit(1);
136+
throw new Error(`Input file not found: ${opts.input}`);
160137
}
161138

162-
// Get roots: from --roots if provided, otherwise from service file
139+
// Get roots
163140
let roots: string[];
164141
if (opts.roots && opts.roots.length > 0) {
165142
roots = opts.roots;
166-
logger.info(`Using manually specified roots: ${roots.join(', ')}`);
167-
} else if (existsSync(opts.service)) {
143+
} else if (opts.service && existsSync(opts.service)) {
168144
roots = extractRootsFromServices(opts.service);
169-
logger.info(`Auto-detected roots from ${opts.service}: ${roots.join(', ')}`);
170145
} else {
171-
logger.error(`Service file not found: ${opts.service}. Specify --roots manually.`);
172-
process.exit(1);
146+
throw new Error(`Service file not found: ${opts.service}. Specify roots manually.`);
173147
}
174148

175149
const parsed = parseProtoFile(opts.input);
@@ -178,8 +152,7 @@ if (require.main === module) {
178152
const messageNames = new Set(parsed.messages.map(m => m.name));
179153
for (const rootMsg of roots) {
180154
if (!messageNames.has(rootMsg)) {
181-
logger.error(`Root message not found: ${rootMsg}`);
182-
process.exit(1);
155+
throw new Error(`Root message not found: ${rootMsg}`);
183156
}
184157
}
185158

@@ -193,39 +166,35 @@ if (require.main === module) {
193166
const removedMessages = parsed.messages.length - keptMessages.length;
194167
const removedEnums = parsed.enums.length - keptEnums.length;
195168

196-
if (removedMessages === 0 && removedEnums === 0) {
197-
logger.info('No unused messages or enums found.');
198-
process.exit(0);
199-
}
169+
// Write output
170+
const outputPath = opts.output || opts.input;
171+
writeProtoFile(keptMessages, keptEnums, outputPath);
200172

201-
logger.info(`Removing ${removedMessages} unused messages, ${removedEnums} unused enums.`);
173+
return { removedMessages, removedEnums };
174+
}
202175

203-
// Generate output
204-
const outputParts: string[] = [];
176+
// ==================== CLI ====================
177+
/* istanbul ignore next -- CLI entry point */
205178

206-
// Use fixed header from template
207-
outputParts.push(PROTO_HEADER.trim());
179+
if (require.main === module) {
180+
const command = new Command()
181+
.description('Remove unused messages and enums from a proto file.')
182+
.addOption(new Option('-i, --input <path>', 'input proto file').default('protos/generated/models/aggregated_models.proto'))
183+
.addOption(new Option('-o, --output <path>', 'output proto file (defaults to input)'))
184+
.addOption(new Option('-s, --service <path>', 'service proto file to auto-detect roots')
185+
.default('protos/generated/services/default_service.proto'))
186+
.addOption(new Option('-r, --roots <names>', 'root message names (comma-separated, overrides --service)')
187+
.argParser((val: string) => val.split(',').map(s => s.trim())))
188+
.allowExcessArguments(false)
189+
.parse();
208190

209-
// Generate messages (excluding custom ones - they're added from template)
210-
for (const msg of keptMessages) {
211-
if (!CUSTOM_MESSAGE_NAMES.has(msg.name)) {
212-
outputParts.push(generateMessage(msg));
213-
}
214-
}
191+
const opts = command.opts() as CleanupOptions;
215192

216-
// Generate enums (excluding custom ones)
217-
for (const e of keptEnums) {
218-
if (!CUSTOM_ENUM_NAMES.has(e.name)) {
219-
outputParts.push(generateEnum(e));
220-
}
193+
try {
194+
const { removedMessages, removedEnums } = cleanupUnusedMessages(opts);
195+
logger.info(`Removed ${removedMessages} messages, ${removedEnums} enums. Updated: ${opts.output || opts.input}`);
196+
} catch (error) {
197+
logger.error((error as Error).message);
198+
process.exit(1);
221199
}
222-
223-
// Append custom messages from template
224-
outputParts.push('');
225-
outputParts.push(CUSTOM_MESSAGES.trim());
226-
227-
const outputPath = opts.output || opts.input;
228-
writeFileSync(outputPath, outputParts.join('\n'));
229-
230-
logger.info(`Updated: ${outputPath}`);
231200
}

0 commit comments

Comments
 (0)