parsing service definition to support auto-detect root messages for cleanup

Signed-off-by: xil <fridalu66@gmail.com>

Author: lucy66hw

CHANGELOG.md

@@ -5,6 +5,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 ## [Unreleased]
 ### Added
 - Add unit test workflow with coverage reporting ([#346](https://github.com/opensearch-project/opensearch-protobufs/pull/346))
+- parsing service definition to support auto-detect root messages for cleanup ([#347](https://github.com/opensearch-project/opensearch-protobufs/pull/347))
 
 ### Changed
 

package.json

@@ -7,8 +7,8 @@
   "scripts": {
     "preprocessing": "ts-node tools/proto-convert/src/PreProcessing.ts",
     "backward-compat": "ts-node tools/proto-convert/src/postprocessing/BackwardCompatibleWriter.ts",
-    "cleanup-common": "ts-node tools/proto-convert/src/postprocessing/CleanupUnusedMessages.ts -i protos/schemas/common.proto",
-    "postprocessing": "npm run backward-compat && npm run cleanup-common",
+    "cleanup-unused": "ts-node tools/proto-convert/src/postprocessing/CleanupUnusedMessages.ts -i protos/schemas/common.proto",
+    "postprocessing": "npm run backward-compat && npm run cleanup-unused",
     "test": "npx jest --no-watchman"
   },
   "dependencies": {

protos/generated/services/default_service.proto

@@ -12,6 +12,13 @@ option java_multiple_files = true;
 option java_outer_classname = "CommonProto";
 option java_package = "org.opensearch.protobufs";
 
+service DefaultService {
+  rpc Bulk (BulkRequest) returns (BulkResponse);
+
+  rpc Search (SearchRequest) returns (SearchResponse);
+
+}
+
 message BulkRequest {
   repeated BulkRequestBody bulk_request_body = 1;
 

tools/proto-convert/src/config/protobuf-schema-template/api.mustache

@@ -6,6 +6,18 @@ import "{{{.}}}.proto";
 {{/import}}
 {{/imports}}
 
+service {{classname}} {
+{{#operations}}
+{{#operation}}
+  {{#description}}
+  // {{{.}}}
+  {{/description}}
+  rpc {{operationId}} ({{#hasParams}}{{operationId}}Request{{/hasParams}}{{^hasParams}}google.protobuf.Empty{{/hasParams}}) returns ({{#vendorExtensions.x-grpc-response}}{{.}}{{/vendorExtensions.x-grpc-response}}{{^vendorExtensions.x-grpc-response}}{{operationId}}Response{{/vendorExtensions.x-grpc-response}});
+
+{{/operation}}
+{{/operations}}
+}
+
 {{#operations}}
 {{#operation}}
 {{#hasParams}}

tools/proto-convert/src/postprocessing/CleanupUnusedMessages.ts

@@ -114,23 +114,42 @@ export function filterEnums(
     );
 }
 
+/**
+ * Extract root message names from service definitions (all request/response types).
+ */
+export function extractRootsFromServices(servicePath: string): string[] {
+    const parsed = parseProtoFile(servicePath);
+    const roots = new Set<string>();
+
+    for (const service of parsed.services) {
+        for (const rpc of service.rpcs) {
+            roots.add(rpc.requestType);
+            roots.add(rpc.responseType);
+        }
+    }
+
+    return Array.from(roots);
+}
+
 // ==================== CLI ====================
 
 if (require.main === module) {
     const command = new Command()
         .description('Remove unused messages and enums from a proto file.')
-        .addOption(new Option('-i, --input <path>', 'input proto file').default('protos/schemas/common.proto'))
+        .addOption(new Option('-i, --input <path>', 'input proto file').default('protos/generated/models/aggregated_models.proto'))
         .addOption(new Option('-o, --output <path>', 'output proto file (defaults to input)'))
-        .addOption(new Option('-r, --roots <names>', 'root message names (comma-separated)')
-            .argParser((val: string) => val.split(',').map(s => s.trim()))
-            .default(['SearchRequest', 'SearchResponse', 'BulkRequest', 'BulkResponse']))
+        .addOption(new Option('-s, --service <path>', 'service proto file to auto-detect roots')
+            .default('protos/generated/services/default_service.proto'))
+        .addOption(new Option('-r, --roots <names>', 'root message names (comma-separated, overrides --service)')
+            .argParser((val: string) => val.split(',').map(s => s.trim())))
         .allowExcessArguments(false)
         .parse();
 
     type CleanupOpts = {
         input: string;
         output?: string;
-        roots: string[];
+        service: string;
+        roots?: string[];
     };
 
     const opts = command.opts() as CleanupOpts;
@@ -140,19 +159,32 @@ if (require.main === module) {
         process.exit(1);
     }
 
+    // Get roots: from --roots if provided, otherwise from service file
+    let roots: string[];
+    if (opts.roots && opts.roots.length > 0) {
+        roots = opts.roots;
+        logger.info(`Using manually specified roots: ${roots.join(', ')}`);
+    } else if (existsSync(opts.service)) {
+        roots = extractRootsFromServices(opts.service);
+        logger.info(`Auto-detected roots from ${opts.service}: ${roots.join(', ')}`);
+    } else {
+        logger.error(`Service file not found: ${opts.service}. Specify --roots manually.`);
+        process.exit(1);
+    }
+
     const parsed = parseProtoFile(opts.input);
 
     // Verify root messages exist
     const messageNames = new Set(parsed.messages.map(m => m.name));
-    for (const rootMsg of opts.roots) {
+    for (const rootMsg of roots) {
         if (!messageNames.has(rootMsg)) {
             logger.error(`Root message not found: ${rootMsg}`);
             process.exit(1);
         }
     }
 
     // Find reachable types
-    const reachable = findReachableTypes(opts.roots, parsed.messages);
+    const reachable = findReachableTypes(roots, parsed.messages);
 
     // Filter to keep only reachable
     const keptMessages = filterMessages(parsed.messages, reachable);

tools/proto-convert/src/postprocessing/parser.ts

@@ -3,13 +3,15 @@
  */
 
 import { readFileSync } from 'fs';
-import { parse, Field, Enum, Type, Namespace, NamespaceBase } from 'protobufjs';
+import { parse, Field, Enum, Type, Namespace, NamespaceBase, Service, Method } from 'protobufjs';
 import {
     ProtoField,
     ProtoMessage,
     ProtoEnum,
     ProtoEnumValue,
     ProtoOneof,
+    ProtoService,
+    ProtoRpc,
     ParsedProtoFile,
     Annotation
 } from './types';
@@ -157,6 +159,28 @@ export function convertMessage(msgDef: Type): ProtoMessage {
     };
 }
 
+/**
+ * Convert a protobufjs Service to internal ProtoService type.
+ */
+export function convertService(serviceDef: Service): ProtoService {
+    const rpcs: ProtoRpc[] = [];
+
+    for (const method of serviceDef.methodsArray) {
+        rpcs.push({
+            name: method.name,
+            requestType: method.requestType,
+            responseType: method.responseType,
+            comment: method.comment || undefined
+        });
+    }
+
+    return {
+        name: serviceDef.name,
+        comment: serviceDef.comment || undefined,
+        rpcs
+    };
+}
+
 /**
  * Parse a .proto file and return internal types.
  */
@@ -166,6 +190,7 @@ export function parseProtoFile(filePath: string): ParsedProtoFile {
 
     const messages: ProtoMessage[] = [];
     const enums: ProtoEnum[] = [];
+    const services: ProtoService[] = [];
 
     function traverse(ns: NamespaceBase) {
         if (ns.nestedArray) {
@@ -174,6 +199,8 @@ export function parseProtoFile(filePath: string): ParsedProtoFile {
                     messages.push(convertMessage(nested));
                 } else if (nested instanceof Enum) {
                     enums.push(convertEnum(nested));
+                } else if (nested instanceof Service) {
+                    services.push(convertService(nested));
                 } else if (nested instanceof Namespace) {
                     traverse(nested);
                 }
@@ -183,5 +210,5 @@ export function parseProtoFile(filePath: string): ParsedProtoFile {
 
     traverse(parsed.root);
 
-    return { messages, enums };
+    return { messages, enums, services };
 }

tools/proto-convert/src/postprocessing/types.ts

@@ -42,9 +42,23 @@ export interface ProtoMessage {
     oneofs?: ProtoOneof[];
 }
 
+export interface ProtoRpc {
+    name: string;
+    requestType: string;
+    responseType: string;
+    comment?: string;
+}
+
+export interface ProtoService {
+    name: string;
+    comment?: string;
+    rpcs: ProtoRpc[];
+}
+
 export interface ParsedProtoFile {
     messages: ProtoMessage[];
     enums: ProtoEnum[];
+    services: ProtoService[];
 }
 
 export class BackwardCompatibilityError extends Error {

tools/proto-convert/test/fixtures/proto/test_service.proto

@@ -0,0 +1,25 @@
+syntax = "proto3";
+
+package test;
+
+// Test service for extractRootsFromServices
+service TestService {
+  rpc Search (SearchRequest) returns (SearchResponse);
+  rpc GetDocument (GetDocumentRequest) returns (GetDocumentResponse);
+}
+
+message SearchRequest {
+  string query = 1;
+}
+
+message SearchResponse {
+  repeated string results = 1;
+}
+
+message GetDocumentRequest {
+  string id = 1;
+}
+
+message GetDocumentResponse {
+  string content = 1;
+}