Preprocessing - x-protobuf-type override origin type (opensearch-project#297)

lucy66hw · web-flow · commit 2ae37e1554e3 · 2025-11-16T15:56:51.000-05:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,6 +12,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Preprocessing - Add filter to not convert additionalProperties when only one key allowed ([#292](https://github.com/opensearch-project/opensearch-protobufs/pull/292))
 - Add HybridQuery protos ([#294](https://github.com/opensearch-project/opensearch-protobufs/pull/294))
 - Preprocessing - Consolidate global parameters into GlobalParams schema ([#295](https://github.com/opensearch-project/opensearch-protobufs/pull/295))
+- Preprocessing - x-protobuf-type override origin type ([#297](https://github.com/opensearch-project/opensearch-protobufs/pull/297))
 
 ### Changed
 - Update preprocessing for x-protobuf-excluded ([#266](https://github.com/opensearch-project/opensearch-protobufs/pull/266))
diff --git a/tools/proto-convert/src/GlobalParamWrapper.ts b/tools/proto-convert/src/GlobalParamWrapper.ts
@@ -47,10 +47,13 @@ export class GlobalParameterConsolidator {
                     const propertyObj: any = {
                         ...param
                     };
+                    delete propertyObj.schema;
+                    if (param.schema) {
+                        Object.assign(propertyObj, param.schema);
+                    }
 
                     properties[paramName] = propertyObj;
                     addedParams.add(paramName);
-                    console.log(`Found global parameter: ${paramKey}`);
                 }
             }
         }
diff --git a/tools/proto-convert/src/VendorExtensionProcessor.ts b/tools/proto-convert/src/VendorExtensionProcessor.ts
@@ -1,13 +1,15 @@
 import { OpenAPIV3 } from 'openapi-types';
 import { traverse } from './utils/OpenApiTraverser';
+import { resolveRef } from './utils/helper';
 import Logger from './utils/logger';
 
 /**
  * VendorExtensionProcessor class:
  * Handles processing of vendor extensions in OpenAPI specifications.
  */
 export class VendorExtensionProcessor {
-    private static readonly GRPC_REMOVED_EXTENSION = 'x-protobuf-excluded';
+    private static readonly PROTOBUF_EXCLUDED_EXTENSION = 'x-protobuf-excluded';
+    private static readonly PROTOBUF_TYPE_EXTENSION = 'x-protobuf-type';
 
     private root: OpenAPIV3.Document;
     private logger: Logger;
@@ -22,62 +24,50 @@ export class VendorExtensionProcessor {
      * Direct path-level handling + traverse for schemas only
      */
     public process(): OpenAPIV3.Document {
-        this.logger.info(`Processing vendor extensions (${VendorExtensionProcessor.GRPC_REMOVED_EXTENSION})...`);
 
-        this.removeGrpcRemovedFromPaths();
+        this.removeProtobufExcludedFromPaths();
         traverse(this.root, {
             onSchema: (schema: any, name: string) => {
                 if ('$ref' in schema) return;
-                this.removeGrpcRemovedProperties(schema);
+                this.removeProtobufExcludedProperties(schema);
+                this.applyTypeOverride(schema);
             },
             onResponseSchema: (schema: any, name: string) => {
-                this.removeGrpcRemovedProperties(schema);
+                this.removeProtobufExcludedProperties(schema);
+                this.applyTypeOverride(schema);
             },
             onRequestSchema: (schema: any, name: string) => {
-                this.removeGrpcRemovedProperties(schema);
+                this.removeProtobufExcludedProperties(schema);
+                this.applyTypeOverride(schema);
+            },
+            onSchemaProperty: (schema: any, name: string) => {
+                this.applyTypeOverride(schema);
             }
         });
 
         return this.root;
     }
 
-    private hasGrpcRemoved(item: any): boolean {
+    private hasProtobufExcluded(item: any): boolean {
         if (!item || typeof item !== 'object') return false;
 
-        if (VendorExtensionProcessor.GRPC_REMOVED_EXTENSION in item && !!item[VendorExtensionProcessor.GRPC_REMOVED_EXTENSION]) {
+        if (VendorExtensionProcessor.PROTOBUF_EXCLUDED_EXTENSION in item && !!item[VendorExtensionProcessor.PROTOBUF_EXCLUDED_EXTENSION]) {
             return true;
         }
         if ('$ref' in item && typeof item.$ref === 'string') {
-            const resolved = this.resolveRef(item.$ref);
-            if (resolved && VendorExtensionProcessor.GRPC_REMOVED_EXTENSION in resolved && !!resolved[VendorExtensionProcessor.GRPC_REMOVED_EXTENSION]) {
+            const resolved = resolveRef(item.$ref, this.root);
+            if (resolved && VendorExtensionProcessor.PROTOBUF_EXCLUDED_EXTENSION in resolved && !!resolved[VendorExtensionProcessor.PROTOBUF_EXCLUDED_EXTENSION]) {
                 return true;
             }
         }
 
         return false;
     }
 
-    /**
-     * Resolve a $ref string to the actual object
-     */
-    private resolveRef(ref: string): any {
-        if (!ref.startsWith('#/')) return null;
-
-        const parts = ref.substring(2).split('/');
-        let current: any = this.root;
-
-        for (const part of parts) {
-            if (!current || typeof current !== 'object') return null;
-            current = current[part];
-        }
-
-        return current;
-    }
-
     /**
      * Remove x-protobuf-excluded items from path-level elements directly
      */
-    private removeGrpcRemovedFromPaths(): void {
+    private removeProtobufExcludedFromPaths(): void {
         if (!this.root.paths) return;
 
         for (const pathKey in this.root.paths) {
@@ -95,37 +85,63 @@ export class VendorExtensionProcessor {
                 // Remove parameters with x-protobuf-excluded
                 if (Array.isArray(operation.parameters)) {
                     const originalLength = operation.parameters.length;
-                    operation.parameters = operation.parameters.filter((p: any) => !this.hasGrpcRemoved(p));
+                    operation.parameters = operation.parameters.filter((p: any) => !this.hasProtobufExcluded(p));
                     const removedCount = originalLength - operation.parameters.length;
                     if (removedCount > 0) {
-                        this.logger.info(`Removed ${removedCount} parameter(s) from ${method.toUpperCase()} ${pathKey} (${VendorExtensionProcessor.GRPC_REMOVED_EXTENSION})`);
+                        this.logger.info(`Removed ${removedCount} parameter(s) from ${method.toUpperCase()} ${pathKey} (${VendorExtensionProcessor.PROTOBUF_EXCLUDED_EXTENSION})`);
                     }
                 }
 
                 // Remove responses with x-protobuf-excluded
                 if (operation.responses) {
                     for (const status in operation.responses) {
-                        if (this.hasGrpcRemoved(operation.responses[status])) {
+                        if (this.hasProtobufExcluded(operation.responses[status])) {
                             delete operation.responses[status];
-                            this.logger.info(`Removed response ${status} from ${method.toUpperCase()} ${pathKey} (${VendorExtensionProcessor.GRPC_REMOVED_EXTENSION})`);
+                            this.logger.info(`Removed response ${status} from ${method.toUpperCase()} ${pathKey} (${VendorExtensionProcessor.PROTOBUF_EXCLUDED_EXTENSION})`);
                         }
                     }
                 }
             }
         }
     }
 
-    private removeGrpcRemovedProperties(schema: OpenAPIV3.SchemaObject): void {
+    private removeProtobufExcludedProperties(schema: OpenAPIV3.SchemaObject): void {
         if (!schema?.properties) return;
 
         for (const prop in schema.properties) {
             const propSchema = schema.properties[prop];
             if (propSchema && typeof propSchema === 'object' && !('$ref' in propSchema)) {
-                if (this.hasGrpcRemoved(propSchema)) {
+                if (this.hasProtobufExcluded(propSchema)) {
                     delete schema.properties[prop];
-                    this.logger.info(`Removed schema property ${prop} (${VendorExtensionProcessor.GRPC_REMOVED_EXTENSION})`);
+                    this.logger.info(`Removed schema property ${prop} (${VendorExtensionProcessor.PROTOBUF_EXCLUDED_EXTENSION})`);
                 }
             }
         }
     }
+
+    /**
+     * Apply type override to a schema if it has x-protobuf-type
+     */
+    private applyTypeOverride(schema: any): void {
+        if (!schema) return;
+
+        if (VendorExtensionProcessor.PROTOBUF_TYPE_EXTENSION in schema) {
+            const protoType = schema[VendorExtensionProcessor.PROTOBUF_TYPE_EXTENSION];
+
+            // Clear structural properties that might conflict
+            if ('$ref' in schema) {
+                delete schema.$ref;
+            }
+            if ('properties' in schema) {
+                delete schema.properties;
+            }
+            if ('additionalProperties' in schema) {
+                delete schema.additionalProperties;
+            }
+
+            schema.type = protoType;
+            delete schema[VendorExtensionProcessor.PROTOBUF_TYPE_EXTENSION];
+            this.logger.info(`Applied ${VendorExtensionProcessor.PROTOBUF_TYPE_EXTENSION}: ${protoType}`);
+        }
+    }
 }
diff --git a/tools/proto-convert/src/config/protobuf-generator-config.yaml b/tools/proto-convert/src/config/protobuf-generator-config.yaml
@@ -16,5 +16,6 @@ typeMappings:
   object: "ObjectMap"
   AnyType: "ObjectMap"
   number: "GeneralNumber"
+  bytes: "bytes"
 openapiGeneratorIgnoreList:
   - "README.md"
diff --git a/tools/proto-convert/src/utils/OpenApiTraverser.ts b/tools/proto-convert/src/utils/OpenApiTraverser.ts
@@ -47,11 +47,14 @@ export function traverse(
     if (components.requestBodies) {
         for (const requestName in components.requestBodies) {
             const request = components.requestBodies[requestName];
-            if (!('$ref' in request) && request.content?.['application/json']?.schema) {
-                const schema = request.content['application/json'].schema;
-                if (!('$ref' in schema)) {
-                    visitors.onRequestSchema?.(schema, requestName);
-                    traverseSchema(schema, visitors);
+            if (!('$ref' in request)) {
+                for (const contentType in request.content || {}) {
+                    const content = (request.content as any)?.[contentType];
+                    if (content?.schema && !('$ref' in content.schema)) {
+                        const schema = content.schema;
+                        visitors.onRequestSchema?.(schema, requestName);
+                        traverseSchema(schema as OpenAPIV3.SchemaObject, visitors);
+                    }
                 }
             }
         }

Original file line number	Diff line number	Diff line change
`@@ -47,10 +47,13 @@ export class GlobalParameterConsolidator {`
`47`	`47`	`const propertyObj: any = {`
`48`	`48`	`...param`
`49`	`49`	`};`
	`50`	`+ delete propertyObj.schema;`
	`51`	`+ if (param.schema) {`
	`52`	`+ Object.assign(propertyObj, param.schema);`
	`53`	`+ }`
`50`	`54`
`51`	`55`	`properties[paramName] = propertyObj;`
`52`	`56`	`addedParams.add(paramName);`
`53`		- console.log(`Found global parameter: ${paramKey}`);
`54`	`57`	`}`
`55`	`58`	`}`
`56`	`59`	`}`