generate backward incompatible report

Author: lucy66hw

.github/workflows/backward-compatible-report.yml

@@ -0,0 +1,133 @@
+name: Backward Compatible Report
+
+on:
+  workflow_call:
+    inputs:
+      opensearch_version:
+        description: 'OpenSearch version. Leave empty to fetch latest.'
+        required: false
+        type: string
+      spec_artifact_name:
+        description: 'Name of the uploaded artifact containing opensearch-openapi.yaml'
+        required: false
+        type: string
+        default: 'openapi-spec'
+    outputs:
+      report:
+        description: 'The compatibility report in markdown format'
+        value: ${{ jobs.generate-report.outputs.report }}
+
+  workflow_dispatch:
+    inputs:
+      opensearch_version:
+        description: 'OpenSearch version. Leave empty to fetch latest.'
+        required: false
+        type: string
+
+jobs:
+  generate-report:
+    runs-on: ubuntu-latest
+    outputs:
+      report: ${{ steps.merge_report.outputs.report }}
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ github.repository_owner }}/opensearch-protobufs
+          ref: main
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20
+
+      - name: Setup Java
+        uses: actions/setup-java@v3
+        with:
+          distribution: temurin
+          java-version: 17
+
+      - name: Download OpenAPI spec artifact
+        if: ${{ inputs.spec_artifact_name != '' }}
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ inputs.spec_artifact_name }}
+          path: .
+
+      - name: Get Latest OpenSearch Core Version
+        id: get_opensearch_version
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            // Check if version was provided as input
+            const inputVersion = "${{ inputs.opensearch_version }}";
+            if (inputVersion && inputVersion.trim() !== "") {
+              core.setOutput("version", inputVersion.trim());
+              console.log("Using provided OpenSearch version: " + inputVersion.trim());
+              return;
+            }
+
+            // Otherwise fetch version from buildSrc/version.properties on main branch
+            try {
+              const response = await github.request(
+                'GET /repos/{owner}/{repo}/contents/{path}',
+                {
+                  owner: 'opensearch-project',
+                  repo: 'OpenSearch',
+                  path: 'buildSrc/version.properties',
+                  ref: 'main'
+                }
+              );
+
+              // Decode the file content
+              const content = Buffer.from(response.data.content, 'base64').toString('utf-8');
+
+              // Extract opensearch version (e.g., "opensearch = 3.4.0")
+              const match = content.match(/opensearch\s*=\s*([^\s]+)/);
+
+              if (match && match[1]) {
+                const version = match[1].trim();
+                core.setOutput("version", version);
+                console.log("Fetched OpenSearch version from source: " + version);
+              } else {
+                core.setFailed("Could not find opensearch version in version.properties");
+              }
+            } catch (error) {
+              core.setFailed("Could not fetch OpenSearch version: " + error.message);
+            }
+
+      - name: Run Proto Conversion
+        env:
+          OPENSEARCH_VERSION: ${{ steps.get_opensearch_version.outputs.version }}
+        run: npm ci && npm run preprocessing -- --opensearch-version "$OPENSEARCH_VERSION"
+
+      - name: Clone Protobuf Generator Repository
+        run: |
+          git clone https://github.com/OpenAPITools/openapi-generator cloned-repo
+          cd cloned-repo
+          git checkout 6699ecd9d2f4e0868f23bb36566ea03cd1230e6a
+
+      - name: Build Protobuf Generator Tool
+        run: |
+          cd cloned-repo
+          ./mvnw clean package
+
+      - name: Convert protobuf
+        run: |
+          java -jar cloned-repo/modules/openapi-generator-cli/target/openapi-generator-cli.jar generate -c tools/proto-convert/src/config/protobuf-generator-config.yaml
+
+      - name: Post Process Protobuf (dry-run for report)
+        id: merge_report
+        run: |
+          npm run postprocessing:dry-run
+          REPORT_PATH="/tmp/merge-report.md"
+
+          echo "=== Compatibility Report ==="
+          cat "$REPORT_PATH"
+          echo "============================="
+
+          REPORT=$(cat "$REPORT_PATH")
+          echo "report<<EOF" >> $GITHUB_OUTPUT
+          echo "$REPORT" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT

.github/workflows/convert-proto.yml

@@ -7,10 +7,13 @@ on:
         description: 'OpenSearch version. Leave empty to fetch latest.'
         required: false
         type: string
+  repository_dispatch:
+    types: [spec-updated]
+
 jobs:
   auto-proto-convert:
     runs-on: ubuntu-latest
-    if: github.repository == 'opensearch-project/opensearch-protobufs'
+    # if: github.repository == 'opensearch-project/opensearch-protobufs'
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -26,15 +29,10 @@ jobs:
           distribution: temurin
           java-version: 17
 
-      - name: Install buf
-        run: |
-          npm install -g @bufbuild/buf
-          buf --version
-
       - name: Download Release Assets
         uses: robinraju/release-downloader@v1
         with:
-         repository: 'opensearch-project/opensearch-api-specification'
+         repository: 'lucy66hw/opensearch-api-specification'
          latest: true
          fileName: 'opensearch-openapi.yaml'
          tag: 'main-latest'
@@ -121,12 +119,19 @@ jobs:
         run: |
           java -jar cloned-repo/modules/openapi-generator-cli/target/openapi-generator-cli.jar generate -c tools/proto-convert/src/config/protobuf-generator-config.yaml
 
-      - name: Reformat proto files
-        run: |
-          buf format -w protos/generated
-
       - name: Post Process Protobuf
-        run: npm run postprocessing
+        id: merge_report
+        run: |
+          npm run postprocessing
+          REPORT_PATH="/tmp/merge-report.md"
+          if [ -f "$REPORT_PATH" ]; then
+            REPORT=$(cat "$REPORT_PATH")
+            echo "report<<EOF" >> $GITHUB_OUTPUT
+            echo "$REPORT" >> $GITHUB_OUTPUT
+            echo "EOF" >> $GITHUB_OUTPUT
+          else
+            echo "report=No changes detected." >> $GITHUB_OUTPUT
+          fi
 
       - name: Configure Git User
         run: |
@@ -161,3 +166,7 @@ jobs:
 
             **OpenSearch Version**: ${{ steps.get_opensearch_version.outputs.version }}
             **API Spec Commit**: ${{ steps.get_commit.outputs.latest_commit }}
+
+            ---
+
+            ${{ steps.merge_report.outputs.report }}

package.json

@@ -9,6 +9,7 @@
     "backward-compat": "ts-node tools/proto-convert/src/postprocessing/BackwardCompatibleWriter.ts",
     "cleanup-unused": "ts-node tools/proto-convert/src/postprocessing/CleanupUnusedMessages.ts -i protos/schemas/common.proto",
     "postprocessing": "npm run backward-compat && npm run cleanup-unused",
+    "postprocessing:dry-run": "npm run backward-compat -- --dry-run",
     "test": "npx jest --no-watchman"
   },
   "dependencies": {

tools/proto-convert/src/postprocessing/BackwardCompatibleWriter.ts

@@ -1,22 +1,21 @@
-import { existsSync } from 'fs';
+import { existsSync, writeFileSync } from 'fs';
+import { tmpdir } from 'os';
+import { join } from 'path';
 import { Command, Option } from '@commander-js/extra-typings';
-import {
-    ProtoMessage,
-    ProtoEnum,
-    BackwardCompatibilityError
-} from './types';
+import { ProtoMessage, ProtoEnum } from './types';
 import { parseProtoFile } from './parser';
 import { mergeMessage, mergeEnum } from './CompatibilityMerger';
 import { writeProtoFile, CUSTOM_MESSAGE_NAMES, CUSTOM_ENUM_NAMES } from './writer';
+import { CompatibilityReporter } from './CompatibilityReporter';
 import logger from '../utils/logger';
 
 export class BackwardCompatibleWriter {
     private existingMessages: ProtoMessage[];
     private existingEnums: ProtoEnum[];
     private incomingMessageMap: Map<string, ProtoMessage> = new Map();
     private incomingEnumMap: Map<string, ProtoEnum> = new Map();
-    private errors: string[] = [];
     private outputPath: string;
+    private reporter: CompatibilityReporter = new CompatibilityReporter();
 
     constructor(existingPath: string, incomingPaths: string[], outputPath: string) {
         this.outputPath = outputPath;
@@ -46,7 +45,7 @@ export class BackwardCompatibleWriter {
         }
     }
 
-    process(): void {
+    process(dryRun: boolean = false): void {
         const finalMessages: ProtoMessage[] = [];
         const finalEnums: ProtoEnum[] = [];
 
@@ -59,7 +58,7 @@ export class BackwardCompatibleWriter {
 
             const incomingMsg = this.incomingMessageMap.get(existingMsg.name);
             if (incomingMsg) {
-                finalMessages.push(mergeMessage(existingMsg, incomingMsg, this.errors));
+                finalMessages.push(mergeMessage(existingMsg, incomingMsg, this.reporter));
                 this.incomingMessageMap.delete(existingMsg.name);
             } else {
                 finalMessages.push(existingMsg);
@@ -75,7 +74,7 @@ export class BackwardCompatibleWriter {
 
             const incomingEnum = this.incomingEnumMap.get(existingEnum.name);
             if (incomingEnum) {
-                finalEnums.push(mergeEnum(existingEnum, incomingEnum));
+                finalEnums.push(mergeEnum(existingEnum, incomingEnum, this.reporter));
                 this.incomingEnumMap.delete(existingEnum.name);
             } else {
                 finalEnums.push(existingEnum);
@@ -96,20 +95,20 @@ export class BackwardCompatibleWriter {
             }
         }
 
-        // Check for errors before writing
-        if (this.errors.length > 0) {
-            logger.error('Backward compatibility errors:');
-            for (const error of this.errors) {
-                logger.error(`  ${error}`);
-            }
-            throw new BackwardCompatibilityError(
-                `Found ${this.errors.length} backward compatibility violation(s).`
-            );
+        // Write output using shared function (skip if dry-run)
+        if (dryRun) {
+            logger.info(`Dry run: would update ${this.outputPath}`);
+        } else {
+            writeProtoFile(finalMessages, finalEnums, this.outputPath);
+            logger.info(`Updated: ${this.outputPath}`);
         }
+    }
 
-        // Write output using shared function
-        writeProtoFile(finalMessages, finalEnums, this.outputPath);
-        logger.info(`Updated: ${this.outputPath}`);
+    /**
+     * Get the merge reporter for accessing change reports.
+     */
+    getReporter(): CompatibilityReporter {
+        return this.reporter;
     }
 }
 
@@ -124,13 +123,15 @@ if (require.main === module) {
             .argParser((val: string) => val.split(',').map(s => s.trim()))
             .default(['protos/generated/models/aggregated_models.proto', 'protos/generated/services/default_service.proto']))
         .addOption(new Option('-o, --output <path>', 'output proto file').default('protos/schemas/common.proto'))
+        .addOption(new Option('-d, --dry-run', 'preview changes without writing output file').default(false))
         .allowExcessArguments(false)
         .parse();
 
     type BackwardCompatOpts = {
         existing: string;
         incoming: string[];
         output: string;
+        dryRun: boolean;
     };
 
     const opts = command.opts() as BackwardCompatOpts;
@@ -146,17 +147,17 @@ if (require.main === module) {
         process.exit(1);
     }
 
-    try {
-        const writer = new BackwardCompatibleWriter(
-            opts.existing,
-            opts.incoming,
-            opts.output
-        );
-        writer.process();
-    } catch (error) {
-        if (error instanceof BackwardCompatibilityError) {
-            process.exit(1);
-        }
-        throw error;
-    }
+    const writer = new BackwardCompatibleWriter(
+        opts.existing,
+        opts.incoming,
+        opts.output
+    );
+
+    // Process and merge
+    writer.process(opts.dryRun);
+
+    // write report to temp directory
+    const reportPath = join(tmpdir(), 'merge-report.md');
+    writeFileSync(reportPath, writer.getReporter().toMarkdown());
+    logger.info(`Report written: ${reportPath}`);
 }