Merge pull request #53 from lucy66hw/Add_version

add version

Author: lucy66hw

.github/workflows/convert-proto.yml

@@ -3,13 +3,14 @@ name: Auto Proto Convert
 on:
   workflow_dispatch:
     inputs:
-      input_param:
-        description: ''
+      opensearch_version:
+        description: 'OpenSearch version (e.g., 3.4, 3.3.2). Leave empty to fetch latest.'
         required: false
+        type: string
 jobs:
   auto-proto-convert:
     runs-on: ubuntu-latest
-    if: github.repository == 'opensearch-project/opensearch-protobufs'
+    # if: github.repository == 'opensearch-project/opensearch-protobufs'
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -57,8 +58,55 @@ jobs:
             core.setOutput("latest_commit", latestCommit);
             console.log("Latest commit: " + latestCommit);
 
+      - name: Get Latest OpenSearch Core Version
+        id: get_opensearch_version
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            // Check if version was provided as input
+            const inputVersion = "${{ inputs.opensearch_version }}";
+            if (inputVersion && inputVersion.trim() !== "") {
+              core.setOutput("version", inputVersion.trim());
+              console.log("Using provided OpenSearch version: " + inputVersion.trim());
+              return;
+            }
+
+            // Otherwise fetch version from buildSrc/version.properties on main branch
+            try {
+              const response = await github.request(
+                'GET /repos/{owner}/{repo}/contents/{path}',
+                {
+                  owner: 'opensearch-project',
+                  repo: 'OpenSearch',
+                  path: 'buildSrc/version.properties',
+                  ref: 'main'
+                }
+              );
+
+              // Decode the file content
+              const content = Buffer.from(response.data.content, 'base64').toString('utf-8');
+
+              // Extract opensearch version (e.g., "opensearch = 3.4.0")
+              const match = content.match(/opensearch\s*=\s*([^\s]+)/);
+
+              if (match && match[1]) {
+                const version = match[1].trim();
+                core.setOutput("version", version);
+                console.log("Fetched OpenSearch version from source: " + version);
+              } else {
+                console.log("Warning: Could not parse version from buildSrc/version.properties");
+                core.setOutput("version", "unknown");
+              }
+            } catch (error) {
+              console.log("Warning: Could not fetch OpenSearch version: " + error.message);
+              core.setOutput("version", "unknown");
+            }
+
       - name: Run Proto Conversion
-        run: npm ci && npm run preprocessing
+        env:
+          OPENSEARCH_VERSION: ${{ steps.get_opensearch_version.outputs.version }}
+        run: npm ci && npm run preprocessing -- --opensearch-version "$OPENSEARCH_VERSION"
 
       - name: Clone Protobuf Generator Repository
         run: |
@@ -105,10 +153,13 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           branch: auto-pr-branch
           commit-message: "Protobuf schema change detected"
-          title: "[Automated PR]: Update generated protobuf schema (spec commit: ${{ steps.get_commit.outputs.latest_commit }})"
+          title: "[Automated PR]: Update generated protobuf schema (OpenSearch: ${{ steps.get_opensearch_version.outputs.version }}, spec commit: ${{ steps.get_commit.outputs.latest_commit }})"
           signoff: true
           base: main
           delete-branch: true
           labels: skip-changelog
           body: |
             This pull request was automatically generated by GitHub Actions.
+
+            **OpenSearch Version**: ${{ steps.get_opensearch_version.outputs.version }}
+            **API Spec Commit**: ${{ steps.get_commit.outputs.latest_commit }}

package-lock.json

@@ -48,6 +48,7 @@
     "lodash.isequal": "^4.5.0",
     "protobufjs": "^6.11.4",
     "qs": "^6.12.1",
+    "semver": "^7.6.0",
     "smile-js": "^0.7.0",
     "titlecase": "^1.1.3",
     "tmp": "^0.2.4",

package.json

@@ -7,6 +7,7 @@ import * as path from 'path';
 import {SchemaModifier} from "./SchemaModifier";
 import {VendorExtensionProcessor} from "./VendorExtensionProcessor";
 import {GlobalParameterConsolidator} from "./GlobalParamWrapper";
+import {VersionProcessor} from "./VersionProcessor";
 import type {OpenAPIV3} from "openapi-types";
 
 let config_filtered_path: string[] | undefined;
@@ -30,6 +31,7 @@ const command = new Command()
       .default(default_api_to_proto)
   )
   .addOption(new Option('--verbose', 'show merge details').default(false))
+  .addOption(new Option('--opensearch-version <version>', 'current OpenSearch version for deprecation removal').default('3.4'))
   .allowExcessArguments(false)
   .parse();
 
@@ -39,6 +41,7 @@ type PreprocessingOpts = {
   output: string;
   filtered_path: string[];
   verbose: boolean;
+  opensearchVersion: string;
 };
 
 const opts = command.opts() as PreprocessingOpts;
@@ -49,7 +52,8 @@ try {
   logger.info(`PreProcessing ${opts.filtered_path.join(', ')} into ${opts.output} ...`)
   const original_spec = read_yaml(opts.input)
   const filtered_spec = new Filter().filter_spec(original_spec, opts.filtered_path);
-  const sanitized_spec = new Sanitizer().sanitize(filtered_spec);
+  const version_processed_spec = new VersionProcessor(filtered_spec, logger).process(opts.opensearchVersion);
+  const sanitized_spec = new Sanitizer().sanitize(version_processed_spec);
   const consolidated_spec = new GlobalParameterConsolidator(sanitized_spec).consolidate();
   const vendor_processed_spec = new VendorExtensionProcessor(consolidated_spec, logger).process();
   const schema_modified_spec = new SchemaModifier(vendor_processed_spec, logger).modify();

tools/proto-convert/src/PreProcessing.ts

@@ -0,0 +1,62 @@
+import _ from 'lodash';
+import * as semver from 'semver';
+import Logger from './utils/logger';
+import { deleteMatchingKeys } from './utils/helper';
+import type { OpenAPIV3 } from 'openapi-types';
+
+/**
+ * Processes version-related vendor extensions:
+ * - x-version-added: Removes fields added after current version
+ * - x-version-deprecated: Removes fields deprecated in current version or earlier
+ * - x-version-removed: Removes fields removed before current version
+ */
+export class VersionProcessor {
+  private _logger: Logger;
+  private _spec: OpenAPIV3.Document;
+  private _target_version: string;
+
+  constructor(spec: OpenAPIV3.Document, logger: Logger) {
+    this._spec = spec;
+    this._logger = logger;
+    this._target_version = '';
+  }
+
+
+  process(currentVersion: string): OpenAPIV3.Document {
+    this._target_version = currentVersion;
+    this._logger.info(`Processing version constraints for OpenSearch ${currentVersion} ...`);
+    deleteMatchingKeys(this._spec, (item: any) => {
+      if (_.isObject(item) && this.#exclude_per_semver(item)) {
+        return true;
+      }
+      return false;
+    });
+    this._logger.info('Version processing complete');
+    return this._spec;
+  }
+
+  #exclude_per_semver(obj: any): boolean {
+    if (this._target_version == undefined) return false
+
+    const x_version_added = semver.coerce(obj['x-version-added'] as string)
+    const x_version_deprecated = semver.coerce(obj['x-version-deprecated'] as string)
+    const x_version_removed = semver.coerce(obj['x-version-removed'] as string)
+
+    // If field was added in a future version, exclude it
+    if (x_version_added !== null && x_version_added !== undefined && !semver.satisfies(this._target_version, `>=${x_version_added.toString()}`)) {
+      return true
+    }
+
+    // If field was deprecated in current version or earlier, exclude it
+    if (x_version_deprecated !== null && x_version_deprecated !== undefined && !semver.satisfies(this._target_version, `<${x_version_deprecated.toString()}`)) {
+      return true
+    }
+
+    // If field was removed in current version or earlier, exclude it
+    if (x_version_removed !== null && x_version_removed !== undefined && !semver.satisfies(this._target_version, `<${x_version_removed.toString()}`)) {
+      return true
+    }
+
+    return false
+  }
+}