enable postprocessing and optimize the log

Signed-off-by: xil <fridalu66@gmail.com>

Author: lucy66hw

.github/workflows/convert-proto.yml

@@ -121,12 +121,8 @@ jobs:
         run: |
           java -jar cloned-repo/modules/openapi-generator-cli/target/openapi-generator-cli.jar generate -c tools/proto-convert/src/config/protobuf-generator-config.yaml
 
-      #      - name: Post Process Protobuf
-      #        run: npm run postprocessing
-
-      - name: Reformat proto files
-        run: |
-         buf format -w protos/generated
+      - name: Post Process Protobuf
+        run: npm run postprocessing
 
       - name: Configure Git User
         run: |

package.json

@@ -6,10 +6,10 @@
   "license": "Apache-2.0",
   "scripts": {
     "preprocessing": "ts-node tools/proto-convert/src/PreProcessing.ts",
-    "backward-compat": "ts-node tools/proto-convert/src/backward-compat/BackwardCompatibleWriter.ts",
-    "cleanup-common": "ts-node tools/proto-convert/src/backward-compat/CleanupUnusedMessages.ts -i protos/schemas/common.proto",
+    "backward-compat": "ts-node tools/proto-convert/src/postprocessing/BackwardCompatibleWriter.ts",
+    "cleanup-common": "ts-node tools/proto-convert/src/postprocessing/CleanupUnusedMessages.ts -i protos/schemas/common.proto",
     "postprocessing": "npm run backward-compat && npm run cleanup-common",
-    "test": "npx jest --testMatch='**/*.test.ts --no-watchman'"
+    "test": "npx jest --testMatch='**/*.test.ts' --no-watchman"
   },
   "dependencies": {
     "@apidevtools/swagger-parser": "^12.1.0",

protos/schemas/common.proto

@@ -1433,6 +1433,9 @@ message QueryContainer {
     // To combine relevance scores from multiple queries into one score for a given document.
     HybridQuery hybrid = 32;
 
+    // Return documents using a template query contains placeholders that are resolved by search request processors during query execution.
+    ObjectMap template = 33;
+
   }
 }
 
@@ -3194,6 +3197,7 @@ enum VersionType {
   VERSION_TYPE_EXTERNAL = 1;
   VERSION_TYPE_EXTERNAL_GTE = 2;
   VERSION_TYPE_INTERNAL = 3;
+  VERSION_TYPE_FORCE = 4;
 }
 
 enum Refresh {
@@ -3393,6 +3397,7 @@ enum FieldType {
   FIELD_TYPE_WILDCARD = 44;
   FIELD_TYPE_XY_POINT = 45;
   FIELD_TYPE_XY_SHAPE = 46;
+  FIELD_TYPE_SEMANTIC = 47;
 }
 
 enum SortOrder {
@@ -3444,6 +3449,7 @@ enum HighlighterOrder {
 enum HighlighterTagsSchema {
   HIGHLIGHTER_TAGS_SCHEMA_UNSPECIFIED = 0;
   HIGHLIGHTER_TAGS_SCHEMA_STYLED = 1;
+  HIGHLIGHTER_TAGS_SCHEMA_DEFAULT = 2;
 }
 
 enum DistanceUnit {

tools/proto-convert/src/Filter.ts

@@ -1,6 +1,6 @@
 import { type OpenAPIV3 } from 'openapi-types'
 import _ from "lodash";
-import Logger from "./utils/logger"
+import logger from "./utils/logger"
 import { getSchemaNames } from "./utils/helper"
 
 /**
@@ -38,18 +38,18 @@ function traverse_and_enqueue(node: any, queue: string[], visited: Set<string>,
  * Schemas in the excluded set are skipped.
  */
 export default class Filter {
-  logger: Logger
   protected _spec: Record<string, any>
+  protected sourceSpec: Record<string, any>
   protected targetPaths: string[]
   protected excludedSchemas: Set<string>
   paths: Record<string, Record<string, OpenAPIV3.PathItemObject>> = {} // namespace -> path -> path_item_object
 
-  constructor(logger: Logger, targetPaths: string[], excludedSchemas: Set<string> = new Set()) {
-    this.logger = logger
+  constructor(sourceSpec: Record<string, any>, targetPaths: string[], excludedSchemas: Set<string> = new Set()) {
+    this.sourceSpec = sourceSpec;
     this.targetPaths = targetPaths;
     this.excludedSchemas = excludedSchemas;
     if (this.excludedSchemas.size > 0) {
-      this.logger.info(`Loaded ${this.excludedSchemas.size} excluded schemas: ${Array.from(this.excludedSchemas).join(', ')}`);
+      logger.info(`Loaded ${this.excludedSchemas.size} excluded schemas: ${Array.from(this.excludedSchemas).join(', ')}`);
     }
     this._spec = {
       openapi: '3.1.0',
@@ -65,14 +65,14 @@ export default class Filter {
   }
 
 
-  filter_spec(spec: Record<string, any>): any {
-    this._spec.info = spec.info;
+  filter(): OpenAPIV3.Document {
+    this._spec.info = this.sourceSpec.info;
     for (const p of this.targetPaths) {
-      if (spec.paths[p] === undefined) {
-        this.logger.error(`Path not found in spec: ${p}`);
+      if (this.sourceSpec.paths[p] === undefined) {
+        logger.error(`Path not found in spec: ${p}`);
         continue;
       }
-      this._spec.paths[p] = spec.paths[p];
+      this._spec.paths[p] = this.sourceSpec.paths[p];
     }
     this.filter_by_max_parameters(this._spec.paths as OpenAPIV3.PathsObject);
     const queue: string[] = [];
@@ -92,13 +92,13 @@ export default class Filter {
         this._spec.components[sub_component] = {};
       }
       if (this._spec.components[sub_component][key] == null) {
-        if (spec.components != null && spec.components[sub_component] != null && spec.components[sub_component][key] != null) {
-          this._spec.components[sub_component][key] = spec.components[sub_component][key];
+        if (this.sourceSpec.components != null && this.sourceSpec.components[sub_component] != null && this.sourceSpec.components[sub_component][key] != null) {
+          this._spec.components[sub_component][key] = this.sourceSpec.components[sub_component][key];
           traverse_and_enqueue(this._spec.components[sub_component][key], queue, visited, this.excludedSchemas);
         }
       }
     }
-    return this._spec;
+    return this._spec as OpenAPIV3.Document;
   }
 
   filter_by_max_parameters(paths: OpenAPIV3.PathsObject): void {

tools/proto-convert/src/OpenSearchVersionExtractor.ts

@@ -1,6 +1,6 @@
 import _ from 'lodash';
 import * as semver from 'semver';
-import Logger from './utils/logger';
+import logger from './utils/logger';
 import { deleteMatchingKeys, remove_unused } from './utils/helper';
 import type { OpenAPIV3 } from 'openapi-types';
 
@@ -10,26 +10,24 @@ import type { OpenAPIV3 } from 'openapi-types';
  * x-version-removed: Removes fields if removed version <= target version.
  */
 export class OpenSearchVersionExtractor {
-  private _logger: Logger;
   private _spec: OpenAPIV3.Document;
   private _target_version: string;
 
-  constructor(spec: OpenAPIV3.Document, logger: Logger) {
+  constructor(spec: OpenAPIV3.Document) {
     this._spec = spec;
-    this._logger = logger;
     this._target_version = '';
   }
 
 
   process(currentVersion: string): OpenAPIV3.Document {
     const coerced = semver.coerce(currentVersion);
     this._target_version = coerced?.toString() || currentVersion;
-    this._logger.info(`Processing version constraints for OpenSearch ${this._target_version} ...`);
+    logger.info(`Processing version constraints for OpenSearch ${this._target_version} ...`);
 
     deleteMatchingKeys(this._spec, this.#exclude_per_semver.bind(this));
     remove_unused(this._spec);
 
-    this._logger.info('Version processing complete');
+    logger.info('Version processing complete');
     return this._spec;
   }
 

tools/proto-convert/src/PreProcessing.ts

@@ -2,7 +2,7 @@ import { Command, Option } from '@commander-js/extra-typings';
 import { read_yaml, write_yaml } from './utils/helper';
 import Filter from './Filter';
 import { Sanitizer } from './Sanitizer';
-import Logger from './utils/logger';
+import logger from './utils/logger';
 import * as path from 'path';
 import {SchemaModifier} from "./SchemaModifier";
 import {VendorExtensionProcessor} from "./VendorExtensionProcessor";
@@ -41,17 +41,15 @@ type PreprocessingOpts = {
 
 const opts = command.opts() as PreprocessingOpts;
 
-const logger = new Logger();
-
 try {
   logger.info(`PreProcessing ${opts.filtered_path.join(', ')} into ${opts.output} ...`)
   const original_spec = read_yaml(opts.input)
-  const filtered_spec = new Filter(logger, opts.filtered_path, excluded_schemas).filter_spec(original_spec);
-  const version_processed_spec = new OpenSearchVersionExtractor(filtered_spec, logger).process(opts.opensearchVersion);
-  const sanitized_spec = new Sanitizer().sanitize(version_processed_spec);
+  const filtered_spec = new Filter(original_spec, opts.filtered_path, excluded_schemas).filter();
+  const version_processed_spec = new OpenSearchVersionExtractor(filtered_spec).process(opts.opensearchVersion);
+  const sanitized_spec = new Sanitizer(version_processed_spec).sanitize();
   const consolidated_spec = new GlobalParameterConsolidator(sanitized_spec).consolidate();
-  const vendor_processed_spec = new VendorExtensionProcessor(consolidated_spec, logger).process();
-  const schema_modified_spec = new SchemaModifier(vendor_processed_spec, logger).modify();
+  const vendor_processed_spec = new VendorExtensionProcessor(consolidated_spec).process();
+  const schema_modified_spec = new SchemaModifier(vendor_processed_spec).modify();
   write_yaml(opts.output, schema_modified_spec);
 
 } catch (err) {

tools/proto-convert/src/Sanitizer.ts

@@ -8,10 +8,16 @@ import {traverse} from "./utils/OpenApiTraverser";
  */
 export class Sanitizer {
   private static readonly META_PREFIX = "x";
-  public sanitize(spec: any): any {
-    this.sanitize_ref(spec);
-    this.sanitize_spec_name(spec as OpenAPIV3.Document);
-    return spec;
+  private spec: OpenAPIV3.Document;
+
+  constructor(spec: OpenAPIV3.Document) {
+    this.spec = spec;
+  }
+
+  public sanitize(): OpenAPIV3.Document {
+    this.sanitize_ref(this.spec);
+    this.sanitize_spec_name(this.spec);
+    return this.spec;
   }
 
   sanitize_ref(obj: any): void {

tools/proto-convert/src/SchemaModifier.ts

@@ -2,18 +2,16 @@ import type {OpenAPIV3} from "openapi-types";
 import {traverse} from './utils/OpenApiTraverser';
 import isEqual from 'lodash.isequal';
 import {compressMultipleUnderscores, isPrimitiveType, resolveObj, isReferenceObject, isEmptyObjectSchema, is_simple_ref} from './utils/helper';
-import Logger from "./utils/logger";
+import logger from "./utils/logger";
 
 
 const DEFAULT_MAP_KEY = 'field' // default key for simplified additionalProperties
 const DEFAULT_MAP_VALUE = 'value' // default value for simplified additionalProperties
 
 export class SchemaModifier {
-    logger: Logger
     root: OpenAPIV3.Document;
-    constructor(root: OpenAPIV3.Document, logger: Logger = new Logger()) {
+    constructor(root: OpenAPIV3.Document) {
         this.root = root;
-        this.logger = logger;
     }
     public modify(): OpenAPIV3.Document {
         traverse(this.root, {
@@ -256,7 +254,7 @@ export class SchemaModifier {
             }
         } else if (complexObject.type === 'object' && complexObject.properties) {
             if (complexObject.properties[DEFAULT_MAP_KEY]) {
-                this.logger.error("Error: additionalProperties key already exists in the schema "+complexObject);
+                logger.error("Error: additionalProperties key already exists in the schema "+complexObject);
             }
             complexObject.properties[DEFAULT_MAP_KEY] = this.createAdditionalPropertySchema().properties?.[DEFAULT_MAP_KEY] as OpenAPIV3.SchemaObject || {};
         } else if (isPrimitiveType(complexObject)) {
@@ -405,7 +403,7 @@ export class SchemaModifier {
         }
 
         if (schema.properties[propertyName]) {
-            this.logger.warn(`Property '${propertyName}' already exists in schema, skipping additionalProperties conversion`);
+            logger.warn(`Property '${propertyName}' already exists in schema, skipping additionalProperties conversion`);
             return;
         }
 
@@ -437,7 +435,7 @@ export class SchemaModifier {
             delete schema.propertyNames;
         }
 
-        this.logger.info(`Converted additionalProperties to named property '${propertyName}' with type: object`);
+        logger.info(`Converted additionalProperties to named property '${propertyName}' with type: object`);
     }
 
     /**
@@ -473,7 +471,7 @@ export class SchemaModifier {
                 schema.additionalProperties = items.additionalProperties;
                 delete (schema as any).items;
 
-                this.logger.info(`Removed array wrapper from array of maps schema`);
+                logger.info(`Removed array wrapper from array of maps schema`);
             }
         }
     }
@@ -514,7 +512,7 @@ export class SchemaModifier {
         if (Array.isArray(schema.oneOf) && schema.oneOf.length > 0) {
             if (this.isOneOfWithSingleProperties(schema.oneOf)) {
                 this.flattenOneOfToProperties(schema);
-                this.logger.info(`Converted oneOf pattern to minProperties/maxProperties`);
+                logger.info(`Converted oneOf pattern to minProperties/maxProperties`);
             }
         }
     }
@@ -587,10 +585,10 @@ export class SchemaModifier {
                 }
             }
             (schema as any)['x-oneof-schema'] = true;
-            this.logger.info(`Added x-oneof-property to properties and marked schema with x-oneof-schema`);
+            logger.info(`Added x-oneof-property to properties and marked schema with x-oneof-schema`);
         } else if (hasNestedPattern) {
             (schema as any)['x-oneof-schema'] = true;
-            this.logger.info(`Marked parent schema with x-oneof-schema (contains nested oneOf pattern)`);
+            logger.info(`Marked parent schema with x-oneof-schema (contains nested oneOf pattern)`);
         }
     }
 

tools/proto-convert/src/VendorExtensionProcessor.ts

@@ -1,7 +1,7 @@
 import { OpenAPIV3 } from 'openapi-types';
 import { traverse } from './utils/OpenApiTraverser';
 import { resolveRef, deleteMatchingKeys, remove_unused } from './utils/helper';
-import Logger from './utils/logger';
+import logger from './utils/logger';
 import _ from 'lodash';
 
 /**
@@ -23,11 +23,9 @@ export class VendorExtensionProcessor {
     };
 
     private root: OpenAPIV3.Document;
-    private logger: Logger;
 
-    constructor(root: OpenAPIV3.Document, logger: Logger = new Logger()) {
+    constructor(root: OpenAPIV3.Document) {
         this.root = root;
-        this.logger = logger;
     }
 
     /**
@@ -90,7 +88,7 @@ export class VendorExtensionProcessor {
             const oldName = param.name;
             param.name = newName;
             delete param[VendorExtensionProcessor.PROTOBUF_NAME_EXTENSION];
-            this.logger.info(`Renamed parameter '${oldName}' -> '${newName}' (${VendorExtensionProcessor.PROTOBUF_NAME_EXTENSION})`);
+            logger.info(`Renamed parameter '${oldName}' -> '${newName}' (${VendorExtensionProcessor.PROTOBUF_NAME_EXTENSION})`);
         }
     }
 
@@ -114,7 +112,7 @@ export class VendorExtensionProcessor {
                         delete schema.properties[prop];
                         delete schema.properties[newName][VendorExtensionProcessor.PROTOBUF_NAME_EXTENSION];
 
-                        this.logger.info(`Renamed property '${prop}' -> '${newName}' (${VendorExtensionProcessor.PROTOBUF_NAME_EXTENSION})`);
+                        logger.info(`Renamed property '${prop}' -> '${newName}' (${VendorExtensionProcessor.PROTOBUF_NAME_EXTENSION})`);
                     }
                 }
             }
@@ -133,7 +131,7 @@ export class VendorExtensionProcessor {
                         const oldTitle = subschema.title;
                         subschema.title = titleValue;
                         delete subschema[VendorExtensionProcessor.PROTOBUF_NAME_EXTENSION];
-                        this.logger.info(`Set title for ${key} sub-schema: '${oldTitle}' -> '${titleValue}' (${VendorExtensionProcessor.PROTOBUF_NAME_EXTENSION})`);
+                        logger.info(`Set title for ${key} sub-schema: '${oldTitle}' -> '${titleValue}' (${VendorExtensionProcessor.PROTOBUF_NAME_EXTENSION})`);
                     }
                 }
             }
@@ -180,7 +178,7 @@ export class VendorExtensionProcessor {
             }
 
             delete schema[VendorExtensionProcessor.PROTOBUF_TYPE_EXTENSION];
-            this.logger.info(`Applied ${VendorExtensionProcessor.PROTOBUF_TYPE_EXTENSION}: ${protoType} -> type: ${schema.type}${schema.format ? `, format: ${schema.format}` : ''}`);
+            logger.info(`Applied ${VendorExtensionProcessor.PROTOBUF_TYPE_EXTENSION}: ${protoType} -> type: ${schema.type}${schema.format ? `, format: ${schema.format}` : ''}`);
         }
     }