Preprocessing - Add schema exclusion list to filter out schemas and their dependencies (#328)

Signed-off-by: xil <fridalu66@gmail.com>

Author: lucy66hw

CHANGELOG.md

@@ -8,6 +8,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Preprocessing - Handling spec added/deprecated versioning.([#309](https://github.com/opensearch-project/opensearch-protobufs/pull/309))
 - preprocessing - Support maxProperties=1 constraints by marking them as `oneof` for protobuf generation ([#317](https://github.com/opensearch-project/opensearch-protobufs/pull/317))
 - Preprocessing - Support x-protobuf-required to enforce required protobuf field and convert oneOf properties pattern to min/max Properties = 1 ([#318](https://github.com/opensearch-project/opensearch-protobufs/pull/318))
+- Preprocessing - Add schema exclusion list to filter out schemas and their dependencies ([#328](https://github.com/opensearch-project/opensearch-protobufs/pull/328))
 
 ### Changed
 - Backward incompatible change for unimplemented query types `ScriptScoreQuery`, `SimpleQueryStringQuery`, `DisMaxQuery`, `IntervalsQuery`, `QueryStringQuery` and `TermsAggregation` ([#324](https://github.com/opensearch-project/opensearch-protobufs/pull/324))

DEVELOPER_GUIDE.md

@@ -75,7 +75,7 @@ dependencies {
 ```
 
 ## Python
-### Generate and install Python Code 
+### Generate and install Python Code
 
 Generate the wheel file with bazel and install the packag with pip:
 ```
@@ -93,7 +93,7 @@ pip install bazel-bin/opensearch_protos-*-py3-none-any.whl
 The [Spec Preprocessing](tools/proto-convert/src/PreProcessing.ts) includes two steps:
 
 1. **Filter**
-    - Filters only the target APIs defined in [target_api.yaml](tools/src/config/target_api.yaml).
+    - Filters only the target APIs defined in [spec-filter.yaml](tools/proto-convert/src/config/spec-filter.yaml).
     - Extract a single API per group from the OpenSearch spec.
 
 2. **Sanitizer**

tools/proto-convert/src/Filter.ts

@@ -1,11 +1,14 @@
 import { type OpenAPIV3 } from 'openapi-types'
 import _ from "lodash";
 import Logger from "./utils/logger"
+import { getSchemaNames } from "./utils/helper"
+
 /**
  * Recursively traverses a node and for every $ref that starts with "#/components/",
- * enqueues the reference string if it hasn’t been visited yet.
+ * enqueues the reference string if it hasn't been visited yet.
+ * Skips schemas in the exclusion list.
  */
-function traverse_and_enqueue(node: any, queue: string[],  visited: Set<string>): void {
+function traverse_and_enqueue(node: any, queue: string[], visited: Set<string>, excluded: Set<string>): void {
   for (const key in node) {
     var item = node[key]
 
@@ -14,22 +17,40 @@ function traverse_and_enqueue(node: any, queue: string[],  visited: Set<string>)
       if (ref == null || ref == "" && _.isString(item)){
         ref = item as string;
       }
+
+      // Check exclusion list - if schema is excluded, don't push ref
+      const names = getSchemaNames(ref);
+      if (names && (excluded.has(names.full) || excluded.has(names.short))) {
+        continue;
+      }
+
       queue.push(ref);
       visited.add(ref);
     }
     if (_.isObject(item) || _.isArray(item) || (_.isString(item) && item.startsWith('#/components/'))) {
-      traverse_and_enqueue(item, queue, visited)
+      traverse_and_enqueue(item, queue, visited, excluded)
     }
   }
 }
 
-//Filter an OpenAPI spec so that only the specified path and all its referenced components (via $ref) are included.
+/**
+ * Filters an OpenAPI spec to include only specified paths and their referenced components.
+ * Schemas in the excluded set are skipped.
+ */
 export default class Filter {
   logger: Logger
   protected _spec: Record<string, any>
+  protected targetPaths: string[]
+  protected excludedSchemas: Set<string>
   paths: Record<string, Record<string, OpenAPIV3.PathItemObject>> = {} // namespace -> path -> path_item_object
-  constructor(logger: Logger = new Logger()) {
+
+  constructor(logger: Logger, targetPaths: string[], excludedSchemas: Set<string> = new Set()) {
     this.logger = logger
+    this.targetPaths = targetPaths;
+    this.excludedSchemas = excludedSchemas;
+    if (this.excludedSchemas.size > 0) {
+      this.logger.info(`Loaded ${this.excludedSchemas.size} excluded schemas: ${Array.from(this.excludedSchemas).join(', ')}`);
+    }
     this._spec = {
       openapi: '3.1.0',
       info: {},
@@ -44,9 +65,9 @@ export default class Filter {
   }
 
 
-  filter_spec(spec: Record<string, any>, paths_to_keep: string[]): any {
+  filter_spec(spec: Record<string, any>): any {
     this._spec.info = spec.info;
-    for (const p of paths_to_keep) {
+    for (const p of this.targetPaths) {
       if (spec.paths[p] === undefined) {
         this.logger.error(`Path not found in spec: ${p}`);
         continue;
@@ -58,7 +79,7 @@ export default class Filter {
     const visited: Set<string> = new Set();
 
     // collect all components that are referenced by the paths
-    traverse_and_enqueue(this._spec.paths , queue, visited);
+    traverse_and_enqueue(this._spec.paths, queue, visited, this.excludedSchemas);
     while (queue.length > 0) {
       const ref_str = queue.shift();
       if (ref_str == null || ref_str == "") continue;
@@ -73,7 +94,7 @@ export default class Filter {
       if (this._spec.components[sub_component][key] == null) {
         if (spec.components != null && spec.components[sub_component] != null && spec.components[sub_component][key] != null) {
           this._spec.components[sub_component][key] = spec.components[sub_component][key];
-          traverse_and_enqueue(this._spec.components[sub_component][key], queue, visited);
+          traverse_and_enqueue(this._spec.components[sub_component][key], queue, visited, this.excludedSchemas);
         }
       }
     }
@@ -141,4 +162,4 @@ export default class Filter {
     }
     this._spec.paths = new_paths;
   }
-}
+}

tools/proto-convert/src/PreProcessing.ts

@@ -8,18 +8,13 @@ import {SchemaModifier} from "./SchemaModifier";
 import {VendorExtensionProcessor} from "./VendorExtensionProcessor";
 import {GlobalParameterConsolidator} from "./GlobalParamWrapper";
 import {OpenSearchVersionExtractor} from "./OpenSearchVersionExtractor";
-import type {OpenAPIV3} from "openapi-types";
 
-let config_filtered_path: string[] | undefined;
-try {
-  const config = read_yaml(path.join(__dirname, 'config','target_api.yaml'));
-  config_filtered_path = config.paths;
-} catch (e) {
-  console.error(e);
-  config_filtered_path = undefined;
-}
-const default_api_to_proto = config_filtered_path ?? ['/_search'];
-const default_api_to_proto_str = default_api_to_proto.join(',');
+// Load config from spec-filter.yaml
+const config = read_yaml<{ paths?: string[]; excluded_schemas?: string[] }>(
+  path.join(__dirname, 'config', 'spec-filter.yaml')
+);
+const target_paths = config.paths ?? ['/_search'];
+const excluded_schemas = new Set(config.excluded_schemas ?? []);
 
 const command = new Command()
   .description('Preprocess an OpenAPI spec by filtering for specific paths and then sanitizing it.')
@@ -28,7 +23,7 @@ const command = new Command()
   .addOption(
     new Option('-p, --filtered_path <paths>', 'the paths to keep (comma-separated, e.g., /_search,)')
       .argParser((val: string) => val.split(',').map(s => s.trim()))
-      .default(default_api_to_proto)
+      .default(target_paths)
   )
   .addOption(new Option('--verbose', 'show merge details').default(false))
   .addOption(new Option('--opensearch-version <version>', 'current OpenSearch version for deprecation removal').default('3.4'))
@@ -51,7 +46,7 @@ const logger = new Logger();
 try {
   logger.info(`PreProcessing ${opts.filtered_path.join(', ')} into ${opts.output} ...`)
   const original_spec = read_yaml(opts.input)
-  const filtered_spec = new Filter().filter_spec(original_spec, opts.filtered_path);
+  const filtered_spec = new Filter(logger, opts.filtered_path, excluded_schemas).filter_spec(original_spec);
   const version_processed_spec = new OpenSearchVersionExtractor(filtered_spec, logger).process(opts.opensearchVersion);
   const sanitized_spec = new Sanitizer().sanitize(version_processed_spec);
   const consolidated_spec = new GlobalParameterConsolidator(sanitized_spec).consolidate();

tools/proto-convert/src/config/spec-filter.yaml

@@ -0,0 +1,37 @@
+# Target API paths to include in proto generation
+paths:
+  - /{index}/_bulk
+  - /{index}/_search
+
+# Schemas to exclude from proto generation
+# These schemas and their nested dependencies will not be included
+excluded_schemas:
+  - AggregationContainer
+  - Aggregate
+  - Suggester
+  - Suggest
+  - CommonTermsQuery
+  - CombinedFieldsQuery
+  - DistanceFeatureQuery
+  - GeoPolygonQuery
+  - GeoShapeQuery
+  - HasChildQuery
+  - HasParentQuery
+  - MoreLikeThisQuery
+  - NeuralQuery
+  - ParentIdQuery
+  - PercolateQuery
+  - RankFeatureQuery
+  - SpanContainingQuery
+  - SpanFieldMaskingQuery
+  - SpanFirstQuery
+  - SpanMultiTermQuery
+  - SpanNearQuery
+  - SpanNotQuery
+  - SpanOrQuery
+  - SpanTermQuery
+  - SpanWithinQuery
+  - ObjectMap
+  - TypeQuery
+  - WrapperQuery
+  - XyShapeQuery

tools/proto-convert/src/config/target_api.yaml

@@ -4,6 +4,16 @@ import {dirname} from "path";
 import {OpenAPIV3} from "openapi-types";
 import _ from 'lodash';
 
+/**
+ * Extracts schema names from a $ref string.
+ */
+export function getSchemaNames(ref: string): { full: string; short: string } | null {
+  if (!ref.startsWith('#/components/schemas/')) return null;
+  const full = ref.split('/').pop() || '';
+  const short = full.includes('___') ? full.split('___').pop() || full : full;
+  return { full, short };
+}
+
 export function read_yaml<T = Record<string, any>> (file_path: string, exclude_schema: boolean = false): T {
     const doc = parse(readFileSync(file_path, 'utf8'))
     if (typeof doc === 'object' && exclude_schema) delete doc.$schema
@@ -172,9 +182,24 @@ export function remove_unused(spec: OpenAPIV3.Document): void {
         (key) => _.keys((spec?.components as any)?.[key]).map((ref) => `#/components/${key}/${ref}`)
     );
 
-    deleteMatchingKeys(spec, (obj: any) =>
-        obj.$ref !== undefined && !_.includes(remaining, obj.$ref)
-    );
+    // Remove properties where $ref is broken (direct or nested in additionalProperties)
+    deleteMatchingKeys(spec, (obj: any) => {
+        // Case 1: Direct broken $ref
+        if (obj.$ref !== undefined && !_.includes(remaining, obj.$ref)) {
+            return true;
+        }
+        // Case 2: additionalProperties.$ref is broken
+        if (obj.additionalProperties?.$ref !== undefined &&
+            !_.includes(remaining, obj.additionalProperties.$ref)) {
+            return true;
+        }
+        // Case 3: additionalProperties.items.$ref is broken (for array types)
+        if (obj.additionalProperties?.items?.$ref !== undefined &&
+            !_.includes(remaining, obj.additionalProperties.items.$ref)) {
+            return true;
+        }
+        return false;
+    });
 }
 
 /**