feat: auto-detect Kubernetes crd schema

README.md

@@ -41,6 +41,8 @@ The following settings are supported:
 - `yaml.schemas`: Helps you associate schemas with files in a glob pattern
 - `yaml.schemaStore.enable`: When set to true the YAML language server will pull in all available schemas from [JSON Schema Store](https://www.schemastore.org/json/)
 - `yaml.schemaStore.url`: URL of a schema store catalog to use when downloading schemas.
+- `yaml.kubernetesCRDStore.enable`: When set to true the YAML language server will parse Kubernetes CRDs automatically and download them from the [CRD store](https://github.com/datreeio/CRDs-catalog).
+- `yaml.kubernetesCRDStore.url`: URL of a crd store catalog to use when downloading schemas. Defaults to `https://raw.githubusercontent.com/datreeio/CRDs-catalog/main`.
 - `yaml.customTags`: Array of custom tags that the parser will validate against. It has two ways to be used. Either an item in the array is a custom tag such as "!Ref" and it will automatically map !Ref to scalar or you can specify the type of the object !Ref should be e.g. "!Ref sequence". The type of object can be either scalar (for strings and booleans), sequence (for arrays), map (for objects).
 - `yaml.maxItemsComputed`: The maximum number of outline symbols and folding regions computed (limited for performance reasons).
 - `[yaml].editor.tabSize`: the number of spaces to use when autocompleting. Takes priority over editor.tabSize.

src/languageserver/handlers/settingsHandlers.ts

@@ -84,6 +84,14 @@ export class SettingsHandler {
           this.yamlSettings.schemaStoreUrl = settings.yaml.schemaStore.url;
         }
       }
+
+      if (settings.yaml.kubernetesCRDStore) {
+        this.yamlSettings.kubernetesCRDStoreEnabled = settings.yaml.kubernetesCRDStore.enable;
+        if (settings.yaml.kubernetesCRDStore.url?.length !== 0) {
+          this.yamlSettings.kubernetesCRDStoreUrl = settings.yaml.kubernetesCRDStore.url;
+        }
+      }
+
       if (settings.files?.associations) {
         for (const [ext, languageId] of Object.entries(settings.files.associations)) {
           if (languageId === 'yaml') {

src/languageservice/services/crdUtil.ts

@@ -0,0 +1,89 @@
+import { SingleYAMLDocument } from '../parser/yamlParser07';
+import { JSONDocument } from '../parser/jsonParser07';
+
+import { ResolvedSchema } from 'vscode-json-languageservice/lib/umd/services/jsonSchemaService';
+import { JSONSchema } from 'vscode-json-languageservice/lib/umd/jsonSchema';
+import { KUBERNETES_SCHEMA_URL } from '../utils/schemaUrls';
+
+/**
+ * Retrieve schema by auto-detecting the Kubernetes GroupVersionKind (GVK) from the document.
+ * If there is no definition for the GVK in the main kubernetes schema,
+ * the schema is then retrieved from the CRD catalog.
+ * Public for testing purpose, not part of the API.
+ * @param doc
+ * @param crdCatalogURI The URL of the CRD catalog to retrieve the schema from
+ * @param kubernetesSchema The main kubernetes schema, if it includes a definition for the GVK it will be used
+ */
+export function autoDetectKubernetesSchemaFromDocument(
+  doc: SingleYAMLDocument | JSONDocument,
+  crdCatalogURI: string,
+  kubernetesSchema: ResolvedSchema
+): string | undefined {
+  const res = getGroupVersionKindFromDocument(doc);
+  if (!res) {
+    return undefined;
+  }
+  const { group, version, kind } = res;
+  if (!group || !version || !kind) {
+    return undefined;
+  }
+
+  const k8sSchema: JSONSchema = kubernetesSchema.schema;
+  let kubernetesBuildIns: string[] = k8sSchema.oneOf
+    .map((s) => {
+      if (typeof s === 'boolean') {
+        return undefined;
+      }
+      // @ts-ignore
+      return s._$ref;
+    })
+    .filter((ref) => ref)
+    .map((ref) => ref.replace('_definitions.json#/definitions/', '').toLowerCase());
+  const k8sTypeName = `io.k8s.api.${group.toLowerCase()}.${version.toLowerCase()}.${kind.toLowerCase()}`;
+
+  if (kubernetesBuildIns.includes(k8sTypeName)) {
+    return KUBERNETES_SCHEMA_URL;
+  }
+
+  const schemaURL = `${crdCatalogURI}/${group.toLowerCase()}/${kind.toLowerCase()}_${version.toLowerCase()}.json`;
+  return schemaURL;
+}
+
+/**
+ * Retrieve the group, version and kind from the document.
+ * Public for testing purpose, not part of the API.
+ * @param doc
+ */
+export function getGroupVersionKindFromDocument(
+  doc: SingleYAMLDocument | JSONDocument
+): { group: string; version: string; kind: string } | undefined {
+  if (doc instanceof SingleYAMLDocument) {
+    try {
+      const rootJSON = doc.root.internalNode.toJSON();
+      if (!rootJSON) {
+        return undefined;
+      }
+
+      const groupVersion = rootJSON['apiVersion'];
+      if (!groupVersion) {
+        return undefined;
+      }
+
+      const [group, version] = groupVersion.split('/');
+      if (!group || !version) {
+        return undefined;
+      }
+
+      const kind = rootJSON['kind'];
+      if (!kind) {
+        return undefined;
+      }
+
+      return { group, version, kind };
+    } catch (error) {
+      console.error('Error parsing YAML document:', error);
+      return undefined;
+    }
+  }
+  return undefined;
+}

src/languageservice/services/yamlSchemaService.ts

@@ -6,6 +6,7 @@
 
 import { JSONSchema, JSONSchemaMap, JSONSchemaRef } from '../jsonSchema';
 import { SchemaPriority, SchemaRequestService, WorkspaceContextService } from '../yamlLanguageService';
+import { SettingsState } from '../../yamlSettings';
 import {
   UnresolvedSchema,
   ResolvedSchema,
@@ -29,6 +30,8 @@ import { SchemaVersions } from '../yamlTypes';
 
 import Ajv, { DefinedError } from 'ajv';
 import { getSchemaTitle } from '../utils/schemaUtils';
+import { autoDetectKubernetesSchemaFromDocument } from './crdUtil';
+import { CRD_CATALOG_URL, KUBERNETES_SCHEMA_URL } from '../utils/schemaUrls';
 
 const ajv = new Ajv();
 
@@ -108,19 +111,22 @@ export class YAMLSchemaService extends JSONSchemaService {
   private filePatternAssociations: JSONSchemaService.FilePatternAssociation[];
   private contextService: WorkspaceContextService;
   private requestService: SchemaRequestService;
+  private yamlSettings: SettingsState;
   public schemaPriorityMapping: Map<string, Set<SchemaPriority>>;
 
   private schemaUriToNameAndDescription = new Map<string, SchemaStoreSchema>();
 
   constructor(
     requestService: SchemaRequestService,
     contextService?: WorkspaceContextService,
-    promiseConstructor?: PromiseConstructor
+    promiseConstructor?: PromiseConstructor,
+    yamlSettings?: SettingsState
   ) {
     super(requestService, contextService, promiseConstructor);
     this.customSchemaProvider = undefined;
     this.requestService = requestService;
     this.schemaPriorityMapping = new Map();
+    this.yamlSettings = yamlSettings;
   }
 
   registerCustomSchemaProvider(customSchemaProvider: CustomSchemaProvider): void {
@@ -416,6 +422,7 @@ export class YAMLSchemaService extends JSONSchemaService {
     if (modelineSchema) {
       return resolveSchemaForResource([modelineSchema]);
     }
+
     if (this.customSchemaProvider) {
       return this.customSchemaProvider(resource)
         .then((schemaUri) => {
@@ -458,9 +465,24 @@ export class YAMLSchemaService extends JSONSchemaService {
             return resolveSchema();
           }
         );
-    } else {
-      return resolveSchema();
     }
+    if (this.yamlSettings?.kubernetesCRDStoreEnabled) {
+      for (const entry of this.filePatternAssociations) {
+        if (entry.uris && entry.uris[0] == KUBERNETES_SCHEMA_URL && entry.matchesPattern(resource)) {
+          return resolveSchemaForResource([KUBERNETES_SCHEMA_URL]).then((schema) => {
+            const kubeSchema = autoDetectKubernetesSchemaFromDocument(
+              doc,
+              this.yamlSettings.kubernetesCRDStoreUrl ?? CRD_CATALOG_URL,
+              schema
+            );
+            if (kubeSchema) {
+              return resolveSchemaForResource([kubeSchema]);
+            }
+          });
+        }
+      }
+    }
+    return resolveSchema();
   }
 
   // Set the priority of a schema in the schema service

src/languageservice/utils/schemaUrls.ts

@@ -8,6 +8,7 @@ import { isRelativePath, relativeToAbsolutePath } from './paths';
 export const KUBERNETES_SCHEMA_URL =
   'https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.32.1-standalone-strict/all.json';
 export const JSON_SCHEMASTORE_URL = 'https://www.schemastore.org/api/json/catalog.json';
+export const CRD_CATALOG_URL = 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main';
 
 export function checkSchemaURI(
   workspaceFolders: WorkspaceFolder[],

src/languageservice/yamlLanguageService.ts

@@ -189,7 +189,7 @@ export function getLanguageService(params: {
   yamlSettings?: SettingsState;
   clientCapabilities?: ClientCapabilities;
 }): LanguageService {
-  const schemaService = new YAMLSchemaService(params.schemaRequestService, params.workspaceContext);
+  const schemaService = new YAMLSchemaService(params.schemaRequestService, params.workspaceContext, null, params.yamlSettings);
   const completer = new YamlCompletion(schemaService, params.clientCapabilities, yamlDocumentsCache, params.telemetry);
   const hover = new YAMLHover(schemaService, params.telemetry);
   const yamlDocumentSymbols = new YAMLDocumentSymbols(schemaService, params.telemetry);

src/yamlSettings.ts

@@ -4,7 +4,7 @@ import { ISchemaAssociations } from './requestTypes';
 import { URI } from 'vscode-uri';
 import { JSONSchema } from './languageservice/jsonSchema';
 import { TextDocument } from 'vscode-languageserver-textdocument';
-import { JSON_SCHEMASTORE_URL } from './languageservice/utils/schemaUrls';
+import { CRD_CATALOG_URL, JSON_SCHEMASTORE_URL } from './languageservice/utils/schemaUrls';
 import { YamlVersion } from './languageservice/parser/yamlParser07';
 
 // Client settings interface to grab settings relevant for the language server
@@ -20,6 +20,10 @@ export interface Settings {
       url: string;
       enable: boolean;
     };
+    kubernetesCRDStore: {
+      url: string;
+      enable: boolean;
+    };
     disableDefaultProperties: boolean;
     disableAdditionalProperties: boolean;
     suggest: {
@@ -77,6 +81,8 @@ export class SettingsState {
   customTags = [];
   schemaStoreEnabled = true;
   schemaStoreUrl = JSON_SCHEMASTORE_URL;
+  kubernetesCRDStoreEnabled = true;
+  kubernetesCRDStoreUrl = CRD_CATALOG_URL;
   indentation: string | undefined = undefined;
   disableAdditionalProperties = false;
   disableDefaultProperties = false;

test/schema.test.ts

@@ -14,6 +14,7 @@ import { LanguageService, SchemaPriority } from '../src';
 import { MarkupContent, Position } from 'vscode-languageserver-types';
 import { LineCounter } from 'yaml';
 import { getSchemaFromModeline } from '../src/languageservice/services/modelineUtil';
+import { getGroupVersionKindFromDocument } from '../src/languageservice/services/crdUtil';
 
 const requestServiceMock = function (uri: string): Promise<string> {
   return Promise.reject<string>(`Resource ${uri} not found.`);
@@ -701,6 +702,60 @@ describe('JSON Schema', () => {
     });
   });
 
+  describe('Test getGroupVersionKindFromDocument', function () {
+    it('builtin kubernetes resource group should not get resolved', async () => {
+      checkReturnGroupVersionKind('apiVersion: v1\nkind: Pod', true, undefined, 'v1', 'Pod');
+    });
+
+    it('custom argo application CRD should get resolved', async () => {
+      checkReturnGroupVersionKind(
+        'apiVersion: argoproj.io/v1alpha1\nkind: Application',
+        false,
+        'argoproj.io',
+        'v1alpha1',
+        'Application'
+      );
+    });
+
+    it('custom argo application CRD with whitespace should get resolved', async () => {
+      checkReturnGroupVersionKind(
+        'apiVersion: argoproj.io/v1alpha1\nkind: Application ',
+        false,
+        'argoproj.io',
+        'v1alpha1',
+        'Application'
+      );
+    });
+
+    it('custom argo application CRD with other fields should get resolved', async () => {
+      checkReturnGroupVersionKind(
+        'someOtherVal: test\napiVersion: argoproj.io/v1alpha1\nkind: Application\nmetadata:\n  name: my-app',
+        false,
+        'argoproj.io',
+        'v1alpha1',
+        'Application'
+      );
+    });
+
+    function checkReturnGroupVersionKind(
+      content: string,
+      error: boolean,
+      expectedGroup: string,
+      expectedVersion: string,
+      expectedKind: string
+    ): void {
+      const yamlDoc = parser.parse(content);
+      const res = getGroupVersionKindFromDocument(yamlDoc.documents[0]);
+      if (error) {
+        assert.strictEqual(res, undefined);
+      } else {
+        assert.strictEqual(res.group, expectedGroup);
+        assert.strictEqual(res.version, expectedVersion);
+        assert.strictEqual(res.kind, expectedKind);
+      }
+    }
+  });
+
   describe('Test getSchemaFromModeline', function () {
     it('simple case', async () => {
       checkReturnSchemaUrl('# yaml-language-server: $schema=expectedUrl', 'expectedUrl');

test/yamlSchemaService.test.ts

@@ -8,6 +8,8 @@ import * as sinonChai from 'sinon-chai';
 import * as path from 'path';
 import * as SchemaService from '../src/languageservice/services/yamlSchemaService';
 import { parse } from '../src/languageservice/parser/yamlParser07';
+import { SettingsState } from '../src/yamlSettings';
+import { KUBERNETES_SCHEMA_URL } from '../src/languageservice/utils/schemaUrls';
 
 const expect = chai.expect;
 chai.use(sinonChai);
@@ -140,5 +142,69 @@ describe('YAML Schema Service', () => {
 
       expect(requestServiceMock).calledOnceWith('https://json-schema.org/draft-07/schema#');
     });
+
+    it('should handle crd catalog for crd', async () => {
+      const documentContent = 'apiVersion: argoproj.io/v1alpha1\nkind: Application';
+      const content = `${documentContent}`;
+      const yamlDock = parse(content);
+
+      const settings = new SettingsState();
+      settings.schemaAssociations = {
+        kubernetes: ['*.yaml'],
+      };
+      settings.kubernetesCRDStoreEnabled = true;
+      requestServiceMock = sandbox.fake.resolves(
+        `
+        {
+          "oneOf": [ {
+              "$ref": "_definitions.json#/definitions/io.k8s.api.admissionregistration.v1.MutatingWebhook"
+            },
+          ]
+        }
+        `
+      );
+      const service = new SchemaService.YAMLSchemaService(requestServiceMock, undefined, undefined, settings);
+      service.registerExternalSchema(KUBERNETES_SCHEMA_URL, ['*.yaml']);
+      await service.getSchemaForResource('test.yaml', yamlDock.documents[0]);
+
+      expect(requestServiceMock).calledWithExactly(KUBERNETES_SCHEMA_URL);
+      expect(requestServiceMock).calledWithExactly('file:///_definitions.json');
+
+      expect(requestServiceMock).calledWithExactly(
+        'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/argoproj.io/application_v1alpha1.json'
+      );
+      expect(requestServiceMock).calledThrice;
+    });
+
+    it('should not get schema from crd catalog if definition in kubernetes schema', async () => {
+      const documentContent = 'apiVersion: admissionregistration.k8s.io/v1\nkind: MutatingWebhook';
+      const content = `${documentContent}`;
+      const yamlDock = parse(content);
+
+      const settings = new SettingsState();
+      settings.schemaAssociations = {
+        kubernetes: ['*.yaml'],
+      };
+      settings.kubernetesCRDStoreEnabled = true;
+      requestServiceMock = sandbox.fake.resolves(
+        `
+        {
+          "oneOf": [ {
+              "$ref": "_definitions.json#/definitions/io.k8s.api.admissionregistration.v1.MutatingWebhook"
+            },
+          ]
+        }
+        `
+      );
+      const service = new SchemaService.YAMLSchemaService(requestServiceMock, undefined, undefined, settings);
+      service.registerExternalSchema(KUBERNETES_SCHEMA_URL, ['*.yaml']);
+      await service.getSchemaForResource('test.yaml', yamlDock.documents[0]);
+
+      expect(requestServiceMock).calledWithExactly(KUBERNETES_SCHEMA_URL);
+      expect(requestServiceMock).calledWithExactly('file:///_definitions.json');
+
+      expect(requestServiceMock).calledWithExactly(KUBERNETES_SCHEMA_URL);
+      expect(requestServiceMock).calledThrice;
+    });
   });
 });