feat(cli/gen-models): Allow generating models from external CRDs through a config file (#37)

Author: Chewie

README.md

@@ -164,10 +164,12 @@ The CLI tool can be used to generate composition manifests from source files:
    functions/
    ├── example1/
    │   ├── composition.fn.ts
+   │   ├── xrd.yaml
    │   ├── package.json (optional)
    │   └── composition.yaml (optional)
    └── example2/
        ├── composition.fn.ts
+       ├── xrd.yaml
        └── package.json
    ```
 
@@ -200,6 +202,33 @@ The CLI handles dependencies and yarn.lock files as follows:
 
 This approach allows you to have function-specific dependencies or share dependencies across all functions.
 
+### Generating models
+
+To help in writing compositions, the CLI tool can generate type-safe models for
+the following:
+* Base Kubernetes resources
+* CRDs derived from the XRDs of your custom resources
+* External extra CRDs defined in a configuration file (optional)
+
+Run the CLI tool:
+
+  ```bash
+  npx @crossplane-js/cli gen-models
+  ```
+
+To generate a `models/` directory containing models that can be imported in
+your composition functions.
+
+To generate models for extra CRDs, create a `config.yaml` in the base
+directory, next to the `functions/` directory with a `extraCrds` field
+containing an array of extra CRDs urls to use in model generation, e.g.:
+
+```yaml
+extraCrds:
+  - https://github.com/fluxcd/source-controller/releases/download/v1.7.0/source-controller.crds.yaml
+```
+
+
 ## Testing
 
 The project includes end-to-end tests that use a Kind cluster to verify functionality:

packages/cli/package.json

@@ -11,6 +11,7 @@
   "dependencies": {
     "@crossplane-js/kubernetes-models-crd-generate": "5.0.2-sha.6721c6987b6a.r5.a1",
     "@crossplane-js/libs": "workspace:^",
+    "@kubernetes-models/read-input": "^3.1.1",
     "commander": "^13.1.0",
     "esbuild": "^0.25.4",
     "fs-extra": "^11.3.0",

packages/cli/src/commands/gen-models/index.ts

@@ -4,7 +4,8 @@ import path from "path"
 import { fileURLToPath } from "url"
 
 import { createLogger } from "@crossplane-js/libs"
-import { Command } from "commander"
+import { readInput } from "@kubernetes-models/read-input"
+import { type Command } from "commander"
 import fs from "fs-extra"
 import YAML from "yaml"
 
@@ -19,6 +20,20 @@ const __dirname = path.dirname(__filename)
 // packages/cli/src/commands/gen-models -> packages/cli
 const cliRoot = path.resolve(__dirname, "../../..")
 
+/**
+ * Fetch extra CRDs
+ * @param urls Array of CRD urls
+ * @returns Promise<string[]> Array of CRD paths
+ */
+async function fetchExtraCRDs(paths: string[]): Promise<string[]> {
+  const extraDocuments: string[] = []
+  for (const path of paths) {
+    moduleLogger.info("Reading extra CRD: %s", path)
+    extraDocuments.push(await readInput(path))
+  }
+  return extraDocuments
+}
+
 /**
  * Find all XRD files in the functions directory
  * @returns Promise<string[]> Array of XRD file paths
@@ -72,7 +87,10 @@ async function runCrdGenerate(crdYaml: string, outputPath: string): Promise<void
         "--output",
         outputAbs,
       ]
-      const child = spawn("yarn", args, { cwd: cliRoot, stdio: ["ignore", "inherit", "inherit"] })
+      const child = spawn("yarn", args, {
+        cwd: cliRoot,
+        stdio: ["ignore", "inherit", "inherit"],
+      })
       child.on("error", err => reject(err))
       child.on("exit", code => {
         if (code === 0) resolve()
@@ -94,6 +112,8 @@ async function genModelsAction(): Promise<void> {
   try {
     moduleLogger.info("Starting model generation...")
 
+    const documents: string[] = []
+
     // Find all XRD files
     const xrdFiles = await findXRDFiles()
     moduleLogger.info(`Found ${xrdFiles.length} XRD file(s): ${xrdFiles.join(", ")}`)
@@ -119,9 +139,7 @@ async function genModelsAction(): Promise<void> {
         // Convert XRD to CRD
         const crd = convertXRDtoCRD(xrd)
         const crdYaml = YAML.stringify(crd)
-
-        // Generate models using crd-generate (via child process)
-        await runCrdGenerate(crdYaml, modelsDir)
+        documents.push(crdYaml)
 
         moduleLogger.info(`✓ Generated models for ${xrdPath}`)
       } catch (error) {
@@ -130,6 +148,27 @@ async function genModelsAction(): Promise<void> {
       }
     }
 
+    // Get extra CRD urls from config file
+    if (await fs.exists("config.yaml")) {
+      moduleLogger.info("Config file exists, searching extra CRDs...")
+      const configFile = await fs.readFile("config.yaml", "utf8")
+      const config = YAML.parse(configFile)
+      if (!config.extraCrds) {
+        moduleLogger.info("Config file has no extra CRDs")
+      } else if (!Array.isArray(config.extraCrds)) {
+        moduleLogger.warn("Config file extraCrds field is not an array!")
+      } else {
+        const crds = await fetchExtraCRDs(config.extraCrds)
+        for (const crd of crds) {
+          documents.push(crd)
+        }
+      }
+    }
+
+    // Generate models using crd-generate (via child process)
+    const allCrds = documents.join("\n---\n")
+    await runCrdGenerate(allCrds, modelsDir)
+
     moduleLogger.info(`✓ Model generation completed. Models saved to '${modelsDir}/' directory.`)
   } catch (error) {
     moduleLogger.error(`Error generating models: ${error}`)

tests/fixtures/domain-sdk/config.yaml

@@ -0,0 +1,2 @@
+extraCrds:
+  - https://github.com/fluxcd/source-controller/releases/download/v1.7.0/source-controller.crds.yaml

…tions/simpleconfigmaps/composition.fn.ts …aps.test.crossplane.io/composition.fn.tstests/fixtures/domain-sdk/functions/simpleconfigmaps/composition.fn.ts renamed to tests/fixtures/domain-sdk/functions/simpleconfigmaps.test.crossplane.io/composition.fn.ts tests/fixtures/domain-sdk/functions/simpleconfigmaps/composition.fn.ts renamed to tests/fixtures/domain-sdk/functions/simpleconfigmaps.test.crossplane.io/composition.fn.ts

@@ -0,0 +1,53 @@
+import {
+  logger,
+  // FieldRef,
+  v1,
+} from '@crossplane-js/sdk'
+import type {
+  CrossplaneDesiredResources,
+  // CrossplaneObservedResources
+} from '@crossplane-js/sdk'
+
+import type { SimpleConfigMap } from '../../models/test.crossplane.io/v1beta1'
+
+export default function (
+  composite: SimpleConfigMap
+  // resources: CrossplaneObservedResources
+): CrossplaneDesiredResources {
+  logger.info('SimpleConfigMap composition function started')
+  logger.info(`composite:${JSON.stringify(composite)}`)
+
+  const namespace = composite.getNamespace()
+  const isReady = composite.isReady()
+
+  logger.info(`Namespace: ${namespace}, Ready: ${isReady}`)
+  logger.info(composite)
+
+  // Transform data to uppercase for testing (same as original test logic)
+  const transformedData: Record<string, string> = {}
+  Object.entries(composite.spec.data).forEach(([key, value]) => {
+    transformedData[key.toUpperCase()] = value.toUpperCase()
+  })
+
+  const testConfigMap = new v1.ConfigMap({
+    metadata: {
+      name: 'generated-configmap',
+      namespace: namespace || 'test-xfuncjs',
+    },
+    data: transformedData,
+  })
+
+  const desired: CrossplaneDesiredResources = {
+    resources: {
+      configmap: {
+        resource: testConfigMap,
+        ready: true,
+      },
+    },
+  }
+
+  logger.info('SimpleConfigMap composition function completed')
+  logger.debug({ desired }, 'Generated output')
+
+  return desired
+}

…-sdk/functions/simpleconfigmaps/xrd.yaml …leconfigmaps.test.crossplane.io/xrd.yamltests/fixtures/domain-sdk/functions/simpleconfigmaps/xrd.yaml renamed to tests/fixtures/domain-sdk/functions/simpleconfigmaps.test.crossplane.io/xrd.yaml tests/fixtures/domain-sdk/functions/simpleconfigmaps/xrd.yaml renamed to tests/fixtures/domain-sdk/functions/simpleconfigmaps.test.crossplane.io/xrd.yaml

@@ -0,0 +1,32 @@
+apiVersion: apiextensions.crossplane.io/v2
+kind: CompositeResourceDefinition
+metadata:
+  name: uppercaseconfigmaps.test.crossplane.io
+spec:
+  scope: Namespaced
+  defaultCompositeDeletePolicy: Foreground
+  group: test2.crossplane.io
+  names:
+    categories:
+      - crossplane
+    kind: UppercaseConfigMap
+    plural: uppercaseconfigmaps
+    singular: uppsercaseconfigmap
+  versions:
+    - name: v1beta1
+      served: true
+      referenceable: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                dataToUppercase:
+                  type: object
+                  description: Key-value data to be uppercased and stored in ConfigMap
+                  additionalProperties:
+                    type: string
+              required:
+                - data

tests/fixtures/domain-sdk/functions/uppercaseconfigmaps.test2.crossplane.io/composition.fn.ts

@@ -53,7 +53,7 @@ yarn --cwd tests/fixtures/domain-sdk gen-manifests
 
 # Apply CRDs and Compositions
 echo "Applying XRD and Compositions..."
-kubectl apply -f tests/fixtures/domain-sdk/manifests/simpleconfigmaps
+kubectl apply --server-side=true -f tests/fixtures/domain-sdk/manifests/simpleconfigmaps.test.crossplane.io
 
 # Wait for XRD to be established
 echo "Waiting for XRD to be established..."

tests/fixtures/domain-sdk/functions/uppercaseconfigmaps.test2.crossplane.io/xrd.yaml

@@ -269,6 +269,7 @@ __metadata:
   dependencies:
     "@crossplane-js/kubernetes-models-crd-generate": "npm:5.0.2-sha.6721c6987b6a.r5.a1"
     "@crossplane-js/libs": "workspace:^"
+    "@kubernetes-models/read-input": "npm:^3.1.1"
     "@types/lodash": "npm:^4"
     "@types/uuid": "npm:^10.0.0"
     commander: "npm:^13.1.0"