Merge pull request #3862 from ndebuhr/feat/slinky-scheduler

Create a new community scheduler module for Slinky (Slurm on Kubernetes)

Author: nick-stroud

community/examples/hpc-slinky/exporter-pod-monitoring.yaml

@@ -0,0 +1,27 @@
+# Copyright 2025 "Google LLC"
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: monitoring.googleapis.com/v1
+kind: PodMonitoring
+metadata:
+  name: slurm-exporter
+  namespace: slurm
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: slurm-exporter
+      app.kubernetes.io/name: slurm-exporter
+  endpoints:
+  - port: metrics
+    interval: 30s

community/examples/hpc-slinky/hpc-slinky.yaml

@@ -0,0 +1,166 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+blueprint_name: hpc-slinky
+
+vars:
+  project_id:  ## Set GCP Project ID Here ##
+  deployment_name: slinky-01
+  region: us-central1
+  zones:
+  - us-central1-a
+  authorized_cidr:  # <your-ip-address>/32
+  gcp_public_cidrs_access_enabled: false
+  exporter_pod_monitoring_path: $(ghpc_stage("./exporter-pod-monitoring.yaml"))
+  debug_nodeset_replicas: 2
+  h3_nodeset_replicas: 2
+
+deployment_groups:
+- group: primary
+  modules:
+  - id: network
+    source: modules/network/vpc
+    settings:
+      subnetwork_name: $(vars.deployment_name)-subnet
+      secondary_ranges_list:
+      - subnetwork_name: $(vars.deployment_name)-subnet
+        ranges:
+        - range_name: pods
+          ip_cidr_range: 10.4.0.0/14
+        - range_name: services
+          ip_cidr_range: 10.0.32.0/20
+
+  - id: node_pool_service_account
+    source: community/modules/project/service-account
+    settings:
+      name: gke-np-sa
+      project_roles:
+      - logging.logWriter
+      - monitoring.metricWriter
+      - monitoring.viewer
+      - stackdriver.resourceMetadata.writer
+      - storage.objectAdmin
+      - artifactregistry.reader
+
+  - id: workload_service_account
+    source: community/modules/project/service-account
+    settings:
+      name: gke-wl-sa
+      project_roles:
+      - logging.logWriter
+      - monitoring.metricWriter
+      - monitoring.viewer
+      - stackdriver.resourceMetadata.writer
+      - storage.objectAdmin
+      - artifactregistry.reader
+
+  - id: gke_cluster
+    source: modules/scheduler/gke-cluster
+    use: [network, workload_service_account]
+    settings:
+      enable_private_endpoint: false
+      gcp_public_cidrs_access_enabled: $(vars.gcp_public_cidrs_access_enabled)
+      master_authorized_networks:
+      - display_name: deployment-machine
+        cidr_block: $(vars.authorized_cidr)
+      system_node_pool_enabled: false
+      configure_workload_identity_sa: true
+    outputs: [instructions]
+
+  - id: base_pool
+    source: modules/compute/gke-node-pool
+    use: [gke_cluster, node_pool_service_account]
+    settings:
+      initial_node_count: 1
+      disk_type: pd-balanced
+      machine_type: e2-standard-4
+      zones: $(vars.zones)
+
+  - id: h3_pool
+    source: modules/compute/gke-node-pool
+    use: [gke_cluster, node_pool_service_account]
+    settings:
+      initial_node_count: 2
+      disk_type: pd-balanced
+      machine_type: h3-standard-88
+      zones: $(vars.zones)
+
+  - id: slinky
+    source: community/modules/scheduler/slinky
+    use:
+    - gke_cluster
+    - base_pool  # Optionally specify nodepool(s) to avoid operator components running on HPC hardware
+    settings:
+      slurm_values:
+        compute:
+          nodesets:
+          - name: debug
+            enabled: true
+            replicas: $(vars.debug_nodeset_replicas)
+            image:
+              # Use the default nodeset image
+              repository: ""
+              tag: ""
+            resources:
+              requests:
+                cpu: 500m
+                memory: 4Gi
+              limits:
+                cpu: 500m
+                memory: 4Gi
+            affinity:
+              nodeAffinity:
+                requiredDuringSchedulingIgnoredDuringExecution:
+                  nodeSelectorTerms:
+                  - matchExpressions:
+                    - key: "node.kubernetes.io/instance-type"
+                      operator: In
+                      values:
+                      - e2-standard-4
+            partition:
+              enabled: true
+          - name: h3
+            enabled: true
+            replicas: $(vars.h3_nodeset_replicas)
+            image:
+              # Use the default nodeset image
+              repository: ""
+              tag: ""
+            resources:
+              requests:
+                cpu: 86
+                memory: 324Gi
+              limits:
+                cpu: 86
+                memory: 324Gi
+            affinity:
+              nodeAffinity:
+                requiredDuringSchedulingIgnoredDuringExecution:
+                  nodeSelectorTerms:
+                  - matchExpressions:
+                    - key: "node.kubernetes.io/instance-type"
+                      operator: In
+                      values:
+                      - h3-standard-88
+            partition:
+              enabled: true
+    outputs: [instructions]
+
+  - id: slurm_exporter_monitoring
+    source: modules/management/kubectl-apply
+    use: [gke_cluster]
+    settings:
+      apply_manifests:
+      - source: $(vars.exporter_pod_monitoring_path)

community/modules/scheduler/slinky/README.md

@@ -0,0 +1,129 @@
+## Description
+
+This module creates a [Slinky](https://slinky.ai) cluster and nodesets, for a [Slurm](https://slurm.schedmd.com/documentation.html)-on-Kubernetes HPC setup.
+
+The setup closely follows the [documented quickstart installation](https://github.com/SlinkyProject/slurm-operator/blob/main/docs/quickstart.md), with the exception of a more lightweight monitoring/metrics setup. Consider scraping the Slurm Exporter with [Google Managed Prometheus](https://cloud.google.com/stackdriver/docs/managed-prometheus) and a [PodMonitoring resource](https://cloud.google.com/stackdriver/docs/managed-prometheus/setup-managed#gmp-pod-monitoring), rather than a cluster-local Kube Prometheus Stack (although both are possible with module parameterizations).
+
+Through `cert_manager_values`, `prometheus_values`, `slurm_operator_values`, and `slurm_values`, you can customize the Helm releases that constitute Slinky. The Cert Manager, Slurm Operator, and Slurm Helm installations are required, whereas the Prometheus Helm chart is optional (and not included by default). Set `install_kube_prometheus_stack=true` to install Prometheus.
+
+### Example
+
+```yaml
+- id: slinky
+  source: community/modules/scheduler/slinky
+  use: [gke_cluster, base_pool]
+  settings:
+    slurm_values:
+      compute:
+        nodesets:
+        - name: h3
+          enabled: true
+          replicas: 2
+          image:
+            # Use the default nodeset image
+            repository: ""
+            tag: ""
+          resources:
+            requests:
+              cpu: 86
+              memory: 324Gi
+            limits:
+              cpu: 86
+              memory: 324Gi
+          affinity:
+            nodeAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                nodeSelectorTerms:
+                - matchExpressions:
+                  - key: "node.kubernetes.io/instance-type"
+                    operator: In
+                    values:
+                    - h3-standard-88
+          partition:
+            enabled: true
+```
+
+This creates a Slinky cluster with the following attributes:
+
+* Slinky Helm releases are installed atop the `gke_cluster` (from the `gke-cluster` module).
+* Slinky system components are scheduled on the `base_pool` (from the `gke-node-pool` module).
+  * This node affinity specification is recommended, to save HPC hardware for HPC nodesets, and to ensure Helm releases are fully uninstalled before all nodepools are deleted during a `gcluster destroy`.
+* One Slurm nodeset is provisioned, with resource requests/limits and node affinities aligned to h3-standard-88 VMs.
+
+### Usage
+
+To test Slurm functionality, connect to the controller and use Slurm client commands:
+
+```bash
+gcloud container clusters get-credentials YOUR_CLUSTER --region YOUR_REGION
+```
+
+```bash
+kubectl exec -it statefulsets/slurm-controller \
+  --namespace=slurm \
+  -- bash --login
+```
+
+On the controller pod (e.g. host slurm@slurm-controller-0), run the following commands to quickly test if Slurm is functioning:
+
+```bash
+sinfo
+srun hostname
+sbatch --wrap="sleep 60"
+squeue
+```
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.3 |
+| <a name="requirement_google"></a> [google](#requirement\_google) | >= 6.16 |
+| <a name="requirement_helm"></a> [helm](#requirement\_helm) | ~> 2.17 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_google"></a> [google](#provider\_google) | >= 6.16 |
+| <a name="provider_helm"></a> [helm](#provider\_helm) | ~> 2.17 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [helm_release.cert_manager](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
+| [helm_release.prometheus](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
+| [helm_release.slurm](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
+| [helm_release.slurm_operator](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
+| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source |
+| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_cert_manager_chart_version"></a> [cert\_manager\_chart\_version](#input\_cert\_manager\_chart\_version) | Version of the Cert Manager chart to install. | `string` | `"v1.17.1"` | no |
+| <a name="input_cert_manager_values"></a> [cert\_manager\_values](#input\_cert\_manager\_values) | Value overrides for the Cert Manager release | `any` | <pre>{<br/>  "crds": {<br/>    "enabled": true<br/>  }<br/>}</pre> | no |
+| <a name="input_cluster_id"></a> [cluster\_id](#input\_cluster\_id) | An identifier for the GKE cluster resource with format projects/<project\_id>/locations/<region>/clusters/<name>. | `string` | n/a | yes |
+| <a name="input_install_kube_prometheus_stack"></a> [install\_kube\_prometheus\_stack](#input\_install\_kube\_prometheus\_stack) | Install the Kube Prometheus Stack. | `bool` | `false` | no |
+| <a name="input_node_pool_names"></a> [node\_pool\_names](#input\_node\_pool\_names) | Names of node pools, for use in node affinities (Slinky system components). | `list(string)` | `null` | no |
+| <a name="input_project_id"></a> [project\_id](#input\_project\_id) | The project ID that hosts the GKE cluster. | `string` | n/a | yes |
+| <a name="input_prometheus_chart_version"></a> [prometheus\_chart\_version](#input\_prometheus\_chart\_version) | Version of the Kube Prometheus Stack chart to install. | `string` | `"70.4.1"` | no |
+| <a name="input_prometheus_values"></a> [prometheus\_values](#input\_prometheus\_values) | Value overrides for the Prometheus release | `any` | <pre>{<br/>  "installCRDs": true<br/>}</pre> | no |
+| <a name="input_slurm_chart_version"></a> [slurm\_chart\_version](#input\_slurm\_chart\_version) | Version of the Slurm chart to install. | `string` | `"0.2.1"` | no |
+| <a name="input_slurm_operator_chart_version"></a> [slurm\_operator\_chart\_version](#input\_slurm\_operator\_chart\_version) | Version of the Slurm Operator chart to install. | `string` | `"0.2.1"` | no |
+| <a name="input_slurm_operator_values"></a> [slurm\_operator\_values](#input\_slurm\_operator\_values) | Value overrides for the Slinky release | `any` | `{}` | no |
+| <a name="input_slurm_values"></a> [slurm\_values](#input\_slurm\_values) | Value overrides for the Slurm release | `any` | `{}` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_instructions"></a> [instructions](#output\_instructions) | Post deployment instructions. |
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->