Merge branch 'develop' into eda-blueprints-new

okrause · web-flow · commit be17c0b34187 · 2025-12-05T13:56:08.000+01:00
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -3,6 +3,15 @@
 We'd love to accept your patches and contributions to this project. There are
 just a few small guidelines you need to follow.
 
+## Branching Strategy
+To maintain a stable release history, this repository follows a specific branching workflow:
+
+`main`: Contains the latest stable release. This branch is only updated during a new release cycle.
+
+`develop`: The active development branch. All new features, bug fixes, and community contributions are merged here first.
+
+**Important**: All Pull Requests must be targeted at the `develop` branch. PRs targeting `main` will be closed or redirected.
+
 ## Contributor License Agreement
 
 Contributions to this project must be accompanied by a Contributor License
diff --git a/community/examples/client-google-cloud-storage.yaml b/community/examples/client-google-cloud-storage.yaml
@@ -31,7 +31,7 @@ deployment_groups:
     source: modules/network/vpc
 
   - id: new-bucket
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings:
       name_prefix: new-bucket
       random_suffix: true
diff --git a/community/examples/fsi-montecarlo-on-batch.yaml b/community/examples/fsi-montecarlo-on-batch.yaml
@@ -45,7 +45,7 @@ deployment_groups:
   modules:
 
   - id: fsi_bucket
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings:
       name_prefix: fsi_bucket
       random_suffix: true
diff --git a/community/modules/compute/notebook/README.md b/community/modules/compute/notebook/README.md
@@ -12,7 +12,7 @@ This is a simple usage, using the default network:
 
 ```yaml
   - id: bucket
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings: 
       name_prefix: my-bucket
       local_mount: /home/jupyter/my-bucket
@@ -30,7 +30,7 @@ If the user wants do specify a custom subnetwork, or specific external IP restri
 
 ```yaml
   - id: bucket
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings: 
       name_prefix: my-bucket
       local_mount: /home/jupyter/my-bucket
diff --git a/community/modules/compute/notebook/main.tf b/community/modules/compute/notebook/main.tf
@@ -26,7 +26,7 @@ locals {
   bucket               = replace(var.gcs_bucket_path, "gs://", "")
   post_script_filename = "mount-${local.suffix}.sh"
 
-  # mount_runner_args is defined here: https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/3abddcfbd245b0e6747917a4e55b30658414ffd7/community/modules/file-system/cloud-storage-bucket/outputs.tf#L40
+  # mount_runner_args is defined in the file: cluster-toolkit/modules/file-system/cloud-storage-bucket/outputs.tf
   mount_args = split(" ", var.mount_runner.args)
 
   unused       = local.mount_args[0]
diff --git a/community/modules/scheduler/htcondor-setup/README.md b/community/modules/scheduler/htcondor-setup/README.md
@@ -91,7 +91,7 @@ No providers.
 | Name | Source | Version |
 |------|--------|---------|
 | <a name="module_health_check_firewall_rule"></a> [health\_check\_firewall\_rule](#module\_health\_check\_firewall\_rule) | ../../../../modules/network/firewall-rules | n/a |
-| <a name="module_htcondor_bucket"></a> [htcondor\_bucket](#module\_htcondor\_bucket) | ../../../../community/modules/file-system/cloud-storage-bucket | n/a |
+| <a name="module_htcondor_bucket"></a> [htcondor\_bucket](#module\_htcondor\_bucket) | ../../../../modules/file-system/cloud-storage-bucket | n/a |
 
 ## Resources
 
diff --git a/community/modules/scheduler/htcondor-setup/main.tf b/community/modules/scheduler/htcondor-setup/main.tf
@@ -54,7 +54,7 @@ module "health_check_firewall_rule" {
 }
 
 module "htcondor_bucket" {
-  source = "../../../../community/modules/file-system/cloud-storage-bucket"
+  source = "../../../../modules/file-system/cloud-storage-bucket"
 
   project_id      = var.project_id
   deployment_name = var.deployment_name
diff --git a/examples/gke-a3-ultragpu/gke-a3-ultragpu.yaml b/examples/gke-a3-ultragpu/gke-a3-ultragpu.yaml
@@ -146,7 +146,7 @@ deployment_groups:
       - container.admin
 
   - id: training_bucket
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings:
       local_mount: /training-data
       name_prefix: training
@@ -155,7 +155,7 @@ deployment_groups:
       enable_hierarchical_namespace: true
 
   - id: checkpoint_bucket
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings:
       local_mount: /checkpoint-data
       name_prefix: checkpoint
diff --git a/examples/gke-a4/gke-a4.yaml b/examples/gke-a4/gke-a4.yaml
@@ -161,7 +161,7 @@ deployment_groups:
       - container.admin
 
   - id: training_bucket
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings:
       local_mount: /training-data
       name_prefix: training
@@ -170,7 +170,7 @@ deployment_groups:
       enable_hierarchical_namespace: true
 
   - id: checkpoint_bucket
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings:
       local_mount: /checkpoint-data
       name_prefix: checkpoint
diff --git a/examples/gke-a4x/gke-a4x.yaml b/examples/gke-a4x/gke-a4x.yaml
@@ -179,7 +179,7 @@ deployment_groups:
       - artifactregistry.reader
 
   - id: training_bucket
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings:
       local_mount: /training-data
       name_prefix: training
@@ -188,7 +188,7 @@ deployment_groups:
       enable_hierarchical_namespace: true
 
   - id: checkpoint_bucket
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings:
       local_mount: /checkpoint-data
       name_prefix: checkpoint
diff --git a/examples/gke-tpu-7x/gke-tpu-7x-advanced.yaml b/examples/gke-tpu-7x/gke-tpu-7x-advanced.yaml
@@ -137,7 +137,7 @@ deployment_groups:
       - container.admin
 
   - id: training_bucket
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings:
       local_mount: /training-data
       name_prefix: training-data
@@ -146,7 +146,7 @@ deployment_groups:
       enable_hierarchical_namespace: true
 
   - id: checkpoint_bucket
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings:
       local_mount: /checkpoint-data
       name_prefix: checkpoint-data
diff --git a/examples/gke-tpu-v6/gke-tpu-v6-advanced.yaml b/examples/gke-tpu-v6/gke-tpu-v6-advanced.yaml
@@ -142,7 +142,7 @@ deployment_groups:
       - container.admin
 
   - id: training_bucket
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings:
       local_mount: /training-data
       name_prefix: training-data
@@ -151,7 +151,7 @@ deployment_groups:
       enable_hierarchical_namespace: true
 
   - id: checkpoint_bucket
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings:
       local_mount: /checkpoint-data
       name_prefix: checkpoint-data
diff --git a/examples/hcls-blueprint.yaml b/examples/hcls-blueprint.yaml
@@ -85,7 +85,7 @@ deployment_groups:
       local_mount: /apps
 
   - id: bucket-software
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings:
       name_prefix: hcls-user-provided-software
       random_suffix: true
@@ -94,7 +94,7 @@ deployment_groups:
     outputs: [gcs_bucket_path]
 
   - id: bucket-input
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings:
       name_prefix: hcls-inputs
       random_suffix: true
@@ -103,7 +103,7 @@ deployment_groups:
       force_destroy: $(vars.bucket_force_destroy)
 
   - id: bucket-output
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings:
       name_prefix: hcls-outputs
       random_suffix: true
diff --git a/examples/machine-learning/a3-megagpu-8g/a3mega-slurm-blueprint.yaml b/examples/machine-learning/a3-megagpu-8g/a3mega-slurm-blueprint.yaml
@@ -375,7 +375,7 @@ deployment_groups:
   modules:
   # if an existing bucket is desired, follow modules/file-system/pre-existing-network-storage/README.md
   - id: data-bucket
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings:
       local_mount: /gcs
       mount_options: defaults,rw,_netdev,implicit_dirs,allow_other,implicit_dirs,file_mode=777,dir_mode=777
diff --git a/examples/machine-learning/a3-megagpu-8g/a3mega-slurm-gcsfuse-lssd-blueprint.yaml b/examples/machine-learning/a3-megagpu-8g/a3mega-slurm-gcsfuse-lssd-blueprint.yaml
@@ -659,7 +659,7 @@ deployment_groups:
   modules:
   # if an existing bucket is desired, follow modules/file-system/pre-existing-network-storage/README.md
   - id: data-bucket
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings:
       local_mount: /gcs
       mount_options: defaults,rw,_netdev,implicit_dirs,allow_other,implicit_dirs,file_mode=777,dir_mode=777
diff --git a/examples/machine-learning/a3-ultragpu-8g/a3ultra-slurm-blueprint.yaml b/examples/machine-learning/a3-ultragpu-8g/a3ultra-slurm-blueprint.yaml
@@ -343,7 +343,7 @@ deployment_groups:
   # using gcsfuse. If you prefer to use a pre-existing bucket, use the
   # modules/file-system/pre-existing-network-storage module.
   - id: gcs_bucket
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings:
       enable_hierarchical_namespace: true
       local_mount: /gcs
diff --git a/examples/machine-learning/a4-highgpu-8g/a4high-slurm-blueprint.yaml b/examples/machine-learning/a4-highgpu-8g/a4high-slurm-blueprint.yaml
@@ -347,7 +347,7 @@ deployment_groups:
   # using gcsfuse. If you prefer to use a pre-existing bucket, use the
   # modules/file-system/pre-existing-network-storage module.
   - id: gcs_bucket
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings:
       enable_hierarchical_namespace: true
       local_mount: /gcs
diff --git a/examples/machine-learning/a4x-highgpu-4g/a4xhigh-slurm-blueprint.yaml b/examples/machine-learning/a4x-highgpu-4g/a4xhigh-slurm-blueprint.yaml
@@ -358,7 +358,7 @@ deployment_groups:
   # using gcsfuse. If you prefer to use a pre-existing bucket, use the
   # modules/file-system/pre-existing-network-storage module.
   - id: gcs_bucket
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings:
       enable_hierarchical_namespace: true
       local_mount: /gcs
diff --git a/examples/storage-gke.yaml b/examples/storage-gke.yaml
@@ -88,7 +88,7 @@ deployment_groups:
   ### Google Cloud Storage ###
 
   - id: data-bucket
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings:
       local_mount: /data
       random_suffix: true
diff --git a/modules/README.md b/modules/README.md
@@ -101,8 +101,7 @@ Modules that are still in development and less stable are labeled with the
 * **[DDN-EXAScaler]** ![community-badge] ![deprecated-badge] : Creates
   a [DDN EXAscaler lustre](https://www.ddn.com/partners/google-cloud-platform/)
   file system. This module is deprecated and will be removed by July 1, 2025. Consider migrating to managed-lustre.
-* **[cloud-storage-bucket]** ![community-badge] ![experimental-badge] : Creates
-  a Google Cloud Storage (GCS) bucket.
+* **[cloud-storage-bucket]** ![core-badge] : Creates a Google Cloud Storage (GCS) bucket.
 * **[gke-persistent-volume]** ![core-badge] ![experimental-badge] : Creates
   persistent volumes and persistent volume claims for shared storage.
 * **[nfs-server]** ![community-badge] ![experimental-badge] : Creates a VM and
@@ -117,8 +116,8 @@ Modules that are still in development and less stable are labeled with the
 [managed-lustre]: file-system/managed-lustre/README.md
 [ddn-exascaler]: ../community/modules/file-system/DDN-EXAScaler/README.md
 [nfs-server]: ../community/modules/file-system/nfs-server/README.md
-[cloud-storage-bucket]: ../community/modules/file-system/cloud-storage-bucket/README.md
-[gke-persistent-volume]: ../modules/file-system/gke-persistent-volume/README.md
+[cloud-storage-bucket]: file-system/cloud-storage-bucket/README.md
+[gke-persistent-volume]: file-system/gke-persistent-volume/README.md
 [weka-client]: ../community/modules/file-system/weka-client/README.md
 
 ### Monitoring
diff --git a/modules/file-system/cloud-storage-bucket/README.md b/modules/file-system/cloud-storage-bucket/README.md
@@ -12,7 +12,7 @@ where `xxxxxxxx` is a randomly generated id.
 
 ```yaml
   - id: bucket
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings:
       name_prefix: simulation-results
       random_suffix: true
@@ -73,7 +73,7 @@ following example:
 
 ```yaml
   - id: bucket
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings: {local_mount: /data}
 
   - id: mount-at-startup
diff --git a/modules/file-system/cloud-storage-bucket/main.tf b/modules/file-system/cloud-storage-bucket/main.tf
diff --git a/modules/file-system/cloud-storage-bucket/metadata.yaml b/modules/file-system/cloud-storage-bucket/metadata.yaml
diff --git a/modules/file-system/cloud-storage-bucket/outputs.tf b/modules/file-system/cloud-storage-bucket/outputs.tf
diff --git a/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh b/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh
diff --git a/modules/file-system/cloud-storage-bucket/scripts/mount.sh b/modules/file-system/cloud-storage-bucket/scripts/mount.sh
diff --git a/modules/file-system/cloud-storage-bucket/variables.tf b/modules/file-system/cloud-storage-bucket/variables.tf
diff --git a/modules/file-system/cloud-storage-bucket/versions.tf b/modules/file-system/cloud-storage-bucket/versions.tf
diff --git a/modules/file-system/gke-persistent-volume/README.md b/modules/file-system/gke-persistent-volume/README.md
@@ -56,7 +56,7 @@ The following example creates a GCS bucket and then uses the
         cidr_block: <your-ip-address>/32
 
   - id: data-bucket
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings:
       local_mount: /data
 
diff --git a/pkg/config/config.go b/pkg/config/config.go
@@ -45,6 +45,7 @@ var movedModules = map[string]string{
 	"community/modules/scheduler/cloud-batch-login-node": "modules/scheduler/batch-login-node",
 	"community/modules/scheduler/htcondor-configure":     "community/modules/scheduler/htcondor-setup",
 	"community/modules/scripts/spack-install":            "community/modules/scripts/spack-setup",
+	"community/modules/file-system/cloud-storage-bucket": "modules/file-system/cloud-storage-bucket",
 }
 
 // GroupName is the name of a deployment group
diff --git a/tools/cloud-build/daily-tests/blueprints/monitoring.yaml b/tools/cloud-build/daily-tests/blueprints/monitoring.yaml
@@ -37,7 +37,7 @@ deployment_groups:
       local_mounts: [/home]
 
   - id: bucket-for-startup-script
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
 
   - id: startup-ops
     source: modules/scripts/startup-script
diff --git a/tools/duplicate-diff.py b/tools/duplicate-diff.py
@@ -20,7 +20,7 @@
         "modules/file-system/pre-existing-network-storage/scripts/mount.sh",
         "modules/file-system/filestore/scripts/mount.sh",
         "modules/file-system/managed-lustre/scripts/mount.sh",
-        "community/modules/file-system/cloud-storage-bucket/scripts/mount.sh",
+        "modules/file-system/cloud-storage-bucket/scripts/mount.sh",
         "community/modules/file-system/nfs-server/scripts/mount.sh",
     ],
     [
@@ -30,7 +30,7 @@
     ],
     [
         "modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh",
-        "community/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh",
+        "modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh",
     ],
     [
         "modules/scheduler/batch-job-template/startup_from_network_storage.tf",
diff --git a/tools/validate_configs/test_configs/cloud-storage-bucket.yaml b/tools/validate_configs/test_configs/cloud-storage-bucket.yaml
@@ -28,12 +28,12 @@ deployment_groups:
     source: modules/network/pre-existing-vpc
 
   - id: bucket-w-suffix
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings:
       random_suffix: true
 
   - id: bucket-wo-suffix
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings:
       name_prefix: simulation-results
       use_deployment_name_in_bucket_name: false
diff --git a/tools/validate_configs/test_configs/slurm-gcp-v6-startup-scripts.yaml b/tools/validate_configs/test_configs/slurm-gcp-v6-startup-scripts.yaml
@@ -35,7 +35,7 @@ deployment_groups:
       local_mounts: [/home]
 
   - id: bucket
-    source: community/modules/file-system/cloud-storage-bucket
+    source: modules/file-system/cloud-storage-bucket
     settings:
       name_prefix: input-data
       local_mount: /data

Original file line number	Diff line number	Diff line change
`@@ -54,7 +54,7 @@ module "health_check_firewall_rule" {`
`54`	`54`	`}`
`55`	`55`
`56`	`56`	`module "htcondor_bucket" {`
`57`		`- source = "../../../../community/modules/file-system/cloud-storage-bucket"`
	`57`	`+ source = "../../../../modules/file-system/cloud-storage-bucket"`
`58`	`58`
`59`	`59`	`project_id = var.project_id`
`60`	`60`	`deployment_name = var.deployment_name`