Merge pull request #2702 from GoogleCloudPlatform/release-candidate

Release v1.35.0

Author: alyssa-sm

.github/workflows/pr-precommit.yml

@@ -24,7 +24,9 @@ on:
     - labeled
     - synchronize
     branches:
+    - main
     - develop
+    - release-candidate
 
 jobs:
   pre-commit:

cmd/deploy.go

@@ -28,9 +28,10 @@ import (
 )
 
 func addDeployFlags(c *cobra.Command) *cobra.Command {
-	return addAutoApproveFlag(
-		addArtifactsDirFlag(
-			addCreateFlags(c)))
+	return addGroupSelectionFlags(
+		addAutoApproveFlag(
+			addArtifactsDirFlag(
+				addCreateFlags(c))))
 }
 
 func init() {
@@ -71,10 +72,16 @@ func doDeploy(deplRoot string) {
 	checkErr(shell.CheckWritableDir(artDir), nil)
 	bp, ctx := artifactBlueprintOrDie(artDir)
 	groups := bp.Groups
+	checkErr(validateGroupSelectionFlags(bp), ctx)
 	checkErr(validateRuntimeDependencies(deplRoot, groups), ctx)
 	checkErr(shell.ValidateDeploymentDirectory(groups, deplRoot), ctx)
 
 	for ig, group := range groups {
+		if !isGroupSelected(group.Name) {
+			logging.Info("skipping group %q", group.Name)
+			continue
+		}
+
 		groupDir := filepath.Join(deplRoot, string(group.Name))
 		checkErr(shell.ImportInputs(groupDir, artDir, bp), ctx)
 

cmd/destroy.go

@@ -31,8 +31,9 @@ import (
 
 func init() {
 	rootCmd.AddCommand(
-		addAutoApproveFlag(
-			addArtifactsDirFlag(destroyCmd)))
+		addGroupSelectionFlags(
+			addAutoApproveFlag(
+				addArtifactsDirFlag(destroyCmd))))
 }
 
 var (
@@ -56,13 +57,17 @@ func runDestroyCmd(cmd *cobra.Command, args []string) {
 	}
 
 	bp, ctx := artifactBlueprintOrDie(artifactsDir)
-
+	checkErr(validateGroupSelectionFlags(bp), ctx)
 	checkErr(shell.ValidateDeploymentDirectory(bp.Groups, deplRoot), ctx)
 
 	// destroy in reverse order of creation!
 	packerManifests := []string{}
 	for i := len(bp.Groups) - 1; i >= 0; i-- {
 		group := bp.Groups[i]
+		if !isGroupSelected(group.Name) {
+			logging.Info("skipping group %q", group.Name)
+			continue
+		}
 		groupDir := filepath.Join(deplRoot, string(group.Name))
 
 		if err := shell.ImportInputs(groupDir, artifactsDir, bp); err != nil {

cmd/root.go

@@ -52,7 +52,7 @@ HPC deployments on the Google Cloud Platform.`,
 				logging.Fatal("cmd.Help function failed: %s", err)
 			}
 		},
-		Version:     "v1.34.1",
+		Version:     "v1.35.0",
 		Annotations: annotation,
 	}
 )

cmd/utils.go

@@ -15,10 +15,13 @@
 package cmd
 
 import (
+	"errors"
 	"fmt"
+	"hpc-toolkit/pkg/config"
 	"hpc-toolkit/pkg/modulewriter"
 	"hpc-toolkit/pkg/shell"
 	"os"
+	"slices"
 
 	"github.com/spf13/cobra"
 )
@@ -78,3 +81,41 @@ func filterYaml(cmd *cobra.Command, args []string, toComplete string) ([]string,
 	}
 	return []string{"yaml", "yml"}, cobra.ShellCompDirectiveFilterFileExt
 }
+
+var flagSkipGroups []string
+var flagOnlyGroups []string
+
+func addGroupSelectionFlags(c *cobra.Command) *cobra.Command {
+	c.Flags().StringSliceVar(&flagSkipGroups, "skip", nil, "Skip groups with the given names")
+	c.Flags().StringSliceVar(&flagOnlyGroups, "only", nil, "Only apply to groups with the given names")
+	return c
+}
+
+func validateGroupSelectionFlags(bp config.Blueprint) error {
+	if flagOnlyGroups != nil && flagSkipGroups != nil {
+		return errors.New("cannot specify both --only and --skip")
+	}
+
+	dict := []string{}
+	for _, group := range bp.Groups {
+		dict = append(dict, string(group.Name))
+	}
+
+	for _, g := range append(flagOnlyGroups, flagSkipGroups...) {
+		if !slices.Contains(dict, g) {
+			return config.HintSpelling(g, dict, fmt.Errorf("group %q not found", g))
+		}
+	}
+
+	return nil
+}
+
+func isGroupSelected(g config.GroupName) bool {
+	if flagOnlyGroups != nil {
+		return slices.Contains(flagOnlyGroups, string(g))
+	}
+	if flagSkipGroups != nil {
+		return !slices.Contains(flagSkipGroups, string(g))
+	}
+	return true
+}

cmd/utils_test.go

@@ -0,0 +1,83 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"fmt"
+	"hpc-toolkit/pkg/config"
+	"testing"
+)
+
+func TestIsGroupSelected(t *testing.T) {
+	type test struct {
+		only  []string
+		skip  []string
+		group config.GroupName
+		want  bool
+	}
+	tests := []test{
+		{nil, nil, "green", true},
+		{[]string{"green"}, nil, "green", true},
+		{[]string{"green"}, nil, "blue", false},
+		{nil, []string{"green"}, "green", false},
+		{nil, []string{"green"}, "blue", true},
+	}
+
+	for _, tc := range tests {
+		t.Run(fmt.Sprintf("%v;%v;%q", tc.only, tc.skip, tc.group), func(t *testing.T) {
+			flagOnlyGroups, flagSkipGroups = tc.only, tc.skip
+			got := isGroupSelected(tc.group)
+			if got != tc.want {
+				t.Errorf("isGroupSelected(%v) = %v; want %v", tc.group, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestValidateGroupSelectionFlags(t *testing.T) {
+	type test struct {
+		only   []string
+		skip   []string
+		groups []string
+		err    bool
+	}
+	tests := []test{
+		{nil, nil, []string{"green"}, false},
+		{[]string{"green"}, []string{"blue"}, []string{"green", "blue"}, true},
+		{[]string{"green"}, nil, []string{"green"}, false},
+		{[]string{"green"}, nil, []string{"blue"}, true},
+		{nil, []string{"green"}, []string{"green"}, false},
+		{nil, []string{"green"}, []string{"blue"}, true},
+	}
+
+	for _, tc := range tests {
+		t.Run(fmt.Sprintf("%v;%v;%v", tc.only, tc.skip, tc.groups), func(t *testing.T) {
+			flagOnlyGroups, flagSkipGroups = tc.only, tc.skip
+			bp := config.Blueprint{}
+			for _, g := range tc.groups {
+				bp.Groups = append(bp.Groups, config.Group{Name: config.GroupName(g)})
+			}
+
+			err := validateGroupSelectionFlags(bp)
+			if tc.err && err == nil {
+				t.Errorf("validateGroupSelectionFlags(%v) = nil; want error", tc.groups)
+			}
+			if !tc.err && err != nil {
+				t.Errorf("validateGroupSelectionFlags(%v) = %v; want nil", tc.groups, err)
+			}
+		})
+	}
+
+}

community/examples/hpc-build-slurm-image.yaml

@@ -23,7 +23,7 @@ vars:
   image_build_machine_type: n2d-standard-16
   build_from_image_family: hpc-rocky-linux-8
   build_from_image_project: cloud-hpc-image-public
-  build_from_git_ref: 6.5.6
+  build_from_git_ref: 6.5.8
   built_image_family: my-custom-slurm
   built_instance_image:
     family: $(vars.built_image_family)

community/examples/ml-gke.yaml

@@ -18,7 +18,9 @@ blueprint_name: ml-gke
 vars:
   project_id:  ## Set GCP Project ID Here ##
   deployment_name: ml-01
-  region: us-central1
+  region: asia-southeast1
+  zones:
+  - asia-southeast1-b  # g2 machine has better availability in this zone
 
   # Cidr block containing the IP of the machine calling terraform.
   # The following line must be updated for this example to work.
@@ -48,22 +50,23 @@ deployment_groups:
         cidr_block: $(vars.authorized_cidr)
     outputs: [instructions]
 
-  # Docs at https://github.com/GoogleCloudPlatform/hpc-toolkit/tree/main/community/modules/scripts/kubernetes-operations
-  - id: install-nvidia-drivers
-    source: github.com/GoogleCloudPlatform/ai-infra-cluster-provisioning//aiinfra-cluster/modules/kubernetes-operations?ref=v0.6.0
-    use: [gke_cluster]
-    settings:
-      install_nvidia_driver: true
-
-  - id: a2-pool
+  - id: g2-pool
     source: community/modules/compute/gke-node-pool
     use: [gke_cluster]
     settings:
-      machine_type: a2-highgpu-8g
+      disk_type: pd-balanced
+      machine_type: g2-standard-4
+      guest_accelerator:
+      - type: nvidia-l4
+        count: 1
+        gpu_partition_size: null
+        gpu_sharing_config: null
+        gpu_driver_installation_config:
+        - gpu_driver_version: "DEFAULT"
 
   - id: job-template
     source: community/modules/compute/gke-job-template
-    use: [a2-pool]
+    use: [g2-pool]
     settings:
       image: nvidia/cuda:11.0.3-runtime-ubuntu20.04
       command:

community/front-end/ofe/deploy.sh

@@ -57,6 +57,8 @@ PRJ_API['bigqueryconnection.googleapis.com']='BigQuery Connection API'
 PRJ_API['sqladmin.googleapis.com']='Cloud SQL Admin API'
 PRJ_API['servicenetworking.googleapis.com']='Service Networking API'
 PRJ_API['secretmanager.googleapis.com']='Secret Manager API'
+PRJ_API['serviceusage.googleapis.com']='Service Usage API'
+PRJ_API['storage.googleapis.com']='Cloud Storage API'
 
 # Location for output credential file = pwd/credential.json
 #

community/front-end/ofe/infrastructure_files/gcs_bucket/webserver/startup.sh

@@ -249,6 +249,7 @@ autostart=true
 autorestart=true
 user=gcluster
 redirect_stderr=true
+environment=HOME=/opt/gcluster
 stdout_logfile=/opt/gcluster/run/supvisor.log" >/etc/supervisord.d/gcluster.ini
 
 printf "Creating systemd service..."