Skip to content

Commit f1e2ec1

Browse files
authored
Merge pull request #1354 from GoogleCloudPlatform/release-candidate
Release v1.18.0
2 parents f7abfbe + 10d80e3 commit f1e2ec1

File tree

278 files changed

+7805
-4319
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

278 files changed

+7805
-4319
lines changed

.ansible-lint

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,20 @@ skip_list:
33
- jinja[invalid]
44

55
exclude_paths:
6-
- .cache/ # implicit unless exclude_paths is defined in config
7-
- .github/
8-
- cmd/
9-
- docs/
10-
- examples/
11-
- community/examples/
12-
- pkg/
6+
- .cache/ # implicit unless exclude_paths is defined in config
7+
- .github/
8+
- cmd/
9+
- docs/
10+
- examples/
11+
- community/examples/
12+
- pkg/
1313

1414
mock_roles:
1515
- googlecloudplatform.google_cloud_ops_agents
1616

1717
kinds:
18-
- playbook: "**/ansible_playbooks/*test.{yml,yaml}"
19-
- playbook: "**/files/*.{yml,yaml}"
20-
- playbook: "**/scripts/*.{yml,yaml}"
21-
- tasks: "**/ansible_playbooks/test*.{yml,yaml}"
22-
- tasks: "**/tasks/*"
18+
- playbook: "**/ansible_playbooks/*test.{yml,yaml}"
19+
- playbook: "**/files/*.{yml,yaml}"
20+
- playbook: "**/scripts/*.{yml,yaml}"
21+
- tasks: "**/ansible_playbooks/test*.{yml,yaml}"
22+
- tasks: "**/tasks/*"

.gitignore

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,5 +59,5 @@ packer-manifest.json
5959
*.auto.pkrvars.hcl
6060

6161
#### Exclude from gitingore
62-
!tools/validate_configs/golden_copies/*/*/*/defaults.auto.pkrvars.hcl
63-
!tools/validate_configs/golden_copies/*/*/terraform.tfvars
62+
!tools/validate_configs/golden_copies/expectations/*/*/*/defaults.auto.pkrvars.hcl
63+
!tools/validate_configs/golden_copies/expectations/*/*/terraform.tfvars

.pre-commit-config.yaml

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -66,15 +66,6 @@ repos:
6666
hooks:
6767
- id: go-critic
6868
args: [-disable, "#experimental,sloppyTypeAssert"]
69-
- repo: https://github.com/ansible/ansible-lint
70-
rev: v6.11.0
71-
hooks:
72-
- id: ansible-lint
73-
always_run: false
74-
exclude: '^(docs/|examples/|community/examples/)'
75-
types: [yaml]
76-
additional_dependencies:
77-
- ansible==6.*
7869
- repo: https://github.com/adrienverge/yamllint
7970
rev: v1.29.0
8071
hooks:

Makefile

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# PREAMBLE
2-
MIN_PACKER_VERSION=1.6 # for building images
3-
MIN_TERRAFORM_VERSION=1.0 # for deploying modules
2+
MIN_PACKER_VERSION=1.7.9 # for building images
3+
MIN_TERRAFORM_VERSION=1.2 # for deploying modules
44
MIN_GOLANG_VERSION=1.18 # for building ghpc
55

66
.PHONY: install install-user tests format add-google-license install-dev-deps \
@@ -117,8 +117,9 @@ else
117117
## GO IS PRESENT
118118
warn-go-missing:
119119

120-
GO_VERSION_CHECK=$(shell expr `go version | cut -f 3 -d ' ' | cut -f 1 -d '-' | cut -c 3-` \>= $(MIN_GOLANG_VERSION))
121-
ifneq ("$(GO_VERSION_CHECK)", "1")
120+
GO_VERSION=$(shell go version | cut -f 3 -d ' ')
121+
GO_VERSION_CHECK=$(shell ./tools/version_check.sh $(GO_VERSION) $(MIN_GOLANG_VERSION))
122+
ifneq ("$(GO_VERSION_CHECK)", "yes")
122123
warn-go-version:
123124
$(warning WARNING: Go version must be greater than $(MIN_GOLANG_VERSION), update at https://go.dev/doc/install)
124125
else
@@ -145,8 +146,9 @@ else
145146
## TERRAFORM IS PRESENT
146147
warn-terraform-missing:
147148

148-
TF_VERSION_CHECK=$(shell expr `terraform version | cut -f 2- -d ' ' | cut -c 2- | head -n1` \>= $(MIN_TERRAFORM_VERSION))
149-
ifneq ("$(TF_VERSION_CHECK)", "1")
149+
TF_VERSION=$(shell terraform version | cut -f 2- -d ' ' | head -n1)
150+
TF_VERSION_CHECK=$(shell ./tools/version_check.sh $(TF_VERSION) $(MIN_TERRAFORM_VERSION))
151+
ifneq ("$(TF_VERSION_CHECK)", "yes")
150152
warn-terraform-version:
151153
$(warning WARNING: terraform version must be greater than $(MIN_TERRAFORM_VERSION), update at https://learn.hashicorp.com/tutorials/terraform/install-cli)
152154
else
@@ -159,7 +161,7 @@ validate_configs: ghpc
159161

160162
validate_golden_copy: ghpc
161163
$(info *********** running "Golden copy" tests ***********)
162-
tools/validate_configs/validate_golden_copy.sh
164+
tools/validate_configs/golden_copies/validate.sh
163165

164166
terraform-format:
165167
$(info *********** cleaning terraform files syntax and generating terraform documentation ***********)
@@ -176,7 +178,7 @@ endif
176178
# END OF TERRAFORM SECTION
177179
###################################
178180
# PACKER SECTION
179-
ifeq (, $(shell which packer))
181+
ifneq (yes, $(shell ./tools/detect_packer.sh ))
180182
## PACKER IS NOT PRESENT
181183
warn-packer-missing:
182184
$(warning WARNING: packer not installed, visit https://learn.hashicorp.com/tutorials/packer/get-started-install-cli)
@@ -193,8 +195,9 @@ else
193195
## PACKER IS PRESENT
194196
warn-packer-missing:
195197

196-
PK_VERSION_CHECK=$(shell expr `packer version | cut -f 2- -d ' ' | cut -c 2- | head -n1` \>= $(MIN_PACKER_VERSION))
197-
ifneq ("$(PK_VERSION_CHECK)", "1")
198+
PK_VERSION=$(shell packer version | cut -f 2- -d ' ' | head -n1)
199+
PK_VERSION_CHECK=$(shell ./tools/version_check.sh $(PK_VERSION) $(MIN_PACKER_VERSION))
200+
ifneq ("$(PK_VERSION_CHECK)", "yes")
198201
### WRONG PACKER VERSION, MAY ALSO MEAN THE USER HAS SOME OTHER PACKER TOOL
199202
warn-packer-version:
200203
$(warning WARNING: packer version must be greater than $(MIN_PACKER_VERSION), update at https://learn.hashicorp.com/tutorials/packer/get-started-install-cli)

README.md

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ If a self directed path is preferred, you can use the following commands to
3333
build the `ghpc` binary:
3434

3535
```shell
36-
git clone git@github.com:GoogleCloudPlatform/hpc-toolkit.git
36+
git clone https://github.com/GoogleCloudPlatform/hpc-toolkit
3737
cd hpc-toolkit
3838
make
3939
./ghpc --version
@@ -247,22 +247,19 @@ the same as the source module, for example the
247247
A hidden directory containing meta information and backups is also created and
248248
named `.ghpc`.
249249

250-
From the [hpc-cluster-small.yaml example](./examples/hpc-cluster-small.yaml), we
250+
From the [hpc-slurm.yaml example](./examples/hpc-slurm.yaml), we
251251
get the following deployment directory:
252252

253253
```text
254-
hpc-small/
254+
hpc-slurm/
255255
primary/
256256
main.tf
257-
variables.tf
258-
terraform.tfvars
259257
modules/
260-
filestore/
261-
SchedMD-slurm-on-gcp-controller/
262-
SchedMD-slurm-on-gcp-login-node/
263-
SchedMD-slurm-on-gcp-partition/
264-
vpc/
265-
.ghpc/
258+
providers.tf
259+
terraform.tfvars
260+
variables.tf
261+
versions.tf
262+
.ghpc/
266263
```
267264

268265
## Dependencies

cmd/create.go

Lines changed: 100 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -24,16 +24,18 @@ import (
2424
"hpc-toolkit/pkg/modulewriter"
2525
"log"
2626
"os"
27+
"strings"
2728

2829
"github.com/spf13/cobra"
30+
"github.com/zclconf/go-cty/cty"
31+
"gopkg.in/yaml.v3"
2932
)
3033

3134
const msgCLIVars = "Comma-separated list of name=value variables to override YAML configuration. Can be used multiple times."
3235
const msgCLIBackendConfig = "Comma-separated list of name=value variables to set Terraform backend configuration. Can be used multiple times."
3336

3437
func init() {
35-
createCmd.Flags().StringVarP(&bpFilename, "config", "c", "",
36-
"HPC Environment Blueprint file to be used to create an HPC deployment dir.")
38+
createCmd.Flags().StringVarP(&bpFilenameDeprecated, "config", "c", "", "")
3739
cobra.CheckErr(createCmd.Flags().MarkDeprecated("config",
3840
"please see the command usage for more details."))
3941

@@ -52,9 +54,9 @@ func init() {
5254
}
5355

5456
var (
55-
bpFilename string
56-
outputDir string
57-
cliVariables []string
57+
bpFilenameDeprecated string
58+
outputDir string
59+
cliVariables []string
5860

5961
cliBEConfigVars []string
6062
overwriteDeployment bool
@@ -64,52 +66,114 @@ var (
6466
skipValidatorsDesc = "Validators to skip"
6567

6668
createCmd = &cobra.Command{
67-
Use: "create BLUEPRINT_NAME",
68-
Short: "Create a new deployment.",
69-
Long: "Create a new deployment based on a provided blueprint.",
70-
Run: runCreateCmd,
71-
Args: cobra.ExactArgs(1),
69+
Use: "create BLUEPRINT_NAME",
70+
Short: "Create a new deployment.",
71+
Long: "Create a new deployment based on a provided blueprint.",
72+
Run: runCreateCmd,
73+
Args: cobra.ExactArgs(1),
74+
ValidArgsFunction: filterYaml,
7275
}
7376
)
7477

7578
func runCreateCmd(cmd *cobra.Command, args []string) {
76-
if bpFilename == "" {
77-
if len(args) == 0 {
78-
fmt.Println(cmd.UsageString())
79-
return
79+
dc := expandOrDie(args[0])
80+
if err := modulewriter.WriteDeployment(dc, outputDir, overwriteDeployment); err != nil {
81+
var target *modulewriter.OverwriteDeniedError
82+
if errors.As(err, &target) {
83+
fmt.Printf("\n%s\n", err.Error())
84+
os.Exit(1)
85+
} else {
86+
log.Fatal(err)
8087
}
81-
82-
bpFilename = args[0]
8388
}
89+
}
8490

85-
deploymentConfig, err := config.NewDeploymentConfig(bpFilename)
91+
func expandOrDie(path string) config.DeploymentConfig {
92+
dc, err := config.NewDeploymentConfig(path)
8693
if err != nil {
8794
log.Fatal(err)
8895
}
89-
if err := deploymentConfig.SetCLIVariables(cliVariables); err != nil {
96+
// Set properties from CLI
97+
if err := setCLIVariables(&dc.Config, cliVariables); err != nil {
9098
log.Fatalf("Failed to set the variables at CLI: %v", err)
9199
}
92-
if err := deploymentConfig.SetBackendConfig(cliBEConfigVars); err != nil {
100+
if err := setBackendConfig(&dc.Config, cliBEConfigVars); err != nil {
93101
log.Fatalf("Failed to set the backend config at CLI: %v", err)
94102
}
95-
if err := deploymentConfig.SetValidationLevel(validationLevel); err != nil {
103+
if err := setValidationLevel(&dc.Config, validationLevel); err != nil {
96104
log.Fatal(err)
97105
}
98-
if err := skipValidators(&deploymentConfig); err != nil {
106+
if err := skipValidators(&dc); err != nil {
99107
log.Fatal(err)
100108
}
101-
if err := deploymentConfig.ExpandConfig(); err != nil {
109+
if dc.Config.GhpcVersion != "" {
110+
fmt.Printf("ghpc_version setting is ignored.")
111+
}
112+
dc.Config.GhpcVersion = GitCommitInfo
113+
114+
// Expand the blueprint
115+
if err := dc.ExpandConfig(); err != nil {
102116
log.Fatal(err)
103117
}
104-
if err := modulewriter.WriteDeployment(deploymentConfig, outputDir, overwriteDeployment); err != nil {
105-
var target *modulewriter.OverwriteDeniedError
106-
if errors.As(err, &target) {
107-
fmt.Printf("\n%s\n", err.Error())
108-
os.Exit(1)
109-
} else {
110-
log.Fatal(err)
118+
119+
return dc
120+
}
121+
122+
func setCLIVariables(bp *config.Blueprint, s []string) error {
123+
for _, cliVar := range s {
124+
arr := strings.SplitN(cliVar, "=", 2)
125+
126+
if len(arr) != 2 {
127+
return fmt.Errorf("invalid format: '%s' should follow the 'name=value' format", cliVar)
128+
}
129+
// Convert the variable's string literal to its equivalent default type.
130+
key := arr[0]
131+
var v config.YamlValue
132+
if err := yaml.Unmarshal([]byte(arr[1]), &v); err != nil {
133+
return fmt.Errorf("invalid input: unable to convert '%s' value '%s' to known type", key, arr[1])
111134
}
135+
bp.Vars.Set(key, v.Unwrap())
112136
}
137+
return nil
138+
}
139+
140+
func setBackendConfig(bp *config.Blueprint, s []string) error {
141+
if len(s) == 0 {
142+
return nil // no op
143+
}
144+
be := config.TerraformBackend{Type: "gcs"}
145+
for _, config := range s {
146+
arr := strings.SplitN(config, "=", 2)
147+
148+
if len(arr) != 2 {
149+
return fmt.Errorf("invalid format: '%s' should follow the 'name=value' format", config)
150+
}
151+
152+
key, value := arr[0], arr[1]
153+
switch key {
154+
case "type":
155+
be.Type = value
156+
default:
157+
be.Configuration.Set(key, cty.StringVal(value))
158+
}
159+
}
160+
bp.TerraformBackendDefaults = be
161+
return nil
162+
}
163+
164+
// SetValidationLevel allows command-line tools to set the validation level
165+
func setValidationLevel(bp *config.Blueprint, s string) error {
166+
switch s {
167+
case "ERROR":
168+
bp.ValidationLevel = config.ValidationError
169+
case "WARNING":
170+
bp.ValidationLevel = config.ValidationWarning
171+
case "IGNORE":
172+
bp.ValidationLevel = config.ValidationIgnore
173+
default:
174+
return fmt.Errorf("invalid validation level (\"ERROR\", \"WARNING\", \"IGNORE\")")
175+
}
176+
return nil
113177
}
114178

115179
func skipValidators(dc *config.DeploymentConfig) error {
@@ -123,3 +187,10 @@ func skipValidators(dc *config.DeploymentConfig) error {
123187
}
124188
return nil
125189
}
190+
191+
func filterYaml(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
192+
if len(args) != 0 {
193+
return nil, cobra.ShellCompDirectiveNoFileComp
194+
}
195+
return []string{"yaml", "yml"}, cobra.ShellCompDirectiveFilterFileExt
196+
}

0 commit comments

Comments
 (0)