Merge pull request #1065 from tpdownes/release-candidate

Release v1.15.0

Author: tpdownes

.github/dependabot.yml

@@ -27,8 +27,10 @@ updates:
 - package-ecosystem: pip
   directory: /community/front-end/ofe/
   schedule:
-    interval: weekly
-    day: monday
+    interval: monthly
     time: "03:00"
     timezone: America/Los_Angeles
   target-branch: develop
+  reviewers:
+  - ek-nag
+  - mattstreet-nag

cmd/root.go

@@ -50,7 +50,7 @@ HPC deployments on the Google Cloud Platform.`,
 				log.Fatalf("cmd.Help function failed: %s", err)
 			}
 		},
-		Version:     "v1.14.1",
+		Version:     "v1.15.0",
 		Annotations: annotation,
 	}
 )

community/examples/fluent-tutorial.yaml

@@ -25,9 +25,12 @@ vars:
 deployment_groups:
 - group: primary
   modules:
-  - source: modules/scripts/startup-script
-    kind: terraform
-    id: startup_login
+
+  - id: network1
+    source: modules/network/pre-existing-vpc
+
+  - id: startup_login
+    source: modules/scripts/startup-script
     settings:
       runners:
       - type: shell
@@ -89,9 +92,8 @@ deployment_groups:
 
         destination: /tmp/install-mpi.sh
 
-  - source: modules/scripts/startup-script
-    kind: terraform
-    id: startup
+  - id: startup
+    source: modules/scripts/startup-script
     settings:
       runners:
       - type: shell
@@ -108,28 +110,26 @@ deployment_groups:
           hostname >> /home/hosts.txt
         destination: /tmp/install-hosts.sh
 
-  - source: modules/file-system/filestore
-    kind: terraform
-    id: homefs
+  - id: homefs
+    source: modules/file-system/filestore
+    use: [network1]
     settings:
-      network_name: "default"
       size_gb: 1024
       filestore_tier: "BASIC_HDD"
       local_mount: /home
 
-  - source: modules/file-system/filestore
-    kind: terraform
-    id: sharefs
+  - id: sharefs
+    source: modules/file-system/filestore
+    use: [network1]
     settings:
-      network_name: "default"
       size_gb: 1024
       filestore_tier: "BASIC_HDD"
       local_mount: /shared
 
-  - source: modules/compute/vm-instance
-    kind: terraform
-    id: compute_1
+  - id: compute_1
+    source: modules/compute/vm-instance
     use:
+    - network1
     - homefs
     - sharefs
     - startup
@@ -145,10 +145,10 @@ deployment_groups:
         collocation: "COLLOCATED"
         availability_domain_count: null
 
-  - source: modules/compute/vm-instance
-    kind: terraform
-    id: login
+  - id: login
+    source: modules/compute/vm-instance
     use:
+    - network1
     - homefs
     - sharefs
     - startup_login
@@ -160,8 +160,7 @@ deployment_groups:
       instance_count: 1
       name_prefix: fluent-login
 
-  - source: community/modules/scripts/wait-for-startup
-    kind: terraform
-    id: wait
+  - id: wait
+    source: community/modules/scripts/wait-for-startup
     settings:
       instance_name: ((module.compute_1.name[0]))

community/examples/hpc-cluster-localssd.yaml

@@ -30,7 +30,6 @@ deployment_groups:
   modules:
   - id: network1
     source: modules/network/pre-existing-vpc
-    kind: terraform
 
   - id: homefs
     source: modules/file-system/filestore

community/examples/hpc-cluster-small-sharedvpc.yaml

@@ -54,7 +54,6 @@ deployment_groups:
     settings:
       local_mount: /home
       connect_mode: PRIVATE_SERVICE_ACCESS
-      network_name: $(network1.network_id)
 
   # This debug_partition will work out of the box without requesting additional GCP quota.
   - id: debug_partition

community/examples/htcondor-pool.yaml

@@ -43,7 +43,7 @@ deployment_groups:
     use:
     - network1
 
-  - id: htcondor_configure_central_manager
+  - id: htcondor_startup_central_manager
     source: modules/scripts/startup-script
     settings:
       runners:
@@ -54,7 +54,7 @@ deployment_groups:
     source: modules/compute/vm-instance
     use:
     - network1
-    - htcondor_configure_central_manager
+    - htcondor_startup_central_manager
     settings:
       name_prefix: central-manager
       machine_type: c2-standard-4
@@ -77,25 +77,40 @@ deployment_groups:
     outputs:
     - internal_ip
 
-  - id: htcondor_configure_execute_point
+  - id: htcondor_startup_execute_point
     source: modules/scripts/startup-script
     settings:
       runners:
       - $(htcondor_install.install_htcondor_runner)
       - $(htcondor_configure.execute_point_runner)
 
+  # the HTCondor modules support up to 2 execute points per blueprint
+  # if using 1, it may use Spot or On-demand pricing
+  # if using 2, one must use Spot and the other must use On-demand (default)
   - id: htcondor_execute_point
     source: community/modules/compute/htcondor-execute-point
     use:
     - network1
-    - htcondor_configure_execute_point
+    - htcondor_startup_execute_point
     settings:
       service_account:
         email: $(htcondor_configure.execute_point_service_account)
         scopes:
         - cloud-platform
 
-  - id: htcondor_configure_access_point
+  - id: htcondor_execute_point_spot
+    source: community/modules/compute/htcondor-execute-point
+    use:
+    - network1
+    - htcondor_startup_execute_point
+    settings:
+      spot: true
+      service_account:
+        email: $(htcondor_configure.execute_point_service_account)
+        scopes:
+        - cloud-platform
+
+  - id: htcondor_startup_access_point
     source: modules/scripts/startup-script
     settings:
       runners:
@@ -104,23 +119,26 @@ deployment_groups:
       - $(htcondor_install.install_autoscaler_runner)
       - $(htcondor_configure.access_point_runner)
       - $(htcondor_execute_point.configure_autoscaler_runner)
+      - $(htcondor_execute_point_spot.configure_autoscaler_runner)
       - type: data
         destination: /var/tmp/helloworld.sub
         content: |
-          universe       = docker
-          docker_image   = hello-world
-          output         = out
-          error          = err
-          log            = log
+          universe       = vanilla
+          executable     = /bin/echo
+          arguments      = "Hello, World!"
+          output         = out.\$(ClusterId).\$(ProcId)
+          error          = err.\$(ClusterId).\$(ProcId)
+          log            = log.\$(ClusterId).\$(ProcId)
           request_cpus   = 1
           request_memory = 100MB
+          +RequireSpot   = true # if unset, defaults to false
           queue
 
   - id: htcondor_access
     source: modules/compute/vm-instance
     use:
     - network1
-    - htcondor_configure_access_point
+    - htcondor_startup_access_point
     settings:
       name_prefix: access-point
       machine_type: c2-standard-4

community/front-end/ofe/deploy.sh

@@ -451,7 +451,7 @@ deploy() {
 		sdir=${SCRIPT_DIR#"${basedir}"}
 		tdir=/tmp/hpc-toolkit
 
-		cp --recursive "${basedir}" ${tdir}/
+		cp -R "${basedir}" ${tdir}/
 		(
 			cd ${tdir}
 			#
@@ -465,9 +465,10 @@ deploy() {
 				--exclude=.terraform \
 				--exclude=.terraform.lock.hcl \
 				--exclude=tf \
-				../hpc-toolkit 2>/dev/null
+				--directory=/tmp \
+				./hpc-toolkit 2>/dev/null
 		)
-		rm --force --recursive ${tdir}
+		rm -rf ${tdir}
 	fi
 
 	# -- All Terraform operations to be done in tf subdir

community/front-end/ofe/requirements.txt

@@ -1,15 +1,15 @@
 archspec==0.1.3
 argcomplete==2.0.0
 asgiref==3.5.0
-astroid==2.9.3
+astroid==2.15.0
 cachetools==5.0.0
 certifi==2022.12.7
 cffi==1.15.0
 charset-normalizer==2.0.12
 click==7.1.2
 cryptography==39.0.1
 defusedxml==0.7.1
-dill==0.3.4
+dill==0.3.6
 Django==3.2.18
 django-allauth==0.48.0
 django-extensions==3.1.5
@@ -21,7 +21,7 @@ google-api-python-client==2.37.0
 google-auth==2.6.0
 google-auth-httplib2==0.1.0
 google-cloud-billing==1.4.1
-google-cloud-core==2.2.2
+google-cloud-core==2.3.2
 google-cloud-pubsub==2.9.0
 google-cloud-storage==2.1.0
 google-crc32c==1.3.0
@@ -34,15 +34,15 @@ grpcio-status==1.43.0
 h11==0.13.0
 httplib2==0.20.4
 idna==3.3
-isort==5.10.1
+isort==5.12.0
 lazy-object-proxy==1.7.1
 libcst==0.4.1
 mccabe==0.6.1
 mypy-extensions==0.4.3
 oauthlib==3.2.2
 platformdirs==2.5.0
 pre-commit==2.17.0
-proto-plus==1.20.1
+proto-plus==1.22.2
 protobuf==3.19.5
 pyasn1==0.4.8
 pyasn1-modules==0.2.8