diff --git a/terraform/gcp/.terraform.lock.hcl b/terraform/gcp/.terraform.lock.hcl index c5936f67f3..5bb08c6646 100644 --- a/terraform/gcp/.terraform.lock.hcl +++ b/terraform/gcp/.terraform.lock.hcl @@ -2,78 +2,98 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/google" { - version = "6.13.0" + version = "6.25.0" constraints = ">= 4.51.0, < 7.0.0" hashes = [ - "h1:jakE2O6gqqQHIZYnr4K7zCNqlMDO/VMgVbCPbbLT/Oc=", - "zh:05067a572a2fcec7ab2e613611db68ff9fdaac18869f76ca53ce308f8f4b904a", - "zh:1122fae676ba52d4842267ce8a9e391c262d2468c095b6ad7b0550c1bf292b05", - "zh:1cde30b75d4e5ecd8eabcf2f9119302611a2b757f9cd50474c21f30dba77dd60", - "zh:444d333a8af09d4f561a09c56eee48cd23e97fca94c427ae5e4a24745418a097", - "zh:471303b121adf8b9225de70127f62f3b96033a9ab6e14f9dc7fe19f6d224186d", - "zh:4caa73ea78c85dc4abde022d114d4d2ff770636def8f308472e3679ac5195d9a", - "zh:6257a064015ed1c277cfb39027b91de2b7bc6f1ce6d42058b9df3d5aa37ecb4f", - "zh:8a45fe7f3e4b10ba6e8c5e44fecd0c7389e6f9c8cca11d052a357fbaae615c1a", - "zh:8ce6902473efdf92724679c2b63905263d83dcf77bd8d9e9f45701cdc9dcc229", - "zh:ce8951ef5ce96d694f7fbcea6d7dd593f6102b3ac1c686e03f08316dd7923d2d", - "zh:ef009b465067b8d3646d0c6ca8a6ba860d4591ddc427f005cfa5d4fcac0c4fce", + "h1:rvsmJc5J5OMMFXj9th8Yqfwm7cLZdRM/stPo0B8Datg=", + "zh:115ce3e84a02412a9a42c7c8f1c4568ab470bdd93c9a3a1f9202d4b77d7bc236", + "zh:623a32acea92d98a8bb66481fa9489e0a1e4dd6fc57ab70dbb53fae5732c1c63", + "zh:6440fc959b5e316152e26c916d55566311dafbe5b64e45d4b9c9931a5b29fa13", + "zh:91edb056638e723b1c7802d0e6e293611208e2825f645572ddba98122723f11c", + "zh:9b57ee44172677d2c2df03b22cf4b40e184ac8bd8facdd456ccbfdb7fee4684a", + "zh:9e2a97dd09b78b36caecdd990fff80bdb47a6a6d45c8b2dd4d23885d6bc89e65", + "zh:b33e40e9ba745f15b1c2e9ebcdccbac185788a4eb57450820cec7b9585858522", + "zh:d9e051bb703597384d83d924c49770e3bcdc1b68583d3def33a607ab168c634b", + "zh:e84253140fc3b0bd5cf7a1ebb54f993bacc6d5ee33c7b5e714ad834d5b2d35d0", + "zh:ebb624504c6f4297e691b6e3c00f789a6729461e5aa80fc165ef2ecd878e2d87", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f8c3e8192e98eff09a9453b52424c0d9727ae284de4fafbc498f2d527e026850", ] } provider "registry.terraform.io/hashicorp/google-beta" { - version = "6.13.0" + version = "6.25.0" hashes = [ - "h1:ydG2kjp6wP4/pLAFn9L9qz9HAPZd0z0SwhiWcXc3d1Q=", - "zh:02f3366f1063ae89b5618bbabab138ac0a2ca1a75f6708fe28d8d23a3e2db98e", - "zh:0c160f494c7eb629954940e4b26e62d37cbc55faf7dd761f80384434a720706b", - "zh:23076c21aca55b4f22f5e253438f9302f90a64e78a416629f95e07890376637b", - "zh:27561ee2350fc747f93d08ff63127223b8a2137c325bc7672637af41067e8b52", - "zh:3e354d40dbd3d0c012d4f5ba1f30aaf9d6287e1c2eff6d88344b5c0d4078effa", - "zh:6a795cc3b358f70f2628785b2cda2b6ea272d8ff804231119a45f88f227c5cc7", - "zh:789b5e3d51387d498d0cbf626c3cb77d70efbaa2acfaed415b4787eba8287b8d", - "zh:8ff948cedfe7e6864d1a5b974e5e16ce00959383f0a7c10543b27f8cf4268010", - "zh:bc92280a45ed4e198684e377bbc36bcd21d0af4c8e4922302f74c42ff1204c20", - "zh:d3243028ee7ca2e2d747e95a212ab3985ecb8719ff7481747b8b45417e12107f", - "zh:e83f1baddc6fc0c6ea7755dc9c219a8e7664b0a1132ed3b7ace1179625a7f8b0", + "h1:KlIctHHQkbToSXa41DWFQUsTKsfaWe+vjs/CFbnj3bc=", + "zh:101a8a679274cae57e8c816164fa28b6d271a8a53b3e007b84153fc63f0f06f7", + "zh:413c41e4ee14b10cb815cd01f8b880c12474ecbf63ecfc6415512fb0c18c473b", + "zh:4260143c8584a8f338a4d45f65c2f08f12b28a746c43d43a1e3a575f96ce4110", + "zh:4785080dd28b79a94ba6e277d247b6bb678d676d99244a8d664ea60802338df7", + "zh:673b09d67cb59487095fe7176c194e8c7ced62fc7bb5084016f3c87dd89e685e", + "zh:7de0dd9367c723cc203fdc4e70d316c779224483637039de5a3c9ee807c3fc55", + "zh:aa49c5955652b21ff2d0b7b2db0e18f59097d3a539efc98ef65deed4293d7327", + "zh:b4d73704d88deee367568fd68303878e6e60924c2a65df397f37b868e6aecaac", + "zh:cae0c5eff7c37c3f9857de08ca35b3a86d44855ae9c0ec2fab1eda604ff501c8", + "zh:e115f6b0c25bdad624fafbb5d685c8a48d9028f9525918274a5e5469f7dfde3d", + "zh:e536197371458b0b177a203f8367f99e74770b3dfce597034e880ea5f3f798b8", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", ] } +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.3" + hashes = [ + "h1:I0Um8UkrMUb81Fxq/dxbr3HLP2cecTH2WMJiwKSrwQY=", + "zh:22d062e5278d872fe7aed834f5577ba0a5afe34a3bdac2b81f828d8d3e6706d2", + "zh:23dead00493ad863729495dc212fd6c29b8293e707b055ce5ba21ee453ce552d", + "zh:28299accf21763ca1ca144d8f660688d7c2ad0b105b7202554ca60b02a3856d3", + "zh:55c9e8a9ac25a7652df8c51a8a9a422bd67d784061b1de2dc9fe6c3cb4e77f2f", + "zh:756586535d11698a216291c06b9ed8a5cc6a4ec43eee1ee09ecd5c6a9e297ac1", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:9d5eea62fdb587eeb96a8c4d782459f4e6b73baeece4d04b4a40e44faaee9301", + "zh:a6355f596a3fb8fc85c2fb054ab14e722991533f87f928e7169a486462c74670", + "zh:b5a65a789cff4ada58a5baffc76cb9767dc26ec6b45c00d2ec8b1b027f6db4ed", + "zh:db5ab669cf11d0e9f81dc380a6fdfcac437aea3d69109c7aef1a5426639d2d65", + "zh:de655d251c470197bcbb5ac45d289595295acb8f829f6c781d4a75c8c8b7c7dd", + "zh:f5c68199f2e6076bce92a12230434782bf768103a427e9bb9abee99b116af7b5", + ] +} + provider "registry.terraform.io/hashicorp/random" { - version = "3.6.3" + version = "3.7.1" + constraints = "~> 3.0" hashes = [ - "h1:zG9uFP8l9u+yGZZvi5Te7PV62j50azpgwPunq2vTm1E=", - "zh:04ceb65210251339f07cd4611885d242cd4d0c7306e86dda9785396807c00451", - "zh:448f56199f3e99ff75d5c0afacae867ee795e4dfda6cb5f8e3b2a72ec3583dd8", - "zh:4b4c11ccfba7319e901df2dac836b1ae8f12185e37249e8d870ee10bb87a13fe", - "zh:4fa45c44c0de582c2edb8a2e054f55124520c16a39b2dfc0355929063b6395b1", - "zh:588508280501a06259e023b0695f6a18149a3816d259655c424d068982cbdd36", - "zh:737c4d99a87d2a4d1ac0a54a73d2cb62974ccb2edbd234f333abd079a32ebc9e", + "h1:t152MY0tQH4a8fLzTtEWx70ITd3azVOrFDn/pQblbto=", + "zh:3193b89b43bf5805493e290374cdda5132578de6535f8009547c8b5d7a351585", + "zh:3218320de4be943e5812ed3de995946056db86eb8d03aa3f074e0c7316599bef", + "zh:419861805a37fa443e7d63b69fb3279926ccf98a79d256c422d5d82f0f387d1d", + "zh:4df9bd9d839b8fc11a3b8098a604b9b46e2235eb65ef15f4432bde0e175f9ca6", + "zh:5814be3f9c9cc39d2955d6f083bae793050d75c572e70ca11ccceb5517ced6b1", + "zh:63c6548a06de1231c8ee5570e42ca09c4b3db336578ded39b938f2156f06dd2e", + "zh:697e434c6bdee0502cc3deb098263b8dcd63948e8a96d61722811628dce2eba1", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:a357ab512e5ebc6d1fda1382503109766e21bbfdfaa9ccda43d313c122069b30", - "zh:c51bfb15e7d52cc1a2eaec2a903ac2aff15d162c172b1b4c17675190e8147615", - "zh:e0951ee6fa9df90433728b96381fb867e3db98f66f735e0c3e24f8f16903f0ad", - "zh:e3cdcb4e73740621dabd82ee6a37d6cfce7fee2a03d8074df65086760f5cf556", - "zh:eff58323099f1bd9a0bec7cb04f717e7f1b2774c7d612bf7581797e1622613a0", + "zh:a0b8e44927e6327852bbfdc9d408d802569367f1e22a95bcdd7181b1c3b07601", + "zh:b7d3af018683ef22794eea9c218bc72d7c35a2b3ede9233b69653b3c782ee436", + "zh:d63b911d618a6fe446c65bfc21e793a7663e934b2fef833d42d3ccd38dd8d68d", + "zh:fa985cd0b11e6d651f47cff3055f0a9fd085ec190b6dbe99bf5448174434cdea", ] } provider "registry.terraform.io/hashicorp/time" { - version = "0.12.1" + version = "0.13.0" hashes = [ - "h1:JzYsPugN8Fb7C4NlfLoFu7BBPuRVT2/fCOdCaxshveI=", - "zh:090023137df8effe8804e81c65f636dadf8f9d35b79c3afff282d39367ba44b2", - "zh:26f1e458358ba55f6558613f1427dcfa6ae2be5119b722d0b3adb27cd001efea", - "zh:272ccc73a03384b72b964918c7afeb22c2e6be22460d92b150aaf28f29a7d511", - "zh:438b8c74f5ed62fe921bd1078abe628a6675e44912933100ea4fa26863e340e9", + "h1:iwR4JouIoeVPDabb8XCqsiaZlZ28IcB3tDD9MuPeSXE=", + "zh:3776dd78ef3053562ccb2f8916d5d3f21a28f05e78859f0f1e4510525f891ecb", + "zh:541ca0b56f808c15d208b9396f149563b133223c4b66cdefbcfe2d8f1c23497e", + "zh:67ed315f3572eb20ce6778423b14fbb6faba3090f454bc20ec4146489b4738c0", + "zh:69dc375845bcfc451426480119f2941ee28b9ef01273d228bb66918180863b3a", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:85c8bd8eefc4afc33445de2ee7fbf33a7807bc34eb3734b8eefa4e98e4cddf38", - "zh:98bbe309c9ff5b2352de6a047e0ec6c7e3764b4ed3dfd370839c4be2fbfff869", - "zh:9c7bf8c56da1b124e0e2f3210a1915e778bab2be924481af684695b52672891e", - "zh:d2200f7f6ab8ecb8373cda796b864ad4867f5c255cff9d3b032f666e4c78f625", - "zh:d8c7926feaddfdc08d5ebb41b03445166df8c125417b28d64712dccd9feef136", - "zh:e2412a192fc340c61b373d6c20c9d805d7d3dee6c720c34db23c2a8ff0abd71b", - "zh:e6ac6bba391afe728a099df344dbd6481425b06d61697522017b8f7a59957d44", + "zh:93c24b7c87b5db9721f60782ac784152599aa78b30fdea2fc9c594d46d92767c", + "zh:95441cf14312041ae0b34640ff33975c09540125b01f9131358fca50e7be239d", + "zh:a294103aeed868c58987e131357a3ec259316c937c909e8a726b862d5a227b82", + "zh:adf6ded3f2e2f318e8aebf1040bc2791b448d006af7d12f7ddc3e8d40b22047a", + "zh:b2d9c16b7acd20d3813060c4d3647dc5f40598ebbdf59f642d53d189e4e3870a", + "zh:bc76a5161e9bcf74cadd76b3d4a51de508aa0c62e7f7ae536a87cd7595d81ebf", + "zh:ce6df2c1052c60b4432cb5c0ead471d7cdb4b285b807c265328a358631fc3610", ] } diff --git a/terraform/gcp/apis.tf b/terraform/gcp/apis.tf index 2436322019..c285f15048 100644 --- a/terraform/gcp/apis.tf +++ b/terraform/gcp/apis.tf @@ -13,7 +13,10 @@ locals { "servicenetworking.googleapis.com", "cloudkms.googleapis.com", "binaryauthorization.googleapis.com", - "cloudbuild.googleapis.com" + "cloudbuild.googleapis.com", + "run.googleapis.com", + "cloudscheduler.googleapis.com", + "eventarc.googleapis.com" ] } diff --git a/terraform/gcp/buckets.tf b/terraform/gcp/buckets.tf new file mode 100644 index 0000000000..b0edfee8c2 --- /dev/null +++ b/terraform/gcp/buckets.tf @@ -0,0 +1,51 @@ +resource "google_storage_bucket" "unscanned_participant_documents" { + name = "${var.documents_bucket_name}-unscanned" + location = var.region + # no public access allowed + public_access_prevention = "enforced" + + # only allow access if you have iam perms + uniform_bucket_level_access = true + versioning { + enabled = true + } +} + +resource "google_storage_bucket" "clean_participant_documents" { + name = "${var.documents_bucket_name}-clean" + location = var.region + # no public access allowed + public_access_prevention = "enforced" + + # only allow access if you have iam perms + uniform_bucket_level_access = true + versioning { + enabled = true + } +} + +resource "google_storage_bucket" "quarantined_participant_documents" { + name = "${var.documents_bucket_name}-quarantined" + location = var.region + # no public access allowed + public_access_prevention = "enforced" + + # only allow access if you have iam perms + uniform_bucket_level_access = true + versioning { + enabled = true + } +} + +resource "google_storage_bucket" "cvd_mirror_bucket" { + name = "${var.documents_bucket_name}-cvd-mirror" + location = var.region + # no public access allowed + public_access_prevention = "enforced" + + # only allow access if you have iam perms + uniform_bucket_level_access = true + versioning { + enabled = true + } +} diff --git a/terraform/gcp/cluster_service_account.tf b/terraform/gcp/cluster_service_account.tf index 6da9ea4221..21b3f56d5c 100644 --- a/terraform/gcp/cluster_service_account.tf +++ b/terraform/gcp/cluster_service_account.tf @@ -8,7 +8,8 @@ resource "google_project_iam_binding" "cluster-metric-writer" { project = var.project role = "roles/monitoring.metricWriter" members = [ - "serviceAccount:${google_service_account.cluster_service_account.email}" + "serviceAccount:${google_service_account.cluster_service_account.email}", + "serviceAccount:${google_service_account.malware_scanner_sa.email}" ] } @@ -25,7 +26,8 @@ resource "google_project_iam_binding" "cluster-log-writer" { role = "roles/logging.logWriter" members = [ "serviceAccount:${google_service_account.cluster_service_account.email}", - "serviceAccount:${google_service_account.juniper_cloudbuild_service_account.email}" + "serviceAccount:${google_service_account.juniper_cloudbuild_service_account.email}", + "serviceAccount:${google_service_account.build_service_account.email}" ] } diff --git a/terraform/gcp/envs/dev.tfvars b/terraform/gcp/envs/dev.tfvars index 1e1367b63f..a809949100 100644 --- a/terraform/gcp/envs/dev.tfvars +++ b/terraform/gcp/envs/dev.tfvars @@ -9,6 +9,9 @@ environment = "dev" portals = ["demo", "atcp", "ourhealth", "hearthive", "rgp", "cmi"] k8s_namespace = "juniper-dev" +malware_scanner_image_name = "juniper-malware-scanner" +documents_bucket_name = "juniper-participant-documents-dev" + # creates DNS records for these customer URLs customer_urls = { demo = { @@ -19,3 +22,4 @@ customer_urls = { } slack_notification_channel = "projects/broad-juniper-dev/notificationChannels/13069356383599666729" + diff --git a/terraform/gcp/envs/prod.tfvars b/terraform/gcp/envs/prod.tfvars index 6d79a44a8c..03a61be2d0 100644 --- a/terraform/gcp/envs/prod.tfvars +++ b/terraform/gcp/envs/prod.tfvars @@ -105,3 +105,6 @@ customer_urls = { } slack_notification_channel = "projects/broad-juniper-prod/notificationChannels/9072110396476167224" + +malware_scanner_image_name = "juniper-malware-scanner" +documents_bucket_name = "juniper-participant-documents-prod" diff --git a/terraform/gcp/k8s/environments/dev.yaml b/terraform/gcp/k8s/environments/dev.yaml index 1aae9228be..d477579b03 100644 --- a/terraform/gcp/k8s/environments/dev.yaml +++ b/terraform/gcp/k8s/environments/dev.yaml @@ -5,7 +5,6 @@ deploymentZone: dev replicas: 1 dsmUrl: https://dsm-dev.datadonationplatform.org/dsm dsmIssuer: admin-d2p.ddp-dev.envs.broadinstitute.org -gcsFileStorageBucketName: juniper-participant-documents-dev # "portals" adds certificates for each portal - both for the juniper-cmi.dev subdomains and the custom domain portals: - name: demo diff --git a/terraform/gcp/k8s/environments/prod.yaml b/terraform/gcp/k8s/environments/prod.yaml index 60510e5ce2..a233e3818d 100644 --- a/terraform/gcp/k8s/environments/prod.yaml +++ b/terraform/gcp/k8s/environments/prod.yaml @@ -5,7 +5,6 @@ deploymentZone: prod replicas: 3 dsmUrl: https://dsm.datadonationplatform.org/dsm dsmIssuer: juniper.terra.bio -gcsFileStorageBucketName: juniper-participant-documents-prod # "portals" adds certificates for each portal - both for the admin subdomains and the custom domain portals: - name: demo @@ -57,3 +56,7 @@ b2c: clientId: 882e287c-8586-456e-8004-b7ff16da3578 policyName: B2C_1A_ddp_participant_signup_signin_trcc-prod changePasswordPolicyName: B2C_1A_ddp_participant_signup_signin_trcc-prod +gcsFileStorageBuckets: + unscanned: juniper-participant-documents-prod-unscanned + clean: juniper-participant-documents-prod-clean + quarantined: juniper-participant-documents-prod-quarantined diff --git a/terraform/gcp/variables.tf b/terraform/gcp/variables.tf index 4a500c79e5..95ab87df29 100644 --- a/terraform/gcp/variables.tf +++ b/terraform/gcp/variables.tf @@ -93,3 +93,40 @@ variable "slack_notification_channel" { default = "" description = "Slack notification channel" } + +variable "documents_bucket_name" { + type = string + description = "The name of the GCP bucket for storing participant documents" +} + +variable "artifact_registry" { + type = string + default = "juniper" +} + +variable "artifact_registry_project" { + type = string + default = "broad-juniper-eng-infra" +} + +variable "artifact_registry_location" { + type = string + default = "us-central1" +} + + +# build image from https://github.com/GoogleCloudPlatform/docker-clamav-malware-scanner/tree/main/cloudrun-malware-scanner +# steps: +# cd cloudrun-malware-scanner +# docker build --tag=us-central1-docker.pkg.dev/broad-juniper-eng-infra/juniper/juniper-malware-scanner:latest -f Dockerfile . --platform linux/amd64 +# docker push us-central1-docker.pkg.dev/broad-juniper-eng-infra/juniper/juniper-malware-scanner:latest + +# if standing up for first time, you might also need to update the cvd mirror +# before deploying. run: +# pip3 install crcmod cvdupdate +# ./updateCvdMirror.sh +# from https://github.com/GoogleCloudPlatform/docker-clamav-malware-scanner + +variable "malware_scanner_image_name" { + type = string +} diff --git a/terraform/gcp/virus_scanning.tf b/terraform/gcp/virus_scanning.tf new file mode 100644 index 0000000000..e5f469448a --- /dev/null +++ b/terraform/gcp/virus_scanning.tf @@ -0,0 +1,191 @@ + +# adapted for our use from https://github.com/GoogleCloudPlatform/docker-clamav-malware-scanner/tree/main/terraform + + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +## Lookup the hash of the latest image +## +data "google_artifact_registry_docker_image" "scanner-service-image" { + location = var.artifact_registry_location + repository_id = var.artifact_registry + project = var.artifact_registry_project + image_name = "${var.malware_scanner_image_name}:latest" +} + +## Deploy the Cloud Run Service +# +resource "google_cloud_run_v2_service" "malware_scanner" { + name = "juniper-malware-scanner" + location = var.region + ingress = "INGRESS_TRAFFIC_INTERNAL_ONLY" + + template { + scaling { + max_instance_count = 5 + min_instance_count = 1 + } + service_account = google_service_account.malware_scanner_sa.email + timeout = "300s" + max_instance_request_concurrency = 20 + + containers { + image = data.google_artifact_registry_docker_image.scanner-service-image.self_link + resources { + limits = { + cpu = "1" + memory = "4Gi" + } + cpu_idle = false # CPU is still allocated outside of requests + startup_cpu_boost = true + } + env { + name = "CONFIG_JSON" + value = jsonencode({ + "buckets": [ + { + "unscanned": google_storage_bucket.unscanned_participant_documents.name, + "clean": google_storage_bucket.clean_participant_documents.name, + "quarantined": google_storage_bucket.quarantined_participant_documents.name, + } + ], + "ClamCvdMirrorBucket": google_storage_bucket.cvd_mirror_bucket.name, + "fileExclusionPatterns": [["\\\\.tmp$","i"]], + "ignoreZeroLengthFiles": true, + "quarantine": { + "encryptedFiles": true, + "fileExtensionAllowList": [], + "fileExtensionDenyList": [] + } + }) + } + + startup_probe { + # Allow 90 secs before we start probing + # Then allow up to 15*10 = 150s before we give up + # (total possible startup time = 240s) + initial_delay_seconds = 90 + failure_threshold = 15 + period_seconds = 10 + timeout_seconds = 1 + + http_get { + path = "/ready" + } + } + + liveness_probe { + # Poll every 30 secs, allowing for 3 failures + # + period_seconds = 30 + failure_threshold = 3 + timeout_seconds = 30 + + http_get { + path = "/ready" + } + } + } + } + traffic { + type = "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST" + percent = 100 + } + + depends_on = [ + google_storage_bucket.unscanned_participant_documents, + google_storage_bucket.clean_participant_documents, + google_storage_bucket.quarantined_participant_documents, + google_storage_bucket.cvd_mirror_bucket, + time_sleep.enable_all_services_with_timeout + ] +} + + +## Create EventArc Triggers on unscanned bucket(s) +# +resource "google_eventarc_trigger" "gcs-object-written" { + name = "gcs-trigger-${google_storage_bucket.unscanned_participant_documents.name}" + location = var.region + matching_criteria { + attribute = "type" + value = "google.cloud.storage.object.v1.finalized" + } + matching_criteria { + attribute = "bucket" + value = google_storage_bucket.unscanned_participant_documents.name + } + destination { + cloud_run_service { + service = google_cloud_run_v2_service.malware_scanner.name + region = google_cloud_run_v2_service.malware_scanner.location + } + } + service_account = google_service_account.malware_scanner_sa.email + + depends_on = [ + time_sleep.enable_all_services_with_timeout + ] +} + +## Update pubsub subscriptions to increase deadlines +# +resource "null_resource" "update-subscription-ack-deadline" { + provisioner "local-exec" { + command = "gcloud pubsub subscriptions update \"${google_eventarc_trigger.gcs-object-written.transport[0].pubsub[0].subscription}\" --ack-deadline=300" + } +} + +## Deploy scheduled task to refresh the CVD Mirror + +# To avoid having too many clients use the same time slot, +# ClamAV requires that updates are scheduled at a random minute between 3 +# and 57 avoiding multiples of 10. +resource "random_integer" "cvd_mirror_update_schedule_minutes" { + min = 3 + max = 57 +} + +locals { + # Avoid multiples of 10 by subtracting 3. + cvd_mirror_update_schedule_minutes = ( + random_integer.cvd_mirror_update_schedule_minutes.result % 10 == 0 + ? random_integer.cvd_mirror_update_schedule_minutes.result - 3 + : random_integer.cvd_mirror_update_schedule_minutes.result + ) +} + +resource "google_cloud_scheduler_job" "cvd_mirror_update" { + name = "juniper-malware-scanner-cvd-mirror-update" + schedule = "${local.cvd_mirror_update_schedule_minutes} */2 * * *" + attempt_deadline = "320s" + region = var.region + http_target { + http_method = "POST" + uri = google_cloud_run_v2_service.malware_scanner.uri + body = base64encode("{\"kind\":\"schedule#cvd_update\"}") + headers = { + "Content-Type" = "application/json" + } + oidc_token { + service_account_email = google_service_account.malware_scanner_sa.email + } + } + + depends_on = [ + time_sleep.enable_all_services_with_timeout + ] +} diff --git a/terraform/gcp/virus_scanning_infra.tf b/terraform/gcp/virus_scanning_infra.tf new file mode 100644 index 0000000000..948ca2f238 --- /dev/null +++ b/terraform/gcp/virus_scanning_infra.tf @@ -0,0 +1,87 @@ + +resource "google_service_account" "malware_scanner_sa" { + account_id = "juniper-malware-scanner" + project = var.project + display_name = "juniper-malware-scanner" +} + + + +resource "google_project_iam_member" "malware_scanner_iam" { + for_each = toset(["roles/run.invoker", "roles/eventarc.eventReceiver"]) + project = var.project + role = each.value + member = "serviceAccount:${google_service_account.malware_scanner_sa.email}" +} + +resource "google_service_account" "build_service_account" { + account_id = "juniper-malware-scanner-build" + display_name = "Service Account for malware scanner cloud run service" +} + +resource "google_project_iam_binding" "malware-scanner-object-viewer" { + project = var.project + role = "roles/storage.objectViewer" + members = ["serviceAccount:${google_service_account.build_service_account.email}"] +} + +## Allow GCS to publish to pubsub +# +data "google_storage_project_service_account" "gcs_account" { +} + +resource "google_project_iam_binding" "gcs_sa_pubsub_publish" { + project = var.project + role = "roles/pubsub.publisher" + members = ["serviceAccount:${data.google_storage_project_service_account.gcs_account.email_address}"] +} + +## Allow service account to admin the scanner buckets. +# +# They may not have been created by TF, so use a data resource +# to verify their existence. +# + + +resource "google_storage_bucket_iam_binding" "unscanned_buckets_sa_binding" { + bucket = google_storage_bucket.unscanned_participant_documents.name + role = "roles/storage.admin" + members = [ + "serviceAccount:${google_service_account.malware_scanner_sa.email}", + ] +} + +resource "google_storage_bucket_iam_binding" "clean_buckets_sa_binding" { + bucket = google_storage_bucket.clean_participant_documents.name + role = "roles/storage.admin" + members = [ + "serviceAccount:${google_service_account.malware_scanner_sa.email}", + ] +} + +resource "google_storage_bucket_iam_binding" "quarantined_buckets_sa_binding" { + bucket = google_storage_bucket.quarantined_participant_documents.name + role = "roles/storage.admin" + members = [ + "serviceAccount:${google_service_account.malware_scanner_sa.email}", + ] +} + +resource "google_storage_bucket_iam_binding" "cvd_buckets_sa_binding" { + bucket = google_storage_bucket.cvd_mirror_bucket.name + role = "roles/storage.admin" + members = [ + "serviceAccount:${google_service_account.malware_scanner_sa.email}", + ] +} + +## Create the CVD Mirror bucket and allow service account admin access. +# + +resource "google_storage_bucket_iam_binding" "cvd_mirror_bucket_sa_binding" { + bucket = google_storage_bucket.cvd_mirror_bucket.name + role = "roles/storage.admin" + members = [ + "serviceAccount:${google_service_account.malware_scanner_sa.email}", + ] +} diff --git a/terraform/infra/.terraform.lock.hcl b/terraform/infra/.terraform.lock.hcl index 5fb78ed475..1f2f8c7269 100644 --- a/terraform/infra/.terraform.lock.hcl +++ b/terraform/infra/.terraform.lock.hcl @@ -2,39 +2,58 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/google" { - version = "6.3.0" + version = "6.25.0" hashes = [ - "h1:SpVI9aF/7wyalm5GhRUTGzwUA3Jv4d/lEnu/59X4U5k=", - "zh:2d5099f33ceaaa74d6d6860b98287ffdc723a6fb3f0a48ae02e9ba11e387f8b0", - "zh:38d937bedaeb9429663eff661b3087e66d7b88b3516eb1dc895eed160283878d", - "zh:5a9a7da6e578fc8448defb87f07616f8745700bd0d17ed458909d2561c08353b", - "zh:60b047acf4e4b2140885608ac37914d10947505dd8afd2da7718af27ff0e7b64", - "zh:84d457c92224f59d6ac7d2cc43581c7fe7dd3500d47251798fed21c7fb816d64", - "zh:b12a32dac4816d5cae9aa2b491d7510160e5f157150e8f814bcc9414a68db1f0", - "zh:cd65223bf3523f3d217e4b2b8c795149be53cfc545055a34e1fbd94f252c52d9", - "zh:d55de710eab8d31da97bb6ff7e5c4fe8afee7bde84ba80ac778d74ced1b7cd74", - "zh:e817d9ec4a7ff3e096dfc1c2ea88eb6eb725ae8c827e88cb77af9905ed2e3aec", - "zh:ebaa5cdf7beb94dec01cc6a2df8b8086bedeaf40cf009ffd27b051ca3770ebc0", - "zh:f3b5177362fa3ce8835daadeea9dd10299d76db667208128d619eb0786348ba4", + "h1:rvsmJc5J5OMMFXj9th8Yqfwm7cLZdRM/stPo0B8Datg=", + "zh:115ce3e84a02412a9a42c7c8f1c4568ab470bdd93c9a3a1f9202d4b77d7bc236", + "zh:623a32acea92d98a8bb66481fa9489e0a1e4dd6fc57ab70dbb53fae5732c1c63", + "zh:6440fc959b5e316152e26c916d55566311dafbe5b64e45d4b9c9931a5b29fa13", + "zh:91edb056638e723b1c7802d0e6e293611208e2825f645572ddba98122723f11c", + "zh:9b57ee44172677d2c2df03b22cf4b40e184ac8bd8facdd456ccbfdb7fee4684a", + "zh:9e2a97dd09b78b36caecdd990fff80bdb47a6a6d45c8b2dd4d23885d6bc89e65", + "zh:b33e40e9ba745f15b1c2e9ebcdccbac185788a4eb57450820cec7b9585858522", + "zh:d9e051bb703597384d83d924c49770e3bcdc1b68583d3def33a607ab168c634b", + "zh:e84253140fc3b0bd5cf7a1ebb54f993bacc6d5ee33c7b5e714ad834d5b2d35d0", + "zh:ebb624504c6f4297e691b6e3c00f789a6729461e5aa80fc165ef2ecd878e2d87", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f8c3e8192e98eff09a9453b52424c0d9727ae284de4fafbc498f2d527e026850", ] } provider "registry.terraform.io/hashicorp/google-beta" { - version = "6.3.0" + version = "6.25.0" hashes = [ - "h1:n9gyWvYz/1O5ecRfB1cV9/OlYTlSILvMy9knO4lvdXY=", - "zh:14911d94a78ee2a5d459a1e246e3722e04c5774f7aa8aeb415b11da89cd4ada8", - "zh:1a937609af8a837bdd181dad37e285c271bcb73971cc355c8f103730d1640aa4", - "zh:2032fa8f207ce06b5acf1e190f1090129b9fb4bae1b312165e5235af6cd61455", - "zh:3803fef01fe0c01e53f5d3628d8c8129d6443305cef458dea29d488661bbc5c0", - "zh:4c4434af7e2eb84c3a99a758e405b7f7f84c311ae6ef911d061d2d01fd4c0e76", - "zh:655eba025e121524ed95031c68b85d72ae2128eb748d68350fa4dd6569ce6770", - "zh:6678f186c550c6bb7ace33af0dd17a8d2ee6c61bcef2ff1bd8f603e0f7a5ce11", - "zh:7fa7de3cc6bf48d80ce3b8a5f3b23563d1f693b6febf20dd7b4a50a098ab6ee1", - "zh:a33d0f8deda840bc3f0263559118f632ddbb0609335739ab7329b517f156bc63", - "zh:c79b8109b3d7e42c1d1b5ebce29319bf69f4360be6f03f4fb0578fdc7f628e3f", + "h1:KlIctHHQkbToSXa41DWFQUsTKsfaWe+vjs/CFbnj3bc=", + "zh:101a8a679274cae57e8c816164fa28b6d271a8a53b3e007b84153fc63f0f06f7", + "zh:413c41e4ee14b10cb815cd01f8b880c12474ecbf63ecfc6415512fb0c18c473b", + "zh:4260143c8584a8f338a4d45f65c2f08f12b28a746c43d43a1e3a575f96ce4110", + "zh:4785080dd28b79a94ba6e277d247b6bb678d676d99244a8d664ea60802338df7", + "zh:673b09d67cb59487095fe7176c194e8c7ced62fc7bb5084016f3c87dd89e685e", + "zh:7de0dd9367c723cc203fdc4e70d316c779224483637039de5a3c9ee807c3fc55", + "zh:aa49c5955652b21ff2d0b7b2db0e18f59097d3a539efc98ef65deed4293d7327", + "zh:b4d73704d88deee367568fd68303878e6e60924c2a65df397f37b868e6aecaac", + "zh:cae0c5eff7c37c3f9857de08ca35b3a86d44855ae9c0ec2fab1eda604ff501c8", + "zh:e115f6b0c25bdad624fafbb5d685c8a48d9028f9525918274a5e5469f7dfde3d", + "zh:e536197371458b0b177a203f8367f99e74770b3dfce597034e880ea5f3f798b8", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - "zh:f9002acf0ce39e1f3f2afc28112343c846a6249868ca567477cbbcc3a4a77517", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.7.1" + hashes = [ + "h1:t152MY0tQH4a8fLzTtEWx70ITd3azVOrFDn/pQblbto=", + "zh:3193b89b43bf5805493e290374cdda5132578de6535f8009547c8b5d7a351585", + "zh:3218320de4be943e5812ed3de995946056db86eb8d03aa3f074e0c7316599bef", + "zh:419861805a37fa443e7d63b69fb3279926ccf98a79d256c422d5d82f0f387d1d", + "zh:4df9bd9d839b8fc11a3b8098a604b9b46e2235eb65ef15f4432bde0e175f9ca6", + "zh:5814be3f9c9cc39d2955d6f083bae793050d75c572e70ca11ccceb5517ced6b1", + "zh:63c6548a06de1231c8ee5570e42ca09c4b3db336578ded39b938f2156f06dd2e", + "zh:697e434c6bdee0502cc3deb098263b8dcd63948e8a96d61722811628dce2eba1", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:a0b8e44927e6327852bbfdc9d408d802569367f1e22a95bcdd7181b1c3b07601", + "zh:b7d3af018683ef22794eea9c218bc72d7c35a2b3ede9233b69653b3c782ee436", + "zh:d63b911d618a6fe446c65bfc21e793a7663e934b2fef833d42d3ccd38dd8d68d", + "zh:fa985cd0b11e6d651f47cff3055f0a9fd085ec190b6dbe99bf5448174434cdea", ] } diff --git a/terraform/infra/artfiact_registry_access.tf b/terraform/infra/artfiact_registry_access.tf index ade08c1078..8d03bb3d3f 100644 --- a/terraform/infra/artfiact_registry_access.tf +++ b/terraform/infra/artfiact_registry_access.tf @@ -8,6 +8,8 @@ resource "google_artifact_registry_repository_iam_binding" "cluster-artifact-reg "serviceAccount:juniper-cluster@broad-juniper-dev.iam.gserviceaccount.com", "serviceAccount:juniper-cluster@broad-juniper-prod.iam.gserviceaccount.com", "serviceAccount:juniper-cloudbuild-sa@broad-juniper-dev.iam.gserviceaccount.com", - "serviceAccount:juniper-cloudbuild-sa@broad-juniper-prod.iam.gserviceaccount.com" + "serviceAccount:juniper-cloudbuild-sa@broad-juniper-prod.iam.gserviceaccount.com", + "serviceAccount:service-663573365422@serverless-robot-prod.iam.gserviceaccount.com", # dev cloud run service account + "serviceAccount:service-849235144342@serverless-robot-prod.iam.gserviceaccount.com", # prod cloud run service account ] }