From 3196ce64ae143473d48b97adaedc9e6fd56cee7f Mon Sep 17 00:00:00 2001 From: Connor Barker Date: Mon, 17 Mar 2025 13:59:19 -0400 Subject: [PATCH 1/8] working in dev; try moving to infra --- terraform/gcp/.terraform.lock.hcl | 124 +++++++------ terraform/gcp/apis.tf | 5 +- terraform/gcp/buckets.tf | 51 ++++++ terraform/gcp/cluster_service_account.tf | 6 +- terraform/gcp/envs/dev.tfvars | 6 + terraform/gcp/variables.tf | 37 ++++ terraform/gcp/virus_scanning.tf | 191 ++++++++++++++++++++ terraform/gcp/virus_scanning_infra.tf | 88 +++++++++ terraform/infra/artfiact_registry_access.tf | 3 +- 9 files changed, 455 insertions(+), 56 deletions(-) create mode 100644 terraform/gcp/buckets.tf create mode 100644 terraform/gcp/virus_scanning.tf create mode 100644 terraform/gcp/virus_scanning_infra.tf diff --git a/terraform/gcp/.terraform.lock.hcl b/terraform/gcp/.terraform.lock.hcl index c5936f67f3..5bb08c6646 100644 --- a/terraform/gcp/.terraform.lock.hcl +++ b/terraform/gcp/.terraform.lock.hcl @@ -2,78 +2,98 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/google" { - version = "6.13.0" + version = "6.25.0" constraints = ">= 4.51.0, < 7.0.0" hashes = [ - "h1:jakE2O6gqqQHIZYnr4K7zCNqlMDO/VMgVbCPbbLT/Oc=", - "zh:05067a572a2fcec7ab2e613611db68ff9fdaac18869f76ca53ce308f8f4b904a", - "zh:1122fae676ba52d4842267ce8a9e391c262d2468c095b6ad7b0550c1bf292b05", - "zh:1cde30b75d4e5ecd8eabcf2f9119302611a2b757f9cd50474c21f30dba77dd60", - "zh:444d333a8af09d4f561a09c56eee48cd23e97fca94c427ae5e4a24745418a097", - "zh:471303b121adf8b9225de70127f62f3b96033a9ab6e14f9dc7fe19f6d224186d", - "zh:4caa73ea78c85dc4abde022d114d4d2ff770636def8f308472e3679ac5195d9a", - "zh:6257a064015ed1c277cfb39027b91de2b7bc6f1ce6d42058b9df3d5aa37ecb4f", - "zh:8a45fe7f3e4b10ba6e8c5e44fecd0c7389e6f9c8cca11d052a357fbaae615c1a", - "zh:8ce6902473efdf92724679c2b63905263d83dcf77bd8d9e9f45701cdc9dcc229", - "zh:ce8951ef5ce96d694f7fbcea6d7dd593f6102b3ac1c686e03f08316dd7923d2d", - "zh:ef009b465067b8d3646d0c6ca8a6ba860d4591ddc427f005cfa5d4fcac0c4fce", + "h1:rvsmJc5J5OMMFXj9th8Yqfwm7cLZdRM/stPo0B8Datg=", + "zh:115ce3e84a02412a9a42c7c8f1c4568ab470bdd93c9a3a1f9202d4b77d7bc236", + "zh:623a32acea92d98a8bb66481fa9489e0a1e4dd6fc57ab70dbb53fae5732c1c63", + "zh:6440fc959b5e316152e26c916d55566311dafbe5b64e45d4b9c9931a5b29fa13", + "zh:91edb056638e723b1c7802d0e6e293611208e2825f645572ddba98122723f11c", + "zh:9b57ee44172677d2c2df03b22cf4b40e184ac8bd8facdd456ccbfdb7fee4684a", + "zh:9e2a97dd09b78b36caecdd990fff80bdb47a6a6d45c8b2dd4d23885d6bc89e65", + "zh:b33e40e9ba745f15b1c2e9ebcdccbac185788a4eb57450820cec7b9585858522", + "zh:d9e051bb703597384d83d924c49770e3bcdc1b68583d3def33a607ab168c634b", + "zh:e84253140fc3b0bd5cf7a1ebb54f993bacc6d5ee33c7b5e714ad834d5b2d35d0", + "zh:ebb624504c6f4297e691b6e3c00f789a6729461e5aa80fc165ef2ecd878e2d87", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f8c3e8192e98eff09a9453b52424c0d9727ae284de4fafbc498f2d527e026850", ] } provider "registry.terraform.io/hashicorp/google-beta" { - version = "6.13.0" + version = "6.25.0" hashes = [ - "h1:ydG2kjp6wP4/pLAFn9L9qz9HAPZd0z0SwhiWcXc3d1Q=", - "zh:02f3366f1063ae89b5618bbabab138ac0a2ca1a75f6708fe28d8d23a3e2db98e", - "zh:0c160f494c7eb629954940e4b26e62d37cbc55faf7dd761f80384434a720706b", - "zh:23076c21aca55b4f22f5e253438f9302f90a64e78a416629f95e07890376637b", - "zh:27561ee2350fc747f93d08ff63127223b8a2137c325bc7672637af41067e8b52", - "zh:3e354d40dbd3d0c012d4f5ba1f30aaf9d6287e1c2eff6d88344b5c0d4078effa", - "zh:6a795cc3b358f70f2628785b2cda2b6ea272d8ff804231119a45f88f227c5cc7", - "zh:789b5e3d51387d498d0cbf626c3cb77d70efbaa2acfaed415b4787eba8287b8d", - "zh:8ff948cedfe7e6864d1a5b974e5e16ce00959383f0a7c10543b27f8cf4268010", - "zh:bc92280a45ed4e198684e377bbc36bcd21d0af4c8e4922302f74c42ff1204c20", - "zh:d3243028ee7ca2e2d747e95a212ab3985ecb8719ff7481747b8b45417e12107f", - "zh:e83f1baddc6fc0c6ea7755dc9c219a8e7664b0a1132ed3b7ace1179625a7f8b0", + "h1:KlIctHHQkbToSXa41DWFQUsTKsfaWe+vjs/CFbnj3bc=", + "zh:101a8a679274cae57e8c816164fa28b6d271a8a53b3e007b84153fc63f0f06f7", + "zh:413c41e4ee14b10cb815cd01f8b880c12474ecbf63ecfc6415512fb0c18c473b", + "zh:4260143c8584a8f338a4d45f65c2f08f12b28a746c43d43a1e3a575f96ce4110", + "zh:4785080dd28b79a94ba6e277d247b6bb678d676d99244a8d664ea60802338df7", + "zh:673b09d67cb59487095fe7176c194e8c7ced62fc7bb5084016f3c87dd89e685e", + "zh:7de0dd9367c723cc203fdc4e70d316c779224483637039de5a3c9ee807c3fc55", + "zh:aa49c5955652b21ff2d0b7b2db0e18f59097d3a539efc98ef65deed4293d7327", + "zh:b4d73704d88deee367568fd68303878e6e60924c2a65df397f37b868e6aecaac", + "zh:cae0c5eff7c37c3f9857de08ca35b3a86d44855ae9c0ec2fab1eda604ff501c8", + "zh:e115f6b0c25bdad624fafbb5d685c8a48d9028f9525918274a5e5469f7dfde3d", + "zh:e536197371458b0b177a203f8367f99e74770b3dfce597034e880ea5f3f798b8", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", ] } +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.3" + hashes = [ + "h1:I0Um8UkrMUb81Fxq/dxbr3HLP2cecTH2WMJiwKSrwQY=", + "zh:22d062e5278d872fe7aed834f5577ba0a5afe34a3bdac2b81f828d8d3e6706d2", + "zh:23dead00493ad863729495dc212fd6c29b8293e707b055ce5ba21ee453ce552d", + "zh:28299accf21763ca1ca144d8f660688d7c2ad0b105b7202554ca60b02a3856d3", + "zh:55c9e8a9ac25a7652df8c51a8a9a422bd67d784061b1de2dc9fe6c3cb4e77f2f", + "zh:756586535d11698a216291c06b9ed8a5cc6a4ec43eee1ee09ecd5c6a9e297ac1", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:9d5eea62fdb587eeb96a8c4d782459f4e6b73baeece4d04b4a40e44faaee9301", + "zh:a6355f596a3fb8fc85c2fb054ab14e722991533f87f928e7169a486462c74670", + "zh:b5a65a789cff4ada58a5baffc76cb9767dc26ec6b45c00d2ec8b1b027f6db4ed", + "zh:db5ab669cf11d0e9f81dc380a6fdfcac437aea3d69109c7aef1a5426639d2d65", + "zh:de655d251c470197bcbb5ac45d289595295acb8f829f6c781d4a75c8c8b7c7dd", + "zh:f5c68199f2e6076bce92a12230434782bf768103a427e9bb9abee99b116af7b5", + ] +} + provider "registry.terraform.io/hashicorp/random" { - version = "3.6.3" + version = "3.7.1" + constraints = "~> 3.0" hashes = [ - "h1:zG9uFP8l9u+yGZZvi5Te7PV62j50azpgwPunq2vTm1E=", - "zh:04ceb65210251339f07cd4611885d242cd4d0c7306e86dda9785396807c00451", - "zh:448f56199f3e99ff75d5c0afacae867ee795e4dfda6cb5f8e3b2a72ec3583dd8", - "zh:4b4c11ccfba7319e901df2dac836b1ae8f12185e37249e8d870ee10bb87a13fe", - "zh:4fa45c44c0de582c2edb8a2e054f55124520c16a39b2dfc0355929063b6395b1", - "zh:588508280501a06259e023b0695f6a18149a3816d259655c424d068982cbdd36", - "zh:737c4d99a87d2a4d1ac0a54a73d2cb62974ccb2edbd234f333abd079a32ebc9e", + "h1:t152MY0tQH4a8fLzTtEWx70ITd3azVOrFDn/pQblbto=", + "zh:3193b89b43bf5805493e290374cdda5132578de6535f8009547c8b5d7a351585", + "zh:3218320de4be943e5812ed3de995946056db86eb8d03aa3f074e0c7316599bef", + "zh:419861805a37fa443e7d63b69fb3279926ccf98a79d256c422d5d82f0f387d1d", + "zh:4df9bd9d839b8fc11a3b8098a604b9b46e2235eb65ef15f4432bde0e175f9ca6", + "zh:5814be3f9c9cc39d2955d6f083bae793050d75c572e70ca11ccceb5517ced6b1", + "zh:63c6548a06de1231c8ee5570e42ca09c4b3db336578ded39b938f2156f06dd2e", + "zh:697e434c6bdee0502cc3deb098263b8dcd63948e8a96d61722811628dce2eba1", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:a357ab512e5ebc6d1fda1382503109766e21bbfdfaa9ccda43d313c122069b30", - "zh:c51bfb15e7d52cc1a2eaec2a903ac2aff15d162c172b1b4c17675190e8147615", - "zh:e0951ee6fa9df90433728b96381fb867e3db98f66f735e0c3e24f8f16903f0ad", - "zh:e3cdcb4e73740621dabd82ee6a37d6cfce7fee2a03d8074df65086760f5cf556", - "zh:eff58323099f1bd9a0bec7cb04f717e7f1b2774c7d612bf7581797e1622613a0", + "zh:a0b8e44927e6327852bbfdc9d408d802569367f1e22a95bcdd7181b1c3b07601", + "zh:b7d3af018683ef22794eea9c218bc72d7c35a2b3ede9233b69653b3c782ee436", + "zh:d63b911d618a6fe446c65bfc21e793a7663e934b2fef833d42d3ccd38dd8d68d", + "zh:fa985cd0b11e6d651f47cff3055f0a9fd085ec190b6dbe99bf5448174434cdea", ] } provider "registry.terraform.io/hashicorp/time" { - version = "0.12.1" + version = "0.13.0" hashes = [ - "h1:JzYsPugN8Fb7C4NlfLoFu7BBPuRVT2/fCOdCaxshveI=", - "zh:090023137df8effe8804e81c65f636dadf8f9d35b79c3afff282d39367ba44b2", - "zh:26f1e458358ba55f6558613f1427dcfa6ae2be5119b722d0b3adb27cd001efea", - "zh:272ccc73a03384b72b964918c7afeb22c2e6be22460d92b150aaf28f29a7d511", - "zh:438b8c74f5ed62fe921bd1078abe628a6675e44912933100ea4fa26863e340e9", + "h1:iwR4JouIoeVPDabb8XCqsiaZlZ28IcB3tDD9MuPeSXE=", + "zh:3776dd78ef3053562ccb2f8916d5d3f21a28f05e78859f0f1e4510525f891ecb", + "zh:541ca0b56f808c15d208b9396f149563b133223c4b66cdefbcfe2d8f1c23497e", + "zh:67ed315f3572eb20ce6778423b14fbb6faba3090f454bc20ec4146489b4738c0", + "zh:69dc375845bcfc451426480119f2941ee28b9ef01273d228bb66918180863b3a", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:85c8bd8eefc4afc33445de2ee7fbf33a7807bc34eb3734b8eefa4e98e4cddf38", - "zh:98bbe309c9ff5b2352de6a047e0ec6c7e3764b4ed3dfd370839c4be2fbfff869", - "zh:9c7bf8c56da1b124e0e2f3210a1915e778bab2be924481af684695b52672891e", - "zh:d2200f7f6ab8ecb8373cda796b864ad4867f5c255cff9d3b032f666e4c78f625", - "zh:d8c7926feaddfdc08d5ebb41b03445166df8c125417b28d64712dccd9feef136", - "zh:e2412a192fc340c61b373d6c20c9d805d7d3dee6c720c34db23c2a8ff0abd71b", - "zh:e6ac6bba391afe728a099df344dbd6481425b06d61697522017b8f7a59957d44", + "zh:93c24b7c87b5db9721f60782ac784152599aa78b30fdea2fc9c594d46d92767c", + "zh:95441cf14312041ae0b34640ff33975c09540125b01f9131358fca50e7be239d", + "zh:a294103aeed868c58987e131357a3ec259316c937c909e8a726b862d5a227b82", + "zh:adf6ded3f2e2f318e8aebf1040bc2791b448d006af7d12f7ddc3e8d40b22047a", + "zh:b2d9c16b7acd20d3813060c4d3647dc5f40598ebbdf59f642d53d189e4e3870a", + "zh:bc76a5161e9bcf74cadd76b3d4a51de508aa0c62e7f7ae536a87cd7595d81ebf", + "zh:ce6df2c1052c60b4432cb5c0ead471d7cdb4b285b807c265328a358631fc3610", ] } diff --git a/terraform/gcp/apis.tf b/terraform/gcp/apis.tf index 2436322019..c285f15048 100644 --- a/terraform/gcp/apis.tf +++ b/terraform/gcp/apis.tf @@ -13,7 +13,10 @@ locals { "servicenetworking.googleapis.com", "cloudkms.googleapis.com", "binaryauthorization.googleapis.com", - "cloudbuild.googleapis.com" + "cloudbuild.googleapis.com", + "run.googleapis.com", + "cloudscheduler.googleapis.com", + "eventarc.googleapis.com" ] } diff --git a/terraform/gcp/buckets.tf b/terraform/gcp/buckets.tf new file mode 100644 index 0000000000..b0edfee8c2 --- /dev/null +++ b/terraform/gcp/buckets.tf @@ -0,0 +1,51 @@ +resource "google_storage_bucket" "unscanned_participant_documents" { + name = "${var.documents_bucket_name}-unscanned" + location = var.region + # no public access allowed + public_access_prevention = "enforced" + + # only allow access if you have iam perms + uniform_bucket_level_access = true + versioning { + enabled = true + } +} + +resource "google_storage_bucket" "clean_participant_documents" { + name = "${var.documents_bucket_name}-clean" + location = var.region + # no public access allowed + public_access_prevention = "enforced" + + # only allow access if you have iam perms + uniform_bucket_level_access = true + versioning { + enabled = true + } +} + +resource "google_storage_bucket" "quarantined_participant_documents" { + name = "${var.documents_bucket_name}-quarantined" + location = var.region + # no public access allowed + public_access_prevention = "enforced" + + # only allow access if you have iam perms + uniform_bucket_level_access = true + versioning { + enabled = true + } +} + +resource "google_storage_bucket" "cvd_mirror_bucket" { + name = "${var.documents_bucket_name}-cvd-mirror" + location = var.region + # no public access allowed + public_access_prevention = "enforced" + + # only allow access if you have iam perms + uniform_bucket_level_access = true + versioning { + enabled = true + } +} diff --git a/terraform/gcp/cluster_service_account.tf b/terraform/gcp/cluster_service_account.tf index 6da9ea4221..21b3f56d5c 100644 --- a/terraform/gcp/cluster_service_account.tf +++ b/terraform/gcp/cluster_service_account.tf @@ -8,7 +8,8 @@ resource "google_project_iam_binding" "cluster-metric-writer" { project = var.project role = "roles/monitoring.metricWriter" members = [ - "serviceAccount:${google_service_account.cluster_service_account.email}" + "serviceAccount:${google_service_account.cluster_service_account.email}", + "serviceAccount:${google_service_account.malware_scanner_sa.email}" ] } @@ -25,7 +26,8 @@ resource "google_project_iam_binding" "cluster-log-writer" { role = "roles/logging.logWriter" members = [ "serviceAccount:${google_service_account.cluster_service_account.email}", - "serviceAccount:${google_service_account.juniper_cloudbuild_service_account.email}" + "serviceAccount:${google_service_account.juniper_cloudbuild_service_account.email}", + "serviceAccount:${google_service_account.build_service_account.email}" ] } diff --git a/terraform/gcp/envs/dev.tfvars b/terraform/gcp/envs/dev.tfvars index 1e1367b63f..3a986523d9 100644 --- a/terraform/gcp/envs/dev.tfvars +++ b/terraform/gcp/envs/dev.tfvars @@ -9,6 +9,12 @@ environment = "dev" portals = ["demo", "atcp", "ourhealth", "hearthive", "rgp", "cmi"] k8s_namespace = "juniper-dev" +artifact_registry = "juniper" +artifact_registry_project = "broad-juniper-eng-infra" +artifact_registry_location = "us-central1" +malware_scanner_image_name = "juniper-malware-scanner" +deploy_virus_scanning = true + # creates DNS records for these customer URLs customer_urls = { demo = { diff --git a/terraform/gcp/variables.tf b/terraform/gcp/variables.tf index 4a500c79e5..95ab87df29 100644 --- a/terraform/gcp/variables.tf +++ b/terraform/gcp/variables.tf @@ -93,3 +93,40 @@ variable "slack_notification_channel" { default = "" description = "Slack notification channel" } + +variable "documents_bucket_name" { + type = string + description = "The name of the GCP bucket for storing participant documents" +} + +variable "artifact_registry" { + type = string + default = "juniper" +} + +variable "artifact_registry_project" { + type = string + default = "broad-juniper-eng-infra" +} + +variable "artifact_registry_location" { + type = string + default = "us-central1" +} + + +# build image from https://github.com/GoogleCloudPlatform/docker-clamav-malware-scanner/tree/main/cloudrun-malware-scanner +# steps: +# cd cloudrun-malware-scanner +# docker build --tag=us-central1-docker.pkg.dev/broad-juniper-eng-infra/juniper/juniper-malware-scanner:latest -f Dockerfile . --platform linux/amd64 +# docker push us-central1-docker.pkg.dev/broad-juniper-eng-infra/juniper/juniper-malware-scanner:latest + +# if standing up for first time, you might also need to update the cvd mirror +# before deploying. run: +# pip3 install crcmod cvdupdate +# ./updateCvdMirror.sh +# from https://github.com/GoogleCloudPlatform/docker-clamav-malware-scanner + +variable "malware_scanner_image_name" { + type = string +} diff --git a/terraform/gcp/virus_scanning.tf b/terraform/gcp/virus_scanning.tf new file mode 100644 index 0000000000..c9c314234f --- /dev/null +++ b/terraform/gcp/virus_scanning.tf @@ -0,0 +1,191 @@ + +# adapted for our use from https://github.com/GoogleCloudPlatform/docker-clamav-malware-scanner/tree/main/terraform + + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +## Lookup the hash of the latest image +## +data "google_artifact_registry_docker_image" "scanner-service-image" { + location = var.artifact_registry_location + repository_id = var.artifact_registry + project = var.artifact_registry_project + image_name = "${var.malware_scanner_image_name}:latest" +} + +## Deploy the Cloud Run Service +# +resource "google_cloud_run_v2_service" "malware_scanner" { + name = "juniper-malware-scanner" + location = var.region + ingress = "INGRESS_TRAFFIC_ALL" + + template { + scaling { + max_instance_count = 5 + min_instance_count = 1 + } + service_account = google_service_account.malware_scanner_sa.email + timeout = "300s" + max_instance_request_concurrency = 20 + + containers { + image = data.google_artifact_registry_docker_image.scanner-service-image.self_link + resources { + limits = { + cpu = "1" + memory = "4Gi" + } + cpu_idle = false # CPU is still allocated outside of requests + startup_cpu_boost = true + } + env { + name = "CONFIG_JSON" + value = jsonencode({ + "buckets": [ + { + "unscanned": google_storage_bucket.unscanned_participant_documents.name, + "clean": google_storage_bucket.clean_participant_documents.name, + "quarantined": google_storage_bucket.quarantined_participant_documents.name, + } + ], + "ClamCvdMirrorBucket": google_storage_bucket.cvd_mirror_bucket.name, + "fileExclusionPatterns": [["\\\\.tmp$","i"]], + "ignoreZeroLengthFiles": true, + "quarantine": { + "encryptedFiles": true, + "fileExtensionAllowList": [], + "fileExtensionDenyList": [] + } + }) + } + + startup_probe { + # Allow 90 secs before we start probing + # Then allow up to 15*10 = 150s before we give up + # (total possible startup time = 240s) + initial_delay_seconds = 90 + failure_threshold = 15 + period_seconds = 10 + timeout_seconds = 1 + + http_get { + path = "/ready" + } + } + + liveness_probe { + # Poll every 30 secs, allowing for 3 failures + # + period_seconds = 30 + failure_threshold = 3 + timeout_seconds = 30 + + http_get { + path = "/ready" + } + } + } + } + traffic { + type = "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST" + percent = 100 + } + + depends_on = [ + google_storage_bucket.unscanned_participant_documents, + google_storage_bucket.clean_participant_documents, + google_storage_bucket.quarantined_participant_documents, + google_storage_bucket.cvd_mirror_bucket, + time_sleep.enable_all_services_with_timeout + ] +} + + +## Create EventArc Triggers on unscanned bucket(s) +# +resource "google_eventarc_trigger" "gcs-object-written" { + name = "gcs-trigger-${google_storage_bucket.unscanned_participant_documents.name}" + location = var.region + matching_criteria { + attribute = "type" + value = "google.cloud.storage.object.v1.finalized" + } + matching_criteria { + attribute = "bucket" + value = google_storage_bucket.unscanned_participant_documents.name + } + destination { + cloud_run_service { + service = google_cloud_run_v2_service.malware_scanner.name + region = google_cloud_run_v2_service.malware_scanner.location + } + } + service_account = google_service_account.malware_scanner_sa.email + + depends_on = [ + time_sleep.enable_all_services_with_timeout + ] +} + +## Update pubsub subscriptions to increase deadlines +# +resource "null_resource" "update-subscription-ack-deadline" { + provisioner "local-exec" { + command = "gcloud pubsub subscriptions update \"${google_eventarc_trigger.gcs-object-written.transport[0].pubsub[0].subscription}\" --ack-deadline=300" + } +} + +## Deploy scheduled task to refresh the CVD Mirror + +# To avoid having too many clients use the same time slot, +# ClamAV requires that updates are scheduled at a random minute between 3 +# and 57 avoiding multiples of 10. +resource "random_integer" "cvd_mirror_update_schedule_minutes" { + min = 3 + max = 57 +} + +locals { + # Avoid multiples of 10 by subtracting 3. + cvd_mirror_update_schedule_minutes = ( + random_integer.cvd_mirror_update_schedule_minutes.result % 10 == 0 + ? random_integer.cvd_mirror_update_schedule_minutes.result - 3 + : random_integer.cvd_mirror_update_schedule_minutes.result + ) +} + +resource "google_cloud_scheduler_job" "cvd_mirror_update" { + name = "juniper-malware-scanner-cvd-mirror-update" + schedule = "${local.cvd_mirror_update_schedule_minutes} */2 * * *" + attempt_deadline = "320s" + region = var.region + http_target { + http_method = "POST" + uri = google_cloud_run_v2_service.malware_scanner.uri + body = base64encode("{\"kind\":\"schedule#cvd_update\"}") + headers = { + "Content-Type" = "application/json" + } + oidc_token { + service_account_email = google_service_account.malware_scanner_sa.email + } + } + + depends_on = [ + time_sleep.enable_all_services_with_timeout + ] +} diff --git a/terraform/gcp/virus_scanning_infra.tf b/terraform/gcp/virus_scanning_infra.tf new file mode 100644 index 0000000000..81aee6a955 --- /dev/null +++ b/terraform/gcp/virus_scanning_infra.tf @@ -0,0 +1,88 @@ + +resource "google_service_account" "malware_scanner_sa" { + account_id = "juniper-malware-scanner" + project = var.project + display_name = "juniper-malware-scanner" +} + + + +resource "google_project_iam_member" "malware_scanner_iam" { + for_each = toset(["roles/run.invoker", "roles/eventarc.eventReceiver"]) + project = var.project + role = each.value + member = "serviceAccount:${google_service_account.malware_scanner_sa.email}" +} + +resource "google_service_account" "build_service_account" { + account_id = "juniper-malware-scanner-build" + display_name = "Service Account for malware scanner cloud run service" +} + +resource "google_project_iam_binding" "build_iam" { + for_each = toset(["roles/storage.objectViewer", "roles/artifactregistry.writer"]) + project = var.project + role = each.value + members = ["serviceAccount:${google_service_account.build_service_account.email}"] +} + +## Allow GCS to publish to pubsub +# +data "google_storage_project_service_account" "gcs_account" { +} + +resource "google_project_iam_binding" "gcs_sa_pubsub_publish" { + project = var.project + role = "roles/pubsub.publisher" + members = ["serviceAccount:${data.google_storage_project_service_account.gcs_account.email_address}"] +} + +## Allow service account to admin the scanner buckets. +# +# They may not have been created by TF, so use a data resource +# to verify their existence. +# + + +resource "google_storage_bucket_iam_binding" "unscanned_buckets_sa_binding" { + bucket = google_storage_bucket.unscanned_participant_documents.name + role = "roles/storage.admin" + members = [ + "serviceAccount:${google_service_account.malware_scanner_sa.email}", + ] +} + +resource "google_storage_bucket_iam_binding" "clean_buckets_sa_binding" { + bucket = google_storage_bucket.clean_participant_documents.name + role = "roles/storage.admin" + members = [ + "serviceAccount:${google_service_account.malware_scanner_sa.email}", + ] +} + +resource "google_storage_bucket_iam_binding" "quarantined_buckets_sa_binding" { + bucket = google_storage_bucket.quarantined_participant_documents.name + role = "roles/storage.admin" + members = [ + "serviceAccount:${google_service_account.malware_scanner_sa.email}", + ] +} + +resource "google_storage_bucket_iam_binding" "cvd_buckets_sa_binding" { + bucket = google_storage_bucket.cvd_mirror_bucket.name + role = "roles/storage.admin" + members = [ + "serviceAccount:${google_service_account.malware_scanner_sa.email}", + ] +} + +## Create the CVD Mirror bucket and allow service account admin access. +# + +resource "google_storage_bucket_iam_binding" "cvd_mirror_bucket_sa_binding" { + bucket = google_storage_bucket.cvd_mirror_bucket.name + role = "roles/storage.admin" + members = [ + "serviceAccount:${google_service_account.malware_scanner_sa.email}", + ] +} diff --git a/terraform/infra/artfiact_registry_access.tf b/terraform/infra/artfiact_registry_access.tf index ade08c1078..d49a2734ca 100644 --- a/terraform/infra/artfiact_registry_access.tf +++ b/terraform/infra/artfiact_registry_access.tf @@ -8,6 +8,7 @@ resource "google_artifact_registry_repository_iam_binding" "cluster-artifact-reg "serviceAccount:juniper-cluster@broad-juniper-dev.iam.gserviceaccount.com", "serviceAccount:juniper-cluster@broad-juniper-prod.iam.gserviceaccount.com", "serviceAccount:juniper-cloudbuild-sa@broad-juniper-dev.iam.gserviceaccount.com", - "serviceAccount:juniper-cloudbuild-sa@broad-juniper-prod.iam.gserviceaccount.com" + "serviceAccount:juniper-cloudbuild-sa@broad-juniper-prod.iam.gserviceaccount.com", + "serviceAccount:service-663573365422@serverless-robot-prod.iam.gserviceaccount.com", # cloud run service account ] } From ae9a5cf291bcc69aab31bd96593765b5eea4640b Mon Sep 17 00:00:00 2001 From: Connor Barker Date: Mon, 17 Mar 2025 14:08:18 -0400 Subject: [PATCH 2/8] oops --- terraform/gcp/virus_scanning_pubsub.tf | 0 terraform/infra/variables.tf | 12 +++++++++ terraform/{gcp => infra}/virus_scanning.tf | 11 +------- terraform/infra/virus_scanning_buckets.tf | 27 +++++++++++++++++++ .../{gcp => infra}/virus_scanning_infra.tf | 0 5 files changed, 40 insertions(+), 10 deletions(-) create mode 100644 terraform/gcp/virus_scanning_pubsub.tf rename terraform/{gcp => infra}/virus_scanning.tf (93%) create mode 100644 terraform/infra/virus_scanning_buckets.tf rename terraform/{gcp => infra}/virus_scanning_infra.tf (100%) diff --git a/terraform/gcp/virus_scanning_pubsub.tf b/terraform/gcp/virus_scanning_pubsub.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/terraform/infra/variables.tf b/terraform/infra/variables.tf index 249a52bf74..506732fc8d 100644 --- a/terraform/infra/variables.tf +++ b/terraform/infra/variables.tf @@ -7,3 +7,15 @@ variable "region" { description = "The GCP region" type = string } + +variable "virus_scanning_buckets" { + type = set(object({ + unscanned: string, + clean: string, + quarantine: string + })) +} + +variable "virus_scanning_cvd_bucket_name" { + type = string +} diff --git a/terraform/gcp/virus_scanning.tf b/terraform/infra/virus_scanning.tf similarity index 93% rename from terraform/gcp/virus_scanning.tf rename to terraform/infra/virus_scanning.tf index c9c314234f..120c95fc49 100644 --- a/terraform/gcp/virus_scanning.tf +++ b/terraform/infra/virus_scanning.tf @@ -55,13 +55,7 @@ resource "google_cloud_run_v2_service" "malware_scanner" { env { name = "CONFIG_JSON" value = jsonencode({ - "buckets": [ - { - "unscanned": google_storage_bucket.unscanned_participant_documents.name, - "clean": google_storage_bucket.clean_participant_documents.name, - "quarantined": google_storage_bucket.quarantined_participant_documents.name, - } - ], + "buckets": var.virus_scanning_buckets, "ClamCvdMirrorBucket": google_storage_bucket.cvd_mirror_bucket.name, "fileExclusionPatterns": [["\\\\.tmp$","i"]], "ignoreZeroLengthFiles": true, @@ -185,7 +179,4 @@ resource "google_cloud_scheduler_job" "cvd_mirror_update" { } } - depends_on = [ - time_sleep.enable_all_services_with_timeout - ] } diff --git a/terraform/infra/virus_scanning_buckets.tf b/terraform/infra/virus_scanning_buckets.tf new file mode 100644 index 0000000000..27a3edc400 --- /dev/null +++ b/terraform/infra/virus_scanning_buckets.tf @@ -0,0 +1,27 @@ +resource "google_storage_bucket" "virus_cvd_bucket"{ + location = var.region + name = var.virus_scanning_cvd_bucket_name + + uniform_bucket_level_access = true + public_access_prevention = true +} + + +data "google_storage_bucket" "unscanned_buckets" { + for_each = var.virus_scanning_buckets + + name = each.value.unscanned +} + +data "google_storage_bucket" "clean_buckets" { + for_each = var.virus_scanning_buckets + + name = each.value.clean +} + +data "google_storage_bucket" "quarantined_buckets" { + for_each = var.virus_scanning_buckets + + name = each.value.quarantine +} + diff --git a/terraform/gcp/virus_scanning_infra.tf b/terraform/infra/virus_scanning_infra.tf similarity index 100% rename from terraform/gcp/virus_scanning_infra.tf rename to terraform/infra/virus_scanning_infra.tf From ea748891da2ea33ce6239ca9db6852040184d78d Mon Sep 17 00:00:00 2001 From: Connor Barker Date: Mon, 17 Mar 2025 15:44:52 -0400 Subject: [PATCH 3/8] it doesnt work :( --- terraform/gcp/apis.tf | 3 - terraform/gcp/buckets.tf | 33 ++++++--- terraform/gcp/cluster_service_account.tf | 6 +- terraform/gcp/envs/dev.tfvars | 8 +- terraform/gcp/variables.tf | 30 +------- terraform/gcp/virus_scanning_pubsub.tf | 0 terraform/infra/.terraform.lock.hcl | 90 ++++++++++++++++------- terraform/infra/apis.tf | 16 ++++ terraform/infra/infra.tfvars | 17 +++++ terraform/infra/variables.tf | 22 +++++- terraform/infra/virus_scanning.tf | 27 +++---- terraform/infra/virus_scanning_buckets.tf | 28 +++++-- terraform/infra/virus_scanning_infra.tf | 49 ++++-------- 13 files changed, 193 insertions(+), 136 deletions(-) delete mode 100644 terraform/gcp/virus_scanning_pubsub.tf diff --git a/terraform/gcp/apis.tf b/terraform/gcp/apis.tf index c285f15048..5eaf76608a 100644 --- a/terraform/gcp/apis.tf +++ b/terraform/gcp/apis.tf @@ -14,9 +14,6 @@ locals { "cloudkms.googleapis.com", "binaryauthorization.googleapis.com", "cloudbuild.googleapis.com", - "run.googleapis.com", - "cloudscheduler.googleapis.com", - "eventarc.googleapis.com" ] } diff --git a/terraform/gcp/buckets.tf b/terraform/gcp/buckets.tf index b0edfee8c2..caf8ddfcc8 100644 --- a/terraform/gcp/buckets.tf +++ b/terraform/gcp/buckets.tf @@ -37,15 +37,26 @@ resource "google_storage_bucket" "quarantined_participant_documents" { } } -resource "google_storage_bucket" "cvd_mirror_bucket" { - name = "${var.documents_bucket_name}-cvd-mirror" - location = var.region - # no public access allowed - public_access_prevention = "enforced" - - # only allow access if you have iam perms - uniform_bucket_level_access = true - versioning { - enabled = true - } +resource "google_storage_bucket_iam_binding" "unscanned_buckets_sa_binding" { + bucket = google_storage_bucket.unscanned_participant_documents.name + role = "roles/storage.objectUser" + members = [ + "serviceAccount:${var.eng-infra-malware-scanner-sa}" + ] +} + +resource "google_storage_bucket_iam_binding" "clean_buckets_sa_binding" { + bucket = google_storage_bucket.clean_participant_documents.name + role = "roles/storage.objectUser" + members = [ + "serviceAccount:${var.eng-infra-malware-scanner-sa}" + ] +} + +resource "google_storage_bucket_iam_binding" "quarantined_buckets_sa_binding" { + bucket = google_storage_bucket.quarantined_participant_documents.name + role = "roles/storage.objectUser" + members = [ + "serviceAccount:${var.eng-infra-malware-scanner-sa}" + ] } diff --git a/terraform/gcp/cluster_service_account.tf b/terraform/gcp/cluster_service_account.tf index 21b3f56d5c..6da9ea4221 100644 --- a/terraform/gcp/cluster_service_account.tf +++ b/terraform/gcp/cluster_service_account.tf @@ -8,8 +8,7 @@ resource "google_project_iam_binding" "cluster-metric-writer" { project = var.project role = "roles/monitoring.metricWriter" members = [ - "serviceAccount:${google_service_account.cluster_service_account.email}", - "serviceAccount:${google_service_account.malware_scanner_sa.email}" + "serviceAccount:${google_service_account.cluster_service_account.email}" ] } @@ -26,8 +25,7 @@ resource "google_project_iam_binding" "cluster-log-writer" { role = "roles/logging.logWriter" members = [ "serviceAccount:${google_service_account.cluster_service_account.email}", - "serviceAccount:${google_service_account.juniper_cloudbuild_service_account.email}", - "serviceAccount:${google_service_account.build_service_account.email}" + "serviceAccount:${google_service_account.juniper_cloudbuild_service_account.email}" ] } diff --git a/terraform/gcp/envs/dev.tfvars b/terraform/gcp/envs/dev.tfvars index 3a986523d9..65e6c7e02c 100644 --- a/terraform/gcp/envs/dev.tfvars +++ b/terraform/gcp/envs/dev.tfvars @@ -9,12 +9,6 @@ environment = "dev" portals = ["demo", "atcp", "ourhealth", "hearthive", "rgp", "cmi"] k8s_namespace = "juniper-dev" -artifact_registry = "juniper" -artifact_registry_project = "broad-juniper-eng-infra" -artifact_registry_location = "us-central1" -malware_scanner_image_name = "juniper-malware-scanner" -deploy_virus_scanning = true - # creates DNS records for these customer URLs customer_urls = { demo = { @@ -24,4 +18,6 @@ customer_urls = { } } +documents_bucket_name = "juniper-participant-documents-dev" +eng-infra-malware-scanner-sa = "juniper-malware-scanner@broad-juniper-eng-infra.iam.gserviceaccount.com" slack_notification_channel = "projects/broad-juniper-dev/notificationChannels/13069356383599666729" diff --git a/terraform/gcp/variables.tf b/terraform/gcp/variables.tf index 95ab87df29..fa3104e3c4 100644 --- a/terraform/gcp/variables.tf +++ b/terraform/gcp/variables.tf @@ -99,34 +99,6 @@ variable "documents_bucket_name" { description = "The name of the GCP bucket for storing participant documents" } -variable "artifact_registry" { - type = string - default = "juniper" -} - -variable "artifact_registry_project" { - type = string - default = "broad-juniper-eng-infra" -} - -variable "artifact_registry_location" { - type = string - default = "us-central1" -} - - -# build image from https://github.com/GoogleCloudPlatform/docker-clamav-malware-scanner/tree/main/cloudrun-malware-scanner -# steps: -# cd cloudrun-malware-scanner -# docker build --tag=us-central1-docker.pkg.dev/broad-juniper-eng-infra/juniper/juniper-malware-scanner:latest -f Dockerfile . --platform linux/amd64 -# docker push us-central1-docker.pkg.dev/broad-juniper-eng-infra/juniper/juniper-malware-scanner:latest - -# if standing up for first time, you might also need to update the cvd mirror -# before deploying. run: -# pip3 install crcmod cvdupdate -# ./updateCvdMirror.sh -# from https://github.com/GoogleCloudPlatform/docker-clamav-malware-scanner - -variable "malware_scanner_image_name" { +variable "eng-infra-malware-scanner-sa" { type = string } diff --git a/terraform/gcp/virus_scanning_pubsub.tf b/terraform/gcp/virus_scanning_pubsub.tf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/terraform/infra/.terraform.lock.hcl b/terraform/infra/.terraform.lock.hcl index 5fb78ed475..84be92de57 100644 --- a/terraform/infra/.terraform.lock.hcl +++ b/terraform/infra/.terraform.lock.hcl @@ -2,39 +2,77 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/google" { - version = "6.3.0" + version = "6.25.0" hashes = [ - "h1:SpVI9aF/7wyalm5GhRUTGzwUA3Jv4d/lEnu/59X4U5k=", - "zh:2d5099f33ceaaa74d6d6860b98287ffdc723a6fb3f0a48ae02e9ba11e387f8b0", - "zh:38d937bedaeb9429663eff661b3087e66d7b88b3516eb1dc895eed160283878d", - "zh:5a9a7da6e578fc8448defb87f07616f8745700bd0d17ed458909d2561c08353b", - "zh:60b047acf4e4b2140885608ac37914d10947505dd8afd2da7718af27ff0e7b64", - "zh:84d457c92224f59d6ac7d2cc43581c7fe7dd3500d47251798fed21c7fb816d64", - "zh:b12a32dac4816d5cae9aa2b491d7510160e5f157150e8f814bcc9414a68db1f0", - "zh:cd65223bf3523f3d217e4b2b8c795149be53cfc545055a34e1fbd94f252c52d9", - "zh:d55de710eab8d31da97bb6ff7e5c4fe8afee7bde84ba80ac778d74ced1b7cd74", - "zh:e817d9ec4a7ff3e096dfc1c2ea88eb6eb725ae8c827e88cb77af9905ed2e3aec", - "zh:ebaa5cdf7beb94dec01cc6a2df8b8086bedeaf40cf009ffd27b051ca3770ebc0", - "zh:f3b5177362fa3ce8835daadeea9dd10299d76db667208128d619eb0786348ba4", + "h1:rvsmJc5J5OMMFXj9th8Yqfwm7cLZdRM/stPo0B8Datg=", + "zh:115ce3e84a02412a9a42c7c8f1c4568ab470bdd93c9a3a1f9202d4b77d7bc236", + "zh:623a32acea92d98a8bb66481fa9489e0a1e4dd6fc57ab70dbb53fae5732c1c63", + "zh:6440fc959b5e316152e26c916d55566311dafbe5b64e45d4b9c9931a5b29fa13", + "zh:91edb056638e723b1c7802d0e6e293611208e2825f645572ddba98122723f11c", + "zh:9b57ee44172677d2c2df03b22cf4b40e184ac8bd8facdd456ccbfdb7fee4684a", + "zh:9e2a97dd09b78b36caecdd990fff80bdb47a6a6d45c8b2dd4d23885d6bc89e65", + "zh:b33e40e9ba745f15b1c2e9ebcdccbac185788a4eb57450820cec7b9585858522", + "zh:d9e051bb703597384d83d924c49770e3bcdc1b68583d3def33a607ab168c634b", + "zh:e84253140fc3b0bd5cf7a1ebb54f993bacc6d5ee33c7b5e714ad834d5b2d35d0", + "zh:ebb624504c6f4297e691b6e3c00f789a6729461e5aa80fc165ef2ecd878e2d87", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f8c3e8192e98eff09a9453b52424c0d9727ae284de4fafbc498f2d527e026850", ] } provider "registry.terraform.io/hashicorp/google-beta" { - version = "6.3.0" + version = "6.25.0" hashes = [ - "h1:n9gyWvYz/1O5ecRfB1cV9/OlYTlSILvMy9knO4lvdXY=", - "zh:14911d94a78ee2a5d459a1e246e3722e04c5774f7aa8aeb415b11da89cd4ada8", - "zh:1a937609af8a837bdd181dad37e285c271bcb73971cc355c8f103730d1640aa4", - "zh:2032fa8f207ce06b5acf1e190f1090129b9fb4bae1b312165e5235af6cd61455", - "zh:3803fef01fe0c01e53f5d3628d8c8129d6443305cef458dea29d488661bbc5c0", - "zh:4c4434af7e2eb84c3a99a758e405b7f7f84c311ae6ef911d061d2d01fd4c0e76", - "zh:655eba025e121524ed95031c68b85d72ae2128eb748d68350fa4dd6569ce6770", - "zh:6678f186c550c6bb7ace33af0dd17a8d2ee6c61bcef2ff1bd8f603e0f7a5ce11", - "zh:7fa7de3cc6bf48d80ce3b8a5f3b23563d1f693b6febf20dd7b4a50a098ab6ee1", - "zh:a33d0f8deda840bc3f0263559118f632ddbb0609335739ab7329b517f156bc63", - "zh:c79b8109b3d7e42c1d1b5ebce29319bf69f4360be6f03f4fb0578fdc7f628e3f", + "h1:KlIctHHQkbToSXa41DWFQUsTKsfaWe+vjs/CFbnj3bc=", + "zh:101a8a679274cae57e8c816164fa28b6d271a8a53b3e007b84153fc63f0f06f7", + "zh:413c41e4ee14b10cb815cd01f8b880c12474ecbf63ecfc6415512fb0c18c473b", + "zh:4260143c8584a8f338a4d45f65c2f08f12b28a746c43d43a1e3a575f96ce4110", + "zh:4785080dd28b79a94ba6e277d247b6bb678d676d99244a8d664ea60802338df7", + "zh:673b09d67cb59487095fe7176c194e8c7ced62fc7bb5084016f3c87dd89e685e", + "zh:7de0dd9367c723cc203fdc4e70d316c779224483637039de5a3c9ee807c3fc55", + "zh:aa49c5955652b21ff2d0b7b2db0e18f59097d3a539efc98ef65deed4293d7327", + "zh:b4d73704d88deee367568fd68303878e6e60924c2a65df397f37b868e6aecaac", + "zh:cae0c5eff7c37c3f9857de08ca35b3a86d44855ae9c0ec2fab1eda604ff501c8", + "zh:e115f6b0c25bdad624fafbb5d685c8a48d9028f9525918274a5e5469f7dfde3d", + "zh:e536197371458b0b177a203f8367f99e74770b3dfce597034e880ea5f3f798b8", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - "zh:f9002acf0ce39e1f3f2afc28112343c846a6249868ca567477cbbcc3a4a77517", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.3" + hashes = [ + "h1:I0Um8UkrMUb81Fxq/dxbr3HLP2cecTH2WMJiwKSrwQY=", + "zh:22d062e5278d872fe7aed834f5577ba0a5afe34a3bdac2b81f828d8d3e6706d2", + "zh:23dead00493ad863729495dc212fd6c29b8293e707b055ce5ba21ee453ce552d", + "zh:28299accf21763ca1ca144d8f660688d7c2ad0b105b7202554ca60b02a3856d3", + "zh:55c9e8a9ac25a7652df8c51a8a9a422bd67d784061b1de2dc9fe6c3cb4e77f2f", + "zh:756586535d11698a216291c06b9ed8a5cc6a4ec43eee1ee09ecd5c6a9e297ac1", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:9d5eea62fdb587eeb96a8c4d782459f4e6b73baeece4d04b4a40e44faaee9301", + "zh:a6355f596a3fb8fc85c2fb054ab14e722991533f87f928e7169a486462c74670", + "zh:b5a65a789cff4ada58a5baffc76cb9767dc26ec6b45c00d2ec8b1b027f6db4ed", + "zh:db5ab669cf11d0e9f81dc380a6fdfcac437aea3d69109c7aef1a5426639d2d65", + "zh:de655d251c470197bcbb5ac45d289595295acb8f829f6c781d4a75c8c8b7c7dd", + "zh:f5c68199f2e6076bce92a12230434782bf768103a427e9bb9abee99b116af7b5", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.7.1" + hashes = [ + "h1:t152MY0tQH4a8fLzTtEWx70ITd3azVOrFDn/pQblbto=", + "zh:3193b89b43bf5805493e290374cdda5132578de6535f8009547c8b5d7a351585", + "zh:3218320de4be943e5812ed3de995946056db86eb8d03aa3f074e0c7316599bef", + "zh:419861805a37fa443e7d63b69fb3279926ccf98a79d256c422d5d82f0f387d1d", + "zh:4df9bd9d839b8fc11a3b8098a604b9b46e2235eb65ef15f4432bde0e175f9ca6", + "zh:5814be3f9c9cc39d2955d6f083bae793050d75c572e70ca11ccceb5517ced6b1", + "zh:63c6548a06de1231c8ee5570e42ca09c4b3db336578ded39b938f2156f06dd2e", + "zh:697e434c6bdee0502cc3deb098263b8dcd63948e8a96d61722811628dce2eba1", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:a0b8e44927e6327852bbfdc9d408d802569367f1e22a95bcdd7181b1c3b07601", + "zh:b7d3af018683ef22794eea9c218bc72d7c35a2b3ede9233b69653b3c782ee436", + "zh:d63b911d618a6fe446c65bfc21e793a7663e934b2fef833d42d3ccd38dd8d68d", + "zh:fa985cd0b11e6d651f47cff3055f0a9fd085ec190b6dbe99bf5448174434cdea", ] } diff --git a/terraform/infra/apis.tf b/terraform/infra/apis.tf index 2896f2e45f..3e967663be 100644 --- a/terraform/infra/apis.tf +++ b/terraform/infra/apis.tf @@ -12,3 +12,19 @@ resource "google_project_service" "enable_iam_creds" { service = "iamcredentials.googleapis.com" project = var.project } + + +resource "google_project_service" "enable_cloud_run" { + service = "run.googleapis.com" + project = var.project +} + +resource "google_project_service" "enable_cloud_scheduler" { + service = "cloudscheduler.googleapis.com" + project = var.project +} + +resource "google_project_service" "enable_event_arc" { + service = "eventarc.googleapis.com" + project = var.project +} diff --git a/terraform/infra/infra.tfvars b/terraform/infra/infra.tfvars index 4966782f87..04f1f0b022 100644 --- a/terraform/infra/infra.tfvars +++ b/terraform/infra/infra.tfvars @@ -1,2 +1,19 @@ project = "broad-juniper-eng-infra" region = "us-central1" + +juniper_projects = ["broad-juniper-dev", "broad-juniper-prod"] +virus_scanning_cvd_bucket_name = "juniper-eng-infra-virus-scanning-cvd-mirror" +virus_scanning_buckets = [ + { + clean: "juniper-participant-documents-dev-clean", + unscanned: "juniper-participant-documents-dev-unscanned", + quarantined: "juniper-participant-documents-dev-quarantined" + }, + # { + # clean: "juniper-participant-documents-prod-clean", + # unscanned: "juniper-participant-documents-prod-unscanned", + # quarantined: "juniper-participant-documents-prod-quarantined" + # }, +] + +malware_scanner_image_name = "juniper-malware-scanner" diff --git a/terraform/infra/variables.tf b/terraform/infra/variables.tf index 506732fc8d..a6aeca975a 100644 --- a/terraform/infra/variables.tf +++ b/terraform/infra/variables.tf @@ -12,10 +12,30 @@ variable "virus_scanning_buckets" { type = set(object({ unscanned: string, clean: string, - quarantine: string + quarantined: string })) } +variable "juniper_projects" { + type = set(string) +} + variable "virus_scanning_cvd_bucket_name" { type = string } + +# build image from https://github.com/GoogleCloudPlatform/docker-clamav-malware-scanner/tree/main/cloudrun-malware-scanner +# steps: +# cd cloudrun-malware-scanner +# docker build --tag=us-central1-docker.pkg.dev/broad-juniper-eng-infra/juniper/juniper-malware-scanner:latest -f Dockerfile . --platform linux/amd64 +# docker push us-central1-docker.pkg.dev/broad-juniper-eng-infra/juniper/juniper-malware-scanner:latest + +# if standing up for first time, you might also need to update the cvd mirror +# before deploying. run: +# pip3 install crcmod cvdupdate +# ./updateCvdMirror.sh +# from https://github.com/GoogleCloudPlatform/docker-clamav-malware-scanner + +variable "malware_scanner_image_name" { + type = string +} diff --git a/terraform/infra/virus_scanning.tf b/terraform/infra/virus_scanning.tf index 120c95fc49..0dbee67700 100644 --- a/terraform/infra/virus_scanning.tf +++ b/terraform/infra/virus_scanning.tf @@ -20,9 +20,9 @@ ## Lookup the hash of the latest image ## data "google_artifact_registry_docker_image" "scanner-service-image" { - location = var.artifact_registry_location - repository_id = var.artifact_registry - project = var.artifact_registry_project + location = var.region + repository_id = google_artifact_registry_repository.juniper_repo.name + project = var.project image_name = "${var.malware_scanner_image_name}:latest" } @@ -100,11 +100,9 @@ resource "google_cloud_run_v2_service" "malware_scanner" { } depends_on = [ - google_storage_bucket.unscanned_participant_documents, - google_storage_bucket.clean_participant_documents, - google_storage_bucket.quarantined_participant_documents, - google_storage_bucket.cvd_mirror_bucket, - time_sleep.enable_all_services_with_timeout + data.google_storage_bucket.clean_buckets, + data.google_storage_bucket.unscanned_buckets, + data.google_storage_bucket.quarantined_buckets, ] } @@ -112,7 +110,9 @@ resource "google_cloud_run_v2_service" "malware_scanner" { ## Create EventArc Triggers on unscanned bucket(s) # resource "google_eventarc_trigger" "gcs-object-written" { - name = "gcs-trigger-${google_storage_bucket.unscanned_participant_documents.name}" + for_each = data.google_storage_bucket.unscanned_buckets + + name = "gcs-trigger-${each.value.name}" location = var.region matching_criteria { attribute = "type" @@ -120,7 +120,7 @@ resource "google_eventarc_trigger" "gcs-object-written" { } matching_criteria { attribute = "bucket" - value = google_storage_bucket.unscanned_participant_documents.name + value = each.value.name } destination { cloud_run_service { @@ -129,17 +129,14 @@ resource "google_eventarc_trigger" "gcs-object-written" { } } service_account = google_service_account.malware_scanner_sa.email - - depends_on = [ - time_sleep.enable_all_services_with_timeout - ] } ## Update pubsub subscriptions to increase deadlines # resource "null_resource" "update-subscription-ack-deadline" { + for_each = google_eventarc_trigger.gcs-object-written provisioner "local-exec" { - command = "gcloud pubsub subscriptions update \"${google_eventarc_trigger.gcs-object-written.transport[0].pubsub[0].subscription}\" --ack-deadline=300" + command = "gcloud pubsub subscriptions update \"${each.value.transport[0].pubsub[0].subscription}\" --ack-deadline=300" } } diff --git a/terraform/infra/virus_scanning_buckets.tf b/terraform/infra/virus_scanning_buckets.tf index 27a3edc400..48917f43c5 100644 --- a/terraform/infra/virus_scanning_buckets.tf +++ b/terraform/infra/virus_scanning_buckets.tf @@ -1,27 +1,41 @@ -resource "google_storage_bucket" "virus_cvd_bucket"{ +resource "google_storage_bucket" "cvd_mirror_bucket" { + name = var.virus_scanning_cvd_bucket_name location = var.region - name = var.virus_scanning_cvd_bucket_name + # no public access allowed + public_access_prevention = "enforced" + # only allow access if you have iam perms uniform_bucket_level_access = true - public_access_prevention = true + versioning { + enabled = true + } } data "google_storage_bucket" "unscanned_buckets" { - for_each = var.virus_scanning_buckets + for_each = { + for index, buckets in var.virus_scanning_buckets: + buckets.unscanned => buckets + } name = each.value.unscanned } data "google_storage_bucket" "clean_buckets" { - for_each = var.virus_scanning_buckets + for_each = { + for index, buckets in var.virus_scanning_buckets: + buckets.unscanned => buckets + } name = each.value.clean } data "google_storage_bucket" "quarantined_buckets" { - for_each = var.virus_scanning_buckets + for_each = { + for index, buckets in var.virus_scanning_buckets: + buckets.unscanned => buckets + } - name = each.value.quarantine + name = each.value.quarantined } diff --git a/terraform/infra/virus_scanning_infra.tf b/terraform/infra/virus_scanning_infra.tf index 81aee6a955..9489ea6e19 100644 --- a/terraform/infra/virus_scanning_infra.tf +++ b/terraform/infra/virus_scanning_infra.tf @@ -5,10 +5,9 @@ resource "google_service_account" "malware_scanner_sa" { display_name = "juniper-malware-scanner" } - - resource "google_project_iam_member" "malware_scanner_iam" { - for_each = toset(["roles/run.invoker", "roles/eventarc.eventReceiver"]) + # need: log writer and metric writer + for_each = toset(["roles/run.invoker", "roles/eventarc.eventReceiver", "roles/monitoring.metricWriter", "roles/logging.logWriter"]) project = var.project role = each.value member = "serviceAccount:${google_service_account.malware_scanner_sa.email}" @@ -28,45 +27,27 @@ resource "google_project_iam_binding" "build_iam" { ## Allow GCS to publish to pubsub # -data "google_storage_project_service_account" "gcs_account" { +data "google_storage_project_service_account" "juniper_gcs_accounts" { + for_each = var.juniper_projects + project = each.value } -resource "google_project_iam_binding" "gcs_sa_pubsub_publish" { - project = var.project - role = "roles/pubsub.publisher" - members = ["serviceAccount:${data.google_storage_project_service_account.gcs_account.email_address}"] +data "google_storage_project_service_account" "infra-gs" { + project = var.project } -## Allow service account to admin the scanner buckets. -# -# They may not have been created by TF, so use a data resource -# to verify their existence. -# - - -resource "google_storage_bucket_iam_binding" "unscanned_buckets_sa_binding" { - bucket = google_storage_bucket.unscanned_participant_documents.name - role = "roles/storage.admin" - members = [ - "serviceAccount:${google_service_account.malware_scanner_sa.email}", - ] -} +resource "google_project_iam_binding" "gcs_sa_pubsub_publish" { -resource "google_storage_bucket_iam_binding" "clean_buckets_sa_binding" { - bucket = google_storage_bucket.clean_participant_documents.name - role = "roles/storage.admin" - members = [ + project = var.project + role = "roles/pubsub.publisher" + members = concat([ + for sa in data.google_storage_project_service_account.juniper_gcs_accounts: "serviceAccount:${sa.email_address}" + ], [ "serviceAccount:${google_service_account.malware_scanner_sa.email}", - ] + "serviceAccount:${data.google_storage_project_service_account.infra-gs.email_address}" + ]) } -resource "google_storage_bucket_iam_binding" "quarantined_buckets_sa_binding" { - bucket = google_storage_bucket.quarantined_participant_documents.name - role = "roles/storage.admin" - members = [ - "serviceAccount:${google_service_account.malware_scanner_sa.email}", - ] -} resource "google_storage_bucket_iam_binding" "cvd_buckets_sa_binding" { bucket = google_storage_bucket.cvd_mirror_bucket.name From c3977b7c06bbcac727644e0309e14d594035bd96 Mon Sep 17 00:00:00 2001 From: Connor Barker Date: Mon, 17 Mar 2025 15:47:07 -0400 Subject: [PATCH 4/8] reset back --- terraform/gcp/apis.tf | 3 + terraform/gcp/buckets.tf | 33 +++---- terraform/gcp/cluster_service_account.tf | 6 +- terraform/gcp/envs/dev.tfvars | 8 +- terraform/gcp/variables.tf | 30 ++++++- terraform/{infra => gcp}/virus_scanning.tf | 38 +++++--- .../{infra => gcp}/virus_scanning_infra.tf | 49 ++++++---- terraform/infra/.terraform.lock.hcl | 90 ++++++------------- terraform/infra/apis.tf | 16 ---- terraform/infra/infra.tfvars | 17 ---- terraform/infra/variables.tf | 32 ------- terraform/infra/virus_scanning_buckets.tf | 41 --------- 12 files changed, 138 insertions(+), 225 deletions(-) rename terraform/{infra => gcp}/virus_scanning.tf (80%) rename terraform/{infra => gcp}/virus_scanning_infra.tf (61%) delete mode 100644 terraform/infra/virus_scanning_buckets.tf diff --git a/terraform/gcp/apis.tf b/terraform/gcp/apis.tf index 5eaf76608a..c285f15048 100644 --- a/terraform/gcp/apis.tf +++ b/terraform/gcp/apis.tf @@ -14,6 +14,9 @@ locals { "cloudkms.googleapis.com", "binaryauthorization.googleapis.com", "cloudbuild.googleapis.com", + "run.googleapis.com", + "cloudscheduler.googleapis.com", + "eventarc.googleapis.com" ] } diff --git a/terraform/gcp/buckets.tf b/terraform/gcp/buckets.tf index caf8ddfcc8..b0edfee8c2 100644 --- a/terraform/gcp/buckets.tf +++ b/terraform/gcp/buckets.tf @@ -37,26 +37,15 @@ resource "google_storage_bucket" "quarantined_participant_documents" { } } -resource "google_storage_bucket_iam_binding" "unscanned_buckets_sa_binding" { - bucket = google_storage_bucket.unscanned_participant_documents.name - role = "roles/storage.objectUser" - members = [ - "serviceAccount:${var.eng-infra-malware-scanner-sa}" - ] -} - -resource "google_storage_bucket_iam_binding" "clean_buckets_sa_binding" { - bucket = google_storage_bucket.clean_participant_documents.name - role = "roles/storage.objectUser" - members = [ - "serviceAccount:${var.eng-infra-malware-scanner-sa}" - ] -} - -resource "google_storage_bucket_iam_binding" "quarantined_buckets_sa_binding" { - bucket = google_storage_bucket.quarantined_participant_documents.name - role = "roles/storage.objectUser" - members = [ - "serviceAccount:${var.eng-infra-malware-scanner-sa}" - ] +resource "google_storage_bucket" "cvd_mirror_bucket" { + name = "${var.documents_bucket_name}-cvd-mirror" + location = var.region + # no public access allowed + public_access_prevention = "enforced" + + # only allow access if you have iam perms + uniform_bucket_level_access = true + versioning { + enabled = true + } } diff --git a/terraform/gcp/cluster_service_account.tf b/terraform/gcp/cluster_service_account.tf index 6da9ea4221..21b3f56d5c 100644 --- a/terraform/gcp/cluster_service_account.tf +++ b/terraform/gcp/cluster_service_account.tf @@ -8,7 +8,8 @@ resource "google_project_iam_binding" "cluster-metric-writer" { project = var.project role = "roles/monitoring.metricWriter" members = [ - "serviceAccount:${google_service_account.cluster_service_account.email}" + "serviceAccount:${google_service_account.cluster_service_account.email}", + "serviceAccount:${google_service_account.malware_scanner_sa.email}" ] } @@ -25,7 +26,8 @@ resource "google_project_iam_binding" "cluster-log-writer" { role = "roles/logging.logWriter" members = [ "serviceAccount:${google_service_account.cluster_service_account.email}", - "serviceAccount:${google_service_account.juniper_cloudbuild_service_account.email}" + "serviceAccount:${google_service_account.juniper_cloudbuild_service_account.email}", + "serviceAccount:${google_service_account.build_service_account.email}" ] } diff --git a/terraform/gcp/envs/dev.tfvars b/terraform/gcp/envs/dev.tfvars index 65e6c7e02c..3a986523d9 100644 --- a/terraform/gcp/envs/dev.tfvars +++ b/terraform/gcp/envs/dev.tfvars @@ -9,6 +9,12 @@ environment = "dev" portals = ["demo", "atcp", "ourhealth", "hearthive", "rgp", "cmi"] k8s_namespace = "juniper-dev" +artifact_registry = "juniper" +artifact_registry_project = "broad-juniper-eng-infra" +artifact_registry_location = "us-central1" +malware_scanner_image_name = "juniper-malware-scanner" +deploy_virus_scanning = true + # creates DNS records for these customer URLs customer_urls = { demo = { @@ -18,6 +24,4 @@ customer_urls = { } } -documents_bucket_name = "juniper-participant-documents-dev" -eng-infra-malware-scanner-sa = "juniper-malware-scanner@broad-juniper-eng-infra.iam.gserviceaccount.com" slack_notification_channel = "projects/broad-juniper-dev/notificationChannels/13069356383599666729" diff --git a/terraform/gcp/variables.tf b/terraform/gcp/variables.tf index fa3104e3c4..95ab87df29 100644 --- a/terraform/gcp/variables.tf +++ b/terraform/gcp/variables.tf @@ -99,6 +99,34 @@ variable "documents_bucket_name" { description = "The name of the GCP bucket for storing participant documents" } -variable "eng-infra-malware-scanner-sa" { +variable "artifact_registry" { + type = string + default = "juniper" +} + +variable "artifact_registry_project" { + type = string + default = "broad-juniper-eng-infra" +} + +variable "artifact_registry_location" { + type = string + default = "us-central1" +} + + +# build image from https://github.com/GoogleCloudPlatform/docker-clamav-malware-scanner/tree/main/cloudrun-malware-scanner +# steps: +# cd cloudrun-malware-scanner +# docker build --tag=us-central1-docker.pkg.dev/broad-juniper-eng-infra/juniper/juniper-malware-scanner:latest -f Dockerfile . --platform linux/amd64 +# docker push us-central1-docker.pkg.dev/broad-juniper-eng-infra/juniper/juniper-malware-scanner:latest + +# if standing up for first time, you might also need to update the cvd mirror +# before deploying. run: +# pip3 install crcmod cvdupdate +# ./updateCvdMirror.sh +# from https://github.com/GoogleCloudPlatform/docker-clamav-malware-scanner + +variable "malware_scanner_image_name" { type = string } diff --git a/terraform/infra/virus_scanning.tf b/terraform/gcp/virus_scanning.tf similarity index 80% rename from terraform/infra/virus_scanning.tf rename to terraform/gcp/virus_scanning.tf index 0dbee67700..c9c314234f 100644 --- a/terraform/infra/virus_scanning.tf +++ b/terraform/gcp/virus_scanning.tf @@ -20,9 +20,9 @@ ## Lookup the hash of the latest image ## data "google_artifact_registry_docker_image" "scanner-service-image" { - location = var.region - repository_id = google_artifact_registry_repository.juniper_repo.name - project = var.project + location = var.artifact_registry_location + repository_id = var.artifact_registry + project = var.artifact_registry_project image_name = "${var.malware_scanner_image_name}:latest" } @@ -55,7 +55,13 @@ resource "google_cloud_run_v2_service" "malware_scanner" { env { name = "CONFIG_JSON" value = jsonencode({ - "buckets": var.virus_scanning_buckets, + "buckets": [ + { + "unscanned": google_storage_bucket.unscanned_participant_documents.name, + "clean": google_storage_bucket.clean_participant_documents.name, + "quarantined": google_storage_bucket.quarantined_participant_documents.name, + } + ], "ClamCvdMirrorBucket": google_storage_bucket.cvd_mirror_bucket.name, "fileExclusionPatterns": [["\\\\.tmp$","i"]], "ignoreZeroLengthFiles": true, @@ -100,9 +106,11 @@ resource "google_cloud_run_v2_service" "malware_scanner" { } depends_on = [ - data.google_storage_bucket.clean_buckets, - data.google_storage_bucket.unscanned_buckets, - data.google_storage_bucket.quarantined_buckets, + google_storage_bucket.unscanned_participant_documents, + google_storage_bucket.clean_participant_documents, + google_storage_bucket.quarantined_participant_documents, + google_storage_bucket.cvd_mirror_bucket, + time_sleep.enable_all_services_with_timeout ] } @@ -110,9 +118,7 @@ resource "google_cloud_run_v2_service" "malware_scanner" { ## Create EventArc Triggers on unscanned bucket(s) # resource "google_eventarc_trigger" "gcs-object-written" { - for_each = data.google_storage_bucket.unscanned_buckets - - name = "gcs-trigger-${each.value.name}" + name = "gcs-trigger-${google_storage_bucket.unscanned_participant_documents.name}" location = var.region matching_criteria { attribute = "type" @@ -120,7 +126,7 @@ resource "google_eventarc_trigger" "gcs-object-written" { } matching_criteria { attribute = "bucket" - value = each.value.name + value = google_storage_bucket.unscanned_participant_documents.name } destination { cloud_run_service { @@ -129,14 +135,17 @@ resource "google_eventarc_trigger" "gcs-object-written" { } } service_account = google_service_account.malware_scanner_sa.email + + depends_on = [ + time_sleep.enable_all_services_with_timeout + ] } ## Update pubsub subscriptions to increase deadlines # resource "null_resource" "update-subscription-ack-deadline" { - for_each = google_eventarc_trigger.gcs-object-written provisioner "local-exec" { - command = "gcloud pubsub subscriptions update \"${each.value.transport[0].pubsub[0].subscription}\" --ack-deadline=300" + command = "gcloud pubsub subscriptions update \"${google_eventarc_trigger.gcs-object-written.transport[0].pubsub[0].subscription}\" --ack-deadline=300" } } @@ -176,4 +185,7 @@ resource "google_cloud_scheduler_job" "cvd_mirror_update" { } } + depends_on = [ + time_sleep.enable_all_services_with_timeout + ] } diff --git a/terraform/infra/virus_scanning_infra.tf b/terraform/gcp/virus_scanning_infra.tf similarity index 61% rename from terraform/infra/virus_scanning_infra.tf rename to terraform/gcp/virus_scanning_infra.tf index 9489ea6e19..81aee6a955 100644 --- a/terraform/infra/virus_scanning_infra.tf +++ b/terraform/gcp/virus_scanning_infra.tf @@ -5,9 +5,10 @@ resource "google_service_account" "malware_scanner_sa" { display_name = "juniper-malware-scanner" } + + resource "google_project_iam_member" "malware_scanner_iam" { - # need: log writer and metric writer - for_each = toset(["roles/run.invoker", "roles/eventarc.eventReceiver", "roles/monitoring.metricWriter", "roles/logging.logWriter"]) + for_each = toset(["roles/run.invoker", "roles/eventarc.eventReceiver"]) project = var.project role = each.value member = "serviceAccount:${google_service_account.malware_scanner_sa.email}" @@ -27,27 +28,45 @@ resource "google_project_iam_binding" "build_iam" { ## Allow GCS to publish to pubsub # -data "google_storage_project_service_account" "juniper_gcs_accounts" { - for_each = var.juniper_projects - project = each.value -} - -data "google_storage_project_service_account" "infra-gs" { - project = var.project +data "google_storage_project_service_account" "gcs_account" { } resource "google_project_iam_binding" "gcs_sa_pubsub_publish" { - project = var.project role = "roles/pubsub.publisher" - members = concat([ - for sa in data.google_storage_project_service_account.juniper_gcs_accounts: "serviceAccount:${sa.email_address}" - ], [ + members = ["serviceAccount:${data.google_storage_project_service_account.gcs_account.email_address}"] +} + +## Allow service account to admin the scanner buckets. +# +# They may not have been created by TF, so use a data resource +# to verify their existence. +# + + +resource "google_storage_bucket_iam_binding" "unscanned_buckets_sa_binding" { + bucket = google_storage_bucket.unscanned_participant_documents.name + role = "roles/storage.admin" + members = [ "serviceAccount:${google_service_account.malware_scanner_sa.email}", - "serviceAccount:${data.google_storage_project_service_account.infra-gs.email_address}" - ]) + ] +} + +resource "google_storage_bucket_iam_binding" "clean_buckets_sa_binding" { + bucket = google_storage_bucket.clean_participant_documents.name + role = "roles/storage.admin" + members = [ + "serviceAccount:${google_service_account.malware_scanner_sa.email}", + ] } +resource "google_storage_bucket_iam_binding" "quarantined_buckets_sa_binding" { + bucket = google_storage_bucket.quarantined_participant_documents.name + role = "roles/storage.admin" + members = [ + "serviceAccount:${google_service_account.malware_scanner_sa.email}", + ] +} resource "google_storage_bucket_iam_binding" "cvd_buckets_sa_binding" { bucket = google_storage_bucket.cvd_mirror_bucket.name diff --git a/terraform/infra/.terraform.lock.hcl b/terraform/infra/.terraform.lock.hcl index 84be92de57..5fb78ed475 100644 --- a/terraform/infra/.terraform.lock.hcl +++ b/terraform/infra/.terraform.lock.hcl @@ -2,77 +2,39 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/google" { - version = "6.25.0" + version = "6.3.0" hashes = [ - "h1:rvsmJc5J5OMMFXj9th8Yqfwm7cLZdRM/stPo0B8Datg=", - "zh:115ce3e84a02412a9a42c7c8f1c4568ab470bdd93c9a3a1f9202d4b77d7bc236", - "zh:623a32acea92d98a8bb66481fa9489e0a1e4dd6fc57ab70dbb53fae5732c1c63", - "zh:6440fc959b5e316152e26c916d55566311dafbe5b64e45d4b9c9931a5b29fa13", - "zh:91edb056638e723b1c7802d0e6e293611208e2825f645572ddba98122723f11c", - "zh:9b57ee44172677d2c2df03b22cf4b40e184ac8bd8facdd456ccbfdb7fee4684a", - "zh:9e2a97dd09b78b36caecdd990fff80bdb47a6a6d45c8b2dd4d23885d6bc89e65", - "zh:b33e40e9ba745f15b1c2e9ebcdccbac185788a4eb57450820cec7b9585858522", - "zh:d9e051bb703597384d83d924c49770e3bcdc1b68583d3def33a607ab168c634b", - "zh:e84253140fc3b0bd5cf7a1ebb54f993bacc6d5ee33c7b5e714ad834d5b2d35d0", - "zh:ebb624504c6f4297e691b6e3c00f789a6729461e5aa80fc165ef2ecd878e2d87", + "h1:SpVI9aF/7wyalm5GhRUTGzwUA3Jv4d/lEnu/59X4U5k=", + "zh:2d5099f33ceaaa74d6d6860b98287ffdc723a6fb3f0a48ae02e9ba11e387f8b0", + "zh:38d937bedaeb9429663eff661b3087e66d7b88b3516eb1dc895eed160283878d", + "zh:5a9a7da6e578fc8448defb87f07616f8745700bd0d17ed458909d2561c08353b", + "zh:60b047acf4e4b2140885608ac37914d10947505dd8afd2da7718af27ff0e7b64", + "zh:84d457c92224f59d6ac7d2cc43581c7fe7dd3500d47251798fed21c7fb816d64", + "zh:b12a32dac4816d5cae9aa2b491d7510160e5f157150e8f814bcc9414a68db1f0", + "zh:cd65223bf3523f3d217e4b2b8c795149be53cfc545055a34e1fbd94f252c52d9", + "zh:d55de710eab8d31da97bb6ff7e5c4fe8afee7bde84ba80ac778d74ced1b7cd74", + "zh:e817d9ec4a7ff3e096dfc1c2ea88eb6eb725ae8c827e88cb77af9905ed2e3aec", + "zh:ebaa5cdf7beb94dec01cc6a2df8b8086bedeaf40cf009ffd27b051ca3770ebc0", + "zh:f3b5177362fa3ce8835daadeea9dd10299d76db667208128d619eb0786348ba4", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - "zh:f8c3e8192e98eff09a9453b52424c0d9727ae284de4fafbc498f2d527e026850", ] } provider "registry.terraform.io/hashicorp/google-beta" { - version = "6.25.0" + version = "6.3.0" hashes = [ - "h1:KlIctHHQkbToSXa41DWFQUsTKsfaWe+vjs/CFbnj3bc=", - "zh:101a8a679274cae57e8c816164fa28b6d271a8a53b3e007b84153fc63f0f06f7", - "zh:413c41e4ee14b10cb815cd01f8b880c12474ecbf63ecfc6415512fb0c18c473b", - "zh:4260143c8584a8f338a4d45f65c2f08f12b28a746c43d43a1e3a575f96ce4110", - "zh:4785080dd28b79a94ba6e277d247b6bb678d676d99244a8d664ea60802338df7", - "zh:673b09d67cb59487095fe7176c194e8c7ced62fc7bb5084016f3c87dd89e685e", - "zh:7de0dd9367c723cc203fdc4e70d316c779224483637039de5a3c9ee807c3fc55", - "zh:aa49c5955652b21ff2d0b7b2db0e18f59097d3a539efc98ef65deed4293d7327", - "zh:b4d73704d88deee367568fd68303878e6e60924c2a65df397f37b868e6aecaac", - "zh:cae0c5eff7c37c3f9857de08ca35b3a86d44855ae9c0ec2fab1eda604ff501c8", - "zh:e115f6b0c25bdad624fafbb5d685c8a48d9028f9525918274a5e5469f7dfde3d", - "zh:e536197371458b0b177a203f8367f99e74770b3dfce597034e880ea5f3f798b8", + "h1:n9gyWvYz/1O5ecRfB1cV9/OlYTlSILvMy9knO4lvdXY=", + "zh:14911d94a78ee2a5d459a1e246e3722e04c5774f7aa8aeb415b11da89cd4ada8", + "zh:1a937609af8a837bdd181dad37e285c271bcb73971cc355c8f103730d1640aa4", + "zh:2032fa8f207ce06b5acf1e190f1090129b9fb4bae1b312165e5235af6cd61455", + "zh:3803fef01fe0c01e53f5d3628d8c8129d6443305cef458dea29d488661bbc5c0", + "zh:4c4434af7e2eb84c3a99a758e405b7f7f84c311ae6ef911d061d2d01fd4c0e76", + "zh:655eba025e121524ed95031c68b85d72ae2128eb748d68350fa4dd6569ce6770", + "zh:6678f186c550c6bb7ace33af0dd17a8d2ee6c61bcef2ff1bd8f603e0f7a5ce11", + "zh:7fa7de3cc6bf48d80ce3b8a5f3b23563d1f693b6febf20dd7b4a50a098ab6ee1", + "zh:a33d0f8deda840bc3f0263559118f632ddbb0609335739ab7329b517f156bc63", + "zh:c79b8109b3d7e42c1d1b5ebce29319bf69f4360be6f03f4fb0578fdc7f628e3f", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - ] -} - -provider "registry.terraform.io/hashicorp/null" { - version = "3.2.3" - hashes = [ - "h1:I0Um8UkrMUb81Fxq/dxbr3HLP2cecTH2WMJiwKSrwQY=", - "zh:22d062e5278d872fe7aed834f5577ba0a5afe34a3bdac2b81f828d8d3e6706d2", - "zh:23dead00493ad863729495dc212fd6c29b8293e707b055ce5ba21ee453ce552d", - "zh:28299accf21763ca1ca144d8f660688d7c2ad0b105b7202554ca60b02a3856d3", - "zh:55c9e8a9ac25a7652df8c51a8a9a422bd67d784061b1de2dc9fe6c3cb4e77f2f", - "zh:756586535d11698a216291c06b9ed8a5cc6a4ec43eee1ee09ecd5c6a9e297ac1", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:9d5eea62fdb587eeb96a8c4d782459f4e6b73baeece4d04b4a40e44faaee9301", - "zh:a6355f596a3fb8fc85c2fb054ab14e722991533f87f928e7169a486462c74670", - "zh:b5a65a789cff4ada58a5baffc76cb9767dc26ec6b45c00d2ec8b1b027f6db4ed", - "zh:db5ab669cf11d0e9f81dc380a6fdfcac437aea3d69109c7aef1a5426639d2d65", - "zh:de655d251c470197bcbb5ac45d289595295acb8f829f6c781d4a75c8c8b7c7dd", - "zh:f5c68199f2e6076bce92a12230434782bf768103a427e9bb9abee99b116af7b5", - ] -} - -provider "registry.terraform.io/hashicorp/random" { - version = "3.7.1" - hashes = [ - "h1:t152MY0tQH4a8fLzTtEWx70ITd3azVOrFDn/pQblbto=", - "zh:3193b89b43bf5805493e290374cdda5132578de6535f8009547c8b5d7a351585", - "zh:3218320de4be943e5812ed3de995946056db86eb8d03aa3f074e0c7316599bef", - "zh:419861805a37fa443e7d63b69fb3279926ccf98a79d256c422d5d82f0f387d1d", - "zh:4df9bd9d839b8fc11a3b8098a604b9b46e2235eb65ef15f4432bde0e175f9ca6", - "zh:5814be3f9c9cc39d2955d6f083bae793050d75c572e70ca11ccceb5517ced6b1", - "zh:63c6548a06de1231c8ee5570e42ca09c4b3db336578ded39b938f2156f06dd2e", - "zh:697e434c6bdee0502cc3deb098263b8dcd63948e8a96d61722811628dce2eba1", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:a0b8e44927e6327852bbfdc9d408d802569367f1e22a95bcdd7181b1c3b07601", - "zh:b7d3af018683ef22794eea9c218bc72d7c35a2b3ede9233b69653b3c782ee436", - "zh:d63b911d618a6fe446c65bfc21e793a7663e934b2fef833d42d3ccd38dd8d68d", - "zh:fa985cd0b11e6d651f47cff3055f0a9fd085ec190b6dbe99bf5448174434cdea", + "zh:f9002acf0ce39e1f3f2afc28112343c846a6249868ca567477cbbcc3a4a77517", ] } diff --git a/terraform/infra/apis.tf b/terraform/infra/apis.tf index 3e967663be..2896f2e45f 100644 --- a/terraform/infra/apis.tf +++ b/terraform/infra/apis.tf @@ -12,19 +12,3 @@ resource "google_project_service" "enable_iam_creds" { service = "iamcredentials.googleapis.com" project = var.project } - - -resource "google_project_service" "enable_cloud_run" { - service = "run.googleapis.com" - project = var.project -} - -resource "google_project_service" "enable_cloud_scheduler" { - service = "cloudscheduler.googleapis.com" - project = var.project -} - -resource "google_project_service" "enable_event_arc" { - service = "eventarc.googleapis.com" - project = var.project -} diff --git a/terraform/infra/infra.tfvars b/terraform/infra/infra.tfvars index 04f1f0b022..4966782f87 100644 --- a/terraform/infra/infra.tfvars +++ b/terraform/infra/infra.tfvars @@ -1,19 +1,2 @@ project = "broad-juniper-eng-infra" region = "us-central1" - -juniper_projects = ["broad-juniper-dev", "broad-juniper-prod"] -virus_scanning_cvd_bucket_name = "juniper-eng-infra-virus-scanning-cvd-mirror" -virus_scanning_buckets = [ - { - clean: "juniper-participant-documents-dev-clean", - unscanned: "juniper-participant-documents-dev-unscanned", - quarantined: "juniper-participant-documents-dev-quarantined" - }, - # { - # clean: "juniper-participant-documents-prod-clean", - # unscanned: "juniper-participant-documents-prod-unscanned", - # quarantined: "juniper-participant-documents-prod-quarantined" - # }, -] - -malware_scanner_image_name = "juniper-malware-scanner" diff --git a/terraform/infra/variables.tf b/terraform/infra/variables.tf index a6aeca975a..249a52bf74 100644 --- a/terraform/infra/variables.tf +++ b/terraform/infra/variables.tf @@ -7,35 +7,3 @@ variable "region" { description = "The GCP region" type = string } - -variable "virus_scanning_buckets" { - type = set(object({ - unscanned: string, - clean: string, - quarantined: string - })) -} - -variable "juniper_projects" { - type = set(string) -} - -variable "virus_scanning_cvd_bucket_name" { - type = string -} - -# build image from https://github.com/GoogleCloudPlatform/docker-clamav-malware-scanner/tree/main/cloudrun-malware-scanner -# steps: -# cd cloudrun-malware-scanner -# docker build --tag=us-central1-docker.pkg.dev/broad-juniper-eng-infra/juniper/juniper-malware-scanner:latest -f Dockerfile . --platform linux/amd64 -# docker push us-central1-docker.pkg.dev/broad-juniper-eng-infra/juniper/juniper-malware-scanner:latest - -# if standing up for first time, you might also need to update the cvd mirror -# before deploying. run: -# pip3 install crcmod cvdupdate -# ./updateCvdMirror.sh -# from https://github.com/GoogleCloudPlatform/docker-clamav-malware-scanner - -variable "malware_scanner_image_name" { - type = string -} diff --git a/terraform/infra/virus_scanning_buckets.tf b/terraform/infra/virus_scanning_buckets.tf deleted file mode 100644 index 48917f43c5..0000000000 --- a/terraform/infra/virus_scanning_buckets.tf +++ /dev/null @@ -1,41 +0,0 @@ -resource "google_storage_bucket" "cvd_mirror_bucket" { - name = var.virus_scanning_cvd_bucket_name - location = var.region - # no public access allowed - public_access_prevention = "enforced" - - # only allow access if you have iam perms - uniform_bucket_level_access = true - versioning { - enabled = true - } -} - - -data "google_storage_bucket" "unscanned_buckets" { - for_each = { - for index, buckets in var.virus_scanning_buckets: - buckets.unscanned => buckets - } - - name = each.value.unscanned -} - -data "google_storage_bucket" "clean_buckets" { - for_each = { - for index, buckets in var.virus_scanning_buckets: - buckets.unscanned => buckets - } - - name = each.value.clean -} - -data "google_storage_bucket" "quarantined_buckets" { - for_each = { - for index, buckets in var.virus_scanning_buckets: - buckets.unscanned => buckets - } - - name = each.value.quarantined -} - From 0ab5d1b64e20b447e4e586b7d05d1f0eb2b345af Mon Sep 17 00:00:00 2001 From: Connor Barker Date: Mon, 17 Mar 2025 16:15:18 -0400 Subject: [PATCH 5/8] finalize --- terraform/gcp/envs/dev.tfvars | 3 +- terraform/gcp/envs/prod.tfvars | 3 + terraform/infra/.terraform.lock.hcl | 71 +++++++++++++-------- terraform/infra/artfiact_registry_access.tf | 3 +- 4 files changed, 52 insertions(+), 28 deletions(-) diff --git a/terraform/gcp/envs/dev.tfvars b/terraform/gcp/envs/dev.tfvars index 3a986523d9..55e46dd988 100644 --- a/terraform/gcp/envs/dev.tfvars +++ b/terraform/gcp/envs/dev.tfvars @@ -13,7 +13,7 @@ artifact_registry = "juniper" artifact_registry_project = "broad-juniper-eng-infra" artifact_registry_location = "us-central1" malware_scanner_image_name = "juniper-malware-scanner" -deploy_virus_scanning = true +documents_bucket_name = "juniper-participant-documents-dev" # creates DNS records for these customer URLs customer_urls = { @@ -25,3 +25,4 @@ customer_urls = { } slack_notification_channel = "projects/broad-juniper-dev/notificationChannels/13069356383599666729" + diff --git a/terraform/gcp/envs/prod.tfvars b/terraform/gcp/envs/prod.tfvars index 6d79a44a8c..03a61be2d0 100644 --- a/terraform/gcp/envs/prod.tfvars +++ b/terraform/gcp/envs/prod.tfvars @@ -105,3 +105,6 @@ customer_urls = { } slack_notification_channel = "projects/broad-juniper-prod/notificationChannels/9072110396476167224" + +malware_scanner_image_name = "juniper-malware-scanner" +documents_bucket_name = "juniper-participant-documents-prod" diff --git a/terraform/infra/.terraform.lock.hcl b/terraform/infra/.terraform.lock.hcl index 5fb78ed475..1f2f8c7269 100644 --- a/terraform/infra/.terraform.lock.hcl +++ b/terraform/infra/.terraform.lock.hcl @@ -2,39 +2,58 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/google" { - version = "6.3.0" + version = "6.25.0" hashes = [ - "h1:SpVI9aF/7wyalm5GhRUTGzwUA3Jv4d/lEnu/59X4U5k=", - "zh:2d5099f33ceaaa74d6d6860b98287ffdc723a6fb3f0a48ae02e9ba11e387f8b0", - "zh:38d937bedaeb9429663eff661b3087e66d7b88b3516eb1dc895eed160283878d", - "zh:5a9a7da6e578fc8448defb87f07616f8745700bd0d17ed458909d2561c08353b", - "zh:60b047acf4e4b2140885608ac37914d10947505dd8afd2da7718af27ff0e7b64", - "zh:84d457c92224f59d6ac7d2cc43581c7fe7dd3500d47251798fed21c7fb816d64", - "zh:b12a32dac4816d5cae9aa2b491d7510160e5f157150e8f814bcc9414a68db1f0", - "zh:cd65223bf3523f3d217e4b2b8c795149be53cfc545055a34e1fbd94f252c52d9", - "zh:d55de710eab8d31da97bb6ff7e5c4fe8afee7bde84ba80ac778d74ced1b7cd74", - "zh:e817d9ec4a7ff3e096dfc1c2ea88eb6eb725ae8c827e88cb77af9905ed2e3aec", - "zh:ebaa5cdf7beb94dec01cc6a2df8b8086bedeaf40cf009ffd27b051ca3770ebc0", - "zh:f3b5177362fa3ce8835daadeea9dd10299d76db667208128d619eb0786348ba4", + "h1:rvsmJc5J5OMMFXj9th8Yqfwm7cLZdRM/stPo0B8Datg=", + "zh:115ce3e84a02412a9a42c7c8f1c4568ab470bdd93c9a3a1f9202d4b77d7bc236", + "zh:623a32acea92d98a8bb66481fa9489e0a1e4dd6fc57ab70dbb53fae5732c1c63", + "zh:6440fc959b5e316152e26c916d55566311dafbe5b64e45d4b9c9931a5b29fa13", + "zh:91edb056638e723b1c7802d0e6e293611208e2825f645572ddba98122723f11c", + "zh:9b57ee44172677d2c2df03b22cf4b40e184ac8bd8facdd456ccbfdb7fee4684a", + "zh:9e2a97dd09b78b36caecdd990fff80bdb47a6a6d45c8b2dd4d23885d6bc89e65", + "zh:b33e40e9ba745f15b1c2e9ebcdccbac185788a4eb57450820cec7b9585858522", + "zh:d9e051bb703597384d83d924c49770e3bcdc1b68583d3def33a607ab168c634b", + "zh:e84253140fc3b0bd5cf7a1ebb54f993bacc6d5ee33c7b5e714ad834d5b2d35d0", + "zh:ebb624504c6f4297e691b6e3c00f789a6729461e5aa80fc165ef2ecd878e2d87", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f8c3e8192e98eff09a9453b52424c0d9727ae284de4fafbc498f2d527e026850", ] } provider "registry.terraform.io/hashicorp/google-beta" { - version = "6.3.0" + version = "6.25.0" hashes = [ - "h1:n9gyWvYz/1O5ecRfB1cV9/OlYTlSILvMy9knO4lvdXY=", - "zh:14911d94a78ee2a5d459a1e246e3722e04c5774f7aa8aeb415b11da89cd4ada8", - "zh:1a937609af8a837bdd181dad37e285c271bcb73971cc355c8f103730d1640aa4", - "zh:2032fa8f207ce06b5acf1e190f1090129b9fb4bae1b312165e5235af6cd61455", - "zh:3803fef01fe0c01e53f5d3628d8c8129d6443305cef458dea29d488661bbc5c0", - "zh:4c4434af7e2eb84c3a99a758e405b7f7f84c311ae6ef911d061d2d01fd4c0e76", - "zh:655eba025e121524ed95031c68b85d72ae2128eb748d68350fa4dd6569ce6770", - "zh:6678f186c550c6bb7ace33af0dd17a8d2ee6c61bcef2ff1bd8f603e0f7a5ce11", - "zh:7fa7de3cc6bf48d80ce3b8a5f3b23563d1f693b6febf20dd7b4a50a098ab6ee1", - "zh:a33d0f8deda840bc3f0263559118f632ddbb0609335739ab7329b517f156bc63", - "zh:c79b8109b3d7e42c1d1b5ebce29319bf69f4360be6f03f4fb0578fdc7f628e3f", + "h1:KlIctHHQkbToSXa41DWFQUsTKsfaWe+vjs/CFbnj3bc=", + "zh:101a8a679274cae57e8c816164fa28b6d271a8a53b3e007b84153fc63f0f06f7", + "zh:413c41e4ee14b10cb815cd01f8b880c12474ecbf63ecfc6415512fb0c18c473b", + "zh:4260143c8584a8f338a4d45f65c2f08f12b28a746c43d43a1e3a575f96ce4110", + "zh:4785080dd28b79a94ba6e277d247b6bb678d676d99244a8d664ea60802338df7", + "zh:673b09d67cb59487095fe7176c194e8c7ced62fc7bb5084016f3c87dd89e685e", + "zh:7de0dd9367c723cc203fdc4e70d316c779224483637039de5a3c9ee807c3fc55", + "zh:aa49c5955652b21ff2d0b7b2db0e18f59097d3a539efc98ef65deed4293d7327", + "zh:b4d73704d88deee367568fd68303878e6e60924c2a65df397f37b868e6aecaac", + "zh:cae0c5eff7c37c3f9857de08ca35b3a86d44855ae9c0ec2fab1eda604ff501c8", + "zh:e115f6b0c25bdad624fafbb5d685c8a48d9028f9525918274a5e5469f7dfde3d", + "zh:e536197371458b0b177a203f8367f99e74770b3dfce597034e880ea5f3f798b8", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - "zh:f9002acf0ce39e1f3f2afc28112343c846a6249868ca567477cbbcc3a4a77517", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.7.1" + hashes = [ + "h1:t152MY0tQH4a8fLzTtEWx70ITd3azVOrFDn/pQblbto=", + "zh:3193b89b43bf5805493e290374cdda5132578de6535f8009547c8b5d7a351585", + "zh:3218320de4be943e5812ed3de995946056db86eb8d03aa3f074e0c7316599bef", + "zh:419861805a37fa443e7d63b69fb3279926ccf98a79d256c422d5d82f0f387d1d", + "zh:4df9bd9d839b8fc11a3b8098a604b9b46e2235eb65ef15f4432bde0e175f9ca6", + "zh:5814be3f9c9cc39d2955d6f083bae793050d75c572e70ca11ccceb5517ced6b1", + "zh:63c6548a06de1231c8ee5570e42ca09c4b3db336578ded39b938f2156f06dd2e", + "zh:697e434c6bdee0502cc3deb098263b8dcd63948e8a96d61722811628dce2eba1", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:a0b8e44927e6327852bbfdc9d408d802569367f1e22a95bcdd7181b1c3b07601", + "zh:b7d3af018683ef22794eea9c218bc72d7c35a2b3ede9233b69653b3c782ee436", + "zh:d63b911d618a6fe446c65bfc21e793a7663e934b2fef833d42d3ccd38dd8d68d", + "zh:fa985cd0b11e6d651f47cff3055f0a9fd085ec190b6dbe99bf5448174434cdea", ] } diff --git a/terraform/infra/artfiact_registry_access.tf b/terraform/infra/artfiact_registry_access.tf index d49a2734ca..181635f5b4 100644 --- a/terraform/infra/artfiact_registry_access.tf +++ b/terraform/infra/artfiact_registry_access.tf @@ -9,6 +9,7 @@ resource "google_artifact_registry_repository_iam_binding" "cluster-artifact-reg "serviceAccount:juniper-cluster@broad-juniper-prod.iam.gserviceaccount.com", "serviceAccount:juniper-cloudbuild-sa@broad-juniper-dev.iam.gserviceaccount.com", "serviceAccount:juniper-cloudbuild-sa@broad-juniper-prod.iam.gserviceaccount.com", - "serviceAccount:service-663573365422@serverless-robot-prod.iam.gserviceaccount.com", # cloud run service account + "serviceAccount:service-663573365422@serverless-robot-prod.iam.gserviceaccount.com", # dev cloud run service account + # "serviceAccount:service-849235144342@serverless-robot-prod.iam.gserviceaccount.com", # prod cloud run service account (ENABLE FOR PROD VIRUS SCANNING) ] } From 5a4b7f16b95cd967d83694cf93c6e65e82393c73 Mon Sep 17 00:00:00 2001 From: Connor Barker Date: Tue, 18 Mar 2025 09:50:26 -0400 Subject: [PATCH 6/8] make more secure --- terraform/gcp/virus_scanning.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/gcp/virus_scanning.tf b/terraform/gcp/virus_scanning.tf index c9c314234f..e5f469448a 100644 --- a/terraform/gcp/virus_scanning.tf +++ b/terraform/gcp/virus_scanning.tf @@ -31,7 +31,7 @@ data "google_artifact_registry_docker_image" "scanner-service-image" { resource "google_cloud_run_v2_service" "malware_scanner" { name = "juniper-malware-scanner" location = var.region - ingress = "INGRESS_TRAFFIC_ALL" + ingress = "INGRESS_TRAFFIC_INTERNAL_ONLY" template { scaling { From daef0e226e114b6cd74b33cca6d8f1520df97600 Mon Sep 17 00:00:00 2001 From: Connor Barker Date: Tue, 25 Mar 2025 10:04:09 -0400 Subject: [PATCH 7/8] cleanup --- terraform/gcp/envs/dev.tfvars | 3 --- terraform/gcp/k8s/environments/dev.yaml | 1 - terraform/gcp/k8s/environments/prod.yaml | 5 ++++- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/terraform/gcp/envs/dev.tfvars b/terraform/gcp/envs/dev.tfvars index 55e46dd988..a809949100 100644 --- a/terraform/gcp/envs/dev.tfvars +++ b/terraform/gcp/envs/dev.tfvars @@ -9,9 +9,6 @@ environment = "dev" portals = ["demo", "atcp", "ourhealth", "hearthive", "rgp", "cmi"] k8s_namespace = "juniper-dev" -artifact_registry = "juniper" -artifact_registry_project = "broad-juniper-eng-infra" -artifact_registry_location = "us-central1" malware_scanner_image_name = "juniper-malware-scanner" documents_bucket_name = "juniper-participant-documents-dev" diff --git a/terraform/gcp/k8s/environments/dev.yaml b/terraform/gcp/k8s/environments/dev.yaml index 1aae9228be..d477579b03 100644 --- a/terraform/gcp/k8s/environments/dev.yaml +++ b/terraform/gcp/k8s/environments/dev.yaml @@ -5,7 +5,6 @@ deploymentZone: dev replicas: 1 dsmUrl: https://dsm-dev.datadonationplatform.org/dsm dsmIssuer: admin-d2p.ddp-dev.envs.broadinstitute.org -gcsFileStorageBucketName: juniper-participant-documents-dev # "portals" adds certificates for each portal - both for the juniper-cmi.dev subdomains and the custom domain portals: - name: demo diff --git a/terraform/gcp/k8s/environments/prod.yaml b/terraform/gcp/k8s/environments/prod.yaml index 60510e5ce2..a233e3818d 100644 --- a/terraform/gcp/k8s/environments/prod.yaml +++ b/terraform/gcp/k8s/environments/prod.yaml @@ -5,7 +5,6 @@ deploymentZone: prod replicas: 3 dsmUrl: https://dsm.datadonationplatform.org/dsm dsmIssuer: juniper.terra.bio -gcsFileStorageBucketName: juniper-participant-documents-prod # "portals" adds certificates for each portal - both for the admin subdomains and the custom domain portals: - name: demo @@ -57,3 +56,7 @@ b2c: clientId: 882e287c-8586-456e-8004-b7ff16da3578 policyName: B2C_1A_ddp_participant_signup_signin_trcc-prod changePasswordPolicyName: B2C_1A_ddp_participant_signup_signin_trcc-prod +gcsFileStorageBuckets: + unscanned: juniper-participant-documents-prod-unscanned + clean: juniper-participant-documents-prod-clean + quarantined: juniper-participant-documents-prod-quarantined From e2490cceca7fdc4b4bba57d27397ed03f2d940c5 Mon Sep 17 00:00:00 2001 From: Connor Barker Date: Tue, 25 Mar 2025 12:47:12 -0400 Subject: [PATCH 8/8] apply --- terraform/gcp/virus_scanning_infra.tf | 5 ++--- terraform/infra/artfiact_registry_access.tf | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/terraform/gcp/virus_scanning_infra.tf b/terraform/gcp/virus_scanning_infra.tf index 81aee6a955..948ca2f238 100644 --- a/terraform/gcp/virus_scanning_infra.tf +++ b/terraform/gcp/virus_scanning_infra.tf @@ -19,10 +19,9 @@ resource "google_service_account" "build_service_account" { display_name = "Service Account for malware scanner cloud run service" } -resource "google_project_iam_binding" "build_iam" { - for_each = toset(["roles/storage.objectViewer", "roles/artifactregistry.writer"]) +resource "google_project_iam_binding" "malware-scanner-object-viewer" { project = var.project - role = each.value + role = "roles/storage.objectViewer" members = ["serviceAccount:${google_service_account.build_service_account.email}"] } diff --git a/terraform/infra/artfiact_registry_access.tf b/terraform/infra/artfiact_registry_access.tf index 181635f5b4..8d03bb3d3f 100644 --- a/terraform/infra/artfiact_registry_access.tf +++ b/terraform/infra/artfiact_registry_access.tf @@ -10,6 +10,6 @@ resource "google_artifact_registry_repository_iam_binding" "cluster-artifact-reg "serviceAccount:juniper-cloudbuild-sa@broad-juniper-dev.iam.gserviceaccount.com", "serviceAccount:juniper-cloudbuild-sa@broad-juniper-prod.iam.gserviceaccount.com", "serviceAccount:service-663573365422@serverless-robot-prod.iam.gserviceaccount.com", # dev cloud run service account - # "serviceAccount:service-849235144342@serverless-robot-prod.iam.gserviceaccount.com", # prod cloud run service account (ENABLE FOR PROD VIRUS SCANNING) + "serviceAccount:service-849235144342@serverless-robot-prod.iam.gserviceaccount.com", # prod cloud run service account ] }