[JN-1644] virus scanning infrastructure (#1547)

Author: connorlbark

terraform/gcp/.terraform.lock.hcl

@@ -13,7 +13,10 @@ locals {
     "servicenetworking.googleapis.com",
     "cloudkms.googleapis.com",
     "binaryauthorization.googleapis.com",
-    "cloudbuild.googleapis.com"
+    "cloudbuild.googleapis.com",
+    "run.googleapis.com",
+    "cloudscheduler.googleapis.com",
+    "eventarc.googleapis.com"
   ]
 }
 

terraform/gcp/apis.tf

@@ -0,0 +1,51 @@
+resource "google_storage_bucket" "unscanned_participant_documents" {
+  name     = "${var.documents_bucket_name}-unscanned"
+  location = var.region
+  # no public access allowed
+  public_access_prevention    = "enforced"
+
+  # only allow access if you have iam perms
+  uniform_bucket_level_access = true
+  versioning {
+    enabled = true
+  }
+}
+
+resource "google_storage_bucket" "clean_participant_documents" {
+  name     = "${var.documents_bucket_name}-clean"
+  location = var.region
+  # no public access allowed
+  public_access_prevention    = "enforced"
+
+  # only allow access if you have iam perms
+  uniform_bucket_level_access = true
+  versioning {
+    enabled = true
+  }
+}
+
+resource "google_storage_bucket" "quarantined_participant_documents" {
+  name     = "${var.documents_bucket_name}-quarantined"
+  location = var.region
+  # no public access allowed
+  public_access_prevention    = "enforced"
+
+  # only allow access if you have iam perms
+  uniform_bucket_level_access = true
+  versioning {
+    enabled = true
+  }
+}
+
+resource "google_storage_bucket" "cvd_mirror_bucket" {
+    name     = "${var.documents_bucket_name}-cvd-mirror"
+    location = var.region
+    # no public access allowed
+    public_access_prevention    = "enforced"
+
+    # only allow access if you have iam perms
+    uniform_bucket_level_access = true
+    versioning {
+        enabled = true
+    }
+}

terraform/gcp/buckets.tf

@@ -8,7 +8,8 @@ resource "google_project_iam_binding" "cluster-metric-writer" {
   project = var.project
   role    = "roles/monitoring.metricWriter"
   members = [
-    "serviceAccount:${google_service_account.cluster_service_account.email}"
+    "serviceAccount:${google_service_account.cluster_service_account.email}",
+    "serviceAccount:${google_service_account.malware_scanner_sa.email}"
   ]
 }
 
@@ -25,7 +26,8 @@ resource "google_project_iam_binding" "cluster-log-writer" {
   role    = "roles/logging.logWriter"
   members = [
     "serviceAccount:${google_service_account.cluster_service_account.email}",
-    "serviceAccount:${google_service_account.juniper_cloudbuild_service_account.email}"
+    "serviceAccount:${google_service_account.juniper_cloudbuild_service_account.email}",
+    "serviceAccount:${google_service_account.build_service_account.email}"
   ]
 }
 

terraform/gcp/cluster_service_account.tf

@@ -9,6 +9,9 @@ environment = "dev"
 portals = ["demo", "atcp", "ourhealth", "hearthive", "rgp", "cmi"]
 k8s_namespace = "juniper-dev"
 
+malware_scanner_image_name = "juniper-malware-scanner"
+documents_bucket_name = "juniper-participant-documents-dev"
+
 # creates DNS records for these customer URLs
 customer_urls = {
   demo = {
@@ -19,3 +22,4 @@ customer_urls = {
 }
 
 slack_notification_channel = "projects/broad-juniper-dev/notificationChannels/13069356383599666729"
+

terraform/gcp/envs/dev.tfvars

@@ -105,3 +105,6 @@ customer_urls = {
 }
 
 slack_notification_channel = "projects/broad-juniper-prod/notificationChannels/9072110396476167224"
+
+malware_scanner_image_name = "juniper-malware-scanner"
+documents_bucket_name = "juniper-participant-documents-prod"

terraform/gcp/envs/prod.tfvars

@@ -5,7 +5,6 @@ deploymentZone: dev
 replicas: 1
 dsmUrl: https://dsm-dev.datadonationplatform.org/dsm
 dsmIssuer: admin-d2p.ddp-dev.envs.broadinstitute.org
-gcsFileStorageBucketName: juniper-participant-documents-dev
 # "portals" adds certificates for each portal - both for the juniper-cmi.dev subdomains and the custom domain
 portals:
   - name: demo

terraform/gcp/k8s/environments/dev.yaml

@@ -5,7 +5,6 @@ deploymentZone: prod
 replicas: 3
 dsmUrl: https://dsm.datadonationplatform.org/dsm
 dsmIssuer: juniper.terra.bio
-gcsFileStorageBucketName: juniper-participant-documents-prod
 # "portals" adds certificates for each portal - both for the admin subdomains and the custom domain
 portals:
   - name: demo
@@ -57,3 +56,7 @@ b2c:
       clientId: 882e287c-8586-456e-8004-b7ff16da3578
       policyName: B2C_1A_ddp_participant_signup_signin_trcc-prod
       changePasswordPolicyName: B2C_1A_ddp_participant_signup_signin_trcc-prod
+gcsFileStorageBuckets:
+  unscanned: juniper-participant-documents-prod-unscanned
+  clean: juniper-participant-documents-prod-clean
+  quarantined: juniper-participant-documents-prod-quarantined

terraform/gcp/k8s/environments/prod.yaml

@@ -93,3 +93,40 @@ variable "slack_notification_channel" {
   default = ""
   description = "Slack notification channel"
 }
+
+variable "documents_bucket_name" {
+  type = string
+  description = "The name of the GCP bucket for storing participant documents"
+}
+
+variable "artifact_registry" {
+  type = string
+  default = "juniper"
+}
+
+variable "artifact_registry_project" {
+  type = string
+  default = "broad-juniper-eng-infra"
+}
+
+variable "artifact_registry_location" {
+  type = string
+  default = "us-central1"
+}
+
+
+# build image from https://github.com/GoogleCloudPlatform/docker-clamav-malware-scanner/tree/main/cloudrun-malware-scanner
+# steps:
+# cd cloudrun-malware-scanner
+# docker build --tag=us-central1-docker.pkg.dev/broad-juniper-eng-infra/juniper/juniper-malware-scanner:latest -f Dockerfile .  --platform linux/amd64
+# docker push us-central1-docker.pkg.dev/broad-juniper-eng-infra/juniper/juniper-malware-scanner:latest
+
+# if standing up for first time, you might also need to update the cvd mirror
+# before deploying. run:
+# pip3 install crcmod cvdupdate
+# ./updateCvdMirror.sh  <cvd_mirror_bucket_name>
+# from https://github.com/GoogleCloudPlatform/docker-clamav-malware-scanner
+
+variable "malware_scanner_image_name" {
+  type = string
+}