Modernized terraform script for deployment as a module of nasa-pds/registry (#765)

* wip: renew terraform for registry integration

* ready to deploy in dev, not tested yet though

* modernize the terraform script so to integrate it as a module called in the registry main deployment.

* switch error status from 400 ot 404, as expected in integration tests

* remove unused github credentials

---------

Co-authored-by: Thomas Loubrieu <loubrieu@jpl.nasa.gov>

Author: tloubrieu-jpl

.gitignore

@@ -65,6 +65,9 @@ src/test/temp/
 
 # terraform
 .terraform/
+terraform/*.tfvars
+!terraform/*.tfvars.example
+!.terraform.lock.hcl
 
 # other stuff
 *.xpr

service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java

@@ -617,7 +617,7 @@ public ResponseEntity<Object> classList(String propertyClass, List<String> userR
     try {
       pdsProductClass = PdsProductClasses.fromSwaggerName(propertyClass);
     } catch (IllegalArgumentException err) {
-      throw new BadRequestException(err.getMessage());
+      throw new NotFoundException(err.getMessage());
     }
 
     RegistrySearchRequestBuilder searchRequestBuilder =

terraform/README.md

@@ -24,18 +24,22 @@ These interfaces are going to be used a arguments of the terraform scripts.
 
 ## Deploy
 
+Initialize the parameters, starting from the terraform.tfvars.example file provided.
+
+Copy it:
+
+    cp terraform.tfvars.example terraform.tfvars
+
+And update the values.
+
 
 Run the terraform scripts:
 
+
+
+
 ```
-    terraform apply \
-        -var 'ecs_task_role=your-task-role-arn' \
-        -var 'ecs_task_execution_role=your-task-execution-role-arn' \
-        -var 'venue=your-venue' \
-        -var 'aws_fg_vpc=your-vpc-arn' \
-        -var 'aws_fg_security_groups=["your security group, e.g. sg-1223455..."]' \
-        -var 'aws_fg_subnets=["your subnet e.g. subnet-1234..."]' \
-        -var 'aws_fg_image=your-docker-image-available-on-ECR' \
-        -var 'aws_acm_certificate_arn=ssl certificate for the load balancer listener' \
-        -var 'spring_boot_args=--openSearch.host=your-opensearch-url-without-http --openSearch.CCSEnabled=true --openSearch.username=our-username-empty-for-opensearch-serverless --openSearch.disciplineNodes=the-prefixes-of-the-registry-indices-in-opensearch --registry.service.version=the-version-of-the-api-to-be-displayed-in-the-application'
+    terraform init -backend-config=backend-config.tfvars
+    terraform plan
+    terraform apply
 ```

terraform/backend-config.tfvars.example

@@ -0,0 +1,12 @@
+# Example backend configuration for S3
+# Copy this file to backend-config.tfvars and customize the values
+# Initialize with: terraform init -backend-config=backend-config.tfvars
+
+bucket         = "my-terraform-state-bucket"
+key            = "registry/opensearch/terraform.tfstate"
+region         = "us-east-1"
+dynamodb_table = "terraform-state-lock"
+encrypt        = true
+
+# Optional: Use a specific profile
+# profile = "my-aws-profile"

terraform/backend.tf

@@ -0,0 +1,25 @@
+# Backend configuration for S3 state storage
+# Variables are not supported in backend blocks. Instead, provide configuration via:
+#
+# 1. backend-config.tfvars file:
+#    terraform init -backend-config=backend-config.tfvars
+#
+# 2. Command line arguments:
+#    terraform init -backend-config="bucket=${TFSTATE_BUCKET}" -backend-config="key=${TFSTATE_KEY}"
+#
+# 3. Environment variables or interactive prompts
+#
+# See https://stackoverflow.com/questions/63048738/how-to-declare-variables-for-s3-backend-in-terraform
+
+terraform {
+  backend "s3" {
+    # Backend configuration values provided via backend-config.tfvars
+    # Example backend-config.tfvars content:
+    #   bucket         = "pds-infra"
+    #   key            = "registry/opensearch/terraform.tfstate"
+    #   region         = "us-east-1"
+    #   dynamodb_table = "terraform-state-lock"
+    #   encrypt        = true
+    #   profile        = "your-aws-profile"
+  }
+}

terraform/ecs.tf terraform/main.tfterraform/ecs.tf renamed to terraform/main.tf

@@ -1,5 +1,11 @@
+locals {
+
+  # Concatenate the load balancer domain to spring boot args
+  spring_boot_args_with_host = "${var.spring_boot_args} --server.authorizedForwardedHost=${aws_lb.registry-api-lb.dns_name},${var.cloudfront_dns}"
+}
+
 resource "aws_lb" "registry-api-lb" {
-  name               = "registry-api-lb-new"
+  name               = "registry-api-lb"
   internal           = false
   load_balancer_type = "application"
   security_groups    = var.aws_fg_security_groups
@@ -9,26 +15,17 @@ resource "aws_lb" "registry-api-lb" {
 
   access_logs {
     bucket  = var.aws_s3_bucket_logs_id
-    prefix  = "registry-api-lb"
+    prefix  = "registry/registry-api-lb"
     enabled = true
   }
 
-  tags = {
-    Alfa = var.node_name_abbr
-    Bravo = var.venue
-    Charlie = "registry"
-  }
+  tags = var.common_tags
 }
 
-resource "aws_ssm_parameter" "load_balancer_domain" {
-  name  = "/pds/registry/load-balancer-domain"
-  type  = "String"
-  overwrite   = true
-  value = aws_lb.registry-api-lb.dns_name
-}
+
 
 resource "aws_lb_target_group" "pds-registry-api-target-group" {
-  name        = "pds-${var.venue}-registry-tgt"
+  name        = "pds-registry-tg"
   port        = 80
   protocol    = "HTTP"
   target_type = "ip"
@@ -44,6 +41,8 @@ resource "aws_lb_target_group" "pds-registry-api-target-group" {
     matcher = "200"
     interval = 300
   }
+
+  tags = var.common_tags
 }
 
 resource "aws_lb_listener" "registry-api-ld-listener" {
@@ -55,6 +54,7 @@ resource "aws_lb_listener" "registry-api-ld-listener" {
     type             = "forward"
     target_group_arn = aws_lb_target_group.pds-registry-api-target-group.arn
   }
+  tags = var.common_tags
 }
 
 resource "aws_lb_listener_rule" "pds-registry-forward-rule" {
@@ -75,44 +75,64 @@ resource "aws_lb_listener_rule" "pds-registry-forward-rule" {
   }
 }
 
-# Define the cluster
-resource "aws_ecs_cluster" "pds-registry-api-ecs" {
-  name = "pds-${var.venue}-registry-api-ecs"
 
-  tags = {
-    Alfa = var.node_name_abbr
-    Bravo = var.venue
-    Charlie = "registry"
-  }
+# Credentials for ECR pull through cache from GHCR
+resource "aws_secretsmanager_secret" "github_ecr_credentials" {
+  count = var.create_github_secret_credentials
+
+  name = "ecr-pullthroughcache/github-credentials"
+  tags = var.common_tags
 }
 
-# Do we need individual dev/test/prod repositories?
-# I don't think we do, but then we need to use prod account instead of the dev account, would that work ?
-data "aws_ecr_repository" "pds-registry-api-service" {
-  name = "pds-registry-api-service"
+resource "aws_secretsmanager_secret_version" "github_ecr_credentials" {
+  count = var.create_github_secret_credentials
+
+  secret_id     = aws_secretsmanager_secret.github_ecr_credentials[count.index].id
+  secret_string = jsonencode({
+    username = var.github_username
+    accessToken    = var.github_token
+  })
+}
+
+# Look up the secret when it is not created by this script
+data "aws_secretsmanager_secret" "github_ecr_credentials" {
+  count = 1 - var.create_github_secret_credentials
+  name  = "ecr-pullthroughcache/github-credentials"
+}
+
+locals {
+  github_ecr_credentials_arn = var.create_github_secret_credentials == 1 ? aws_secretsmanager_secret.github_ecr_credentials[0].arn : data.aws_secretsmanager_secret.github_ecr_credentials[0].arn
+}
+
+# Add a Pull Through Cache rule for GHCR
+resource "aws_ecr_pull_through_cache_rule" "ghcr" {
+  ecr_repository_prefix = "ghcr"
+  upstream_registry_url = "ghcr.io"
+  credential_arn        = local.github_ecr_credentials_arn
+}
+
+resource "aws_ecr_repository" "ghcr_registry_api" {
+    name = "ghcr/nasa-pds/registry-api"
+    tags = var.common_tags
 }
 
 # Log groups hold logs from our app.
 resource "aws_cloudwatch_log_group" "pds-registry-log-group" {
-  name = "/ecs/pds-${var.venue}-registry-api-svc-task"
+  name = "/ecs/pds-registry-api-task"
 
-  tags = {
-    Alfa = var.node_name_abbr
-    Bravo = var.venue
-    Charlie = "registry"
-  }
+  tags = var.common_tags
 }
 
 
 # The task definition for app.
 resource "aws_ecs_task_definition" "pds-registry-ecs-task" {
-  family = "pds-${var.venue}-registry-api-svc-task"
+  family = "pds-registry-api-task"
 
   container_definitions = <<EOF
   [
     {
-      "name": "pds-${var.venue}-reg-container",
-      "image": "${var.aws_fg_image}",
+      "name": "registry-api-container",
+      "image": "${var.registry_api_docker_image}",
       "portMappings": [
         {
           "containerPort": 80
@@ -138,7 +158,7 @@ resource "aws_ecs_task_definition" "pds-registry-ecs-task" {
       },
       "environment": [
         {"name": "SERVER_PORT", "value": "80"},
-        {"name": "SPRING_BOOT_APP_ARGS", "value": "${var.spring_boot_args}"}
+        {"name": "SPRING_BOOT_APP_ARGS", "value": "${local.spring_boot_args_with_host}"}
       ]
     }
   ]
@@ -149,25 +169,27 @@ EOF
   task_role_arn      = var.ecs_task_role
 
   # These are the minimum values for Fargate containers.
-  cpu                      = 256
-  memory                   = 512
+  cpu                      = var.aws_fg_cpu_units
+  memory                   = var.aws_fg_ram_units
   requires_compatibilities = ["FARGATE"]
 
   # This is required for Fargate containers
   network_mode = "awsvpc"
 
-  tags = {
-    Alfa = var.node_name_abbr
-    Bravo = var.venue
-    Charlie = "registry"
-  }
+  tags = var.common_tags
 }
 
+# Define the cluster
+resource "aws_ecs_cluster" "pds-registry-api-ecs" {
+  name = "pds-registry-api-cluster"
+
+  tags = var.common_tags
+}
 
 
 # The main service.
 resource "aws_ecs_service" "pds-registry-reg-service" {
-  name            = "pds-${var.venue}-registry-api-service"
+  name            = "pds-registry-api-service"
   task_definition = aws_ecs_task_definition.pds-registry-ecs-task.arn
   cluster         = aws_ecs_cluster.pds-registry-api-ecs.id
   launch_type     = "FARGATE"
@@ -176,7 +198,7 @@ resource "aws_ecs_service" "pds-registry-reg-service" {
 
   load_balancer {
     target_group_arn = aws_lb_target_group.pds-registry-api-target-group.arn
-    container_name   = "pds-${var.venue}-reg-container"
+    container_name   = "registry-api-container"
     container_port   = "80"
   }
 
@@ -186,9 +208,8 @@ resource "aws_ecs_service" "pds-registry-reg-service" {
     subnets = var.aws_fg_subnets
   }
 
-  tags = {
-    Alfa = var.node_name_abbr
-    Bravo = var.venue
-    Charlie = "registry"
-  }
+  tags = var.common_tags
+
+  depends_on = [aws_ecr_repository.ghcr_registry_api, aws_ecr_pull_through_cache_rule.ghcr]
 }
+

terraform/output.tf

@@ -0,0 +1,19 @@
+locals {
+
+  module_relative_path = replace(abspath(path.module), "/^.*\\/terraform(\\/|$)/", "")
+  ssm_prefix = "/pds/${var.component_name}${local.module_relative_path}"
+}
+
+
+output "load_balancer_domain" {
+  description = "Registry API load balancer domain"
+  value = aws_lb.registry-api-lb.dns_name
+}
+
+resource "aws_ssm_parameter" "load_balancer_domain" {
+  name  = "${local.ssm_prefix}/api-load-balancer-domain"
+  description = "Registry API load balancer domain"
+  type  = "String"
+  value = aws_lb.registry-api-lb.dns_name
+  tags = var.common_tags
+}

terraform/provider.tf

@@ -1,5 +1,4 @@
 provider "aws" {
-  version = "~> 3.0"
   region  = var.aws_region
   profile = var.aws_profile
 }
@@ -12,7 +11,12 @@ terraform {
   # using environment variables (as shown) or explicit values.
   # See https://stackoverflow.com/questions/63048738/how-to-declare-variables-for-s3-backend-in-terraform
   #
-  backend "s3" {
-    bucket = "pds-infra"
+  required_version = ">= 1.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 6.32.1"
+    }
   }
 }

terraform/terraform.tfvars.example

@@ -0,0 +1,30 @@
+aws_region="<aws-region>"
+spring_boot_args="--openSearch.host=<opensearch-collection-id>.<aws-region>.aoss.amazonaws.com --openSearch.CCSEnabled=true --openSearch.username='' --openSearch.disciplineNodes=en,geo --registry.service.version=1.0.0"
+aws_s3_bucket_logs_id="<s3-logs-bucket-name>"
+
+registry_api_docker_image="<aws-account-id>.dkr.ecr.<aws-region>.amazonaws.com/ghcr/nasa-pds/registry-api:develop"
+
+# network stuff
+# TODO have them in the pds infra, save in SSM parameters
+aws_fg_vpc = "vpc-<vpc-id>"
+aws_fg_security_groups = ["sg-<security-group-id>"]
+aws_fg_subnets = ["subnet-<subnet-id-1>","subnet-<subnet-id-2>"]
+aws_lb_subnets = ["subnet-<lb-subnet-id-1>", "subnet-<lb-subnet-id-2>"]
+aws_acm_certificate_arn = "arn:aws:acm:<aws-region>:<aws-account-id>:certificate/<certificate-id>"
+ecs_task_role = "arn:aws:iam::<aws-account-id>:role/<task-role-name>"
+ecs_task_execution_role = "arn:aws:iam::<aws-account-id>:role/<task-execution-role-name>"
+
+# GitHub credentials for ECR pull through cache
+github_username = "<github-username>"
+github_token    = "<github-personal-access-token>"
+
+cloudfront_dns = "www.example.com"
+
+# Common tags applied to all resources
+common_tags = {
+  tenant      = "<tenant>"
+  venue       = "<venue>"
+  component   = "registry"
+  cicd        = "iac"
+  managedby   = "<your-email>"
+}