Merge branch 'main' into mw/azure-layered-terraform-samples

Author: cmansky

.github/workflows/trivy.yaml

@@ -0,0 +1,97 @@
+name: Trivy Security Scan
+on:
+  pull_request:
+    paths:
+      - 'terraform/**'
+      - '.github/workflows/trivy.yaml'
+  push:
+    branches:
+      - main
+    paths:
+      - 'terraform/**'
+      - '.github/workflows/trivy.yaml'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  trivy-aws:
+    name: Trivy IaC Scan (AWS)
+    runs-on: ubuntu-latest
+    permissions:
+       contents: read
+       security-events: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run Trivy scanner (table output)
+        uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
+        with:
+          scan-type: 'fs'
+          scan-ref: 'terraform/aws/modules/'
+          scanners: 'vuln,secret,misconfig'
+          ignore-unfixed: false
+          exit-code: '0'
+          format: 'table'
+          severity: 'CRITICAL,HIGH'
+
+      - name: Run Trivy scanner (SARIF output)
+        uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
+        with:
+          scan-type: 'fs'
+          scan-ref: 'terraform/aws/modules/'
+          scanners: 'vuln,secret,misconfig'
+          ignore-unfixed: false
+          exit-code: '0'
+          format: 'sarif'
+          output: 'trivy-aws-results.sarif'
+          severity: 'CRITICAL,HIGH'
+
+      - name: Upload AWS scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v3
+        if: always()
+        with:
+          sarif_file: 'trivy-aws-results.sarif'
+          category: 'trivy-iac-aws'
+
+  trivy-azure:
+    name: Trivy IaC Scan (Azure)
+    runs-on: ubuntu-latest
+    permissions:
+       contents: read
+       security-events: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run Trivy scanner (table output)
+        uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
+        with:
+          scan-type: 'fs'
+          scan-ref: 'terraform/azure/modules/'
+          scanners: 'vuln,secret,misconfig'
+          ignore-unfixed: false
+          exit-code: '0'
+          format: 'table'
+          severity: 'CRITICAL,HIGH'
+
+      - name: Run Trivy scanner (SARIF output)
+        uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
+        with:
+          scan-type: 'fs'
+          scan-ref: 'terraform/azure/modules/'
+          scanners: 'vuln,secret,misconfig'
+          ignore-unfixed: false
+          exit-code: '0'
+          format: 'sarif'
+          output: 'trivy-azure-results.sarif'
+          severity: 'CRITICAL,HIGH'
+
+      - name: Upload Azure scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v3
+        if: always()
+        with:
+          sarif_file: 'trivy-azure-results.sarif'
+          category: 'trivy-iac-azure'

.gitignore

@@ -38,5 +38,4 @@ terraform.rc
 __MACOSX
 
 # Configs to Ignore
-provider.tf
 inputs.tfvars

CODEOWNERS

@@ -1,4 +1,3 @@
 # This file details who the code owners are.  Code owners are automatically requested for review when a PR is created.
-# For details on the format, and condfiguration of this file see: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
-
-
+# For details on the format, and configuration of this file see: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
+* @cmansky @EmmanuelNwa247 @jolson490 @mww59 @rin-skylight @szamfir-skylight

terraform/aws/modules/2-nbs7/aws-prometheus-grafana/modules/grafana-dashboard/providers.tf

@@ -0,0 +1,15 @@
+# #https://registry.terraform.io/providers/grafana/grafana/1.30.0
+terraform {
+  required_providers {
+    grafana = {
+      source  = "grafana/grafana"
+      version = ">=4.19.0, < 5.0.0"
+    }
+  }
+}
+
+# provider "grafana" {
+#   alias = "cloud"
+#   url   = var.grafana_workspace_url
+#   auth  = var.amg_api_token #grafana_service_account_token.admin-sa-token.key #
+# }

terraform/aws/modules/2-nbs7/eks-nbs/irsa.tf

@@ -50,7 +50,7 @@ module "otel_collector_irsa_role" {
   }
 
   policies = {
-    policy = aws_iam_policy.otel_collector_irsa_policy[0].arn
+    policy = try(aws_iam_policy.otel_collector_irsa_policy[0].arn, null)
   }
 }
 
@@ -96,7 +96,7 @@ module "datacompare_irsa_role" {
   }
 
   policies = {
-    policy = aws_iam_policy.datacompare_irsa_policy[0].arn
+    policy = try(aws_iam_policy.datacompare_irsa_policy[0].arn, null)
   }
 }
 

terraform/aws/modules/2-nbs7/eks-nbs/outputs.tf

@@ -41,5 +41,5 @@ output "readonly_role_arns" {
 
 output "otel_collector_role_arn" {
   description = "OTEL Collector IRSA role ARN — pass to helm install via --set serviceAccount.annotations"
-  value       = var.create_otel_collector_irsa ? module.otel_collector_irsa_role.iam_role_arn : null
+  value       = var.create_otel_collector_irsa ? module.otel_collector_irsa_role.arn : null
 }

terraform/aws/modules/2-nbs7/msk/README.md

@@ -16,7 +16,7 @@ Below are the input parameter variables for the MSK:
 | modern-cidr | string |  | VPC CIDR to be used with cluster SG |
 | msk_ebs_volume_size | number |  | EBS volume size for the MSK broker nodes in GB |
 | msk_security_groups | list(string) |  | A list of security groups to use for the MSK cluster  |
-| msk_subnet_ids | list(string) |  | A list of subnets to use for the MSK cluster  |
+| msk_subnet_ids | list(string) |  | A list of subnets to use for the MSK cluster. Must contain two subnets for 'development' otherwise three subnets for 'production', in order to match the number of brokers.  |
 | resource_prefix | string |  | Prefix for resource names |
 | vpc_id | string |  | VPC Id to be used with cluster |
 | vpn-cidr | string |  | VPN VPC CIDR to be used with cluster SG |
@@ -32,5 +32,3 @@ Below are the referenceable outputs from this module.
 ## Module Dependencies 
 
 Dependencies are external modules that this module references. A module is considered external if it isn't within the same repository.
-
-

terraform/aws/modules/2-nbs7/msk/main.tf

@@ -1,8 +1,8 @@
 locals {
   module_name          = "msk"
-  module_serial_number = "2023071301" # update with each commit?  Date plus two digit increment
+  module_serial_number = "2026-04-30_01" # Update with each commit? Date plus two digit increment.
   instance_type        = var.environment == "development" ? "kafka.t3.small" : "kafka.m5.large"
-  instance_count       = var.environment == "development" ? 2 : 3
+  number_of_brokers    = var.environment == "development" ? 2 : 3
 }
 
 # Create an IAM role for MSK
@@ -126,7 +126,7 @@ resource "aws_msk_cluster" "this" {
   count                  = var.create_msk ? 1 : 0
   cluster_name           = "${var.resource_prefix}-${var.environment}-msk-cluster"
   kafka_version          = var.kafka_version
-  number_of_broker_nodes = local.instance_count
+  number_of_broker_nodes = local.number_of_brokers
   #iam_instance_profile = aws_iam_role.msk.arn
 
   configuration_info {
@@ -173,23 +173,27 @@ resource "aws_msk_cluster" "this" {
     }
   }
 
-
   tags = {
     Environment   = var.environment
     ModuleVersion = "${local.module_name}-${local.module_serial_number}"
   }
 }
 
-resource "aws_msk_configuration" "msk_configuration_environment" {
-  count          = var.create_msk ? 1 : 0
-  kafka_versions = ["2.8.1"]
-  name           = "${var.resource_prefix}-${var.environment}-msk-cluster-config"
-
+locals {
+  # Reference info: https://docs.confluent.io/platform/current/installation/configuration/index.html
+  #
+  # For production, as noted on https://docs.aws.amazon.com/msk/latest/developerguide/bestpractices.html it's a best practice to set RF (Replication Factor) to 3, and set MinISR (min.insync.replicas) to at most RF - 1.
+  # For non-production, if minimizing resource usage is more of a priority to you than simulating production behavior, then set MinISR=1 and RF=2.
+  # Note that if you are using AWS MSK and if you set MinISR >= RF then you'll receive a notification in AWS User Notifications advising you to change your configuration so that MinISR is at most RF - 1.
+  #
+  # More considerations for production environments:
+  #  * https://repost.aws/knowledge-center/msk-avoid-disruption-during-patching
+  #  * https://docs.aws.amazon.com/securityhub/latest/userguide/msk-controls.html
   server_properties = <<PROPERTIES
 auto.create.topics.enable = true
 delete.topic.enable = true
-default.replication.factor=2
-min.insync.replicas=2
+default.replication.factor=${local.number_of_brokers}
+min.insync.replicas=1
 num.io.threads=8
 num.network.threads=5
 num.partitions=1
@@ -200,12 +204,16 @@ socket.request.max.bytes=104857600
 socket.send.buffer.bytes=102400
 unclean.leader.election.enable=true
 zookeeper.session.timeout.ms=18000
-offsets.topic.replication.factor=2
-transaction.state.log.replication.factor=2
+offsets.topic.replication.factor=${local.number_of_brokers}
+transaction.state.log.replication.factor=${local.number_of_brokers}
 PROPERTIES
 }
 
+# Reference info: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_configuration
+resource "aws_msk_configuration" "msk_configuration_environment" {
+  count          = var.create_msk ? 1 : 0
+  kafka_versions = ["2.8.1"]
+  name           = "${var.resource_prefix}-${var.environment}-msk-cluster-config"
 
-
-
-
+  server_properties = local.server_properties
+}