Add Loki and Monitoring Support in EKS Module

- Introduced support for Loki by adding an IRSA role and IAM policy for S3 access in the comet_eks module.
- Added variables for enabling Loki and configuring the S3 bucket for Loki logs.
- Implemented monitoring setup with a dedicated namespace and Grafana credentials management.
- Updated outputs to include ARNs and names for the new Loki resources.
- Enhanced the comet_s3 module to create an S3 bucket for Loki logs based on configuration settings.

Author: jms200

main.tf

@@ -160,6 +160,16 @@ module "comet_eks" {
   # External Secrets IRSA and Helm chart
   enable_external_secrets        = var.eks_enable_external_secrets
   external_secrets_chart_version = var.eks_external_secrets_chart_version
+
+  # Loki IRSA for S3 access
+  enable_loki        = var.enable_loki_bucket
+  loki_s3_bucket_arn = var.enable_s3 && var.enable_loki_bucket ? module.comet_s3[0].comet_loki_bucket_arn : null
+
+  # Monitoring namespace and Grafana credentials
+  enable_monitoring_setup = var.enable_monitoring_setup
+  monitoring_namespace    = var.monitoring_namespace
+  grafana_admin_user     = var.grafana_admin_user
+  grafana_admin_password = var.grafana_admin_password
 }
 
 module "comet_elasticache" {
@@ -218,7 +228,8 @@ module "comet_s3" {
   comet_s3_bucket  = var.s3_bucket_name
   s3_force_destroy = var.s3_force_destroy
 
-  enable_mpm_infra = var.enable_mpm_infra
+  enable_mpm_infra   = var.enable_mpm_infra
+  enable_loki_bucket = var.enable_loki_bucket
 }
 
 module "comet_secretsmanager" {

modules/comet_eks/main.tf

@@ -471,4 +471,107 @@ resource "kubernetes_manifest" "cluster_secret_store" {
   depends_on = [
     helm_release.external_secrets
   ]
+}
+
+#########################################
+#### Loki IRSA Role and IAM Policy ####
+#########################################
+data "aws_iam_policy_document" "loki" {
+  count = var.enable_loki ? 1 : 0
+
+  statement {
+    actions = [
+      "s3:ListBucket",
+      "s3:PutObject",
+      "s3:GetObject",
+      "s3:DeleteObject",
+    ]
+    resources = [
+      var.loki_s3_bucket_arn,
+      "${var.loki_s3_bucket_arn}/*"
+    ]
+  }
+}
+
+resource "aws_iam_policy" "loki" {
+  count = var.enable_loki ? 1 : 0
+
+  name_prefix = "${var.environment}-loki-"
+  description = "Provides permissions for Loki on ${var.environment} cluster"
+  policy      = data.aws_iam_policy_document.loki[0].json
+
+  tags = merge(
+    var.common_tags,
+    {
+      Name = "${var.environment}-loki"
+    }
+  )
+}
+
+module "loki_irsa_role" {
+  source  = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
+  version = "~> 5.39"
+
+  count = var.enable_loki ? 1 : 0
+
+  role_name = "${var.environment}-loki"
+
+  role_policy_arns = {
+    loki = aws_iam_policy.loki[0].arn
+  }
+
+  oidc_providers = {
+    ex = {
+      provider_arn               = module.eks.oidc_provider_arn
+      namespace_service_accounts = ["monitoring:monitoring-loki"]
+    }
+  }
+
+  depends_on = [
+    module.eks,
+    aws_iam_policy.loki
+  ]
+
+  tags = merge(
+    var.common_tags,
+    {
+      Name        = "${var.environment}-loki"
+      Description = "IRSA role for Loki to access S3 bucket for log storage"
+    }
+  )
+}
+
+#########################################
+#### Monitoring Namespace and Secrets ####
+#########################################
+resource "kubernetes_namespace" "monitoring" {
+  count = var.enable_monitoring_setup ? 1 : 0
+
+  metadata {
+    name = var.monitoring_namespace
+  }
+
+  depends_on = [
+    module.eks,
+    module.eks_blueprints_addons
+  ]
+}
+
+resource "kubernetes_secret" "monitoring" {
+  count = var.enable_monitoring_setup ? 1 : 0
+
+  metadata {
+    name      = "monitoring"
+    namespace = kubernetes_namespace.monitoring[0].metadata[0].name
+  }
+
+  data = {
+    grafana-admin-user     = var.grafana_admin_user
+    grafana-admin-password = var.grafana_admin_password
+  }
+
+  type      = "Opaque"
+  immutable = false
+
+  depends_on = [kubernetes_namespace.monitoring]
 }

modules/comet_eks/outputs.tf

@@ -31,4 +31,14 @@ output "external_secrets_irsa_role_name" {
 output "oidc_provider_arn" {
   description = "ARN of the OIDC provider for the EKS cluster"
   value       = module.eks.oidc_provider_arn
+}
+
+output "loki_irsa_role_arn" {
+  description = "ARN of the Loki IRSA role for S3 access"
+  value       = var.enable_loki ? module.loki_irsa_role[0].iam_role_arn : null
+}
+
+output "loki_irsa_role_name" {
+  description = "Name of the Loki IRSA role"
+  value       = var.enable_loki ? module.loki_irsa_role[0].iam_role_name : null
 }

modules/comet_eks/variables.tf

@@ -221,6 +221,43 @@ variable "enable_external_secrets" {
   default     = true
 }
 
+variable "enable_loki" {
+  description = "Enable Loki IRSA role for accessing S3 bucket for log storage"
+  type        = bool
+  default     = false
+}
+
+variable "loki_s3_bucket_arn" {
+  description = "ARN of the S3 bucket for Loki log storage"
+  type        = string
+  default     = null
+}
+
+variable "enable_monitoring_setup" {
+  description = "Enable monitoring namespace and Grafana credentials secret"
+  type        = bool
+  default     = false
+}
+
+variable "monitoring_namespace" {
+  description = "Kubernetes namespace for monitoring resources"
+  type        = string
+  default     = "monitoring"
+}
+
+variable "grafana_admin_user" {
+  description = "Grafana admin username"
+  type        = string
+  default     = "admin"
+}
+
+variable "grafana_admin_password" {
+  description = "Grafana admin password"
+  type        = string
+  sensitive   = true
+  default     = null
+}
+
 # Druid Node Group Variables
 variable "eks_druid_name" {
   description = "Name for the druid node group"

modules/comet_s3/main.tf

@@ -45,6 +45,21 @@ resource "aws_s3_bucket" "comet_airflow_bucket" {
   )
 }
 
+resource "aws_s3_bucket" "comet_loki_bucket" {
+  count = var.enable_loki_bucket ? 1 : 0
+
+  bucket = "comet-loki-${local.suffix}"
+
+  force_destroy = var.s3_force_destroy
+
+  tags = merge(
+    var.common_tags,
+    {
+      Name = "comet-loki-${local.suffix}"
+    }
+  )
+}
+
 resource "aws_iam_policy" "comet_s3_iam_policy" {
   name        = "comet-s3-access-policy-${local.suffix}"
   description = "Policy for access to comet S3 buckets"
@@ -65,6 +80,10 @@ resource "aws_iam_policy" "comet_s3_iam_policy" {
             "${aws_s3_bucket.comet_druid_bucket[0].arn}/*",
             aws_s3_bucket.comet_airflow_bucket[0].arn,
             "${aws_s3_bucket.comet_airflow_bucket[0].arn}/*"
+          ] : [],
+          var.enable_loki_bucket ? [
+            aws_s3_bucket.comet_loki_bucket[0].arn,
+            "${aws_s3_bucket.comet_loki_bucket[0].arn}/*"
           ] : []
         )
       }

modules/comet_s3/outputs.tf

@@ -1,4 +1,14 @@
 output "comet_s3_iam_policy_arn" {
   description = "ARN of the IAM policy granting access to the provisioned bucket(s)"
   value       = aws_iam_policy.comet_s3_iam_policy.arn
+}
+
+output "comet_loki_bucket_name" {
+  description = "Name of the Loki S3 bucket"
+  value       = var.enable_loki_bucket ? aws_s3_bucket.comet_loki_bucket[0].id : null
+}
+
+output "comet_loki_bucket_arn" {
+  description = "ARN of the Loki S3 bucket"
+  value       = var.enable_loki_bucket ? aws_s3_bucket.comet_loki_bucket[0].arn : null
 }

modules/comet_s3/variables.tf

@@ -18,6 +18,12 @@ variable "enable_mpm_infra" {
   type        = bool
 }
 
+variable "enable_loki_bucket" {
+  description = "Enable creation of S3 bucket for Loki logs"
+  type        = bool
+  default     = false
+}
+
 variable "common_tags" {
   type        = map(string)
   description = "A map of common tags"

outputs.tf

@@ -84,4 +84,24 @@ output "external_secrets_irsa_role_arn" {
 output "external_secrets_irsa_role_name" {
   description = "Name of the External Secrets IRSA role"
   value       = var.enable_eks && var.eks_enable_external_secrets ? module.comet_eks[0].external_secrets_irsa_role_name : null
+}
+
+output "comet_loki_bucket_name" {
+  description = "Name of the Loki S3 bucket"
+  value       = var.enable_s3 && var.enable_loki_bucket ? module.comet_s3[0].comet_loki_bucket_name : null
+}
+
+output "comet_loki_bucket_arn" {
+  description = "ARN of the Loki S3 bucket"
+  value       = var.enable_s3 && var.enable_loki_bucket ? module.comet_s3[0].comet_loki_bucket_arn : null
+}
+
+output "loki_irsa_role_arn" {
+  description = "ARN of the Loki IRSA role for S3 access"
+  value       = var.enable_eks && var.enable_loki_bucket ? module.comet_eks[0].loki_irsa_role_arn : null
+}
+
+output "loki_irsa_role_name" {
+  description = "Name of the Loki IRSA role"
+  value       = var.enable_eks && var.enable_loki_bucket ? module.comet_eks[0].loki_irsa_role_name : null
 }

variables.tf

@@ -42,6 +42,24 @@ variable "enable_mpm_infra" {
   default     = false
 }
 
+variable "enable_loki_bucket" {
+  description = "Enable creation of S3 bucket for Loki logs (used by comet_s3 module)"
+  type        = bool
+  default     = false
+}
+
+variable "enable_monitoring_setup" {
+  description = "Enable monitoring namespace and Grafana credentials secret in EKS (used by comet_eks module)"
+  type        = bool
+  default     = false
+}
+
+variable "monitoring_namespace" {
+  description = "Kubernetes namespace for monitoring resources"
+  type        = string
+  default     = "monitoring"
+}
+
 variable "enable_secretsmanager" {
   description = "Toggles the comet_secretsmanager module for provisioning Comet Secrets Manager secrets. Requires enable_rds and enable_elasticache to be true."
   type        = bool